diff options
Diffstat (limited to 'src/spdk/dpdk/drivers/net/tap')
| -rw-r--r-- | src/spdk/dpdk/drivers/net/tap/Makefile | 99 | ||||
| -rw-r--r-- | src/spdk/dpdk/drivers/net/tap/rte_eth_tap.c | 2140 | ||||
| -rw-r--r-- | src/spdk/dpdk/drivers/net/tap/rte_eth_tap.h | 99 | ||||
| -rw-r--r-- | src/spdk/dpdk/drivers/net/tap/rte_pmd_tap_version.map | 4 | ||||
| -rw-r--r-- | src/spdk/dpdk/drivers/net/tap/tap_bpf.h | 117 | ||||
| -rw-r--r-- | src/spdk/dpdk/drivers/net/tap/tap_bpf_api.c | 190 | ||||
| -rw-r--r-- | src/spdk/dpdk/drivers/net/tap/tap_bpf_insns.h | 1696 | ||||
| -rw-r--r-- | src/spdk/dpdk/drivers/net/tap/tap_bpf_program.c | 224 | ||||
| -rw-r--r-- | src/spdk/dpdk/drivers/net/tap/tap_flow.c | 2191 | ||||
| -rw-r--r-- | src/spdk/dpdk/drivers/net/tap/tap_flow.h | 68 | ||||
| -rw-r--r-- | src/spdk/dpdk/drivers/net/tap/tap_intr.c | 110 | ||||
| -rw-r--r-- | src/spdk/dpdk/drivers/net/tap/tap_log.h | 10 | ||||
| -rw-r--r-- | src/spdk/dpdk/drivers/net/tap/tap_netlink.c | 340 | ||||
| -rw-r--r-- | src/spdk/dpdk/drivers/net/tap/tap_netlink.h | 42 | ||||
| -rw-r--r-- | src/spdk/dpdk/drivers/net/tap/tap_rss.h | 40 | ||||
| -rw-r--r-- | src/spdk/dpdk/drivers/net/tap/tap_tcmsgs.c | 296 | ||||
| -rw-r--r-- | src/spdk/dpdk/drivers/net/tap/tap_tcmsgs.h | 37 |
17 files changed, 7703 insertions, 0 deletions
diff --git a/src/spdk/dpdk/drivers/net/tap/Makefile b/src/spdk/dpdk/drivers/net/tap/Makefile new file mode 100644 index 00000000..32433653 --- /dev/null +++ b/src/spdk/dpdk/drivers/net/tap/Makefile @@ -0,0 +1,99 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright(c) 2016 Intel Corporation + +include $(RTE_SDK)/mk/rte.vars.mk + +# +# library name +# +LIB = librte_pmd_tap.a + +EXPORT_MAP := rte_pmd_tap_version.map + +LIBABIVER := 1 + +# +# TAP_MAX_QUEUES must be a power of 2 +# +ifeq ($(TAP_MAX_QUEUES),) + TAP_MAX_QUEUES = 16 +endif +CFLAGS += -O3 +CFLAGS += -I$(SRCDIR) +CFLAGS += -I. +CFLAGS += $(WERROR_FLAGS) +LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring +LDLIBS += -lrte_ethdev -lrte_net -lrte_kvargs -lrte_hash +LDLIBS += -lrte_bus_vdev -lrte_gso + +CFLAGS += -DTAP_MAX_QUEUES=$(TAP_MAX_QUEUES) + +# +# all source are stored in SRCS-y +# +SRCS-$(CONFIG_RTE_LIBRTE_PMD_TAP) += rte_eth_tap.c +SRCS-$(CONFIG_RTE_LIBRTE_PMD_TAP) += tap_flow.c +SRCS-$(CONFIG_RTE_LIBRTE_PMD_TAP) += tap_netlink.c +SRCS-$(CONFIG_RTE_LIBRTE_PMD_TAP) += tap_tcmsgs.c +SRCS-$(CONFIG_RTE_LIBRTE_PMD_TAP) += tap_bpf_api.c +SRCS-$(CONFIG_RTE_LIBRTE_PMD_TAP) += tap_intr.c + +include $(RTE_SDK)/mk/rte.lib.mk + +# Generate and clean-up tap_autoconf.h. + +export CC CFLAGS CPPFLAGS EXTRA_CFLAGS EXTRA_CPPFLAGS +export AUTO_CONFIG_CFLAGS = -Wno-error + +ifndef V +AUTOCONF_OUTPUT := >/dev/null +endif + +tap_autoconf.h.new: FORCE + +tap_autoconf.h.new: $(RTE_SDK)/buildtools/auto-config-h.sh + $Q $(RM) -f -- '$@' + $Q sh -- '$<' '$@' \ + HAVE_TC_FLOWER \ + linux/pkt_cls.h \ + enum TCA_FLOWER_UNSPEC \ + $(AUTOCONF_OUTPUT) + $Q sh -- '$<' '$@' \ + HAVE_TC_VLAN_ID \ + linux/pkt_cls.h \ + enum TCA_FLOWER_KEY_VLAN_PRIO \ + $(AUTOCONF_OUTPUT) + $Q sh -- '$<' '$@' \ + HAVE_TC_BPF \ + linux/pkt_cls.h \ + enum TCA_BPF_UNSPEC \ + $(AUTOCONF_OUTPUT) + $Q sh -- '$<' '$@' \ + HAVE_TC_BPF_FD \ + linux/pkt_cls.h \ + enum TCA_BPF_FD \ + $(AUTOCONF_OUTPUT) + $Q sh -- '$<' '$@' \ + HAVE_TC_ACT_BPF \ + linux/tc_act/tc_bpf.h \ + enum TCA_ACT_BPF_UNSPEC \ + $(AUTOCONF_OUTPUT) + $Q sh -- '$<' '$@' \ + HAVE_TC_ACT_BPF_FD \ + linux/tc_act/tc_bpf.h \ + enum TCA_ACT_BPF_FD \ + $(AUTOCONF_OUTPUT) + +# Create tap_autoconf.h or update it in case it differs from the new one. + +tap_autoconf.h: tap_autoconf.h.new + $Q [ -f '$@' ] && \ + cmp '$<' '$@' $(AUTOCONF_OUTPUT) || \ + mv '$<' '$@' + +$(SRCS-$(CONFIG_RTE_LIBRTE_PMD_TAP):.c=.o): tap_autoconf.h + +clean_tap: FORCE + $Q rm -f -- tap_autoconf.h tap_autoconf.h.new + +clean: clean_tap diff --git a/src/spdk/dpdk/drivers/net/tap/rte_eth_tap.c b/src/spdk/dpdk/drivers/net/tap/rte_eth_tap.c new file mode 100644 index 00000000..feb92b48 --- /dev/null +++ b/src/spdk/dpdk/drivers/net/tap/rte_eth_tap.c @@ -0,0 +1,2140 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2016-2017 Intel Corporation + */ + +#include <rte_atomic.h> +#include <rte_branch_prediction.h> +#include <rte_byteorder.h> +#include <rte_common.h> +#include <rte_mbuf.h> +#include <rte_ethdev_driver.h> +#include <rte_ethdev_vdev.h> +#include <rte_malloc.h> +#include <rte_bus_vdev.h> +#include <rte_kvargs.h> +#include <rte_net.h> +#include <rte_debug.h> +#include <rte_ip.h> +#include <rte_string_fns.h> + +#include <assert.h> +#include <sys/types.h> +#include <sys/stat.h> +#include <sys/socket.h> +#include <sys/ioctl.h> +#include <sys/utsname.h> +#include <sys/mman.h> +#include <errno.h> +#include <signal.h> +#include <stdbool.h> +#include <stdint.h> +#include <sys/uio.h> +#include <unistd.h> +#include <arpa/inet.h> +#include <net/if.h> +#include <linux/if_tun.h> +#include <linux/if_ether.h> +#include <fcntl.h> + +#include <tap_rss.h> +#include <rte_eth_tap.h> +#include <tap_flow.h> +#include <tap_netlink.h> +#include <tap_tcmsgs.h> + +/* Linux based path to the TUN device */ +#define TUN_TAP_DEV_PATH "/dev/net/tun" +#define DEFAULT_TAP_NAME "dtap" +#define DEFAULT_TUN_NAME "dtun" + +#define ETH_TAP_IFACE_ARG "iface" +#define ETH_TAP_REMOTE_ARG "remote" +#define ETH_TAP_MAC_ARG "mac" +#define ETH_TAP_MAC_FIXED "fixed" + +#define ETH_TAP_USR_MAC_FMT "xx:xx:xx:xx:xx:xx" +#define ETH_TAP_CMP_MAC_FMT "0123456789ABCDEFabcdef" +#define ETH_TAP_MAC_ARG_FMT ETH_TAP_MAC_FIXED "|" ETH_TAP_USR_MAC_FMT + +#define TAP_GSO_MBUFS_PER_CORE 128 +#define TAP_GSO_MBUF_SEG_SIZE 128 +#define TAP_GSO_MBUF_CACHE_SIZE 4 +#define TAP_GSO_MBUFS_NUM \ + (TAP_GSO_MBUFS_PER_CORE * TAP_GSO_MBUF_CACHE_SIZE) + +static struct rte_vdev_driver pmd_tap_drv; +static struct rte_vdev_driver pmd_tun_drv; + +static const char *valid_arguments[] = { + ETH_TAP_IFACE_ARG, + ETH_TAP_REMOTE_ARG, + ETH_TAP_MAC_ARG, + NULL +}; + +static unsigned int tap_unit; +static unsigned int tun_unit; + +static char tuntap_name[8]; + +static volatile uint32_t tap_trigger; /* Rx trigger */ + +static struct rte_eth_link pmd_link = { + .link_speed = ETH_SPEED_NUM_10G, + .link_duplex = ETH_LINK_FULL_DUPLEX, + .link_status = ETH_LINK_DOWN, + .link_autoneg = ETH_LINK_FIXED, +}; + +static void +tap_trigger_cb(int sig __rte_unused) +{ + /* Valid trigger values are nonzero */ + tap_trigger = (tap_trigger + 1) | 0x80000000; +} + +/* Specifies on what netdevices the ioctl should be applied */ +enum ioctl_mode { + LOCAL_AND_REMOTE, + LOCAL_ONLY, + REMOTE_ONLY, +}; + +static int tap_intr_handle_set(struct rte_eth_dev *dev, int set); + +/** + * Tun/Tap allocation routine + * + * @param[in] pmd + * Pointer to private structure. + * + * @param[in] is_keepalive + * Keepalive flag + * + * @return + * -1 on failure, fd on success + */ +static int +tun_alloc(struct pmd_internals *pmd, int is_keepalive) +{ + struct ifreq ifr; +#ifdef IFF_MULTI_QUEUE + unsigned int features; +#endif + int fd; + + memset(&ifr, 0, sizeof(struct ifreq)); + + /* + * Do not set IFF_NO_PI as packet information header will be needed + * to check if a received packet has been truncated. + */ + ifr.ifr_flags = (pmd->type == ETH_TUNTAP_TYPE_TAP) ? + IFF_TAP : IFF_TUN | IFF_POINTOPOINT; + snprintf(ifr.ifr_name, IFNAMSIZ, "%s", pmd->name); + + TAP_LOG(DEBUG, "ifr_name '%s'", ifr.ifr_name); + + fd = open(TUN_TAP_DEV_PATH, O_RDWR); + if (fd < 0) { + TAP_LOG(ERR, "Unable to create %s interface", tuntap_name); + goto error; + } + +#ifdef IFF_MULTI_QUEUE + /* Grab the TUN features to verify we can work multi-queue */ + if (ioctl(fd, TUNGETFEATURES, &features) < 0) { + TAP_LOG(ERR, "%s unable to get TUN/TAP features", + tuntap_name); + goto error; + } + TAP_LOG(DEBUG, "%s Features %08x", tuntap_name, features); + + if (features & IFF_MULTI_QUEUE) { + TAP_LOG(DEBUG, " Multi-queue support for %d queues", + RTE_PMD_TAP_MAX_QUEUES); + ifr.ifr_flags |= IFF_MULTI_QUEUE; + } else +#endif + { + ifr.ifr_flags |= IFF_ONE_QUEUE; + TAP_LOG(DEBUG, " Single queue only support"); + } + + /* Set the TUN/TAP configuration and set the name if needed */ + if (ioctl(fd, TUNSETIFF, (void *)&ifr) < 0) { + TAP_LOG(WARNING, "Unable to set TUNSETIFF for %s: %s", + ifr.ifr_name, strerror(errno)); + goto error; + } + + if (is_keepalive) { + /* + * Detach the TUN/TAP keep-alive queue + * to avoid traffic through it + */ + ifr.ifr_flags = IFF_DETACH_QUEUE; + if (ioctl(fd, TUNSETQUEUE, (void *)&ifr) < 0) { + TAP_LOG(WARNING, + "Unable to detach keep-alive queue for %s: %s", + ifr.ifr_name, strerror(errno)); + goto error; + } + } + + /* Always set the file descriptor to non-blocking */ + if (fcntl(fd, F_SETFL, O_NONBLOCK) < 0) { + TAP_LOG(WARNING, + "Unable to set %s to nonblocking: %s", + ifr.ifr_name, strerror(errno)); + goto error; + } + + /* Set up trigger to optimize empty Rx bursts */ + errno = 0; + do { + struct sigaction sa; + int flags = fcntl(fd, F_GETFL); + + if (flags == -1 || sigaction(SIGIO, NULL, &sa) == -1) + break; + if (sa.sa_handler != tap_trigger_cb) { + /* + * Make sure SIGIO is not already taken. This is done + * as late as possible to leave the application a + * chance to set up its own signal handler first. + */ + if (sa.sa_handler != SIG_IGN && + sa.sa_handler != SIG_DFL) { + errno = EBUSY; + break; + } + sa = (struct sigaction){ + .sa_flags = SA_RESTART, + .sa_handler = tap_trigger_cb, + }; + if (sigaction(SIGIO, &sa, NULL) == -1) + break; + } + /* Enable SIGIO on file descriptor */ + fcntl(fd, F_SETFL, flags | O_ASYNC); + fcntl(fd, F_SETOWN, getpid()); + } while (0); + + if (errno) { + /* Disable trigger globally in case of error */ + tap_trigger = 0; + TAP_LOG(WARNING, "Rx trigger disabled: %s", + strerror(errno)); + } + + return fd; + +error: + if (fd > 0) + close(fd); + return -1; +} + +static void +tap_verify_csum(struct rte_mbuf *mbuf) +{ + uint32_t l2 = mbuf->packet_type & RTE_PTYPE_L2_MASK; + uint32_t l3 = mbuf->packet_type & RTE_PTYPE_L3_MASK; + uint32_t l4 = mbuf->packet_type & RTE_PTYPE_L4_MASK; + unsigned int l2_len = sizeof(struct ether_hdr); + unsigned int l3_len; + uint16_t cksum = 0; + void *l3_hdr; + void *l4_hdr; + + if (l2 == RTE_PTYPE_L2_ETHER_VLAN) + l2_len += 4; + else if (l2 == RTE_PTYPE_L2_ETHER_QINQ) + l2_len += 8; + /* Don't verify checksum for packets with discontinuous L2 header */ + if (unlikely(l2_len + sizeof(struct ipv4_hdr) > + rte_pktmbuf_data_len(mbuf))) + return; + l3_hdr = rte_pktmbuf_mtod_offset(mbuf, void *, l2_len); + if (l3 == RTE_PTYPE_L3_IPV4 || l3 == RTE_PTYPE_L3_IPV4_EXT) { + struct ipv4_hdr *iph = l3_hdr; + + /* ihl contains the number of 4-byte words in the header */ + l3_len = 4 * (iph->version_ihl & 0xf); + if (unlikely(l2_len + l3_len > rte_pktmbuf_data_len(mbuf))) + return; + + cksum = ~rte_raw_cksum(iph, l3_len); + mbuf->ol_flags |= cksum ? + PKT_RX_IP_CKSUM_BAD : + PKT_RX_IP_CKSUM_GOOD; + } else if (l3 == RTE_PTYPE_L3_IPV6) { + l3_len = sizeof(struct ipv6_hdr); + } else { + /* IPv6 extensions are not supported */ + return; + } + if (l4 == RTE_PTYPE_L4_UDP || l4 == RTE_PTYPE_L4_TCP) { + l4_hdr = rte_pktmbuf_mtod_offset(mbuf, void *, l2_len + l3_len); + /* Don't verify checksum for multi-segment packets. */ + if (mbuf->nb_segs > 1) + return; + if (l3 == RTE_PTYPE_L3_IPV4) + cksum = ~rte_ipv4_udptcp_cksum(l3_hdr, l4_hdr); + else if (l3 == RTE_PTYPE_L3_IPV6) + cksum = ~rte_ipv6_udptcp_cksum(l3_hdr, l4_hdr); + mbuf->ol_flags |= cksum ? + PKT_RX_L4_CKSUM_BAD : + PKT_RX_L4_CKSUM_GOOD; + } +} + +static uint64_t +tap_rx_offload_get_port_capa(void) +{ + /* + * No specific port Rx offload capabilities. + */ + return 0; +} + +static uint64_t +tap_rx_offload_get_queue_capa(void) +{ + return DEV_RX_OFFLOAD_SCATTER | + DEV_RX_OFFLOAD_IPV4_CKSUM | + DEV_RX_OFFLOAD_UDP_CKSUM | + DEV_RX_OFFLOAD_TCP_CKSUM | + DEV_RX_OFFLOAD_CRC_STRIP; +} + +/* Callback to handle the rx burst of packets to the correct interface and + * file descriptor(s) in a multi-queue setup. + */ +static uint16_t +pmd_rx_burst(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts) +{ + struct rx_queue *rxq = queue; + uint16_t num_rx; + unsigned long num_rx_bytes = 0; + uint32_t trigger = tap_trigger; + + if (trigger == rxq->trigger_seen) + return 0; + if (trigger) + rxq->trigger_seen = trigger; + rte_compiler_barrier(); + for (num_rx = 0; num_rx < nb_pkts; ) { + struct rte_mbuf *mbuf = rxq->pool; + struct rte_mbuf *seg = NULL; + struct rte_mbuf *new_tail = NULL; + uint16_t data_off = rte_pktmbuf_headroom(mbuf); + int len; + + len = readv(rxq->fd, *rxq->iovecs, + 1 + + (rxq->rxmode->offloads & DEV_RX_OFFLOAD_SCATTER ? + rxq->nb_rx_desc : 1)); + if (len < (int)sizeof(struct tun_pi)) + break; + + /* Packet couldn't fit in the provided mbuf */ + if (unlikely(rxq->pi.flags & TUN_PKT_STRIP)) { + rxq->stats.ierrors++; + continue; + } + + len -= sizeof(struct tun_pi); + + mbuf->pkt_len = len; + mbuf->port = rxq->in_port; + while (1) { + struct rte_mbuf *buf = rte_pktmbuf_alloc(rxq->mp); + + if (unlikely(!buf)) { + rxq->stats.rx_nombuf++; + /* No new buf has been allocated: do nothing */ + if (!new_tail || !seg) + goto end; + + seg->next = NULL; + rte_pktmbuf_free(mbuf); + + goto end; + } + seg = seg ? seg->next : mbuf; + if (rxq->pool == mbuf) + rxq->pool = buf; + if (new_tail) + new_tail->next = buf; + new_tail = buf; + new_tail->next = seg->next; + + /* iovecs[0] is reserved for packet info (pi) */ + (*rxq->iovecs)[mbuf->nb_segs].iov_len = + buf->buf_len - data_off; + (*rxq->iovecs)[mbuf->nb_segs].iov_base = + (char *)buf->buf_addr + data_off; + + seg->data_len = RTE_MIN(seg->buf_len - data_off, len); + seg->data_off = data_off; + + len -= seg->data_len; + if (len <= 0) + break; + mbuf->nb_segs++; + /* First segment has headroom, not the others */ + data_off = 0; + } + seg->next = NULL; + mbuf->packet_type = rte_net_get_ptype(mbuf, NULL, + RTE_PTYPE_ALL_MASK); + if (rxq->rxmode->offloads & DEV_RX_OFFLOAD_CHECKSUM) + tap_verify_csum(mbuf); + + /* account for the receive frame */ + bufs[num_rx++] = mbuf; + num_rx_bytes += mbuf->pkt_len; + } +end: + rxq->stats.ipackets += num_rx; + rxq->stats.ibytes += num_rx_bytes; + + return num_rx; +} + +static uint64_t +tap_tx_offload_get_port_capa(void) +{ + /* + * No specific port Tx offload capabilities. + */ + return 0; +} + +static uint64_t +tap_tx_offload_get_queue_capa(void) +{ + return DEV_TX_OFFLOAD_MULTI_SEGS | + DEV_TX_OFFLOAD_IPV4_CKSUM | + DEV_TX_OFFLOAD_UDP_CKSUM | + DEV_TX_OFFLOAD_TCP_CKSUM | + DEV_TX_OFFLOAD_TCP_TSO; +} + +/* Finalize l4 checksum calculation */ +static void +tap_tx_l4_cksum(uint16_t *l4_cksum, uint16_t l4_phdr_cksum, + uint32_t l4_raw_cksum) +{ + if (l4_cksum) { + uint32_t cksum; + + cksum = __rte_raw_cksum_reduce(l4_raw_cksum); + cksum += l4_phdr_cksum; + + cksum = ((cksum & 0xffff0000) >> 16) + (cksum & 0xffff); + cksum = (~cksum) & 0xffff; + if (cksum == 0) + cksum = 0xffff; + *l4_cksum = cksum; + } +} + +/* Accumaulate L4 raw checksums */ +static void +tap_tx_l4_add_rcksum(char *l4_data, unsigned int l4_len, uint16_t *l4_cksum, + uint32_t *l4_raw_cksum) +{ + if (l4_cksum == NULL) + return; + + *l4_raw_cksum = __rte_raw_cksum(l4_data, l4_len, *l4_raw_cksum); +} + +/* L3 and L4 pseudo headers checksum offloads */ +static void +tap_tx_l3_cksum(char *packet, uint64_t ol_flags, unsigned int l2_len, + unsigned int l3_len, unsigned int l4_len, uint16_t **l4_cksum, + uint16_t *l4_phdr_cksum, uint32_t *l4_raw_cksum) +{ + void *l3_hdr = packet + l2_len; + + if (ol_flags & (PKT_TX_IP_CKSUM | PKT_TX_IPV4)) { + struct ipv4_hdr *iph = l3_hdr; + uint16_t cksum; + + iph->hdr_checksum = 0; + cksum = rte_raw_cksum(iph, l3_len); + iph->hdr_checksum = (cksum == 0xffff) ? cksum : ~cksum; + } + if (ol_flags & PKT_TX_L4_MASK) { + void *l4_hdr; + + l4_hdr = packet + l2_len + l3_len; + if ((ol_flags & PKT_TX_L4_MASK) == PKT_TX_UDP_CKSUM) + *l4_cksum = &((struct udp_hdr *)l4_hdr)->dgram_cksum; + else if ((ol_flags & PKT_TX_L4_MASK) == PKT_TX_TCP_CKSUM) + *l4_cksum = &((struct tcp_hdr *)l4_hdr)->cksum; + else + return; + **l4_cksum = 0; + if (ol_flags & PKT_TX_IPV4) + *l4_phdr_cksum = rte_ipv4_phdr_cksum(l3_hdr, 0); + else + *l4_phdr_cksum = rte_ipv6_phdr_cksum(l3_hdr, 0); + *l4_raw_cksum = __rte_raw_cksum(l4_hdr, l4_len, 0); + } +} + +static inline void +tap_write_mbufs(struct tx_queue *txq, uint16_t num_mbufs, + struct rte_mbuf **pmbufs, + uint16_t *num_packets, unsigned long *num_tx_bytes) +{ + int i; + uint16_t l234_hlen; + + for (i = 0; i < num_mbufs; i++) { + struct rte_mbuf *mbuf = pmbufs[i]; + struct iovec iovecs[mbuf->nb_segs + 2]; + struct tun_pi pi = { .flags = 0, .proto = 0x00 }; + struct rte_mbuf *seg = mbuf; + char m_copy[mbuf->data_len]; + int proto; + int n; + int j; + int k; /* current index in iovecs for copying segments */ + uint16_t seg_len; /* length of first segment */ + uint16_t nb_segs; + uint16_t *l4_cksum; /* l4 checksum (pseudo header + payload) */ + uint32_t l4_raw_cksum = 0; /* TCP/UDP payload raw checksum */ + uint16_t l4_phdr_cksum = 0; /* TCP/UDP pseudo header checksum */ + uint16_t is_cksum = 0; /* in case cksum should be offloaded */ + + l4_cksum = NULL; + if (txq->type == ETH_TUNTAP_TYPE_TUN) { + /* + * TUN and TAP are created with IFF_NO_PI disabled. + * For TUN PMD this mandatory as fields are used by + * Kernel tun.c to determine whether its IP or non IP + * packets. + * + * The logic fetches the first byte of data from mbuf + * then compares whether its v4 or v6. If first byte + * is 4 or 6, then protocol field is updated. + */ + char *buff_data = rte_pktmbuf_mtod(seg, void *); + proto = (*buff_data & 0xf0); + pi.proto = (proto == 0x40) ? + rte_cpu_to_be_16(ETHER_TYPE_IPv4) : + ((proto == 0x60) ? + rte_cpu_to_be_16(ETHER_TYPE_IPv6) : + 0x00); + } + + k = 0; + iovecs[k].iov_base = π + iovecs[k].iov_len = sizeof(pi); + k++; + + nb_segs = mbuf->nb_segs; + if (txq->csum && + ((mbuf->ol_flags & (PKT_TX_IP_CKSUM | PKT_TX_IPV4) || + (mbuf->ol_flags & PKT_TX_L4_MASK) == PKT_TX_UDP_CKSUM || + (mbuf->ol_flags & PKT_TX_L4_MASK) == PKT_TX_TCP_CKSUM))) { + is_cksum = 1; + + /* Support only packets with at least layer 4 + * header included in the first segment + */ + seg_len = rte_pktmbuf_data_len(mbuf); + l234_hlen = mbuf->l2_len + mbuf->l3_len + mbuf->l4_len; + if (seg_len < l234_hlen) + break; + + /* To change checksums, work on a * copy of l2, l3 + * headers + l4 pseudo header + */ + rte_memcpy(m_copy, rte_pktmbuf_mtod(mbuf, void *), + l234_hlen); + tap_tx_l3_cksum(m_copy, mbuf->ol_flags, + mbuf->l2_len, mbuf->l3_len, mbuf->l4_len, + &l4_cksum, &l4_phdr_cksum, + &l4_raw_cksum); + iovecs[k].iov_base = m_copy; + iovecs[k].iov_len = l234_hlen; + k++; + + /* Update next iovecs[] beyond l2, l3, l4 headers */ + if (seg_len > l234_hlen) { + iovecs[k].iov_len = seg_len - l234_hlen; + iovecs[k].iov_base = + rte_pktmbuf_mtod(seg, char *) + + l234_hlen; + tap_tx_l4_add_rcksum(iovecs[k].iov_base, + iovecs[k].iov_len, l4_cksum, + &l4_raw_cksum); + k++; + nb_segs++; + } + seg = seg->next; + } + + for (j = k; j <= nb_segs; j++) { + iovecs[j].iov_len = rte_pktmbuf_data_len(seg); + iovecs[j].iov_base = rte_pktmbuf_mtod(seg, void *); + if (is_cksum) + tap_tx_l4_add_rcksum(iovecs[j].iov_base, + iovecs[j].iov_len, l4_cksum, + &l4_raw_cksum); + seg = seg->next; + } + + if (is_cksum) + tap_tx_l4_cksum(l4_cksum, l4_phdr_cksum, l4_raw_cksum); + + /* copy the tx frame data */ + n = writev(txq->fd, iovecs, j); + if (n <= 0) + break; + (*num_packets)++; + (*num_tx_bytes) += rte_pktmbuf_pkt_len(mbuf); + } +} + +/* Callback to handle sending packets from the tap interface + */ +static uint16_t +pmd_tx_burst(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts) +{ + struct tx_queue *txq = queue; + uint16_t num_tx = 0; + uint16_t num_packets = 0; + unsigned long num_tx_bytes = 0; + uint32_t max_size; + int i; + + if (unlikely(nb_pkts == 0)) + return 0; + + struct rte_mbuf *gso_mbufs[MAX_GSO_MBUFS]; + max_size = *txq->mtu + (ETHER_HDR_LEN + ETHER_CRC_LEN + 4); + for (i = 0; i < nb_pkts; i++) { + struct rte_mbuf *mbuf_in = bufs[num_tx]; + struct rte_mbuf **mbuf; + uint16_t num_mbufs = 0; + uint16_t tso_segsz = 0; + int ret; + uint16_t hdrs_len; + int j; + uint64_t tso; + + tso = mbuf_in->ol_flags & PKT_TX_TCP_SEG; + if (tso) { + struct rte_gso_ctx *gso_ctx = &txq->gso_ctx; + + assert(gso_ctx != NULL); + + /* TCP segmentation implies TCP checksum offload */ + mbuf_in->ol_flags |= PKT_TX_TCP_CKSUM; + + /* gso size is calculated without ETHER_CRC_LEN */ + hdrs_len = mbuf_in->l2_len + mbuf_in->l3_len + + mbuf_in->l4_len; + tso_segsz = mbuf_in->tso_segsz + hdrs_len; + if (unlikely(tso_segsz == hdrs_len) || + tso_segsz > *txq->mtu) { + txq->stats.errs++; + break; + } + gso_ctx->gso_size = tso_segsz; + ret = rte_gso_segment(mbuf_in, /* packet to segment */ + gso_ctx, /* gso control block */ + (struct rte_mbuf **)&gso_mbufs, /* out mbufs */ + RTE_DIM(gso_mbufs)); /* max tso mbufs */ + + /* ret contains the number of new created mbufs */ + if (ret < 0) + break; + + mbuf = gso_mbufs; + num_mbufs = ret; + } else { + /* stats.errs will be incremented */ + if (rte_pktmbuf_pkt_len(mbuf_in) > max_size) + break; + + /* ret 0 indicates no new mbufs were created */ + ret = 0; + mbuf = &mbuf_in; + num_mbufs = 1; + } + + tap_write_mbufs(txq, num_mbufs, mbuf, + &num_packets, &num_tx_bytes); + num_tx++; + /* free original mbuf */ + rte_pktmbuf_free(mbuf_in); + /* free tso mbufs */ + for (j = 0; j < ret; j++) + rte_pktmbuf_free(mbuf[j]); + } + + txq->stats.opackets += num_packets; + txq->stats.errs += nb_pkts - num_tx; + txq->stats.obytes += num_tx_bytes; + + return num_tx; +} + +static const char * +tap_ioctl_req2str(unsigned long request) +{ + switch (request) { + case SIOCSIFFLAGS: + return "SIOCSIFFLAGS"; + case SIOCGIFFLAGS: + return "SIOCGIFFLAGS"; + case SIOCGIFHWADDR: + return "SIOCGIFHWADDR"; + case SIOCSIFHWADDR: + return "SIOCSIFHWADDR"; + case SIOCSIFMTU: + return "SIOCSIFMTU"; + } + return "UNKNOWN"; +} + +static int +tap_ioctl(struct pmd_internals *pmd, unsigned long request, + struct ifreq *ifr, int set, enum ioctl_mode mode) +{ + short req_flags = ifr->ifr_flags; + int remote = pmd->remote_if_index && + (mode == REMOTE_ONLY || mode == LOCAL_AND_REMOTE); + + if (!pmd->remote_if_index && mode == REMOTE_ONLY) + return 0; + /* + * If there is a remote netdevice, apply ioctl on it, then apply it on + * the tap netdevice. + */ +apply: + if (remote) + snprintf(ifr->ifr_name, IFNAMSIZ, "%s", pmd->remote_iface); + else if (mode == LOCAL_ONLY || mode == LOCAL_AND_REMOTE) + snprintf(ifr->ifr_name, IFNAMSIZ, "%s", pmd->name); + switch (request) { + case SIOCSIFFLAGS: + /* fetch current flags to leave other flags untouched */ + if (ioctl(pmd->ioctl_sock, SIOCGIFFLAGS, ifr) < 0) + goto error; + if (set) + ifr->ifr_flags |= req_flags; + else + ifr->ifr_flags &= ~req_flags; + break; + case SIOCGIFFLAGS: + case SIOCGIFHWADDR: + case SIOCSIFHWADDR: + case SIOCSIFMTU: + break; + default: + RTE_LOG(WARNING, PMD, "%s: ioctl() called with wrong arg\n", + pmd->name); + return -EINVAL; + } + if (ioctl(pmd->ioctl_sock, request, ifr) < 0) + goto error; + if (remote-- && mode == LOCAL_AND_REMOTE) + goto apply; + return 0; + +error: + TAP_LOG(DEBUG, "%s(%s) failed: %s(%d)", ifr->ifr_name, + tap_ioctl_req2str(request), strerror(errno), errno); + return -errno; +} + +static int +tap_link_set_down(struct rte_eth_dev *dev) +{ + struct pmd_internals *pmd = dev->data->dev_private; + struct ifreq ifr = { .ifr_flags = IFF_UP }; + + dev->data->dev_link.link_status = ETH_LINK_DOWN; + return tap_ioctl(pmd, SIOCSIFFLAGS, &ifr, 0, LOCAL_ONLY); +} + +static int +tap_link_set_up(struct rte_eth_dev *dev) +{ + struct pmd_internals *pmd = dev->data->dev_private; + struct ifreq ifr = { .ifr_flags = IFF_UP }; + + dev->data->dev_link.link_status = ETH_LINK_UP; + return tap_ioctl(pmd, SIOCSIFFLAGS, &ifr, 1, LOCAL_AND_REMOTE); +} + +static int +tap_dev_start(struct rte_eth_dev *dev) +{ + int err, i; + + err = tap_intr_handle_set(dev, 1); + if (err) + return err; + + err = tap_link_set_up(dev); + if (err) + return err; + + for (i = 0; i < dev->data->nb_tx_queues; i++) + dev->data->tx_queue_state[i] = RTE_ETH_QUEUE_STATE_STARTED; + for (i = 0; i < dev->data->nb_rx_queues; i++) + dev->data->rx_queue_state[i] = RTE_ETH_QUEUE_STATE_STARTED; + + return err; +} + +/* This function gets called when the current port gets stopped. + */ +static void +tap_dev_stop(struct rte_eth_dev *dev) +{ + int i; + + for (i = 0; i < dev->data->nb_tx_queues; i++) + dev->data->tx_queue_state[i] = RTE_ETH_QUEUE_STATE_STOPPED; + for (i = 0; i < dev->data->nb_rx_queues; i++) + dev->data->rx_queue_state[i] = RTE_ETH_QUEUE_STATE_STOPPED; + + tap_intr_handle_set(dev, 0); + tap_link_set_down(dev); +} + +static int +tap_dev_configure(struct rte_eth_dev *dev) +{ + if (dev->data->nb_rx_queues > RTE_PMD_TAP_MAX_QUEUES) { + TAP_LOG(ERR, + "%s: number of rx queues %d exceeds max num of queues %d", + dev->device->name, + dev->data->nb_rx_queues, + RTE_PMD_TAP_MAX_QUEUES); + return -1; + } + if (dev->data->nb_tx_queues > RTE_PMD_TAP_MAX_QUEUES) { + TAP_LOG(ERR, + "%s: number of tx queues %d exceeds max num of queues %d", + dev->device->name, + dev->data->nb_tx_queues, + RTE_PMD_TAP_MAX_QUEUES); + return -1; + } + + TAP_LOG(INFO, "%s: %p: TX configured queues number: %u", + dev->device->name, (void *)dev, dev->data->nb_tx_queues); + + TAP_LOG(INFO, "%s: %p: RX configured queues number: %u", + dev->device->name, (void *)dev, dev->data->nb_rx_queues); + + return 0; +} + +static uint32_t +tap_dev_speed_capa(void) +{ + uint32_t speed = pmd_link.link_speed; + uint32_t capa = 0; + + if (speed >= ETH_SPEED_NUM_10M) + capa |= ETH_LINK_SPEED_10M; + if (speed >= ETH_SPEED_NUM_100M) + capa |= ETH_LINK_SPEED_100M; + if (speed >= ETH_SPEED_NUM_1G) + capa |= ETH_LINK_SPEED_1G; + if (speed >= ETH_SPEED_NUM_5G) + capa |= ETH_LINK_SPEED_2_5G; + if (speed >= ETH_SPEED_NUM_5G) + capa |= ETH_LINK_SPEED_5G; + if (speed >= ETH_SPEED_NUM_10G) + capa |= ETH_LINK_SPEED_10G; + if (speed >= ETH_SPEED_NUM_20G) + capa |= ETH_LINK_SPEED_20G; + if (speed >= ETH_SPEED_NUM_25G) + capa |= ETH_LINK_SPEED_25G; + if (speed >= ETH_SPEED_NUM_40G) + capa |= ETH_LINK_SPEED_40G; + if (speed >= ETH_SPEED_NUM_50G) + capa |= ETH_LINK_SPEED_50G; + if (speed >= ETH_SPEED_NUM_56G) + capa |= ETH_LINK_SPEED_56G; + if (speed >= ETH_SPEED_NUM_100G) + capa |= ETH_LINK_SPEED_100G; + + return capa; +} + +static void +tap_dev_info(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info) +{ + struct pmd_internals *internals = dev->data->dev_private; + + dev_info->if_index = internals->if_index; + dev_info->max_mac_addrs = 1; + dev_info->max_rx_pktlen = (uint32_t)ETHER_MAX_VLAN_FRAME_LEN; + dev_info->max_rx_queues = RTE_PMD_TAP_MAX_QUEUES; + dev_info->max_tx_queues = RTE_PMD_TAP_MAX_QUEUES; + dev_info->min_rx_bufsize = 0; + dev_info->speed_capa = tap_dev_speed_capa(); + dev_info->rx_queue_offload_capa = tap_rx_offload_get_queue_capa(); + dev_info->rx_offload_capa = tap_rx_offload_get_port_capa() | + dev_info->rx_queue_offload_capa; + dev_info->tx_queue_offload_capa = tap_tx_offload_get_queue_capa(); + dev_info->tx_offload_capa = tap_tx_offload_get_port_capa() | + dev_info->tx_queue_offload_capa; + dev_info->hash_key_size = TAP_RSS_HASH_KEY_SIZE; + /* + * limitation: TAP supports all of IP, UDP and TCP hash + * functions together and not in partial combinations + */ + dev_info->flow_type_rss_offloads = ~TAP_RSS_HF_MASK; +} + +static int +tap_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *tap_stats) +{ + unsigned int i, imax; + unsigned long rx_total = 0, tx_total = 0, tx_err_total = 0; + unsigned long rx_bytes_total = 0, tx_bytes_total = 0; + unsigned long rx_nombuf = 0, ierrors = 0; + const struct pmd_internals *pmd = dev->data->dev_private; + + /* rx queue statistics */ + imax = (dev->data->nb_rx_queues < RTE_ETHDEV_QUEUE_STAT_CNTRS) ? + dev->data->nb_rx_queues : RTE_ETHDEV_QUEUE_STAT_CNTRS; + for (i = 0; i < imax; i++) { + tap_stats->q_ipackets[i] = pmd->rxq[i].stats.ipackets; + tap_stats->q_ibytes[i] = pmd->rxq[i].stats.ibytes; + rx_total += tap_stats->q_ipackets[i]; + rx_bytes_total += tap_stats->q_ibytes[i]; + rx_nombuf += pmd->rxq[i].stats.rx_nombuf; + ierrors += pmd->rxq[i].stats.ierrors; + } + + /* tx queue statistics */ + imax = (dev->data->nb_tx_queues < RTE_ETHDEV_QUEUE_STAT_CNTRS) ? + dev->data->nb_tx_queues : RTE_ETHDEV_QUEUE_STAT_CNTRS; + + for (i = 0; i < imax; i++) { + tap_stats->q_opackets[i] = pmd->txq[i].stats.opackets; + tap_stats->q_errors[i] = pmd->txq[i].stats.errs; + tap_stats->q_obytes[i] = pmd->txq[i].stats.obytes; + tx_total += tap_stats->q_opackets[i]; + tx_err_total += tap_stats->q_errors[i]; + tx_bytes_total += tap_stats->q_obytes[i]; + } + + tap_stats->ipackets = rx_total; + tap_stats->ibytes = rx_bytes_total; + tap_stats->ierrors = ierrors; + tap_stats->rx_nombuf = rx_nombuf; + tap_stats->opackets = tx_total; + tap_stats->oerrors = tx_err_total; + tap_stats->obytes = tx_bytes_total; + return 0; +} + +static void +tap_stats_reset(struct rte_eth_dev *dev) +{ + int i; + struct pmd_internals *pmd = dev->data->dev_private; + + for (i = 0; i < RTE_PMD_TAP_MAX_QUEUES; i++) { + pmd->rxq[i].stats.ipackets = 0; + pmd->rxq[i].stats.ibytes = 0; + pmd->rxq[i].stats.ierrors = 0; + pmd->rxq[i].stats.rx_nombuf = 0; + + pmd->txq[i].stats.opackets = 0; + pmd->txq[i].stats.errs = 0; + pmd->txq[i].stats.obytes = 0; + } +} + +static void +tap_dev_close(struct rte_eth_dev *dev) +{ + int i; + struct pmd_internals *internals = dev->data->dev_private; + + tap_link_set_down(dev); + tap_flow_flush(dev, NULL); + tap_flow_implicit_flush(internals, NULL); + + for (i = 0; i < RTE_PMD_TAP_MAX_QUEUES; i++) { + if (internals->rxq[i].fd != -1) { + close(internals->rxq[i].fd); + internals->rxq[i].fd = -1; + } + if (internals->txq[i].fd != -1) { + close(internals->txq[i].fd); + internals->txq[i].fd = -1; + } + } + + if (internals->remote_if_index) { + /* Restore initial remote state */ + ioctl(internals->ioctl_sock, SIOCSIFFLAGS, + &internals->remote_initial_flags); + } + + if (internals->ka_fd != -1) { + close(internals->ka_fd); + internals->ka_fd = -1; + } + /* + * Since TUN device has no more opened file descriptors + * it will be removed from kernel + */ +} + +static void +tap_rx_queue_release(void *queue) +{ + struct rx_queue *rxq = queue; + + if (rxq && (rxq->fd > 0)) { + close(rxq->fd); + rxq->fd = -1; + rte_pktmbuf_free(rxq->pool); + rte_free(rxq->iovecs); + rxq->pool = NULL; + rxq->iovecs = NULL; + } +} + +static void +tap_tx_queue_release(void *queue) +{ + struct tx_queue *txq = queue; + + if (txq && (txq->fd > 0)) { + close(txq->fd); + txq->fd = -1; + } +} + +static int +tap_link_update(struct rte_eth_dev *dev, int wait_to_complete __rte_unused) +{ + struct rte_eth_link *dev_link = &dev->data->dev_link; + struct pmd_internals *pmd = dev->data->dev_private; + struct ifreq ifr = { .ifr_flags = 0 }; + + if (pmd->remote_if_index) { + tap_ioctl(pmd, SIOCGIFFLAGS, &ifr, 0, REMOTE_ONLY); + if (!(ifr.ifr_flags & IFF_UP) || + !(ifr.ifr_flags & IFF_RUNNING)) { + dev_link->link_status = ETH_LINK_DOWN; + return 0; + } + } + tap_ioctl(pmd, SIOCGIFFLAGS, &ifr, 0, LOCAL_ONLY); + dev_link->link_status = + ((ifr.ifr_flags & IFF_UP) && (ifr.ifr_flags & IFF_RUNNING) ? + ETH_LINK_UP : + ETH_LINK_DOWN); + return 0; +} + +static void +tap_promisc_enable(struct rte_eth_dev *dev) +{ + struct pmd_internals *pmd = dev->data->dev_private; + struct ifreq ifr = { .ifr_flags = IFF_PROMISC }; + + dev->data->promiscuous = 1; + tap_ioctl(pmd, SIOCSIFFLAGS, &ifr, 1, LOCAL_AND_REMOTE); + if (pmd->remote_if_index && !pmd->flow_isolate) + tap_flow_implicit_create(pmd, TAP_REMOTE_PROMISC); +} + +static void +tap_promisc_disable(struct rte_eth_dev *dev) +{ + struct pmd_internals *pmd = dev->data->dev_private; + struct ifreq ifr = { .ifr_flags = IFF_PROMISC }; + + dev->data->promiscuous = 0; + tap_ioctl(pmd, SIOCSIFFLAGS, &ifr, 0, LOCAL_AND_REMOTE); + if (pmd->remote_if_index && !pmd->flow_isolate) + tap_flow_implicit_destroy(pmd, TAP_REMOTE_PROMISC); +} + +static void +tap_allmulti_enable(struct rte_eth_dev *dev) +{ + struct pmd_internals *pmd = dev->data->dev_private; + struct ifreq ifr = { .ifr_flags = IFF_ALLMULTI }; + + dev->data->all_multicast = 1; + tap_ioctl(pmd, SIOCSIFFLAGS, &ifr, 1, LOCAL_AND_REMOTE); + if (pmd->remote_if_index && !pmd->flow_isolate) + tap_flow_implicit_create(pmd, TAP_REMOTE_ALLMULTI); +} + +static void +tap_allmulti_disable(struct rte_eth_dev *dev) +{ + struct pmd_internals *pmd = dev->data->dev_private; + struct ifreq ifr = { .ifr_flags = IFF_ALLMULTI }; + + dev->data->all_multicast = 0; + tap_ioctl(pmd, SIOCSIFFLAGS, &ifr, 0, LOCAL_AND_REMOTE); + if (pmd->remote_if_index && !pmd->flow_isolate) + tap_flow_implicit_destroy(pmd, TAP_REMOTE_ALLMULTI); +} + +static int +tap_mac_set(struct rte_eth_dev *dev, struct ether_addr *mac_addr) +{ + struct pmd_internals *pmd = dev->data->dev_private; + enum ioctl_mode mode = LOCAL_ONLY; + struct ifreq ifr; + int ret; + + if (pmd->type == ETH_TUNTAP_TYPE_TUN) { + TAP_LOG(ERR, "%s: can't MAC address for TUN", + dev->device->name); + return -ENOTSUP; + } + + if (is_zero_ether_addr(mac_addr)) { + TAP_LOG(ERR, "%s: can't set an empty MAC address", + dev->device->name); + return -EINVAL; + } + /* Check the actual current MAC address on the tap netdevice */ + ret = tap_ioctl(pmd, SIOCGIFHWADDR, &ifr, 0, LOCAL_ONLY); + if (ret < 0) + return ret; + if (is_same_ether_addr((struct ether_addr *)&ifr.ifr_hwaddr.sa_data, + mac_addr)) + return 0; + /* Check the current MAC address on the remote */ + ret = tap_ioctl(pmd, SIOCGIFHWADDR, &ifr, 0, REMOTE_ONLY); + if (ret < 0) + return ret; + if (!is_same_ether_addr((struct ether_addr *)&ifr.ifr_hwaddr.sa_data, + mac_addr)) + mode = LOCAL_AND_REMOTE; + ifr.ifr_hwaddr.sa_family = AF_LOCAL; + rte_memcpy(ifr.ifr_hwaddr.sa_data, mac_addr, ETHER_ADDR_LEN); + ret = tap_ioctl(pmd, SIOCSIFHWADDR, &ifr, 1, mode); + if (ret < 0) + return ret; + rte_memcpy(&pmd->eth_addr, mac_addr, ETHER_ADDR_LEN); + if (pmd->remote_if_index && !pmd->flow_isolate) { + /* Replace MAC redirection rule after a MAC change */ + ret = tap_flow_implicit_destroy(pmd, TAP_REMOTE_LOCAL_MAC); + if (ret < 0) { + TAP_LOG(ERR, + "%s: Couldn't delete MAC redirection rule", + dev->device->name); + return ret; + } + ret = tap_flow_implicit_create(pmd, TAP_REMOTE_LOCAL_MAC); + if (ret < 0) { + TAP_LOG(ERR, + "%s: Couldn't add MAC redirection rule", + dev->device->name); + return ret; + } + } + + return 0; +} + +static int +tap_gso_ctx_setup(struct rte_gso_ctx *gso_ctx, struct rte_eth_dev *dev) +{ + uint32_t gso_types; + char pool_name[64]; + + /* + * Create private mbuf pool with TAP_GSO_MBUF_SEG_SIZE bytes + * size per mbuf use this pool for both direct and indirect mbufs + */ + + struct rte_mempool *mp; /* Mempool for GSO packets */ + + /* initialize GSO context */ + gso_types = DEV_TX_OFFLOAD_TCP_TSO; + snprintf(pool_name, sizeof(pool_name), "mp_%s", dev->device->name); + mp = rte_mempool_lookup((const char *)pool_name); + if (!mp) { + mp = rte_pktmbuf_pool_create(pool_name, TAP_GSO_MBUFS_NUM, + TAP_GSO_MBUF_CACHE_SIZE, 0, + RTE_PKTMBUF_HEADROOM + TAP_GSO_MBUF_SEG_SIZE, + SOCKET_ID_ANY); + if (!mp) { + struct pmd_internals *pmd = dev->data->dev_private; + RTE_LOG(DEBUG, PMD, "%s: failed to create mbuf pool for device %s\n", + pmd->name, dev->device->name); + return -1; + } + } + + gso_ctx->direct_pool = mp; + gso_ctx->indirect_pool = mp; + gso_ctx->gso_types = gso_types; + gso_ctx->gso_size = 0; /* gso_size is set in tx_burst() per packet */ + gso_ctx->flag = 0; + + return 0; +} + +static int +tap_setup_queue(struct rte_eth_dev *dev, + struct pmd_internals *internals, + uint16_t qid, + int is_rx) +{ + int ret; + int *fd; + int *other_fd; + const char *dir; + struct pmd_internals *pmd = dev->data->dev_private; + struct rx_queue *rx = &internals->rxq[qid]; + struct tx_queue *tx = &internals->txq[qid]; + struct rte_gso_ctx *gso_ctx; + + if (is_rx) { + fd = &rx->fd; + other_fd = &tx->fd; + dir = "rx"; + gso_ctx = NULL; + } else { + fd = &tx->fd; + other_fd = &rx->fd; + dir = "tx"; + gso_ctx = &tx->gso_ctx; + } + if (*fd != -1) { + /* fd for this queue already exists */ + TAP_LOG(DEBUG, "%s: fd %d for %s queue qid %d exists", + pmd->name, *fd, dir, qid); + gso_ctx = NULL; + } else if (*other_fd != -1) { + /* Only other_fd exists. dup it */ + *fd = dup(*other_fd); + if (*fd < 0) { + *fd = -1; + TAP_LOG(ERR, "%s: dup() failed.", pmd->name); + return -1; + } + TAP_LOG(DEBUG, "%s: dup fd %d for %s queue qid %d (%d)", + pmd->name, *other_fd, dir, qid, *fd); + } else { + /* Both RX and TX fds do not exist (equal -1). Create fd */ + *fd = tun_alloc(pmd, 0); + if (*fd < 0) { + *fd = -1; /* restore original value */ + TAP_LOG(ERR, "%s: tun_alloc() failed.", pmd->name); + return -1; + } + TAP_LOG(DEBUG, "%s: add %s queue for qid %d fd %d", + pmd->name, dir, qid, *fd); + } + + tx->mtu = &dev->data->mtu; + rx->rxmode = &dev->data->dev_conf.rxmode; + if (gso_ctx) { + ret = tap_gso_ctx_setup(gso_ctx, dev); + if (ret) + return -1; + } + + tx->type = pmd->type; + + return *fd; +} + +static int +tap_rx_queue_setup(struct rte_eth_dev *dev, + uint16_t rx_queue_id, + uint16_t nb_rx_desc, + unsigned int socket_id, + const struct rte_eth_rxconf *rx_conf __rte_unused, + struct rte_mempool *mp) +{ + struct pmd_internals *internals = dev->data->dev_private; + struct rx_queue *rxq = &internals->rxq[rx_queue_id]; + struct rte_mbuf **tmp = &rxq->pool; + long iov_max = sysconf(_SC_IOV_MAX); + uint16_t nb_desc = RTE_MIN(nb_rx_desc, iov_max - 1); + struct iovec (*iovecs)[nb_desc + 1]; + int data_off = RTE_PKTMBUF_HEADROOM; + int ret = 0; + int fd; + int i; + + if (rx_queue_id >= dev->data->nb_rx_queues || !mp) { + TAP_LOG(WARNING, + "nb_rx_queues %d too small or mempool NULL", + dev->data->nb_rx_queues); + return -1; + } + + rxq->mp = mp; + rxq->trigger_seen = 1; /* force initial burst */ + rxq->in_port = dev->data->port_id; + rxq->nb_rx_desc = nb_desc; + iovecs = rte_zmalloc_socket(dev->device->name, sizeof(*iovecs), 0, + socket_id); + if (!iovecs) { + TAP_LOG(WARNING, + "%s: Couldn't allocate %d RX descriptors", + dev->device->name, nb_desc); + return -ENOMEM; + } + rxq->iovecs = iovecs; + + dev->data->rx_queues[rx_queue_id] = rxq; + fd = tap_setup_queue(dev, internals, rx_queue_id, 1); + if (fd == -1) { + ret = fd; + goto error; + } + + (*rxq->iovecs)[0].iov_len = sizeof(struct tun_pi); + (*rxq->iovecs)[0].iov_base = &rxq->pi; + + for (i = 1; i <= nb_desc; i++) { + *tmp = rte_pktmbuf_alloc(rxq->mp); + if (!*tmp) { + TAP_LOG(WARNING, + "%s: couldn't allocate memory for queue %d", + dev->device->name, rx_queue_id); + ret = -ENOMEM; + goto error; + } + (*rxq->iovecs)[i].iov_len = (*tmp)->buf_len - data_off; + (*rxq->iovecs)[i].iov_base = + (char *)(*tmp)->buf_addr + data_off; + data_off = 0; + tmp = &(*tmp)->next; + } + + TAP_LOG(DEBUG, " RX TUNTAP device name %s, qid %d on fd %d", + internals->name, rx_queue_id, internals->rxq[rx_queue_id].fd); + + return 0; + +error: + rte_pktmbuf_free(rxq->pool); + rxq->pool = NULL; + rte_free(rxq->iovecs); + rxq->iovecs = NULL; + return ret; +} + +static int +tap_tx_queue_setup(struct rte_eth_dev *dev, + uint16_t tx_queue_id, + uint16_t nb_tx_desc __rte_unused, + unsigned int socket_id __rte_unused, + const struct rte_eth_txconf *tx_conf) +{ + struct pmd_internals *internals = dev->data->dev_private; + struct tx_queue *txq; + int ret; + uint64_t offloads; + + if (tx_queue_id >= dev->data->nb_tx_queues) + return -1; + dev->data->tx_queues[tx_queue_id] = &internals->txq[tx_queue_id]; + txq = dev->data->tx_queues[tx_queue_id]; + + offloads = tx_conf->offloads | dev->data->dev_conf.txmode.offloads; + txq->csum = !!(offloads & + (DEV_TX_OFFLOAD_IPV4_CKSUM | + DEV_TX_OFFLOAD_UDP_CKSUM | + DEV_TX_OFFLOAD_TCP_CKSUM)); + + ret = tap_setup_queue(dev, internals, tx_queue_id, 0); + if (ret == -1) + return -1; + TAP_LOG(DEBUG, + " TX TUNTAP device name %s, qid %d on fd %d csum %s", + internals->name, tx_queue_id, internals->txq[tx_queue_id].fd, + txq->csum ? "on" : "off"); + + return 0; +} + +static int +tap_mtu_set(struct rte_eth_dev *dev, uint16_t mtu) +{ + struct pmd_internals *pmd = dev->data->dev_private; + struct ifreq ifr = { .ifr_mtu = mtu }; + int err = 0; + + err = tap_ioctl(pmd, SIOCSIFMTU, &ifr, 1, LOCAL_AND_REMOTE); + if (!err) + dev->data->mtu = mtu; + + return err; +} + +static int +tap_set_mc_addr_list(struct rte_eth_dev *dev __rte_unused, + struct ether_addr *mc_addr_set __rte_unused, + uint32_t nb_mc_addr __rte_unused) +{ + /* + * Nothing to do actually: the tap has no filtering whatsoever, every + * packet is received. + */ + return 0; +} + +static int +tap_nl_msg_handler(struct nlmsghdr *nh, void *arg) +{ + struct rte_eth_dev *dev = arg; + struct pmd_internals *pmd = dev->data->dev_private; + struct ifinfomsg *info = NLMSG_DATA(nh); + + if (nh->nlmsg_type != RTM_NEWLINK || + (info->ifi_index != pmd->if_index && + info->ifi_index != pmd->remote_if_index)) + return 0; + return tap_link_update(dev, 0); +} + +static void +tap_dev_intr_handler(void *cb_arg) +{ + struct rte_eth_dev *dev = cb_arg; + struct pmd_internals *pmd = dev->data->dev_private; + + tap_nl_recv(pmd->intr_handle.fd, tap_nl_msg_handler, dev); +} + +static int +tap_lsc_intr_handle_set(struct rte_eth_dev *dev, int set) +{ + struct pmd_internals *pmd = dev->data->dev_private; + + /* In any case, disable interrupt if the conf is no longer there. */ + if (!dev->data->dev_conf.intr_conf.lsc) { + if (pmd->intr_handle.fd != -1) { + tap_nl_final(pmd->intr_handle.fd); + rte_intr_callback_unregister(&pmd->intr_handle, + tap_dev_intr_handler, dev); + } + return 0; + } + if (set) { + pmd->intr_handle.fd = tap_nl_init(RTMGRP_LINK); + if (unlikely(pmd->intr_handle.fd == -1)) + return -EBADF; + return rte_intr_callback_register( + &pmd->intr_handle, tap_dev_intr_handler, dev); + } + tap_nl_final(pmd->intr_handle.fd); + return rte_intr_callback_unregister(&pmd->intr_handle, + tap_dev_intr_handler, dev); +} + +static int +tap_intr_handle_set(struct rte_eth_dev *dev, int set) +{ + int err; + + err = tap_lsc_intr_handle_set(dev, set); + if (err) + return err; + err = tap_rx_intr_vec_set(dev, set); + if (err && set) + tap_lsc_intr_handle_set(dev, 0); + return err; +} + +static const uint32_t* +tap_dev_supported_ptypes_get(struct rte_eth_dev *dev __rte_unused) +{ + static const uint32_t ptypes[] = { + RTE_PTYPE_INNER_L2_ETHER, + RTE_PTYPE_INNER_L2_ETHER_VLAN, + RTE_PTYPE_INNER_L2_ETHER_QINQ, + RTE_PTYPE_INNER_L3_IPV4, + RTE_PTYPE_INNER_L3_IPV4_EXT, + RTE_PTYPE_INNER_L3_IPV6, + RTE_PTYPE_INNER_L3_IPV6_EXT, + RTE_PTYPE_INNER_L4_FRAG, + RTE_PTYPE_INNER_L4_UDP, + RTE_PTYPE_INNER_L4_TCP, + RTE_PTYPE_INNER_L4_SCTP, + RTE_PTYPE_L2_ETHER, + RTE_PTYPE_L2_ETHER_VLAN, + RTE_PTYPE_L2_ETHER_QINQ, + RTE_PTYPE_L3_IPV4, + RTE_PTYPE_L3_IPV4_EXT, + RTE_PTYPE_L3_IPV6_EXT, + RTE_PTYPE_L3_IPV6, + RTE_PTYPE_L4_FRAG, + RTE_PTYPE_L4_UDP, + RTE_PTYPE_L4_TCP, + RTE_PTYPE_L4_SCTP, + }; + + return ptypes; +} + +static int +tap_flow_ctrl_get(struct rte_eth_dev *dev __rte_unused, + struct rte_eth_fc_conf *fc_conf) +{ + fc_conf->mode = RTE_FC_NONE; + return 0; +} + +static int +tap_flow_ctrl_set(struct rte_eth_dev *dev __rte_unused, + struct rte_eth_fc_conf *fc_conf) +{ + if (fc_conf->mode != RTE_FC_NONE) + return -ENOTSUP; + return 0; +} + +/** + * DPDK callback to update the RSS hash configuration. + * + * @param dev + * Pointer to Ethernet device structure. + * @param[in] rss_conf + * RSS configuration data. + * + * @return + * 0 on success, a negative errno value otherwise and rte_errno is set. + */ +static int +tap_rss_hash_update(struct rte_eth_dev *dev, + struct rte_eth_rss_conf *rss_conf) +{ + if (rss_conf->rss_hf & TAP_RSS_HF_MASK) { + rte_errno = EINVAL; + return -rte_errno; + } + if (rss_conf->rss_key && rss_conf->rss_key_len) { + /* + * Currently TAP RSS key is hard coded + * and cannot be updated + */ + TAP_LOG(ERR, + "port %u RSS key cannot be updated", + dev->data->port_id); + rte_errno = EINVAL; + return -rte_errno; + } + return 0; +} + +static int +tap_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id) +{ + dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED; + + return 0; +} + +static int +tap_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id) +{ + dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED; + + return 0; +} + +static int +tap_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rx_queue_id) +{ + dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED; + + return 0; +} + +static int +tap_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id) +{ + dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED; + + return 0; +} +static const struct eth_dev_ops ops = { + .dev_start = tap_dev_start, + .dev_stop = tap_dev_stop, + .dev_close = tap_dev_close, + .dev_configure = tap_dev_configure, + .dev_infos_get = tap_dev_info, + .rx_queue_setup = tap_rx_queue_setup, + .tx_queue_setup = tap_tx_queue_setup, + .rx_queue_start = tap_rx_queue_start, + .tx_queue_start = tap_tx_queue_start, + .rx_queue_stop = tap_rx_queue_stop, + .tx_queue_stop = tap_tx_queue_stop, + .rx_queue_release = tap_rx_queue_release, + .tx_queue_release = tap_tx_queue_release, + .flow_ctrl_get = tap_flow_ctrl_get, + .flow_ctrl_set = tap_flow_ctrl_set, + .link_update = tap_link_update, + .dev_set_link_up = tap_link_set_up, + .dev_set_link_down = tap_link_set_down, + .promiscuous_enable = tap_promisc_enable, + .promiscuous_disable = tap_promisc_disable, + .allmulticast_enable = tap_allmulti_enable, + .allmulticast_disable = tap_allmulti_disable, + .mac_addr_set = tap_mac_set, + .mtu_set = tap_mtu_set, + .set_mc_addr_list = tap_set_mc_addr_list, + .stats_get = tap_stats_get, + .stats_reset = tap_stats_reset, + .dev_supported_ptypes_get = tap_dev_supported_ptypes_get, + .rss_hash_update = tap_rss_hash_update, + .filter_ctrl = tap_dev_filter_ctrl, +}; + +static int +eth_dev_tap_create(struct rte_vdev_device *vdev, char *tap_name, + char *remote_iface, struct ether_addr *mac_addr, + enum rte_tuntap_type type) +{ + int numa_node = rte_socket_id(); + struct rte_eth_dev *dev; + struct pmd_internals *pmd; + struct rte_eth_dev_data *data; + struct ifreq ifr; + int i; + + TAP_LOG(DEBUG, "%s device on numa %u", + tuntap_name, rte_socket_id()); + + dev = rte_eth_vdev_allocate(vdev, sizeof(*pmd)); + if (!dev) { + TAP_LOG(ERR, "%s Unable to allocate device struct", + tuntap_name); + goto error_exit_nodev; + } + + pmd = dev->data->dev_private; + pmd->dev = dev; + snprintf(pmd->name, sizeof(pmd->name), "%s", tap_name); + pmd->type = type; + + pmd->ioctl_sock = socket(AF_INET, SOCK_DGRAM, 0); + if (pmd->ioctl_sock == -1) { + TAP_LOG(ERR, + "%s Unable to get a socket for management: %s", + tuntap_name, strerror(errno)); + goto error_exit; + } + + /* Setup some default values */ + data = dev->data; + data->dev_private = pmd; + data->dev_flags = RTE_ETH_DEV_INTR_LSC; + data->numa_node = numa_node; + + data->dev_link = pmd_link; + data->mac_addrs = &pmd->eth_addr; + /* Set the number of RX and TX queues */ + data->nb_rx_queues = 0; + data->nb_tx_queues = 0; + + dev->dev_ops = &ops; + dev->rx_pkt_burst = pmd_rx_burst; + dev->tx_pkt_burst = pmd_tx_burst; + + pmd->intr_handle.type = RTE_INTR_HANDLE_EXT; + pmd->intr_handle.fd = -1; + dev->intr_handle = &pmd->intr_handle; + + /* Presetup the fds to -1 as being not valid */ + pmd->ka_fd = -1; + for (i = 0; i < RTE_PMD_TAP_MAX_QUEUES; i++) { + pmd->rxq[i].fd = -1; + pmd->txq[i].fd = -1; + } + + if (pmd->type == ETH_TUNTAP_TYPE_TAP) { + if (is_zero_ether_addr(mac_addr)) + eth_random_addr((uint8_t *)&pmd->eth_addr); + else + rte_memcpy(&pmd->eth_addr, mac_addr, sizeof(*mac_addr)); + } + + /* + * Allocate a TUN device keep-alive file descriptor that will only be + * closed when the TUN device itself is closed or removed. + * This keep-alive file descriptor will guarantee that the TUN device + * exists even when all of its queues are closed + */ + pmd->ka_fd = tun_alloc(pmd, 1); + if (pmd->ka_fd == -1) { + TAP_LOG(ERR, "Unable to create %s interface", tuntap_name); + goto error_exit; + } + + ifr.ifr_mtu = dev->data->mtu; + if (tap_ioctl(pmd, SIOCSIFMTU, &ifr, 1, LOCAL_AND_REMOTE) < 0) + goto error_exit; + + if (pmd->type == ETH_TUNTAP_TYPE_TAP) { + memset(&ifr, 0, sizeof(struct ifreq)); + ifr.ifr_hwaddr.sa_family = AF_LOCAL; + rte_memcpy(ifr.ifr_hwaddr.sa_data, &pmd->eth_addr, + ETHER_ADDR_LEN); + if (tap_ioctl(pmd, SIOCSIFHWADDR, &ifr, 0, LOCAL_ONLY) < 0) + goto error_exit; + } + + /* + * Set up everything related to rte_flow: + * - netlink socket + * - tap / remote if_index + * - mandatory QDISCs + * - rte_flow actual/implicit lists + * - implicit rules + */ + pmd->nlsk_fd = tap_nl_init(0); + if (pmd->nlsk_fd == -1) { + TAP_LOG(WARNING, "%s: failed to create netlink socket.", + pmd->name); + goto disable_rte_flow; + } + pmd->if_index = if_nametoindex(pmd->name); + if (!pmd->if_index) { + TAP_LOG(ERR, "%s: failed to get if_index.", pmd->name); + goto disable_rte_flow; + } + if (qdisc_create_multiq(pmd->nlsk_fd, pmd->if_index) < 0) { + TAP_LOG(ERR, "%s: failed to create multiq qdisc.", + pmd->name); + goto disable_rte_flow; + } + if (qdisc_create_ingress(pmd->nlsk_fd, pmd->if_index) < 0) { + TAP_LOG(ERR, "%s: failed to create ingress qdisc.", + pmd->name); + goto disable_rte_flow; + } + LIST_INIT(&pmd->flows); + + if (strlen(remote_iface)) { + pmd->remote_if_index = if_nametoindex(remote_iface); + if (!pmd->remote_if_index) { + TAP_LOG(ERR, "%s: failed to get %s if_index.", + pmd->name, remote_iface); + goto error_remote; + } + snprintf(pmd->remote_iface, RTE_ETH_NAME_MAX_LEN, + "%s", remote_iface); + + /* Save state of remote device */ + tap_ioctl(pmd, SIOCGIFFLAGS, &pmd->remote_initial_flags, 0, REMOTE_ONLY); + + /* Replicate remote MAC address */ + if (tap_ioctl(pmd, SIOCGIFHWADDR, &ifr, 0, REMOTE_ONLY) < 0) { + TAP_LOG(ERR, "%s: failed to get %s MAC address.", + pmd->name, pmd->remote_iface); + goto error_remote; + } + rte_memcpy(&pmd->eth_addr, ifr.ifr_hwaddr.sa_data, + ETHER_ADDR_LEN); + /* The desired MAC is already in ifreq after SIOCGIFHWADDR. */ + if (tap_ioctl(pmd, SIOCSIFHWADDR, &ifr, 0, LOCAL_ONLY) < 0) { + TAP_LOG(ERR, "%s: failed to get %s MAC address.", + pmd->name, remote_iface); + goto error_remote; + } + + /* + * Flush usually returns negative value because it tries to + * delete every QDISC (and on a running device, one QDISC at + * least is needed). Ignore negative return value. + */ + qdisc_flush(pmd->nlsk_fd, pmd->remote_if_index); + if (qdisc_create_ingress(pmd->nlsk_fd, + pmd->remote_if_index) < 0) { + TAP_LOG(ERR, "%s: failed to create ingress qdisc.", + pmd->remote_iface); + goto error_remote; + } + LIST_INIT(&pmd->implicit_flows); + if (tap_flow_implicit_create(pmd, TAP_REMOTE_TX) < 0 || + tap_flow_implicit_create(pmd, TAP_REMOTE_LOCAL_MAC) < 0 || + tap_flow_implicit_create(pmd, TAP_REMOTE_BROADCAST) < 0 || + tap_flow_implicit_create(pmd, TAP_REMOTE_BROADCASTV6) < 0) { + TAP_LOG(ERR, + "%s: failed to create implicit rules.", + pmd->name); + goto error_remote; + } + } + + rte_eth_dev_probing_finish(dev); + return 0; + +disable_rte_flow: + TAP_LOG(ERR, " Disabling rte flow support: %s(%d)", + strerror(errno), errno); + if (strlen(remote_iface)) { + TAP_LOG(ERR, "Remote feature requires flow support."); + goto error_exit; + } + return 0; + +error_remote: + TAP_LOG(ERR, " Can't set up remote feature: %s(%d)", + strerror(errno), errno); + tap_flow_implicit_flush(pmd, NULL); + +error_exit: + if (pmd->ioctl_sock > 0) + close(pmd->ioctl_sock); + rte_eth_dev_release_port(dev); + +error_exit_nodev: + TAP_LOG(ERR, "%s Unable to initialize %s", + tuntap_name, rte_vdev_device_name(vdev)); + + return -EINVAL; +} + +static int +set_interface_name(const char *key __rte_unused, + const char *value, + void *extra_args) +{ + char *name = (char *)extra_args; + + if (value) + strlcpy(name, value, RTE_ETH_NAME_MAX_LEN - 1); + else + snprintf(name, RTE_ETH_NAME_MAX_LEN - 1, "%s%d", + DEFAULT_TAP_NAME, (tap_unit - 1)); + + return 0; +} + +static int +set_remote_iface(const char *key __rte_unused, + const char *value, + void *extra_args) +{ + char *name = (char *)extra_args; + + if (value) + strlcpy(name, value, RTE_ETH_NAME_MAX_LEN); + + return 0; +} + +static int parse_user_mac(struct ether_addr *user_mac, + const char *value) +{ + unsigned int index = 0; + char mac_temp[strlen(ETH_TAP_USR_MAC_FMT) + 1], *mac_byte = NULL; + + if (user_mac == NULL || value == NULL) + return 0; + + strlcpy(mac_temp, value, sizeof(mac_temp)); + mac_byte = strtok(mac_temp, ":"); + + while ((mac_byte != NULL) && + (strlen(mac_byte) <= 2) && + (strlen(mac_byte) == strspn(mac_byte, + ETH_TAP_CMP_MAC_FMT))) { + user_mac->addr_bytes[index++] = strtoul(mac_byte, NULL, 16); + mac_byte = strtok(NULL, ":"); + } + + return index; +} + +static int +set_mac_type(const char *key __rte_unused, + const char *value, + void *extra_args) +{ + struct ether_addr *user_mac = extra_args; + + if (!value) + return 0; + + if (!strncasecmp(ETH_TAP_MAC_FIXED, value, strlen(ETH_TAP_MAC_FIXED))) { + static int iface_idx; + + /* fixed mac = 00:64:74:61:70:<iface_idx> */ + memcpy((char *)user_mac->addr_bytes, "\0dtap", ETHER_ADDR_LEN); + user_mac->addr_bytes[ETHER_ADDR_LEN - 1] = iface_idx++ + '0'; + goto success; + } + + if (parse_user_mac(user_mac, value) != 6) + goto error; +success: + TAP_LOG(DEBUG, "TAP user MAC param (%s)", value); + return 0; + +error: + TAP_LOG(ERR, "TAP user MAC (%s) is not in format (%s|%s)", + value, ETH_TAP_MAC_FIXED, ETH_TAP_USR_MAC_FMT); + return -1; +} + +/* + * Open a TUN interface device. TUN PMD + * 1) sets tap_type as false + * 2) intakes iface as argument. + * 3) as interface is virtual set speed to 10G + */ +static int +rte_pmd_tun_probe(struct rte_vdev_device *dev) +{ + const char *name, *params; + int ret; + struct rte_kvargs *kvlist = NULL; + char tun_name[RTE_ETH_NAME_MAX_LEN]; + char remote_iface[RTE_ETH_NAME_MAX_LEN]; + struct rte_eth_dev *eth_dev; + + strcpy(tuntap_name, "TUN"); + + name = rte_vdev_device_name(dev); + params = rte_vdev_device_args(dev); + memset(remote_iface, 0, RTE_ETH_NAME_MAX_LEN); + + if (rte_eal_process_type() == RTE_PROC_SECONDARY && + strlen(params) == 0) { + eth_dev = rte_eth_dev_attach_secondary(name); + if (!eth_dev) { + TAP_LOG(ERR, "Failed to probe %s", name); + return -1; + } + eth_dev->dev_ops = &ops; + eth_dev->device = &dev->device; + rte_eth_dev_probing_finish(eth_dev); + return 0; + } + + snprintf(tun_name, sizeof(tun_name), "%s%u", + DEFAULT_TUN_NAME, tun_unit++); + + if (params && (params[0] != '\0')) { + TAP_LOG(DEBUG, "parameters (%s)", params); + + kvlist = rte_kvargs_parse(params, valid_arguments); + if (kvlist) { + if (rte_kvargs_count(kvlist, ETH_TAP_IFACE_ARG) == 1) { + ret = rte_kvargs_process(kvlist, + ETH_TAP_IFACE_ARG, + &set_interface_name, + tun_name); + + if (ret == -1) + goto leave; + } + } + } + pmd_link.link_speed = ETH_SPEED_NUM_10G; + + TAP_LOG(NOTICE, "Initializing pmd_tun for %s as %s", + name, tun_name); + + ret = eth_dev_tap_create(dev, tun_name, remote_iface, 0, + ETH_TUNTAP_TYPE_TUN); + +leave: + if (ret == -1) { + TAP_LOG(ERR, "Failed to create pmd for %s as %s", + name, tun_name); + tun_unit--; /* Restore the unit number */ + } + rte_kvargs_free(kvlist); + + return ret; +} + +/* Open a TAP interface device. + */ +static int +rte_pmd_tap_probe(struct rte_vdev_device *dev) +{ + const char *name, *params; + int ret; + struct rte_kvargs *kvlist = NULL; + int speed; + char tap_name[RTE_ETH_NAME_MAX_LEN]; + char remote_iface[RTE_ETH_NAME_MAX_LEN]; + struct ether_addr user_mac = { .addr_bytes = {0} }; + struct rte_eth_dev *eth_dev; + + strcpy(tuntap_name, "TAP"); + + name = rte_vdev_device_name(dev); + params = rte_vdev_device_args(dev); + + if (rte_eal_process_type() == RTE_PROC_SECONDARY && + strlen(params) == 0) { + eth_dev = rte_eth_dev_attach_secondary(name); + if (!eth_dev) { + TAP_LOG(ERR, "Failed to probe %s", name); + return -1; + } + /* TODO: request info from primary to set up Rx and Tx */ + eth_dev->dev_ops = &ops; + eth_dev->device = &dev->device; + rte_eth_dev_probing_finish(eth_dev); + return 0; + } + + speed = ETH_SPEED_NUM_10G; + snprintf(tap_name, sizeof(tap_name), "%s%u", + DEFAULT_TAP_NAME, tap_unit++); + memset(remote_iface, 0, RTE_ETH_NAME_MAX_LEN); + + if (params && (params[0] != '\0')) { + TAP_LOG(DEBUG, "parameters (%s)", params); + + kvlist = rte_kvargs_parse(params, valid_arguments); + if (kvlist) { + if (rte_kvargs_count(kvlist, ETH_TAP_IFACE_ARG) == 1) { + ret = rte_kvargs_process(kvlist, + ETH_TAP_IFACE_ARG, + &set_interface_name, + tap_name); + if (ret == -1) + goto leave; + } + + if (rte_kvargs_count(kvlist, ETH_TAP_REMOTE_ARG) == 1) { + ret = rte_kvargs_process(kvlist, + ETH_TAP_REMOTE_ARG, + &set_remote_iface, + remote_iface); + if (ret == -1) + goto leave; + } + + if (rte_kvargs_count(kvlist, ETH_TAP_MAC_ARG) == 1) { + ret = rte_kvargs_process(kvlist, + ETH_TAP_MAC_ARG, + &set_mac_type, + &user_mac); + if (ret == -1) + goto leave; + } + } + } + pmd_link.link_speed = speed; + + TAP_LOG(NOTICE, "Initializing pmd_tap for %s as %s", + name, tap_name); + + ret = eth_dev_tap_create(dev, tap_name, remote_iface, &user_mac, + ETH_TUNTAP_TYPE_TAP); + +leave: + if (ret == -1) { + TAP_LOG(ERR, "Failed to create pmd for %s as %s", + name, tap_name); + tap_unit--; /* Restore the unit number */ + } + rte_kvargs_free(kvlist); + + return ret; +} + +/* detach a TUNTAP device. + */ +static int +rte_pmd_tap_remove(struct rte_vdev_device *dev) +{ + struct rte_eth_dev *eth_dev = NULL; + struct pmd_internals *internals; + int i; + + /* find the ethdev entry */ + eth_dev = rte_eth_dev_allocated(rte_vdev_device_name(dev)); + if (!eth_dev) + return 0; + + internals = eth_dev->data->dev_private; + + TAP_LOG(DEBUG, "Closing %s Ethernet device on numa %u", + (internals->type == ETH_TUNTAP_TYPE_TAP) ? "TAP" : "TUN", + rte_socket_id()); + + if (internals->nlsk_fd) { + tap_flow_flush(eth_dev, NULL); + tap_flow_implicit_flush(internals, NULL); + tap_nl_final(internals->nlsk_fd); + } + for (i = 0; i < RTE_PMD_TAP_MAX_QUEUES; i++) { + if (internals->rxq[i].fd != -1) { + close(internals->rxq[i].fd); + internals->rxq[i].fd = -1; + } + if (internals->txq[i].fd != -1) { + close(internals->txq[i].fd); + internals->txq[i].fd = -1; + } + } + + close(internals->ioctl_sock); + rte_free(eth_dev->data->dev_private); + rte_eth_dev_release_port(eth_dev); + + if (internals->ka_fd != -1) { + close(internals->ka_fd); + internals->ka_fd = -1; + } + return 0; +} + +static struct rte_vdev_driver pmd_tun_drv = { + .probe = rte_pmd_tun_probe, + .remove = rte_pmd_tap_remove, +}; + +static struct rte_vdev_driver pmd_tap_drv = { + .probe = rte_pmd_tap_probe, + .remove = rte_pmd_tap_remove, +}; + +RTE_PMD_REGISTER_VDEV(net_tap, pmd_tap_drv); +RTE_PMD_REGISTER_VDEV(net_tun, pmd_tun_drv); +RTE_PMD_REGISTER_ALIAS(net_tap, eth_tap); +RTE_PMD_REGISTER_PARAM_STRING(net_tun, + ETH_TAP_IFACE_ARG "=<string> "); +RTE_PMD_REGISTER_PARAM_STRING(net_tap, + ETH_TAP_IFACE_ARG "=<string> " + ETH_TAP_MAC_ARG "=" ETH_TAP_MAC_ARG_FMT " " + ETH_TAP_REMOTE_ARG "=<string>"); +int tap_logtype; + +RTE_INIT(tap_init_log) +{ + tap_logtype = rte_log_register("pmd.net.tap"); + if (tap_logtype >= 0) + rte_log_set_level(tap_logtype, RTE_LOG_NOTICE); +} diff --git a/src/spdk/dpdk/drivers/net/tap/rte_eth_tap.h b/src/spdk/dpdk/drivers/net/tap/rte_eth_tap.h new file mode 100644 index 00000000..44e2773f --- /dev/null +++ b/src/spdk/dpdk/drivers/net/tap/rte_eth_tap.h @@ -0,0 +1,99 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright 2017 6WIND S.A. + * Copyright 2017 Mellanox Technologies, Ltd + */ + +#ifndef _RTE_ETH_TAP_H_ +#define _RTE_ETH_TAP_H_ + +#include <sys/queue.h> +#include <sys/uio.h> +#include <inttypes.h> +#include <net/if.h> + +#include <linux/if_tun.h> + +#include <rte_ethdev_driver.h> +#include <rte_ether.h> +#include <rte_gso.h> +#include "tap_log.h" + +#ifdef IFF_MULTI_QUEUE +#define RTE_PMD_TAP_MAX_QUEUES TAP_MAX_QUEUES +#else +#define RTE_PMD_TAP_MAX_QUEUES 1 +#endif +#define MAX_GSO_MBUFS 64 + +enum rte_tuntap_type { + ETH_TUNTAP_TYPE_UNKNOWN, + ETH_TUNTAP_TYPE_TUN, + ETH_TUNTAP_TYPE_TAP, + ETH_TUNTAP_TYPE_MAX, +}; + +struct pkt_stats { + uint64_t opackets; /* Number of output packets */ + uint64_t ipackets; /* Number of input packets */ + uint64_t obytes; /* Number of bytes on output */ + uint64_t ibytes; /* Number of bytes on input */ + uint64_t errs; /* Number of TX error packets */ + uint64_t ierrors; /* Number of RX error packets */ + uint64_t rx_nombuf; /* Nb of RX mbuf alloc failures */ +}; + +struct rx_queue { + struct rte_mempool *mp; /* Mempool for RX packets */ + uint32_t trigger_seen; /* Last seen Rx trigger value */ + uint16_t in_port; /* Port ID */ + int fd; + struct pkt_stats stats; /* Stats for this RX queue */ + uint16_t nb_rx_desc; /* max number of mbufs available */ + struct rte_eth_rxmode *rxmode; /* RX features */ + struct rte_mbuf *pool; /* mbufs pool for this queue */ + struct iovec (*iovecs)[]; /* descriptors for this queue */ + struct tun_pi pi; /* packet info for iovecs */ +}; + +struct tx_queue { + int fd; + int type; /* Type field - TUN|TAP */ + uint16_t *mtu; /* Pointer to MTU from dev_data */ + uint16_t csum:1; /* Enable checksum offloading */ + struct pkt_stats stats; /* Stats for this TX queue */ + struct rte_gso_ctx gso_ctx; /* GSO context */ +}; + +struct pmd_internals { + struct rte_eth_dev *dev; /* Ethernet device. */ + char remote_iface[RTE_ETH_NAME_MAX_LEN]; /* Remote netdevice name */ + char name[RTE_ETH_NAME_MAX_LEN]; /* Internal Tap device name */ + int type; /* Type field - TUN|TAP */ + struct ether_addr eth_addr; /* Mac address of the device port */ + struct ifreq remote_initial_flags; /* Remote netdevice flags on init */ + int remote_if_index; /* remote netdevice IF_INDEX */ + int if_index; /* IF_INDEX for the port */ + int ioctl_sock; /* socket for ioctl calls */ + int nlsk_fd; /* Netlink socket fd */ + int flow_isolate; /* 1 if flow isolation is enabled */ + int flower_support; /* 1 if kernel supports, else 0 */ + int flower_vlan_support; /* 1 if kernel supports, else 0 */ + int rss_enabled; /* 1 if RSS is enabled, else 0 */ + /* implicit rules set when RSS is enabled */ + int map_fd; /* BPF RSS map fd */ + int bpf_fd[RTE_PMD_TAP_MAX_QUEUES];/* List of bpf fds per queue */ + LIST_HEAD(tap_rss_flows, rte_flow) rss_flows; + LIST_HEAD(tap_flows, rte_flow) flows; /* rte_flow rules */ + /* implicit rte_flow rules set when a remote device is active */ + LIST_HEAD(tap_implicit_flows, rte_flow) implicit_flows; + struct rx_queue rxq[RTE_PMD_TAP_MAX_QUEUES]; /* List of RX queues */ + struct tx_queue txq[RTE_PMD_TAP_MAX_QUEUES]; /* List of TX queues */ + struct rte_intr_handle intr_handle; /* LSC interrupt handle. */ + int ka_fd; /* keep-alive file descriptor */ +}; + +/* tap_intr.c */ + +int tap_rx_intr_vec_set(struct rte_eth_dev *dev, int set); + +#endif /* _RTE_ETH_TAP_H_ */ diff --git a/src/spdk/dpdk/drivers/net/tap/rte_pmd_tap_version.map b/src/spdk/dpdk/drivers/net/tap/rte_pmd_tap_version.map new file mode 100644 index 00000000..31eca32e --- /dev/null +++ b/src/spdk/dpdk/drivers/net/tap/rte_pmd_tap_version.map @@ -0,0 +1,4 @@ +DPDK_17.02 { + + local: *; +}; diff --git a/src/spdk/dpdk/drivers/net/tap/tap_bpf.h b/src/spdk/dpdk/drivers/net/tap/tap_bpf.h new file mode 100644 index 00000000..9192686a --- /dev/null +++ b/src/spdk/dpdk/drivers/net/tap/tap_bpf.h @@ -0,0 +1,117 @@ +/* SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0 + * Copyright 2017 Mellanox Technologies, Ltd + */ + +#ifndef __TAP_BPF_H__ +#define __TAP_BPF_H__ + +#include <tap_autoconf.h> + +/* Do not #include <linux/bpf.h> since eBPF must compile on different + * distros which may include partial definitions for eBPF (while the + * kernel itself may support eBPF). Instead define here all that is needed + */ + +/* BPF_MAP_UPDATE_ELEM command flags */ +#define BPF_ANY 0 /* create a new element or update an existing */ + +/* BPF architecture instruction struct */ +struct bpf_insn { + __u8 code; + __u8 dst_reg:4; + __u8 src_reg:4; + __s16 off; + __s32 imm; /* immediate value */ +}; + +/* BPF program types */ +enum bpf_prog_type { + BPF_PROG_TYPE_UNSPEC, + BPF_PROG_TYPE_SOCKET_FILTER, + BPF_PROG_TYPE_KPROBE, + BPF_PROG_TYPE_SCHED_CLS, + BPF_PROG_TYPE_SCHED_ACT, +}; + +/* BPF commands types */ +enum bpf_cmd { + BPF_MAP_CREATE, + BPF_MAP_LOOKUP_ELEM, + BPF_MAP_UPDATE_ELEM, + BPF_MAP_DELETE_ELEM, + BPF_MAP_GET_NEXT_KEY, + BPF_PROG_LOAD, +}; + +/* BPF maps types */ +enum bpf_map_type { + BPF_MAP_TYPE_UNSPEC, + BPF_MAP_TYPE_HASH, +}; + +/* union of anonymous structs used with TAP BPF commands */ +union bpf_attr { + /* BPF_MAP_CREATE command */ + struct { + __u32 map_type; + __u32 key_size; + __u32 value_size; + __u32 max_entries; + __u32 map_flags; + __u32 inner_map_fd; + }; + + /* BPF_MAP_UPDATE_ELEM, BPF_MAP_DELETE_ELEM commands */ + struct { + __u32 map_fd; + __aligned_u64 key; + union { + __aligned_u64 value; + __aligned_u64 next_key; + }; + __u64 flags; + }; + + /* BPF_PROG_LOAD command */ + struct { + __u32 prog_type; + __u32 insn_cnt; + __aligned_u64 insns; + __aligned_u64 license; + __u32 log_level; + __u32 log_size; + __aligned_u64 log_buf; + __u32 kern_version; + __u32 prog_flags; + }; +} __attribute__((aligned(8))); + +#ifndef __NR_bpf +# if defined(__i386__) +# define __NR_bpf 357 +# elif defined(__x86_64__) +# define __NR_bpf 321 +# elif defined(__arm__) +# define __NR_bpf 386 +# elif defined(__aarch64__) +# define __NR_bpf 280 +# elif defined(__sparc__) +# define __NR_bpf 349 +# elif defined(__s390__) +# define __NR_bpf 351 +# elif defined(__powerpc__) +# define __NR_bpf 361 +# else +# error __NR_bpf not defined +# endif +#endif + +enum { + BPF_MAP_ID_KEY, + BPF_MAP_ID_SIMPLE, +}; + +static int bpf_load(enum bpf_prog_type type, const struct bpf_insn *insns, + size_t insns_cnt, const char *license); + +#endif /* __TAP_BPF_H__ */ diff --git a/src/spdk/dpdk/drivers/net/tap/tap_bpf_api.c b/src/spdk/dpdk/drivers/net/tap/tap_bpf_api.c new file mode 100644 index 00000000..98f6a760 --- /dev/null +++ b/src/spdk/dpdk/drivers/net/tap/tap_bpf_api.c @@ -0,0 +1,190 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright 2017 Mellanox Technologies, Ltd + */ + +#include <errno.h> +#include <string.h> +#include <unistd.h> +#include <sys/queue.h> + +#include <rte_malloc.h> +#include <rte_eth_tap.h> +#include <tap_flow.h> +#include <tap_autoconf.h> +#include <tap_tcmsgs.h> +#include <tap_bpf.h> +#include <tap_bpf_insns.h> + +/** + * Load BPF program (section cls_q) into the kernel and return a bpf fd + * + * @param queue_idx + * Queue index matching packet cb + * + * @return + * -1 if the BPF program couldn't be loaded. An fd (int) otherwise. + */ +int tap_flow_bpf_cls_q(__u32 queue_idx) +{ + cls_q_insns[1].imm = queue_idx; + + return bpf_load(BPF_PROG_TYPE_SCHED_CLS, + (struct bpf_insn *)cls_q_insns, + RTE_DIM(cls_q_insns), + "Dual BSD/GPL"); +} + +/** + * Load BPF program (section l3_l4) into the kernel and return a bpf fd. + * + * @param[in] key_idx + * RSS MAP key index + * + * @param[in] map_fd + * BPF RSS map file descriptor + * + * @return + * -1 if the BPF program couldn't be loaded. An fd (int) otherwise. + */ +int tap_flow_bpf_calc_l3_l4_hash(__u32 key_idx, int map_fd) +{ + l3_l4_hash_insns[4].imm = key_idx; + l3_l4_hash_insns[9].imm = map_fd; + + return bpf_load(BPF_PROG_TYPE_SCHED_ACT, + (struct bpf_insn *)l3_l4_hash_insns, + RTE_DIM(l3_l4_hash_insns), + "Dual BSD/GPL"); +} + +/** + * Helper function to convert a pointer to unsigned 64 bits + * + * @param[in] ptr + * pointer to address + * + * @return + * 64 bit unsigned long type of pointer address + */ +static inline __u64 ptr_to_u64(const void *ptr) +{ + return (__u64)(unsigned long)ptr; +} + +/** + * Call BPF system call + * + * @param[in] cmd + * BPF command for program loading, map creation, map entry update, etc + * + * @param[in] attr + * System call attributes relevant to system call command + * + * @param[in] size + * size of attr parameter + * + * @return + * -1 if BPF system call failed, 0 otherwise + */ +static inline int sys_bpf(enum bpf_cmd cmd, union bpf_attr *attr, + unsigned int size) +{ + return syscall(__NR_bpf, cmd, attr, size); +} + +/** + * Load BPF instructions to kernel + * + * @param[in] type + * BPF program type: classifieir or action + * + * @param[in] insns + * Array of BPF instructions (equivalent to BPF instructions) + * + * @param[in] insns_cnt + * Number of BPF instructions (size of array) + * + * @param[in] lincense + * License string that must be acknowledged by the kernel + * + * @return + * -1 if the BPF program couldn't be loaded, fd (file descriptor) otherwise + */ +static int bpf_load(enum bpf_prog_type type, + const struct bpf_insn *insns, + size_t insns_cnt, + const char *license) +{ + union bpf_attr attr = {}; + + bzero(&attr, sizeof(attr)); + attr.prog_type = type; + attr.insn_cnt = (__u32)insns_cnt; + attr.insns = ptr_to_u64(insns); + attr.license = ptr_to_u64(license); + attr.log_buf = ptr_to_u64(NULL); + attr.log_level = 0; + attr.kern_version = 0; + + return sys_bpf(BPF_PROG_LOAD, &attr, sizeof(attr)); +} + +/** + * Create BPF map for RSS rules + * + * @param[in] key_size + * map RSS key size + * + * @param[in] value_size + * Map RSS value size + * + * @param[in] max_entries + * Map max number of RSS entries (limit on max RSS rules) + * + * @return + * -1 if BPF map couldn't be created, map fd otherwise + */ +int tap_flow_bpf_rss_map_create(unsigned int key_size, + unsigned int value_size, + unsigned int max_entries) +{ + union bpf_attr attr = {}; + + bzero(&attr, sizeof(attr)); + attr.map_type = BPF_MAP_TYPE_HASH; + attr.key_size = key_size; + attr.value_size = value_size; + attr.max_entries = max_entries; + + return sys_bpf(BPF_MAP_CREATE, &attr, sizeof(attr)); +} + +/** + * Update RSS entry in BPF map + * + * @param[in] fd + * RSS map fd + * + * @param[in] key + * Pointer to RSS key whose entry is updated + * + * @param[in] value + * Pointer to RSS new updated value + * + * @return + * -1 if RSS entry failed to be updated, 0 otherwise + */ +int tap_flow_bpf_update_rss_elem(int fd, void *key, void *value) +{ + union bpf_attr attr = {}; + + bzero(&attr, sizeof(attr)); + + attr.map_type = BPF_MAP_TYPE_HASH; + attr.map_fd = fd; + attr.key = ptr_to_u64(key); + attr.value = ptr_to_u64(value); + attr.flags = BPF_ANY; + + return sys_bpf(BPF_MAP_UPDATE_ELEM, &attr, sizeof(attr)); +} diff --git a/src/spdk/dpdk/drivers/net/tap/tap_bpf_insns.h b/src/spdk/dpdk/drivers/net/tap/tap_bpf_insns.h new file mode 100644 index 00000000..79e3e66b --- /dev/null +++ b/src/spdk/dpdk/drivers/net/tap/tap_bpf_insns.h @@ -0,0 +1,1696 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright 2017 Mellanox Technologies, Ltd + */ + +#include <tap_bpf.h> + +/* bpf_insn array matching cls_q section. See tap_bpf_program.c file */ +struct bpf_insn cls_q_insns[] = { + {0x61, 2, 1, 52, 0x00000000}, + {0x18, 3, 0, 0, 0xdeadbeef}, + {0x00, 0, 0, 0, 0x00000000}, + {0x63, 10, 3, -4, 0x00000000}, + {0xb7, 0, 0, 0, 0x00000000}, + {0x61, 3, 10, -4, 0x00000000}, + {0x07, 3, 0, 0, 0x7cafe800}, + {0x67, 3, 0, 0, 0x00000020}, + {0x77, 3, 0, 0, 0x00000020}, + {0x5d, 2, 3, 4, 0x00000000}, + {0xb7, 2, 0, 0, 0x00000000}, + {0x63, 1, 2, 52, 0x00000000}, + {0x18, 0, 0, 0, 0xffffffff}, + {0x00, 0, 0, 0, 0x00000000}, + {0x95, 0, 0, 0, 0x00000000}, +}; + +/* bpf_insn array matching l3_l4 section. see tap_bpf_program.c file */ +struct bpf_insn l3_l4_hash_insns[] = { + {0xbf, 7, 1, 0, 0x00000000}, + {0x61, 8, 7, 16, 0x00000000}, + {0x61, 6, 7, 76, 0x00000000}, + {0x61, 9, 7, 80, 0x00000000}, + {0x18, 1, 0, 0, 0xdeadbeef}, + {0x00, 0, 0, 0, 0x00000000}, + {0x63, 10, 1, -4, 0x00000000}, + {0xbf, 2, 10, 0, 0x00000000}, + {0x07, 2, 0, 0, 0xfffffffc}, + {0x18, 1, 1, 0, 0x0000cafe}, + {0x00, 0, 0, 0, 0x00000000}, + {0x85, 0, 0, 0, 0x00000001}, + {0x55, 0, 0, 21, 0x00000000}, + {0xb7, 1, 0, 0, 0x00000a64}, + {0x6b, 10, 1, -16, 0x00000000}, + {0x18, 1, 0, 0, 0x69666e6f}, + {0x00, 0, 0, 0, 0x65727567}, + {0x7b, 10, 1, -24, 0x00000000}, + {0x18, 1, 0, 0, 0x6e207369}, + {0x00, 0, 0, 0, 0x6320746f}, + {0x7b, 10, 1, -32, 0x00000000}, + {0x18, 1, 0, 0, 0x20737372}, + {0x00, 0, 0, 0, 0x2079656b}, + {0x7b, 10, 1, -40, 0x00000000}, + {0x18, 1, 0, 0, 0x68736168}, + {0x00, 0, 0, 0, 0x203a2928}, + {0x7b, 10, 1, -48, 0x00000000}, + {0xb7, 7, 0, 0, 0x00000000}, + {0x73, 10, 7, -14, 0x00000000}, + {0xbf, 1, 10, 0, 0x00000000}, + {0x07, 1, 0, 0, 0xffffffd0}, + {0xb7, 2, 0, 0, 0x00000023}, + {0x85, 0, 0, 0, 0x00000006}, + {0x05, 0, 0, 1632, 0x00000000}, + {0xb7, 1, 0, 0, 0x0000000e}, + {0x61, 2, 7, 20, 0x00000000}, + {0x15, 2, 0, 10, 0x00000000}, + {0x61, 2, 7, 28, 0x00000000}, + {0x55, 2, 0, 8, 0x0000a888}, + {0xbf, 2, 7, 0, 0x00000000}, + {0xb7, 7, 0, 0, 0x00000000}, + {0xbf, 1, 6, 0, 0x00000000}, + {0x07, 1, 0, 0, 0x00000012}, + {0x2d, 1, 9, 1622, 0x00000000}, + {0xb7, 1, 0, 0, 0x00000012}, + {0x69, 8, 6, 16, 0x00000000}, + {0xbf, 7, 2, 0, 0x00000000}, + {0x7b, 10, 7, -56, 0x00000000}, + {0x57, 8, 0, 0, 0x0000ffff}, + {0x15, 8, 0, 409, 0x0000dd86}, + {0xb7, 7, 0, 0, 0x00000003}, + {0x55, 8, 0, 1614, 0x00000008}, + {0x0f, 6, 1, 0, 0x00000000}, + {0xb7, 7, 0, 0, 0x00000000}, + {0xbf, 1, 6, 0, 0x00000000}, + {0x07, 1, 0, 0, 0x00000018}, + {0x2d, 1, 9, 1609, 0x00000000}, + {0x71, 3, 6, 12, 0x00000000}, + {0xbf, 1, 3, 0, 0x00000000}, + {0x67, 1, 0, 0, 0x00000038}, + {0xc7, 1, 0, 0, 0x00000020}, + {0x77, 1, 0, 0, 0x0000001f}, + {0x57, 1, 0, 0, 0x2cc681d1}, + {0x67, 3, 0, 0, 0x00000018}, + {0xbf, 4, 3, 0, 0x00000000}, + {0x57, 4, 0, 0, 0x40000000}, + {0xb7, 2, 0, 0, 0x00000000}, + {0x1d, 4, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x598d03a2}, + {0xbf, 4, 3, 0, 0x00000000}, + {0x57, 4, 0, 0, 0x20000000}, + {0x1d, 4, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xb31a0745}, + {0xbf, 4, 3, 0, 0x00000000}, + {0x57, 4, 0, 0, 0x10000000}, + {0x1d, 4, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x66340e8a}, + {0xbf, 4, 3, 0, 0x00000000}, + {0x57, 4, 0, 0, 0x08000000}, + {0x1d, 4, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xcc681d15}, + {0xbf, 4, 3, 0, 0x00000000}, + {0x57, 4, 0, 0, 0x04000000}, + {0x1d, 4, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x98d03a2b}, + {0xbf, 4, 3, 0, 0x00000000}, + {0x57, 4, 0, 0, 0x02000000}, + {0x1d, 4, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x31a07456}, + {0x57, 3, 0, 0, 0x01000000}, + {0x1d, 3, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x6340e8ad}, + {0x71, 3, 6, 13, 0x00000000}, + {0x67, 3, 0, 0, 0x00000010}, + {0xbf, 4, 3, 0, 0x00000000}, + {0x57, 4, 0, 0, 0x00800000}, + {0x1d, 4, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xc681d15b}, + {0xbf, 4, 3, 0, 0x00000000}, + {0x57, 4, 0, 0, 0x00400000}, + {0x1d, 4, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x8d03a2b7}, + {0xbf, 4, 3, 0, 0x00000000}, + {0x57, 4, 0, 0, 0x00200000}, + {0x1d, 4, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x1a07456f}, + {0xbf, 4, 3, 0, 0x00000000}, + {0x57, 4, 0, 0, 0x00100000}, + {0x1d, 4, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x340e8ade}, + {0xbf, 4, 3, 0, 0x00000000}, + {0x57, 4, 0, 0, 0x00080000}, + {0x1d, 4, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x681d15bd}, + {0xbf, 4, 3, 0, 0x00000000}, + {0x57, 4, 0, 0, 0x00040000}, + {0x1d, 4, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xd03a2b7b}, + {0xbf, 4, 3, 0, 0x00000000}, + {0x57, 4, 0, 0, 0x00020000}, + {0x1d, 4, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xa07456f6}, + {0x57, 3, 0, 0, 0x00010000}, + {0x1d, 3, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x40e8aded}, + {0x71, 3, 6, 14, 0x00000000}, + {0x67, 3, 0, 0, 0x00000008}, + {0xbf, 4, 3, 0, 0x00000000}, + {0x57, 4, 0, 0, 0x00008000}, + {0x1d, 4, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x81d15bdb}, + {0xbf, 4, 3, 0, 0x00000000}, + {0x57, 4, 0, 0, 0x00004000}, + {0x1d, 4, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x03a2b7b7}, + {0xbf, 4, 3, 0, 0x00000000}, + {0x57, 4, 0, 0, 0x00002000}, + {0x1d, 4, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x07456f6f}, + {0xbf, 4, 3, 0, 0x00000000}, + {0x57, 4, 0, 0, 0x00001000}, + {0x1d, 4, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x0e8adedf}, + {0xbf, 4, 3, 0, 0x00000000}, + {0x57, 4, 0, 0, 0x00000800}, + {0x1d, 4, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x1d15bdbf}, + {0xbf, 4, 3, 0, 0x00000000}, + {0x57, 4, 0, 0, 0x00000400}, + {0x1d, 4, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x3a2b7b7e}, + {0xbf, 4, 3, 0, 0x00000000}, + {0x57, 4, 0, 0, 0x00000200}, + {0x1d, 4, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x7456f6fd}, + {0x57, 3, 0, 0, 0x00000100}, + {0x1d, 3, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xe8adedfa}, + {0x71, 3, 6, 15, 0x00000000}, + {0xbf, 4, 3, 0, 0x00000000}, + {0x57, 4, 0, 0, 0x00000080}, + {0x1d, 4, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xd15bdbf4}, + {0xbf, 4, 3, 0, 0x00000000}, + {0x57, 4, 0, 0, 0x00000040}, + {0x1d, 4, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xa2b7b7e9}, + {0xbf, 4, 3, 0, 0x00000000}, + {0x57, 4, 0, 0, 0x00000020}, + {0x1d, 4, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x456f6fd3}, + {0xbf, 4, 3, 0, 0x00000000}, + {0x57, 4, 0, 0, 0x00000010}, + {0x1d, 4, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x8adedfa7}, + {0xbf, 4, 3, 0, 0x00000000}, + {0x57, 4, 0, 0, 0x00000008}, + {0x1d, 4, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x15bdbf4f}, + {0xbf, 4, 3, 0, 0x00000000}, + {0x57, 4, 0, 0, 0x00000004}, + {0x1d, 4, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x2b7b7e9e}, + {0xbf, 4, 3, 0, 0x00000000}, + {0x57, 4, 0, 0, 0x00000002}, + {0x1d, 4, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x56f6fd3d}, + {0x57, 3, 0, 0, 0x00000001}, + {0x1d, 3, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xadedfa7b}, + {0x71, 4, 6, 16, 0x00000000}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x67, 5, 0, 0, 0x00000038}, + {0xc7, 5, 0, 0, 0x00000020}, + {0xb7, 3, 0, 0, 0xffffffff}, + {0x6d, 5, 3, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x5bdbf4f7}, + {0x67, 4, 0, 0, 0x00000018}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x40000000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xb7b7e9ef}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x20000000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x6f6fd3df}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x10000000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xdedfa7bf}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x08000000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xbdbf4f7f}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x04000000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x7b7e9eff}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x02000000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xf6fd3dff}, + {0x57, 4, 0, 0, 0x01000000}, + {0x1d, 4, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xedfa7bfe}, + {0x71, 4, 6, 17, 0x00000000}, + {0x67, 4, 0, 0, 0x00000010}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00800000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xdbf4f7fc}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00400000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xb7e9eff9}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00200000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x6fd3dff2}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00100000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xdfa7bfe5}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00080000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xbf4f7fca}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00040000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x7e9eff94}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00020000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xfd3dff28}, + {0x57, 4, 0, 0, 0x00010000}, + {0x1d, 4, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xfa7bfe51}, + {0x71, 4, 6, 18, 0x00000000}, + {0x67, 4, 0, 0, 0x00000008}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00008000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xf4f7fca2}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00004000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xe9eff945}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00002000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xd3dff28a}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00001000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xa7bfe514}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000800}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x4f7fca28}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000400}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x9eff9450}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000200}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x3dff28a0}, + {0x57, 4, 0, 0, 0x00000100}, + {0x1d, 4, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x7bfe5141}, + {0x71, 4, 6, 19, 0x00000000}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000080}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xf7fca283}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000040}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xeff94506}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000020}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xdff28a0c}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000010}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xbfe51418}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000008}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x7fca2831}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000004}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xff945063}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000002}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xff28a0c6}, + {0x57, 4, 0, 0, 0x00000001}, + {0x1d, 4, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xfe51418c}, + {0x71, 4, 6, 20, 0x00000000}, + {0x67, 4, 0, 0, 0x00000008}, + {0x71, 5, 6, 21, 0x00000000}, + {0x4f, 4, 5, 0, 0x00000000}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x67, 5, 0, 0, 0x00000030}, + {0xc7, 5, 0, 0, 0x00000020}, + {0x6d, 5, 3, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xfca28319}, + {0x67, 4, 0, 0, 0x00000010}, + {0xbf, 3, 4, 0, 0x00000000}, + {0x57, 3, 0, 0, 0x40000000}, + {0x1d, 3, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xf9450633}, + {0xbf, 3, 4, 0, 0x00000000}, + {0x57, 3, 0, 0, 0x20000000}, + {0x1d, 3, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xf28a0c67}, + {0xbf, 3, 4, 0, 0x00000000}, + {0x57, 3, 0, 0, 0x10000000}, + {0x1d, 3, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xe51418ce}, + {0xbf, 3, 4, 0, 0x00000000}, + {0x57, 3, 0, 0, 0x08000000}, + {0x1d, 3, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xca28319d}, + {0xbf, 3, 4, 0, 0x00000000}, + {0x57, 3, 0, 0, 0x04000000}, + {0x1d, 3, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x9450633b}, + {0xbf, 3, 4, 0, 0x00000000}, + {0x57, 3, 0, 0, 0x02000000}, + {0x1d, 3, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x28a0c676}, + {0xbf, 3, 4, 0, 0x00000000}, + {0x57, 3, 0, 0, 0x01000000}, + {0x1d, 3, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x51418ced}, + {0xbf, 3, 4, 0, 0x00000000}, + {0x57, 3, 0, 0, 0x00800000}, + {0x1d, 3, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xa28319db}, + {0xbf, 3, 4, 0, 0x00000000}, + {0x57, 3, 0, 0, 0x00400000}, + {0x1d, 3, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x450633b6}, + {0xbf, 3, 4, 0, 0x00000000}, + {0x57, 3, 0, 0, 0x00200000}, + {0x1d, 3, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x8a0c676c}, + {0xbf, 3, 4, 0, 0x00000000}, + {0x57, 3, 0, 0, 0x00100000}, + {0x1d, 3, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x1418ced8}, + {0xbf, 3, 4, 0, 0x00000000}, + {0x57, 3, 0, 0, 0x00080000}, + {0x1d, 3, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x28319db1}, + {0xbf, 3, 4, 0, 0x00000000}, + {0x57, 3, 0, 0, 0x00040000}, + {0x1d, 3, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x50633b63}, + {0xbf, 3, 4, 0, 0x00000000}, + {0x57, 3, 0, 0, 0x00020000}, + {0x1d, 3, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xa0c676c6}, + {0x57, 4, 0, 0, 0x00010000}, + {0x1d, 4, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x418ced8d}, + {0x71, 3, 6, 22, 0x00000000}, + {0x67, 3, 0, 0, 0x00000008}, + {0xbf, 4, 3, 0, 0x00000000}, + {0x57, 4, 0, 0, 0x00008000}, + {0x1d, 4, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x8319db1a}, + {0xbf, 4, 3, 0, 0x00000000}, + {0x57, 4, 0, 0, 0x00004000}, + {0x1d, 4, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x0633b634}, + {0xbf, 4, 3, 0, 0x00000000}, + {0x57, 4, 0, 0, 0x00002000}, + {0x1d, 4, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x0c676c68}, + {0xbf, 4, 3, 0, 0x00000000}, + {0x57, 4, 0, 0, 0x00001000}, + {0x1d, 4, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x18ced8d1}, + {0xbf, 4, 3, 0, 0x00000000}, + {0x57, 4, 0, 0, 0x00000800}, + {0x1d, 4, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x319db1a3}, + {0xbf, 4, 3, 0, 0x00000000}, + {0x57, 4, 0, 0, 0x00000400}, + {0x1d, 4, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x633b6347}, + {0xbf, 4, 3, 0, 0x00000000}, + {0x57, 4, 0, 0, 0x00000200}, + {0x1d, 4, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xc676c68f}, + {0x57, 3, 0, 0, 0x00000100}, + {0x1d, 3, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x8ced8d1f}, + {0x71, 3, 6, 23, 0x00000000}, + {0xbf, 4, 3, 0, 0x00000000}, + {0x57, 4, 0, 0, 0x00000080}, + {0x1d, 4, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x19db1a3e}, + {0xbf, 4, 3, 0, 0x00000000}, + {0x57, 4, 0, 0, 0x00000040}, + {0x79, 5, 10, -56, 0x00000000}, + {0x1d, 4, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x33b6347d}, + {0xbf, 4, 3, 0, 0x00000000}, + {0x57, 4, 0, 0, 0x00000020}, + {0x1d, 4, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x676c68fa}, + {0xbf, 4, 3, 0, 0x00000000}, + {0x57, 4, 0, 0, 0x00000010}, + {0x1d, 4, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xced8d1f4}, + {0xbf, 4, 3, 0, 0x00000000}, + {0x57, 4, 0, 0, 0x00000008}, + {0x1d, 4, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x9db1a3e9}, + {0xbf, 4, 3, 0, 0x00000000}, + {0x57, 4, 0, 0, 0x00000004}, + {0x1d, 4, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x3b6347d2}, + {0xbf, 4, 3, 0, 0x00000000}, + {0x57, 4, 0, 0, 0x00000002}, + {0x1d, 4, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x76c68fa5}, + {0x57, 3, 0, 0, 0x00000001}, + {0x1d, 3, 2, 1177, 0x00000000}, + {0xa7, 1, 0, 0, 0xed8d1f4a}, + {0x05, 0, 0, 1175, 0x00000000}, + {0x0f, 6, 1, 0, 0x00000000}, + {0xb7, 7, 0, 0, 0x00000000}, + {0xbf, 1, 6, 0, 0x00000000}, + {0x07, 1, 0, 0, 0x0000002c}, + {0x2d, 1, 9, 1202, 0x00000000}, + {0x61, 4, 6, 8, 0x00000000}, + {0xbf, 1, 4, 0, 0x00000000}, + {0x67, 1, 0, 0, 0x00000038}, + {0xc7, 1, 0, 0, 0x00000020}, + {0x77, 1, 0, 0, 0x0000001f}, + {0x57, 1, 0, 0, 0x2cc681d1}, + {0xbf, 3, 4, 0, 0x00000000}, + {0x57, 3, 0, 0, 0x00000040}, + {0xb7, 2, 0, 0, 0x00000000}, + {0x1d, 3, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x598d03a2}, + {0xbf, 3, 4, 0, 0x00000000}, + {0x57, 3, 0, 0, 0x00000020}, + {0x1d, 3, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xb31a0745}, + {0xbf, 3, 4, 0, 0x00000000}, + {0x57, 3, 0, 0, 0x00000010}, + {0x1d, 3, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x66340e8a}, + {0xbf, 3, 4, 0, 0x00000000}, + {0x57, 3, 0, 0, 0x00000008}, + {0x1d, 3, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xcc681d15}, + {0xbf, 3, 4, 0, 0x00000000}, + {0x57, 3, 0, 0, 0x00000004}, + {0x1d, 3, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x98d03a2b}, + {0xbf, 3, 4, 0, 0x00000000}, + {0x57, 3, 0, 0, 0x00000002}, + {0x1d, 3, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x31a07456}, + {0xbf, 3, 4, 0, 0x00000000}, + {0x57, 3, 0, 0, 0x00000001}, + {0x1d, 3, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x6340e8ad}, + {0xbf, 3, 4, 0, 0x00000000}, + {0x57, 3, 0, 0, 0x00008000}, + {0x1d, 3, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xc681d15b}, + {0xbf, 3, 4, 0, 0x00000000}, + {0x57, 3, 0, 0, 0x00004000}, + {0x1d, 3, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x8d03a2b7}, + {0xbf, 3, 4, 0, 0x00000000}, + {0x57, 3, 0, 0, 0x00002000}, + {0x1d, 3, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x1a07456f}, + {0xbf, 3, 4, 0, 0x00000000}, + {0x57, 3, 0, 0, 0x00001000}, + {0x1d, 3, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x340e8ade}, + {0xbf, 3, 4, 0, 0x00000000}, + {0x57, 3, 0, 0, 0x00000800}, + {0x1d, 3, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x681d15bd}, + {0xbf, 3, 4, 0, 0x00000000}, + {0x57, 3, 0, 0, 0x00000400}, + {0x1d, 3, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xd03a2b7b}, + {0xbf, 3, 4, 0, 0x00000000}, + {0x57, 3, 0, 0, 0x00000200}, + {0x1d, 3, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xa07456f6}, + {0xbf, 3, 4, 0, 0x00000000}, + {0x57, 3, 0, 0, 0x00000100}, + {0x1d, 3, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x40e8aded}, + {0xbf, 3, 4, 0, 0x00000000}, + {0x57, 3, 0, 0, 0x00800000}, + {0x1d, 3, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x81d15bdb}, + {0xbf, 3, 4, 0, 0x00000000}, + {0x57, 3, 0, 0, 0x00400000}, + {0x1d, 3, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x03a2b7b7}, + {0xbf, 3, 4, 0, 0x00000000}, + {0x57, 3, 0, 0, 0x00200000}, + {0x1d, 3, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x07456f6f}, + {0xbf, 3, 4, 0, 0x00000000}, + {0x57, 3, 0, 0, 0x00100000}, + {0x1d, 3, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x0e8adedf}, + {0xbf, 3, 4, 0, 0x00000000}, + {0x57, 3, 0, 0, 0x00080000}, + {0x1d, 3, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x1d15bdbf}, + {0xbf, 3, 4, 0, 0x00000000}, + {0x57, 3, 0, 0, 0x00040000}, + {0x1d, 3, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x3a2b7b7e}, + {0xbf, 3, 4, 0, 0x00000000}, + {0x57, 3, 0, 0, 0x00020000}, + {0x1d, 3, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x7456f6fd}, + {0xbf, 3, 4, 0, 0x00000000}, + {0x57, 3, 0, 0, 0x00010000}, + {0x1d, 3, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xe8adedfa}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x67, 5, 0, 0, 0x00000020}, + {0xc7, 5, 0, 0, 0x00000020}, + {0xb7, 3, 0, 0, 0xffffffff}, + {0x6d, 5, 3, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xd15bdbf4}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x40000000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xa2b7b7e9}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x20000000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x456f6fd3}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x10000000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x8adedfa7}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x08000000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x15bdbf4f}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x04000000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x2b7b7e9e}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x02000000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x56f6fd3d}, + {0x57, 4, 0, 0, 0x01000000}, + {0x1d, 4, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xadedfa7b}, + {0x61, 4, 6, 12, 0x00000000}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000080}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x5bdbf4f7}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000040}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xb7b7e9ef}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000020}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x6f6fd3df}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000010}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xdedfa7bf}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000008}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xbdbf4f7f}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000004}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x7b7e9eff}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000002}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xf6fd3dff}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000001}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xedfa7bfe}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00008000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xdbf4f7fc}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00004000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xb7e9eff9}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00002000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x6fd3dff2}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00001000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xdfa7bfe5}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000800}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xbf4f7fca}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000400}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x7e9eff94}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000200}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xfd3dff28}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000100}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xfa7bfe51}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00800000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xf4f7fca2}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00400000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xe9eff945}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00200000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xd3dff28a}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00100000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xa7bfe514}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00080000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x4f7fca28}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00040000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x9eff9450}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00020000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x3dff28a0}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00010000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x7bfe5141}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x67, 5, 0, 0, 0x00000020}, + {0xc7, 5, 0, 0, 0x00000020}, + {0x6d, 5, 3, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xf7fca283}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x40000000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xeff94506}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x20000000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xdff28a0c}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x10000000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xbfe51418}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x08000000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x7fca2831}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x04000000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xff945063}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x02000000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xff28a0c6}, + {0x57, 4, 0, 0, 0x01000000}, + {0x1d, 4, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xfe51418c}, + {0x61, 4, 6, 16, 0x00000000}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000080}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xfca28319}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000040}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xf9450633}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000020}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xf28a0c67}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000010}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xe51418ce}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000008}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xca28319d}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000004}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x9450633b}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000002}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x28a0c676}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000001}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x51418ced}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00008000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xa28319db}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00004000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x450633b6}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00002000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x8a0c676c}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00001000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x1418ced8}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000800}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x28319db1}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000400}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x50633b63}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000200}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xa0c676c6}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000100}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x418ced8d}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00800000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x8319db1a}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00400000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x0633b634}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00200000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x0c676c68}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00100000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x18ced8d1}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00080000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x319db1a3}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00040000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x633b6347}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00020000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xc676c68f}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00010000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x8ced8d1f}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x67, 5, 0, 0, 0x00000020}, + {0xc7, 5, 0, 0, 0x00000020}, + {0x6d, 5, 3, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x19db1a3e}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x40000000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x33b6347d}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x20000000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x676c68fa}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x10000000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xced8d1f4}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x08000000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x9db1a3e9}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x04000000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x3b6347d2}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x02000000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x76c68fa5}, + {0x57, 4, 0, 0, 0x01000000}, + {0x1d, 4, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xed8d1f4a}, + {0x61, 4, 6, 20, 0x00000000}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000080}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xdb1a3e94}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000040}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xb6347d28}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000020}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x6c68fa51}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000010}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xd8d1f4a3}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000008}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xb1a3e946}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000004}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x6347d28d}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000002}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xc68fa51a}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000001}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x8d1f4a35}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00008000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x1a3e946b}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00004000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x347d28d7}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00002000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x68fa51ae}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00001000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xd1f4a35c}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000800}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xa3e946b9}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000400}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x47d28d73}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000200}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x8fa51ae7}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000100}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x1f4a35cf}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00800000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x3e946b9e}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00400000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x7d28d73c}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00200000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xfa51ae78}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00100000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xf4a35cf1}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00080000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xe946b9e3}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00040000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xd28d73c7}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00020000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xa51ae78e}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00010000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x4a35cf1c}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x67, 5, 0, 0, 0x00000020}, + {0xc7, 5, 0, 0, 0x00000020}, + {0x6d, 5, 3, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x946b9e38}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x40000000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x28d73c71}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x20000000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x51ae78e3}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x10000000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xa35cf1c6}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x08000000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x46b9e38d}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x04000000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x8d73c71b}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x02000000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x1ae78e36}, + {0x57, 4, 0, 0, 0x01000000}, + {0x1d, 4, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x35cf1c6c}, + {0x61, 4, 6, 24, 0x00000000}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000080}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x6b9e38d9}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000040}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xd73c71b2}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000020}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xae78e364}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000010}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x5cf1c6c9}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000008}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xb9e38d92}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000004}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x73c71b25}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000002}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xe78e364b}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000001}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xcf1c6c96}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00008000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x9e38d92c}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00004000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x3c71b259}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00002000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x78e364b2}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00001000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xf1c6c964}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000800}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xe38d92c9}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000400}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xc71b2593}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000200}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x8e364b27}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000100}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x1c6c964e}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00800000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x38d92c9c}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00400000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x71b25938}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00200000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xe364b270}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00100000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xc6c964e0}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00080000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x8d92c9c0}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00040000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x1b259380}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00020000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x364b2700}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00010000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x6c964e01}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x67, 5, 0, 0, 0x00000020}, + {0xc7, 5, 0, 0, 0x00000020}, + {0x6d, 5, 3, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xd92c9c03}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x40000000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xb2593807}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x20000000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x64b2700f}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x10000000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xc964e01e}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x08000000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x92c9c03d}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x04000000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x2593807a}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x02000000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x4b2700f4}, + {0x57, 4, 0, 0, 0x01000000}, + {0x1d, 4, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x964e01e8}, + {0x61, 4, 6, 28, 0x00000000}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000080}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x2c9c03d1}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000040}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x593807a3}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000020}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xb2700f46}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000010}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x64e01e8d}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000008}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xc9c03d1a}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000004}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x93807a35}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000002}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x2700f46b}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000001}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x4e01e8d6}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00008000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x9c03d1ad}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00004000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x3807a35b}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00002000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x700f46b6}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00001000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xe01e8d6c}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000800}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xc03d1ad9}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000400}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x807a35b3}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000200}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x00f46b66}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000100}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x01e8d6cc}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00800000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x03d1ad99}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00400000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x07a35b32}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00200000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x0f46b665}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00100000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x1e8d6cca}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00080000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x3d1ad994}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00040000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x7a35b328}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00020000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xf46b6651}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00010000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xe8d6cca2}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x67, 5, 0, 0, 0x00000020}, + {0xc7, 5, 0, 0, 0x00000020}, + {0x6d, 5, 3, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xd1ad9944}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x40000000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xa35b3289}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x20000000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x46b66512}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x10000000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x8d6cca25}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x08000000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x1ad9944a}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x04000000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x35b32894}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x02000000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x6b665129}, + {0x57, 4, 0, 0, 0x01000000}, + {0x1d, 4, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xd6cca253}, + {0x61, 4, 6, 32, 0x00000000}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000080}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xad9944a7}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000040}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x5b32894f}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000020}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xb665129f}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000010}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x6cca253e}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000008}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xd9944a7d}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000004}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xb32894fb}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000002}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x665129f6}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000001}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xcca253ec}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00008000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x9944a7d9}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00004000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x32894fb2}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00002000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x65129f65}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00001000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xca253eca}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000800}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x944a7d95}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000400}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x2894fb2a}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000200}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x5129f655}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000100}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xa253ecab}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00800000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x44a7d956}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00400000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x894fb2ac}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00200000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x129f6558}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00100000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x253ecab1}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00080000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x4a7d9563}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00040000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x94fb2ac7}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00020000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x29f6558f}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00010000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x53ecab1e}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x67, 5, 0, 0, 0x00000020}, + {0xc7, 5, 0, 0, 0x00000020}, + {0x6d, 5, 3, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xa7d9563d}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x40000000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x4fb2ac7a}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x20000000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x9f6558f5}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x10000000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x3ecab1ea}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x08000000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x7d9563d5}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x04000000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xfb2ac7ab}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x02000000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xf6558f56}, + {0x57, 4, 0, 0, 0x01000000}, + {0x1d, 4, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xecab1eac}, + {0x61, 4, 6, 36, 0x00000000}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000080}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xd9563d59}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000040}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xb2ac7ab2}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000020}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x6558f564}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000010}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xcab1eac8}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000008}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x9563d590}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000004}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x2ac7ab20}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000002}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x558f5641}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000001}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xab1eac83}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00008000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x563d5906}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00004000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xac7ab20c}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00002000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x58f56418}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00001000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xb1eac831}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000800}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x63d59063}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000400}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xc7ab20c7}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000200}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x8f56418f}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00000100}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x1eac831e}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00800000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x3d59063c}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00400000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x7ab20c78}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00200000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xf56418f0}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00100000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xeac831e1}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00080000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xd59063c2}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00040000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xab20c784}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00020000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x56418f09}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x00010000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xac831e12}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x67, 5, 0, 0, 0x00000020}, + {0xc7, 5, 0, 0, 0x00000020}, + {0x6d, 5, 3, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x59063c25}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x40000000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xb20c784b}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x20000000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x6418f097}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x10000000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xc831e12f}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x08000000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x9063c25f}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x04000000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x20c784be}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x57, 5, 0, 0, 0x02000000}, + {0x1d, 5, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x418f097c}, + {0x57, 4, 0, 0, 0x01000000}, + {0x1d, 4, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x831e12f9}, + {0x71, 4, 6, 40, 0x00000000}, + {0x67, 4, 0, 0, 0x00000008}, + {0x71, 5, 6, 41, 0x00000000}, + {0x4f, 4, 5, 0, 0x00000000}, + {0xbf, 5, 4, 0, 0x00000000}, + {0x67, 5, 0, 0, 0x00000030}, + {0xc7, 5, 0, 0, 0x00000020}, + {0x6d, 5, 3, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x063c25f3}, + {0x67, 4, 0, 0, 0x00000010}, + {0xbf, 3, 4, 0, 0x00000000}, + {0x57, 3, 0, 0, 0x40000000}, + {0x1d, 3, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x0c784be7}, + {0xbf, 3, 4, 0, 0x00000000}, + {0x57, 3, 0, 0, 0x20000000}, + {0x1d, 3, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x18f097cf}, + {0xbf, 3, 4, 0, 0x00000000}, + {0x57, 3, 0, 0, 0x10000000}, + {0x1d, 3, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x31e12f9f}, + {0xbf, 3, 4, 0, 0x00000000}, + {0x57, 3, 0, 0, 0x08000000}, + {0x1d, 3, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x63c25f3f}, + {0xbf, 3, 4, 0, 0x00000000}, + {0x57, 3, 0, 0, 0x04000000}, + {0x1d, 3, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xc784be7f}, + {0xbf, 3, 4, 0, 0x00000000}, + {0x57, 3, 0, 0, 0x02000000}, + {0x1d, 3, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x8f097cff}, + {0xbf, 3, 4, 0, 0x00000000}, + {0x57, 3, 0, 0, 0x01000000}, + {0x1d, 3, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x1e12f9fe}, + {0xbf, 3, 4, 0, 0x00000000}, + {0x57, 3, 0, 0, 0x00800000}, + {0x1d, 3, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x3c25f3fc}, + {0xbf, 3, 4, 0, 0x00000000}, + {0x57, 3, 0, 0, 0x00400000}, + {0x1d, 3, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x784be7f8}, + {0xbf, 3, 4, 0, 0x00000000}, + {0x57, 3, 0, 0, 0x00200000}, + {0x1d, 3, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xf097cff0}, + {0xbf, 3, 4, 0, 0x00000000}, + {0x57, 3, 0, 0, 0x00100000}, + {0x1d, 3, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xe12f9fe0}, + {0xbf, 3, 4, 0, 0x00000000}, + {0x57, 3, 0, 0, 0x00080000}, + {0x1d, 3, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xc25f3fc1}, + {0xbf, 3, 4, 0, 0x00000000}, + {0x57, 3, 0, 0, 0x00040000}, + {0x1d, 3, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x84be7f83}, + {0xbf, 3, 4, 0, 0x00000000}, + {0x57, 3, 0, 0, 0x00020000}, + {0x1d, 3, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x097cff07}, + {0x57, 4, 0, 0, 0x00010000}, + {0x1d, 4, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x12f9fe0f}, + {0x71, 3, 6, 42, 0x00000000}, + {0x67, 3, 0, 0, 0x00000008}, + {0xbf, 4, 3, 0, 0x00000000}, + {0x57, 4, 0, 0, 0x00008000}, + {0x1d, 4, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x25f3fc1f}, + {0xbf, 4, 3, 0, 0x00000000}, + {0x57, 4, 0, 0, 0x00004000}, + {0x1d, 4, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x4be7f83f}, + {0xbf, 4, 3, 0, 0x00000000}, + {0x57, 4, 0, 0, 0x00002000}, + {0x1d, 4, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x97cff07f}, + {0xbf, 4, 3, 0, 0x00000000}, + {0x57, 4, 0, 0, 0x00001000}, + {0x1d, 4, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x2f9fe0fe}, + {0xbf, 4, 3, 0, 0x00000000}, + {0x57, 4, 0, 0, 0x00000800}, + {0x1d, 4, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x5f3fc1fd}, + {0xbf, 4, 3, 0, 0x00000000}, + {0x57, 4, 0, 0, 0x00000400}, + {0x1d, 4, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xbe7f83fb}, + {0xbf, 4, 3, 0, 0x00000000}, + {0x57, 4, 0, 0, 0x00000200}, + {0x1d, 4, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x7cff07f7}, + {0x57, 3, 0, 0, 0x00000100}, + {0x1d, 3, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xf9fe0fee}, + {0x71, 3, 6, 43, 0x00000000}, + {0xbf, 4, 3, 0, 0x00000000}, + {0x57, 4, 0, 0, 0x00000080}, + {0x1d, 4, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xf3fc1fdc}, + {0xbf, 4, 3, 0, 0x00000000}, + {0x57, 4, 0, 0, 0x00000040}, + {0x79, 5, 10, -56, 0x00000000}, + {0x1d, 4, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xe7f83fb8}, + {0xbf, 4, 3, 0, 0x00000000}, + {0x57, 4, 0, 0, 0x00000020}, + {0x1d, 4, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xcff07f70}, + {0xbf, 4, 3, 0, 0x00000000}, + {0x57, 4, 0, 0, 0x00000010}, + {0x1d, 4, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x9fe0fee1}, + {0xbf, 4, 3, 0, 0x00000000}, + {0x57, 4, 0, 0, 0x00000008}, + {0x1d, 4, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x3fc1fdc2}, + {0xbf, 4, 3, 0, 0x00000000}, + {0x57, 4, 0, 0, 0x00000004}, + {0x1d, 4, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0x7f83fb85}, + {0xbf, 4, 3, 0, 0x00000000}, + {0x57, 4, 0, 0, 0x00000002}, + {0x1d, 4, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xff07f70a}, + {0x57, 3, 0, 0, 0x00000001}, + {0x1d, 3, 2, 1, 0x00000000}, + {0xa7, 1, 0, 0, 0xfe0fee15}, + {0x71, 2, 0, 201, 0x00000000}, + {0x67, 2, 0, 0, 0x00000008}, + {0x71, 3, 0, 200, 0x00000000}, + {0x4f, 2, 3, 0, 0x00000000}, + {0x71, 3, 0, 203, 0x00000000}, + {0x67, 3, 0, 0, 0x00000008}, + {0x71, 4, 0, 202, 0x00000000}, + {0x4f, 3, 4, 0, 0x00000000}, + {0x67, 3, 0, 0, 0x00000010}, + {0x4f, 3, 2, 0, 0x00000000}, + {0x67, 1, 0, 0, 0x00000020}, + {0x77, 1, 0, 0, 0x00000020}, + {0xbf, 2, 1, 0, 0x00000000}, + {0x3f, 2, 3, 0, 0x00000000}, + {0x2f, 2, 3, 0, 0x00000000}, + {0x1f, 1, 2, 0, 0x00000000}, + {0x57, 1, 0, 0, 0x0000000f}, + {0x67, 1, 0, 0, 0x00000002}, + {0x0f, 0, 1, 0, 0x00000000}, + {0x71, 1, 0, 137, 0x00000000}, + {0x67, 1, 0, 0, 0x00000008}, + {0x71, 2, 0, 136, 0x00000000}, + {0x4f, 1, 2, 0, 0x00000000}, + {0x71, 2, 0, 138, 0x00000000}, + {0x71, 3, 0, 139, 0x00000000}, + {0x67, 3, 0, 0, 0x00000008}, + {0x4f, 3, 2, 0, 0x00000000}, + {0x67, 3, 0, 0, 0x00000010}, + {0x4f, 3, 1, 0, 0x00000000}, + {0x07, 3, 0, 0, 0x7cafe800}, + {0x63, 5, 3, 52, 0x00000000}, + {0xb7, 7, 0, 0, 0x00000001}, + {0xbf, 0, 7, 0, 0x00000000}, + {0x95, 0, 0, 0, 0x00000000}, +}; diff --git a/src/spdk/dpdk/drivers/net/tap/tap_bpf_program.c b/src/spdk/dpdk/drivers/net/tap/tap_bpf_program.c new file mode 100644 index 00000000..1cb73822 --- /dev/null +++ b/src/spdk/dpdk/drivers/net/tap/tap_bpf_program.c @@ -0,0 +1,224 @@ +/* SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0 + * Copyright 2017 Mellanox Technologies, Ltd + */ + +#include <stdint.h> +#include <stdbool.h> +#include <sys/types.h> +#include <sys/socket.h> +#include <asm/types.h> +#include <linux/in.h> +#include <linux/if.h> +#include <linux/if_ether.h> +#include <linux/ip.h> +#include <linux/ipv6.h> +#include <linux/if_tunnel.h> +#include <linux/filter.h> +#include <linux/bpf.h> + +#include "tap_rss.h" + +/** Create IPv4 address */ +#define IPv4(a, b, c, d) ((__u32)(((a) & 0xff) << 24) | \ + (((b) & 0xff) << 16) | \ + (((c) & 0xff) << 8) | \ + ((d) & 0xff)) + +#define PORT(a, b) ((__u16)(((a) & 0xff) << 8) | \ + ((b) & 0xff)) + +/* + * The queue number is offset by a unique QUEUE_OFFSET, to distinguish + * packets that have gone through this rule (skb->cb[1] != 0) from others. + */ +#define QUEUE_OFFSET 0x7cafe800 +#define PIN_GLOBAL_NS 2 + +#define KEY_IDX 0 +#define BPF_MAP_ID_KEY 1 + +struct vlan_hdr { + __be16 proto; + __be16 tci; +}; + +struct bpf_elf_map __attribute__((section("maps"), used)) +map_keys = { + .type = BPF_MAP_TYPE_HASH, + .id = BPF_MAP_ID_KEY, + .size_key = sizeof(__u32), + .size_value = sizeof(struct rss_key), + .max_elem = 256, + .pinning = PIN_GLOBAL_NS, +}; + +__section("cls_q") int +match_q(struct __sk_buff *skb) +{ + __u32 queue = skb->cb[1]; + volatile __u32 q = 0xdeadbeef; + __u32 match_queue = QUEUE_OFFSET + q; + + /* printt("match_q$i() queue = %d\n", queue); */ + + if (queue != match_queue) + return TC_ACT_OK; + + /* queue match */ + skb->cb[1] = 0; + return TC_ACT_UNSPEC; +} + + +struct ipv4_l3_l4_tuple { + __u32 src_addr; + __u32 dst_addr; + __u16 dport; + __u16 sport; +} __attribute__((packed)); + +struct ipv6_l3_l4_tuple { + __u8 src_addr[16]; + __u8 dst_addr[16]; + __u16 dport; + __u16 sport; +} __attribute__((packed)); + +static const __u8 def_rss_key[TAP_RSS_HASH_KEY_SIZE] = { + 0xd1, 0x81, 0xc6, 0x2c, + 0xf7, 0xf4, 0xdb, 0x5b, + 0x19, 0x83, 0xa2, 0xfc, + 0x94, 0x3e, 0x1a, 0xdb, + 0xd9, 0x38, 0x9e, 0x6b, + 0xd1, 0x03, 0x9c, 0x2c, + 0xa7, 0x44, 0x99, 0xad, + 0x59, 0x3d, 0x56, 0xd9, + 0xf3, 0x25, 0x3c, 0x06, + 0x2a, 0xdc, 0x1f, 0xfc, +}; + +static __u32 __attribute__((always_inline)) +rte_softrss_be(const __u32 *input_tuple, const uint8_t *rss_key, + __u8 input_len) +{ + __u32 i, j, hash = 0; +#pragma unroll + for (j = 0; j < input_len; j++) { +#pragma unroll + for (i = 0; i < 32; i++) { + if (input_tuple[j] & (1 << (31 - i))) { + hash ^= ((const __u32 *)def_rss_key)[j] << i | + (__u32)((uint64_t) + (((const __u32 *)def_rss_key)[j + 1]) + >> (32 - i)); + } + } + } + return hash; +} + +static int __attribute__((always_inline)) +rss_l3_l4(struct __sk_buff *skb) +{ + void *data_end = (void *)(long)skb->data_end; + void *data = (void *)(long)skb->data; + __u16 proto = (__u16)skb->protocol; + __u32 key_idx = 0xdeadbeef; + __u32 hash; + struct rss_key *rsskey; + __u64 off = ETH_HLEN; + int j; + __u8 *key = 0; + __u32 len; + __u32 queue = 0; + + rsskey = map_lookup_elem(&map_keys, &key_idx); + if (!rsskey) { + printt("hash(): rss key is not configured\n"); + return TC_ACT_OK; + } + key = (__u8 *)rsskey->key; + + /* Get correct proto for 802.1ad */ + if (skb->vlan_present && skb->vlan_proto == htons(ETH_P_8021AD)) { + if (data + ETH_ALEN * 2 + sizeof(struct vlan_hdr) + + sizeof(proto) > data_end) + return TC_ACT_OK; + proto = *(__u16 *)(data + ETH_ALEN * 2 + + sizeof(struct vlan_hdr)); + off += sizeof(struct vlan_hdr); + } + + if (proto == htons(ETH_P_IP)) { + if (data + off + sizeof(struct iphdr) + sizeof(__u32) + > data_end) + return TC_ACT_OK; + + __u8 *src_dst_addr = data + off + offsetof(struct iphdr, saddr); + __u8 *src_dst_port = data + off + sizeof(struct iphdr); + struct ipv4_l3_l4_tuple v4_tuple = { + .src_addr = IPv4(*(src_dst_addr + 0), + *(src_dst_addr + 1), + *(src_dst_addr + 2), + *(src_dst_addr + 3)), + .dst_addr = IPv4(*(src_dst_addr + 4), + *(src_dst_addr + 5), + *(src_dst_addr + 6), + *(src_dst_addr + 7)), + .sport = PORT(*(src_dst_port + 0), + *(src_dst_port + 1)), + .dport = PORT(*(src_dst_port + 2), + *(src_dst_port + 3)), + }; + __u8 input_len = sizeof(v4_tuple) / sizeof(__u32); + if (rsskey->hash_fields & (1 << HASH_FIELD_IPV4_L3)) + input_len--; + hash = rte_softrss_be((__u32 *)&v4_tuple, key, 3); + } else if (proto == htons(ETH_P_IPV6)) { + if (data + off + sizeof(struct ipv6hdr) + + sizeof(__u32) > data_end) + return TC_ACT_OK; + __u8 *src_dst_addr = data + off + + offsetof(struct ipv6hdr, saddr); + __u8 *src_dst_port = data + off + + sizeof(struct ipv6hdr); + struct ipv6_l3_l4_tuple v6_tuple; + for (j = 0; j < 4; j++) + *((uint32_t *)&v6_tuple.src_addr + j) = + __builtin_bswap32(*((uint32_t *) + src_dst_addr + j)); + for (j = 0; j < 4; j++) + *((uint32_t *)&v6_tuple.dst_addr + j) = + __builtin_bswap32(*((uint32_t *) + src_dst_addr + 4 + j)); + v6_tuple.sport = PORT(*(src_dst_port + 0), + *(src_dst_port + 1)); + v6_tuple.dport = PORT(*(src_dst_port + 2), + *(src_dst_port + 3)); + + __u8 input_len = sizeof(v6_tuple) / sizeof(__u32); + if (rsskey->hash_fields & (1 << HASH_FIELD_IPV6_L3)) + input_len--; + hash = rte_softrss_be((__u32 *)&v6_tuple, key, 9); + } else { + return TC_ACT_PIPE; + } + + queue = rsskey->queues[(hash % rsskey->nb_queues) & + (TAP_MAX_QUEUES - 1)]; + skb->cb[1] = QUEUE_OFFSET + queue; + /* printt(">>>>> rss_l3_l4 hash=0x%x queue=%u\n", hash, queue); */ + + return TC_ACT_RECLASSIFY; +} + +#define RSS(L) \ + __section(#L) int \ + L ## _hash(struct __sk_buff *skb) \ + { \ + return rss_ ## L (skb); \ + } + +RSS(l3_l4) + +BPF_LICENSE("Dual BSD/GPL"); diff --git a/src/spdk/dpdk/drivers/net/tap/tap_flow.c b/src/spdk/dpdk/drivers/net/tap/tap_flow.c new file mode 100644 index 00000000..0e01af62 --- /dev/null +++ b/src/spdk/dpdk/drivers/net/tap/tap_flow.c @@ -0,0 +1,2191 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright 2017 6WIND S.A. + * Copyright 2017 Mellanox Technologies, Ltd + */ + +#include <errno.h> +#include <string.h> +#include <unistd.h> +#include <sys/queue.h> +#include <sys/resource.h> + +#include <rte_byteorder.h> +#include <rte_jhash.h> +#include <rte_malloc.h> +#include <rte_eth_tap.h> +#include <tap_flow.h> +#include <tap_autoconf.h> +#include <tap_tcmsgs.h> +#include <tap_rss.h> + +#ifndef HAVE_TC_FLOWER +/* + * For kernels < 4.2, this enum is not defined. Runtime checks will be made to + * avoid sending TC messages the kernel cannot understand. + */ +enum { + TCA_FLOWER_UNSPEC, + TCA_FLOWER_CLASSID, + TCA_FLOWER_INDEV, + TCA_FLOWER_ACT, + TCA_FLOWER_KEY_ETH_DST, /* ETH_ALEN */ + TCA_FLOWER_KEY_ETH_DST_MASK, /* ETH_ALEN */ + TCA_FLOWER_KEY_ETH_SRC, /* ETH_ALEN */ + TCA_FLOWER_KEY_ETH_SRC_MASK, /* ETH_ALEN */ + TCA_FLOWER_KEY_ETH_TYPE, /* be16 */ + TCA_FLOWER_KEY_IP_PROTO, /* u8 */ + TCA_FLOWER_KEY_IPV4_SRC, /* be32 */ + TCA_FLOWER_KEY_IPV4_SRC_MASK, /* be32 */ + TCA_FLOWER_KEY_IPV4_DST, /* be32 */ + TCA_FLOWER_KEY_IPV4_DST_MASK, /* be32 */ + TCA_FLOWER_KEY_IPV6_SRC, /* struct in6_addr */ + TCA_FLOWER_KEY_IPV6_SRC_MASK, /* struct in6_addr */ + TCA_FLOWER_KEY_IPV6_DST, /* struct in6_addr */ + TCA_FLOWER_KEY_IPV6_DST_MASK, /* struct in6_addr */ + TCA_FLOWER_KEY_TCP_SRC, /* be16 */ + TCA_FLOWER_KEY_TCP_DST, /* be16 */ + TCA_FLOWER_KEY_UDP_SRC, /* be16 */ + TCA_FLOWER_KEY_UDP_DST, /* be16 */ +}; +#endif +#ifndef HAVE_TC_VLAN_ID +enum { + /* TCA_FLOWER_FLAGS, */ + TCA_FLOWER_KEY_VLAN_ID = TCA_FLOWER_KEY_UDP_DST + 2, /* be16 */ + TCA_FLOWER_KEY_VLAN_PRIO, /* u8 */ + TCA_FLOWER_KEY_VLAN_ETH_TYPE, /* be16 */ +}; +#endif +/* + * For kernels < 4.2 BPF related enums may not be defined. + * Runtime checks will be carried out to gracefully report on TC messages that + * are rejected by the kernel. Rejection reasons may be due to: + * 1. enum is not defined + * 2. enum is defined but kernel is not configured to support BPF system calls, + * BPF classifications or BPF actions. + */ +#ifndef HAVE_TC_BPF +enum { + TCA_BPF_UNSPEC, + TCA_BPF_ACT, + TCA_BPF_POLICE, + TCA_BPF_CLASSID, + TCA_BPF_OPS_LEN, + TCA_BPF_OPS, +}; +#endif +#ifndef HAVE_TC_BPF_FD +enum { + TCA_BPF_FD = TCA_BPF_OPS + 1, + TCA_BPF_NAME, +}; +#endif +#ifndef HAVE_TC_ACT_BPF +#define tc_gen \ + __u32 index; \ + __u32 capab; \ + int action; \ + int refcnt; \ + int bindcnt + +struct tc_act_bpf { + tc_gen; +}; + +enum { + TCA_ACT_BPF_UNSPEC, + TCA_ACT_BPF_TM, + TCA_ACT_BPF_PARMS, + TCA_ACT_BPF_OPS_LEN, + TCA_ACT_BPF_OPS, +}; + +#endif +#ifndef HAVE_TC_ACT_BPF_FD +enum { + TCA_ACT_BPF_FD = TCA_ACT_BPF_OPS + 1, + TCA_ACT_BPF_NAME, +}; +#endif + +/* RSS key management */ +enum bpf_rss_key_e { + KEY_CMD_GET = 1, + KEY_CMD_RELEASE, + KEY_CMD_INIT, + KEY_CMD_DEINIT, +}; + +enum key_status_e { + KEY_STAT_UNSPEC, + KEY_STAT_USED, + KEY_STAT_AVAILABLE, +}; + +#define ISOLATE_HANDLE 1 +#define REMOTE_PROMISCUOUS_HANDLE 2 + +struct rte_flow { + LIST_ENTRY(rte_flow) next; /* Pointer to the next rte_flow structure */ + struct rte_flow *remote_flow; /* associated remote flow */ + int bpf_fd[SEC_MAX]; /* list of bfs fds per ELF section */ + uint32_t key_idx; /* RSS rule key index into BPF map */ + struct nlmsg msg; +}; + +struct convert_data { + uint16_t eth_type; + uint16_t ip_proto; + uint8_t vlan; + struct rte_flow *flow; +}; + +struct remote_rule { + struct rte_flow_attr attr; + struct rte_flow_item items[2]; + struct rte_flow_action actions[2]; + int mirred; +}; + +struct action_data { + char id[16]; + + union { + struct tc_gact gact; + struct tc_mirred mirred; + struct skbedit { + struct tc_skbedit skbedit; + uint16_t queue; + } skbedit; + struct bpf { + struct tc_act_bpf bpf; + int bpf_fd; + const char *annotation; + } bpf; + }; +}; + +static int tap_flow_create_eth(const struct rte_flow_item *item, void *data); +static int tap_flow_create_vlan(const struct rte_flow_item *item, void *data); +static int tap_flow_create_ipv4(const struct rte_flow_item *item, void *data); +static int tap_flow_create_ipv6(const struct rte_flow_item *item, void *data); +static int tap_flow_create_udp(const struct rte_flow_item *item, void *data); +static int tap_flow_create_tcp(const struct rte_flow_item *item, void *data); +static int +tap_flow_validate(struct rte_eth_dev *dev, + const struct rte_flow_attr *attr, + const struct rte_flow_item items[], + const struct rte_flow_action actions[], + struct rte_flow_error *error); + +static struct rte_flow * +tap_flow_create(struct rte_eth_dev *dev, + const struct rte_flow_attr *attr, + const struct rte_flow_item items[], + const struct rte_flow_action actions[], + struct rte_flow_error *error); + +static void +tap_flow_free(struct pmd_internals *pmd, + struct rte_flow *flow); + +static int +tap_flow_destroy(struct rte_eth_dev *dev, + struct rte_flow *flow, + struct rte_flow_error *error); + +static int +tap_flow_isolate(struct rte_eth_dev *dev, + int set, + struct rte_flow_error *error); + +static int bpf_rss_key(enum bpf_rss_key_e cmd, __u32 *key_idx); +static int rss_enable(struct pmd_internals *pmd, + const struct rte_flow_attr *attr, + struct rte_flow_error *error); +static int rss_add_actions(struct rte_flow *flow, struct pmd_internals *pmd, + const struct rte_flow_action_rss *rss, + struct rte_flow_error *error); + +static const struct rte_flow_ops tap_flow_ops = { + .validate = tap_flow_validate, + .create = tap_flow_create, + .destroy = tap_flow_destroy, + .flush = tap_flow_flush, + .isolate = tap_flow_isolate, +}; + +/* Static initializer for items. */ +#define ITEMS(...) \ + (const enum rte_flow_item_type []){ \ + __VA_ARGS__, RTE_FLOW_ITEM_TYPE_END, \ + } + +/* Structure to generate a simple graph of layers supported by the NIC. */ +struct tap_flow_items { + /* Bit-mask corresponding to what is supported for this item. */ + const void *mask; + const unsigned int mask_sz; /* Bit-mask size in bytes. */ + /* + * Bit-mask corresponding to the default mask, if none is provided + * along with the item. + */ + const void *default_mask; + /** + * Conversion function from rte_flow to netlink attributes. + * + * @param item + * rte_flow item to convert. + * @param data + * Internal structure to store the conversion. + * + * @return + * 0 on success, negative value otherwise. + */ + int (*convert)(const struct rte_flow_item *item, void *data); + /** List of possible following items. */ + const enum rte_flow_item_type *const items; +}; + +/* Graph of supported items and associated actions. */ +static const struct tap_flow_items tap_flow_items[] = { + [RTE_FLOW_ITEM_TYPE_END] = { + .items = ITEMS(RTE_FLOW_ITEM_TYPE_ETH), + }, + [RTE_FLOW_ITEM_TYPE_ETH] = { + .items = ITEMS( + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_IPV6), + .mask = &(const struct rte_flow_item_eth){ + .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff", + .src.addr_bytes = "\xff\xff\xff\xff\xff\xff", + .type = -1, + }, + .mask_sz = sizeof(struct rte_flow_item_eth), + .default_mask = &rte_flow_item_eth_mask, + .convert = tap_flow_create_eth, + }, + [RTE_FLOW_ITEM_TYPE_VLAN] = { + .items = ITEMS(RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_IPV6), + .mask = &(const struct rte_flow_item_vlan){ + /* DEI matching is not supported */ +#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN + .tci = 0xffef, +#else + .tci = 0xefff, +#endif + .inner_type = -1, + }, + .mask_sz = sizeof(struct rte_flow_item_vlan), + .default_mask = &rte_flow_item_vlan_mask, + .convert = tap_flow_create_vlan, + }, + [RTE_FLOW_ITEM_TYPE_IPV4] = { + .items = ITEMS(RTE_FLOW_ITEM_TYPE_UDP, + RTE_FLOW_ITEM_TYPE_TCP), + .mask = &(const struct rte_flow_item_ipv4){ + .hdr = { + .src_addr = -1, + .dst_addr = -1, + .next_proto_id = -1, + }, + }, + .mask_sz = sizeof(struct rte_flow_item_ipv4), + .default_mask = &rte_flow_item_ipv4_mask, + .convert = tap_flow_create_ipv4, + }, + [RTE_FLOW_ITEM_TYPE_IPV6] = { + .items = ITEMS(RTE_FLOW_ITEM_TYPE_UDP, + RTE_FLOW_ITEM_TYPE_TCP), + .mask = &(const struct rte_flow_item_ipv6){ + .hdr = { + .src_addr = { + "\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff", + }, + .dst_addr = { + "\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff", + }, + .proto = -1, + }, + }, + .mask_sz = sizeof(struct rte_flow_item_ipv6), + .default_mask = &rte_flow_item_ipv6_mask, + .convert = tap_flow_create_ipv6, + }, + [RTE_FLOW_ITEM_TYPE_UDP] = { + .mask = &(const struct rte_flow_item_udp){ + .hdr = { + .src_port = -1, + .dst_port = -1, + }, + }, + .mask_sz = sizeof(struct rte_flow_item_udp), + .default_mask = &rte_flow_item_udp_mask, + .convert = tap_flow_create_udp, + }, + [RTE_FLOW_ITEM_TYPE_TCP] = { + .mask = &(const struct rte_flow_item_tcp){ + .hdr = { + .src_port = -1, + .dst_port = -1, + }, + }, + .mask_sz = sizeof(struct rte_flow_item_tcp), + .default_mask = &rte_flow_item_tcp_mask, + .convert = tap_flow_create_tcp, + }, +}; + +/* + * TC rules, by growing priority + * + * Remote netdevice Tap netdevice + * +-------------+-------------+ +-------------+-------------+ + * | Ingress | Egress | | Ingress | Egress | + * |-------------|-------------| |-------------|-------------| + * | | \ / | | | REMOTE TX | prio 1 + * | | \ / | | | \ / | prio 2 + * | EXPLICIT | \ / | | EXPLICIT | \ / | . + * | | \ / | | | \ / | . + * | RULES | X | | RULES | X | . + * | . | / \ | | . | / \ | . + * | . | / \ | | . | / \ | . + * | . | / \ | | . | / \ | . + * | . | / \ | | . | / \ | . + * + * .... .... .... .... + * + * | . | \ / | | . | \ / | . + * | . | \ / | | . | \ / | . + * | | \ / | | | \ / | + * | LOCAL_MAC | \ / | | \ / | \ / | last prio - 5 + * | PROMISC | X | | \ / | X | last prio - 4 + * | ALLMULTI | / \ | | X | / \ | last prio - 3 + * | BROADCAST | / \ | | / \ | / \ | last prio - 2 + * | BROADCASTV6 | / \ | | / \ | / \ | last prio - 1 + * | xx | / \ | | ISOLATE | / \ | last prio + * +-------------+-------------+ +-------------+-------------+ + * + * The implicit flow rules are stored in a list in with mandatorily the last two + * being the ISOLATE and REMOTE_TX rules. e.g.: + * + * LOCAL_MAC -> BROADCAST -> BROADCASTV6 -> REMOTE_TX -> ISOLATE -> NULL + * + * That enables tap_flow_isolate() to remove implicit rules by popping the list + * head and remove it as long as it applies on the remote netdevice. The + * implicit rule for TX redirection is not removed, as isolate concerns only + * incoming traffic. + */ + +static struct remote_rule implicit_rte_flows[TAP_REMOTE_MAX_IDX] = { + [TAP_REMOTE_LOCAL_MAC] = { + .attr = { + .group = MAX_GROUP, + .priority = PRIORITY_MASK - TAP_REMOTE_LOCAL_MAC, + .ingress = 1, + }, + .items[0] = { + .type = RTE_FLOW_ITEM_TYPE_ETH, + .mask = &(const struct rte_flow_item_eth){ + .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff", + }, + }, + .items[1] = { + .type = RTE_FLOW_ITEM_TYPE_END, + }, + .mirred = TCA_EGRESS_REDIR, + }, + [TAP_REMOTE_BROADCAST] = { + .attr = { + .group = MAX_GROUP, + .priority = PRIORITY_MASK - TAP_REMOTE_BROADCAST, + .ingress = 1, + }, + .items[0] = { + .type = RTE_FLOW_ITEM_TYPE_ETH, + .mask = &(const struct rte_flow_item_eth){ + .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff", + }, + .spec = &(const struct rte_flow_item_eth){ + .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff", + }, + }, + .items[1] = { + .type = RTE_FLOW_ITEM_TYPE_END, + }, + .mirred = TCA_EGRESS_MIRROR, + }, + [TAP_REMOTE_BROADCASTV6] = { + .attr = { + .group = MAX_GROUP, + .priority = PRIORITY_MASK - TAP_REMOTE_BROADCASTV6, + .ingress = 1, + }, + .items[0] = { + .type = RTE_FLOW_ITEM_TYPE_ETH, + .mask = &(const struct rte_flow_item_eth){ + .dst.addr_bytes = "\x33\x33\x00\x00\x00\x00", + }, + .spec = &(const struct rte_flow_item_eth){ + .dst.addr_bytes = "\x33\x33\x00\x00\x00\x00", + }, + }, + .items[1] = { + .type = RTE_FLOW_ITEM_TYPE_END, + }, + .mirred = TCA_EGRESS_MIRROR, + }, + [TAP_REMOTE_PROMISC] = { + .attr = { + .group = MAX_GROUP, + .priority = PRIORITY_MASK - TAP_REMOTE_PROMISC, + .ingress = 1, + }, + .items[0] = { + .type = RTE_FLOW_ITEM_TYPE_VOID, + }, + .items[1] = { + .type = RTE_FLOW_ITEM_TYPE_END, + }, + .mirred = TCA_EGRESS_MIRROR, + }, + [TAP_REMOTE_ALLMULTI] = { + .attr = { + .group = MAX_GROUP, + .priority = PRIORITY_MASK - TAP_REMOTE_ALLMULTI, + .ingress = 1, + }, + .items[0] = { + .type = RTE_FLOW_ITEM_TYPE_ETH, + .mask = &(const struct rte_flow_item_eth){ + .dst.addr_bytes = "\x01\x00\x00\x00\x00\x00", + }, + .spec = &(const struct rte_flow_item_eth){ + .dst.addr_bytes = "\x01\x00\x00\x00\x00\x00", + }, + }, + .items[1] = { + .type = RTE_FLOW_ITEM_TYPE_END, + }, + .mirred = TCA_EGRESS_MIRROR, + }, + [TAP_REMOTE_TX] = { + .attr = { + .group = 0, + .priority = TAP_REMOTE_TX, + .egress = 1, + }, + .items[0] = { + .type = RTE_FLOW_ITEM_TYPE_VOID, + }, + .items[1] = { + .type = RTE_FLOW_ITEM_TYPE_END, + }, + .mirred = TCA_EGRESS_MIRROR, + }, + [TAP_ISOLATE] = { + .attr = { + .group = MAX_GROUP, + .priority = PRIORITY_MASK - TAP_ISOLATE, + .ingress = 1, + }, + .items[0] = { + .type = RTE_FLOW_ITEM_TYPE_VOID, + }, + .items[1] = { + .type = RTE_FLOW_ITEM_TYPE_END, + }, + }, +}; + +/** + * Make as much checks as possible on an Ethernet item, and if a flow is + * provided, fill it appropriately with Ethernet info. + * + * @param[in] item + * Item specification. + * @param[in, out] data + * Additional data structure to tell next layers we've been here. + * + * @return + * 0 if checks are alright, -1 otherwise. + */ +static int +tap_flow_create_eth(const struct rte_flow_item *item, void *data) +{ + struct convert_data *info = (struct convert_data *)data; + const struct rte_flow_item_eth *spec = item->spec; + const struct rte_flow_item_eth *mask = item->mask; + struct rte_flow *flow = info->flow; + struct nlmsg *msg; + + /* use default mask if none provided */ + if (!mask) + mask = tap_flow_items[RTE_FLOW_ITEM_TYPE_ETH].default_mask; + /* TC does not support eth_type masking. Only accept if exact match. */ + if (mask->type && mask->type != 0xffff) + return -1; + if (!spec) + return 0; + /* store eth_type for consistency if ipv4/6 pattern item comes next */ + if (spec->type & mask->type) + info->eth_type = spec->type; + if (!flow) + return 0; + msg = &flow->msg; + if (!is_zero_ether_addr(&mask->dst)) { + tap_nlattr_add(&msg->nh, TCA_FLOWER_KEY_ETH_DST, ETHER_ADDR_LEN, + &spec->dst.addr_bytes); + tap_nlattr_add(&msg->nh, + TCA_FLOWER_KEY_ETH_DST_MASK, ETHER_ADDR_LEN, + &mask->dst.addr_bytes); + } + if (!is_zero_ether_addr(&mask->src)) { + tap_nlattr_add(&msg->nh, TCA_FLOWER_KEY_ETH_SRC, ETHER_ADDR_LEN, + &spec->src.addr_bytes); + tap_nlattr_add(&msg->nh, + TCA_FLOWER_KEY_ETH_SRC_MASK, ETHER_ADDR_LEN, + &mask->src.addr_bytes); + } + return 0; +} + +/** + * Make as much checks as possible on a VLAN item, and if a flow is provided, + * fill it appropriately with VLAN info. + * + * @param[in] item + * Item specification. + * @param[in, out] data + * Additional data structure to tell next layers we've been here. + * + * @return + * 0 if checks are alright, -1 otherwise. + */ +static int +tap_flow_create_vlan(const struct rte_flow_item *item, void *data) +{ + struct convert_data *info = (struct convert_data *)data; + const struct rte_flow_item_vlan *spec = item->spec; + const struct rte_flow_item_vlan *mask = item->mask; + struct rte_flow *flow = info->flow; + struct nlmsg *msg; + + /* use default mask if none provided */ + if (!mask) + mask = tap_flow_items[RTE_FLOW_ITEM_TYPE_VLAN].default_mask; + /* Outer TPID cannot be matched. */ + if (info->eth_type) + return -1; + /* Double-tagging not supported. */ + if (info->vlan) + return -1; + info->vlan = 1; + if (mask->inner_type) { + /* TC does not support partial eth_type masking */ + if (mask->inner_type != RTE_BE16(0xffff)) + return -1; + info->eth_type = spec->inner_type; + } + if (!flow) + return 0; + msg = &flow->msg; + msg->t.tcm_info = TC_H_MAKE(msg->t.tcm_info, htons(ETH_P_8021Q)); +#define VLAN_PRIO(tci) ((tci) >> 13) +#define VLAN_ID(tci) ((tci) & 0xfff) + if (!spec) + return 0; + if (spec->tci) { + uint16_t tci = ntohs(spec->tci) & mask->tci; + uint16_t prio = VLAN_PRIO(tci); + uint8_t vid = VLAN_ID(tci); + + if (prio) + tap_nlattr_add8(&msg->nh, + TCA_FLOWER_KEY_VLAN_PRIO, prio); + if (vid) + tap_nlattr_add16(&msg->nh, + TCA_FLOWER_KEY_VLAN_ID, vid); + } + return 0; +} + +/** + * Make as much checks as possible on an IPv4 item, and if a flow is provided, + * fill it appropriately with IPv4 info. + * + * @param[in] item + * Item specification. + * @param[in, out] data + * Additional data structure to tell next layers we've been here. + * + * @return + * 0 if checks are alright, -1 otherwise. + */ +static int +tap_flow_create_ipv4(const struct rte_flow_item *item, void *data) +{ + struct convert_data *info = (struct convert_data *)data; + const struct rte_flow_item_ipv4 *spec = item->spec; + const struct rte_flow_item_ipv4 *mask = item->mask; + struct rte_flow *flow = info->flow; + struct nlmsg *msg; + + /* use default mask if none provided */ + if (!mask) + mask = tap_flow_items[RTE_FLOW_ITEM_TYPE_IPV4].default_mask; + /* check that previous eth type is compatible with ipv4 */ + if (info->eth_type && info->eth_type != htons(ETH_P_IP)) + return -1; + /* store ip_proto for consistency if udp/tcp pattern item comes next */ + if (spec) + info->ip_proto = spec->hdr.next_proto_id; + if (!flow) + return 0; + msg = &flow->msg; + if (!info->eth_type) + info->eth_type = htons(ETH_P_IP); + if (!spec) + return 0; + if (mask->hdr.dst_addr) { + tap_nlattr_add32(&msg->nh, TCA_FLOWER_KEY_IPV4_DST, + spec->hdr.dst_addr); + tap_nlattr_add32(&msg->nh, TCA_FLOWER_KEY_IPV4_DST_MASK, + mask->hdr.dst_addr); + } + if (mask->hdr.src_addr) { + tap_nlattr_add32(&msg->nh, TCA_FLOWER_KEY_IPV4_SRC, + spec->hdr.src_addr); + tap_nlattr_add32(&msg->nh, TCA_FLOWER_KEY_IPV4_SRC_MASK, + mask->hdr.src_addr); + } + if (spec->hdr.next_proto_id) + tap_nlattr_add8(&msg->nh, TCA_FLOWER_KEY_IP_PROTO, + spec->hdr.next_proto_id); + return 0; +} + +/** + * Make as much checks as possible on an IPv6 item, and if a flow is provided, + * fill it appropriately with IPv6 info. + * + * @param[in] item + * Item specification. + * @param[in, out] data + * Additional data structure to tell next layers we've been here. + * + * @return + * 0 if checks are alright, -1 otherwise. + */ +static int +tap_flow_create_ipv6(const struct rte_flow_item *item, void *data) +{ + struct convert_data *info = (struct convert_data *)data; + const struct rte_flow_item_ipv6 *spec = item->spec; + const struct rte_flow_item_ipv6 *mask = item->mask; + struct rte_flow *flow = info->flow; + uint8_t empty_addr[16] = { 0 }; + struct nlmsg *msg; + + /* use default mask if none provided */ + if (!mask) + mask = tap_flow_items[RTE_FLOW_ITEM_TYPE_IPV6].default_mask; + /* check that previous eth type is compatible with ipv6 */ + if (info->eth_type && info->eth_type != htons(ETH_P_IPV6)) + return -1; + /* store ip_proto for consistency if udp/tcp pattern item comes next */ + if (spec) + info->ip_proto = spec->hdr.proto; + if (!flow) + return 0; + msg = &flow->msg; + if (!info->eth_type) + info->eth_type = htons(ETH_P_IPV6); + if (!spec) + return 0; + if (memcmp(mask->hdr.dst_addr, empty_addr, 16)) { + tap_nlattr_add(&msg->nh, TCA_FLOWER_KEY_IPV6_DST, + sizeof(spec->hdr.dst_addr), &spec->hdr.dst_addr); + tap_nlattr_add(&msg->nh, TCA_FLOWER_KEY_IPV6_DST_MASK, + sizeof(mask->hdr.dst_addr), &mask->hdr.dst_addr); + } + if (memcmp(mask->hdr.src_addr, empty_addr, 16)) { + tap_nlattr_add(&msg->nh, TCA_FLOWER_KEY_IPV6_SRC, + sizeof(spec->hdr.src_addr), &spec->hdr.src_addr); + tap_nlattr_add(&msg->nh, TCA_FLOWER_KEY_IPV6_SRC_MASK, + sizeof(mask->hdr.src_addr), &mask->hdr.src_addr); + } + if (spec->hdr.proto) + tap_nlattr_add8(&msg->nh, + TCA_FLOWER_KEY_IP_PROTO, spec->hdr.proto); + return 0; +} + +/** + * Make as much checks as possible on a UDP item, and if a flow is provided, + * fill it appropriately with UDP info. + * + * @param[in] item + * Item specification. + * @param[in, out] data + * Additional data structure to tell next layers we've been here. + * + * @return + * 0 if checks are alright, -1 otherwise. + */ +static int +tap_flow_create_udp(const struct rte_flow_item *item, void *data) +{ + struct convert_data *info = (struct convert_data *)data; + const struct rte_flow_item_udp *spec = item->spec; + const struct rte_flow_item_udp *mask = item->mask; + struct rte_flow *flow = info->flow; + struct nlmsg *msg; + + /* use default mask if none provided */ + if (!mask) + mask = tap_flow_items[RTE_FLOW_ITEM_TYPE_UDP].default_mask; + /* check that previous ip_proto is compatible with udp */ + if (info->ip_proto && info->ip_proto != IPPROTO_UDP) + return -1; + /* TC does not support UDP port masking. Only accept if exact match. */ + if ((mask->hdr.src_port && mask->hdr.src_port != 0xffff) || + (mask->hdr.dst_port && mask->hdr.dst_port != 0xffff)) + return -1; + if (!flow) + return 0; + msg = &flow->msg; + tap_nlattr_add8(&msg->nh, TCA_FLOWER_KEY_IP_PROTO, IPPROTO_UDP); + if (!spec) + return 0; + if (mask->hdr.dst_port) + tap_nlattr_add16(&msg->nh, TCA_FLOWER_KEY_UDP_DST, + spec->hdr.dst_port); + if (mask->hdr.src_port) + tap_nlattr_add16(&msg->nh, TCA_FLOWER_KEY_UDP_SRC, + spec->hdr.src_port); + return 0; +} + +/** + * Make as much checks as possible on a TCP item, and if a flow is provided, + * fill it appropriately with TCP info. + * + * @param[in] item + * Item specification. + * @param[in, out] data + * Additional data structure to tell next layers we've been here. + * + * @return + * 0 if checks are alright, -1 otherwise. + */ +static int +tap_flow_create_tcp(const struct rte_flow_item *item, void *data) +{ + struct convert_data *info = (struct convert_data *)data; + const struct rte_flow_item_tcp *spec = item->spec; + const struct rte_flow_item_tcp *mask = item->mask; + struct rte_flow *flow = info->flow; + struct nlmsg *msg; + + /* use default mask if none provided */ + if (!mask) + mask = tap_flow_items[RTE_FLOW_ITEM_TYPE_TCP].default_mask; + /* check that previous ip_proto is compatible with tcp */ + if (info->ip_proto && info->ip_proto != IPPROTO_TCP) + return -1; + /* TC does not support TCP port masking. Only accept if exact match. */ + if ((mask->hdr.src_port && mask->hdr.src_port != 0xffff) || + (mask->hdr.dst_port && mask->hdr.dst_port != 0xffff)) + return -1; + if (!flow) + return 0; + msg = &flow->msg; + tap_nlattr_add8(&msg->nh, TCA_FLOWER_KEY_IP_PROTO, IPPROTO_TCP); + if (!spec) + return 0; + if (mask->hdr.dst_port) + tap_nlattr_add16(&msg->nh, TCA_FLOWER_KEY_TCP_DST, + spec->hdr.dst_port); + if (mask->hdr.src_port) + tap_nlattr_add16(&msg->nh, TCA_FLOWER_KEY_TCP_SRC, + spec->hdr.src_port); + return 0; +} + +/** + * Check support for a given item. + * + * @param[in] item + * Item specification. + * @param size + * Bit-Mask size in bytes. + * @param[in] supported_mask + * Bit-mask covering supported fields to compare with spec, last and mask in + * \item. + * @param[in] default_mask + * Bit-mask default mask if none is provided in \item. + * + * @return + * 0 on success. + */ +static int +tap_flow_item_validate(const struct rte_flow_item *item, + unsigned int size, + const uint8_t *supported_mask, + const uint8_t *default_mask) +{ + int ret = 0; + + /* An empty layer is allowed, as long as all fields are NULL */ + if (!item->spec && (item->mask || item->last)) + return -1; + /* Is the item spec compatible with what the NIC supports? */ + if (item->spec && !item->mask) { + unsigned int i; + const uint8_t *spec = item->spec; + + for (i = 0; i < size; ++i) + if ((spec[i] | supported_mask[i]) != supported_mask[i]) + return -1; + /* Is the default mask compatible with what the NIC supports? */ + for (i = 0; i < size; i++) + if ((default_mask[i] | supported_mask[i]) != + supported_mask[i]) + return -1; + } + /* Is the item last compatible with what the NIC supports? */ + if (item->last && !item->mask) { + unsigned int i; + const uint8_t *spec = item->last; + + for (i = 0; i < size; ++i) + if ((spec[i] | supported_mask[i]) != supported_mask[i]) + return -1; + } + /* Is the item mask compatible with what the NIC supports? */ + if (item->mask) { + unsigned int i; + const uint8_t *spec = item->mask; + + for (i = 0; i < size; ++i) + if ((spec[i] | supported_mask[i]) != supported_mask[i]) + return -1; + } + /** + * Once masked, Are item spec and item last equal? + * TC does not support range so anything else is invalid. + */ + if (item->spec && item->last) { + uint8_t spec[size]; + uint8_t last[size]; + const uint8_t *apply = default_mask; + unsigned int i; + + if (item->mask) + apply = item->mask; + for (i = 0; i < size; ++i) { + spec[i] = ((const uint8_t *)item->spec)[i] & apply[i]; + last[i] = ((const uint8_t *)item->last)[i] & apply[i]; + } + ret = memcmp(spec, last, size); + } + return ret; +} + +/** + * Configure the kernel with a TC action and its configured parameters + * Handled actions: "gact", "mirred", "skbedit", "bpf" + * + * @param[in] flow + * Pointer to rte flow containing the netlink message + * + * @param[in, out] act_index + * Pointer to action sequence number in the TC command + * + * @param[in] adata + * Pointer to struct holding the action parameters + * + * @return + * -1 on failure, 0 on success + */ +static int +add_action(struct rte_flow *flow, size_t *act_index, struct action_data *adata) +{ + struct nlmsg *msg = &flow->msg; + + if (tap_nlattr_nested_start(msg, (*act_index)++) < 0) + return -1; + + tap_nlattr_add(&msg->nh, TCA_ACT_KIND, + strlen(adata->id) + 1, adata->id); + if (tap_nlattr_nested_start(msg, TCA_ACT_OPTIONS) < 0) + return -1; + if (strcmp("gact", adata->id) == 0) { + tap_nlattr_add(&msg->nh, TCA_GACT_PARMS, sizeof(adata->gact), + &adata->gact); + } else if (strcmp("mirred", adata->id) == 0) { + if (adata->mirred.eaction == TCA_EGRESS_MIRROR) + adata->mirred.action = TC_ACT_PIPE; + else /* REDIRECT */ + adata->mirred.action = TC_ACT_STOLEN; + tap_nlattr_add(&msg->nh, TCA_MIRRED_PARMS, + sizeof(adata->mirred), + &adata->mirred); + } else if (strcmp("skbedit", adata->id) == 0) { + tap_nlattr_add(&msg->nh, TCA_SKBEDIT_PARMS, + sizeof(adata->skbedit.skbedit), + &adata->skbedit.skbedit); + tap_nlattr_add16(&msg->nh, TCA_SKBEDIT_QUEUE_MAPPING, + adata->skbedit.queue); + } else if (strcmp("bpf", adata->id) == 0) { + tap_nlattr_add32(&msg->nh, TCA_ACT_BPF_FD, adata->bpf.bpf_fd); + tap_nlattr_add(&msg->nh, TCA_ACT_BPF_NAME, + strlen(adata->bpf.annotation) + 1, + adata->bpf.annotation); + tap_nlattr_add(&msg->nh, TCA_ACT_BPF_PARMS, + sizeof(adata->bpf.bpf), + &adata->bpf.bpf); + } else { + return -1; + } + tap_nlattr_nested_finish(msg); /* nested TCA_ACT_OPTIONS */ + tap_nlattr_nested_finish(msg); /* nested act_index */ + return 0; +} + +/** + * Helper function to send a serie of TC actions to the kernel + * + * @param[in] flow + * Pointer to rte flow containing the netlink message + * + * @param[in] nb_actions + * Number of actions in an array of action structs + * + * @param[in] data + * Pointer to an array of action structs + * + * @param[in] classifier_actions + * The classifier on behave of which the actions are configured + * + * @return + * -1 on failure, 0 on success + */ +static int +add_actions(struct rte_flow *flow, int nb_actions, struct action_data *data, + int classifier_action) +{ + struct nlmsg *msg = &flow->msg; + size_t act_index = 1; + int i; + + if (tap_nlattr_nested_start(msg, classifier_action) < 0) + return -1; + for (i = 0; i < nb_actions; i++) + if (add_action(flow, &act_index, data + i) < 0) + return -1; + tap_nlattr_nested_finish(msg); /* nested TCA_FLOWER_ACT */ + return 0; +} + +/** + * Validate a flow supported by TC. + * If flow param is not NULL, then also fill the netlink message inside. + * + * @param pmd + * Pointer to private structure. + * @param[in] attr + * Flow rule attributes. + * @param[in] pattern + * Pattern specification (list terminated by the END pattern item). + * @param[in] actions + * Associated actions (list terminated by the END action). + * @param[out] error + * Perform verbose error reporting if not NULL. + * @param[in, out] flow + * Flow structure to update. + * @param[in] mirred + * If set to TCA_EGRESS_REDIR, provided actions will be replaced with a + * redirection to the tap netdevice, and the TC rule will be configured + * on the remote netdevice in pmd. + * If set to TCA_EGRESS_MIRROR, provided actions will be replaced with a + * mirroring to the tap netdevice, and the TC rule will be configured + * on the remote netdevice in pmd. Matching packets will thus be duplicated. + * If set to 0, the standard behavior is to be used: set correct actions for + * the TC rule, and apply it on the tap netdevice. + * + * @return + * 0 on success, a negative errno value otherwise and rte_errno is set. + */ +static int +priv_flow_process(struct pmd_internals *pmd, + const struct rte_flow_attr *attr, + const struct rte_flow_item items[], + const struct rte_flow_action actions[], + struct rte_flow_error *error, + struct rte_flow *flow, + int mirred) +{ + const struct tap_flow_items *cur_item = tap_flow_items; + struct convert_data data = { + .eth_type = 0, + .ip_proto = 0, + .flow = flow, + }; + int action = 0; /* Only one action authorized for now */ + + if (attr->transfer) { + rte_flow_error_set( + error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER, + NULL, "transfer is not supported"); + return -rte_errno; + } + if (attr->group > MAX_GROUP) { + rte_flow_error_set( + error, EINVAL, RTE_FLOW_ERROR_TYPE_ATTR_GROUP, + NULL, "group value too big: cannot exceed 15"); + return -rte_errno; + } + if (attr->priority > MAX_PRIORITY) { + rte_flow_error_set( + error, EINVAL, RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY, + NULL, "priority value too big"); + return -rte_errno; + } else if (flow) { + uint16_t group = attr->group << GROUP_SHIFT; + uint16_t prio = group | (attr->priority + + RSS_PRIORITY_OFFSET + PRIORITY_OFFSET); + flow->msg.t.tcm_info = TC_H_MAKE(prio << 16, + flow->msg.t.tcm_info); + } + if (flow) { + if (mirred) { + /* + * If attr->ingress, the rule applies on remote ingress + * to match incoming packets + * If attr->egress, the rule applies on tap ingress (as + * seen from the kernel) to deal with packets going out + * from the DPDK app. + */ + flow->msg.t.tcm_parent = TC_H_MAKE(TC_H_INGRESS, 0); + } else { + /* Standard rule on tap egress (kernel standpoint). */ + flow->msg.t.tcm_parent = + TC_H_MAKE(MULTIQ_MAJOR_HANDLE, 0); + } + /* use flower filter type */ + tap_nlattr_add(&flow->msg.nh, TCA_KIND, sizeof("flower"), "flower"); + if (tap_nlattr_nested_start(&flow->msg, TCA_OPTIONS) < 0) + goto exit_item_not_supported; + } + for (; items->type != RTE_FLOW_ITEM_TYPE_END; ++items) { + const struct tap_flow_items *token = NULL; + unsigned int i; + int err = 0; + + if (items->type == RTE_FLOW_ITEM_TYPE_VOID) + continue; + for (i = 0; + cur_item->items && + cur_item->items[i] != RTE_FLOW_ITEM_TYPE_END; + ++i) { + if (cur_item->items[i] == items->type) { + token = &tap_flow_items[items->type]; + break; + } + } + if (!token) + goto exit_item_not_supported; + cur_item = token; + err = tap_flow_item_validate( + items, cur_item->mask_sz, + (const uint8_t *)cur_item->mask, + (const uint8_t *)cur_item->default_mask); + if (err) + goto exit_item_not_supported; + if (flow && cur_item->convert) { + err = cur_item->convert(items, &data); + if (err) + goto exit_item_not_supported; + } + } + if (flow) { + if (data.vlan) { + tap_nlattr_add16(&flow->msg.nh, TCA_FLOWER_KEY_ETH_TYPE, + htons(ETH_P_8021Q)); + tap_nlattr_add16(&flow->msg.nh, + TCA_FLOWER_KEY_VLAN_ETH_TYPE, + data.eth_type ? + data.eth_type : htons(ETH_P_ALL)); + } else if (data.eth_type) { + tap_nlattr_add16(&flow->msg.nh, TCA_FLOWER_KEY_ETH_TYPE, + data.eth_type); + } + } + if (mirred && flow) { + struct action_data adata = { + .id = "mirred", + .mirred = { + .eaction = mirred, + }, + }; + + /* + * If attr->egress && mirred, then this is a special + * case where the rule must be applied on the tap, to + * redirect packets coming from the DPDK App, out + * through the remote netdevice. + */ + adata.mirred.ifindex = attr->ingress ? pmd->if_index : + pmd->remote_if_index; + if (mirred == TCA_EGRESS_MIRROR) + adata.mirred.action = TC_ACT_PIPE; + else + adata.mirred.action = TC_ACT_STOLEN; + if (add_actions(flow, 1, &adata, TCA_FLOWER_ACT) < 0) + goto exit_action_not_supported; + else + goto end; + } +actions: + for (; actions->type != RTE_FLOW_ACTION_TYPE_END; ++actions) { + int err = 0; + + if (actions->type == RTE_FLOW_ACTION_TYPE_VOID) { + continue; + } else if (actions->type == RTE_FLOW_ACTION_TYPE_DROP) { + if (action) + goto exit_action_not_supported; + action = 1; + if (flow) { + struct action_data adata = { + .id = "gact", + .gact = { + .action = TC_ACT_SHOT, + }, + }; + + err = add_actions(flow, 1, &adata, + TCA_FLOWER_ACT); + } + } else if (actions->type == RTE_FLOW_ACTION_TYPE_PASSTHRU) { + if (action) + goto exit_action_not_supported; + action = 1; + if (flow) { + struct action_data adata = { + .id = "gact", + .gact = { + /* continue */ + .action = TC_ACT_UNSPEC, + }, + }; + + err = add_actions(flow, 1, &adata, + TCA_FLOWER_ACT); + } + } else if (actions->type == RTE_FLOW_ACTION_TYPE_QUEUE) { + const struct rte_flow_action_queue *queue = + (const struct rte_flow_action_queue *) + actions->conf; + + if (action) + goto exit_action_not_supported; + action = 1; + if (!queue || + (queue->index > pmd->dev->data->nb_rx_queues - 1)) + goto exit_action_not_supported; + if (flow) { + struct action_data adata = { + .id = "skbedit", + .skbedit = { + .skbedit = { + .action = TC_ACT_PIPE, + }, + .queue = queue->index, + }, + }; + + err = add_actions(flow, 1, &adata, + TCA_FLOWER_ACT); + } + } else if (actions->type == RTE_FLOW_ACTION_TYPE_RSS) { + const struct rte_flow_action_rss *rss = + (const struct rte_flow_action_rss *) + actions->conf; + + if (action++) + goto exit_action_not_supported; + + if (!pmd->rss_enabled) { + err = rss_enable(pmd, attr, error); + if (err) + goto exit_action_not_supported; + } + if (flow) + err = rss_add_actions(flow, pmd, rss, error); + } else { + goto exit_action_not_supported; + } + if (err) + goto exit_action_not_supported; + } + /* When fate is unknown, drop traffic. */ + if (!action) { + static const struct rte_flow_action drop[] = { + { .type = RTE_FLOW_ACTION_TYPE_DROP, }, + { .type = RTE_FLOW_ACTION_TYPE_END, }, + }; + + actions = drop; + goto actions; + } +end: + if (flow) + tap_nlattr_nested_finish(&flow->msg); /* nested TCA_OPTIONS */ + return 0; +exit_item_not_supported: + rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, + items, "item not supported"); + return -rte_errno; +exit_action_not_supported: + rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, + actions, "action not supported"); + return -rte_errno; +} + + + +/** + * Validate a flow. + * + * @see rte_flow_validate() + * @see rte_flow_ops + */ +static int +tap_flow_validate(struct rte_eth_dev *dev, + const struct rte_flow_attr *attr, + const struct rte_flow_item items[], + const struct rte_flow_action actions[], + struct rte_flow_error *error) +{ + struct pmd_internals *pmd = dev->data->dev_private; + + return priv_flow_process(pmd, attr, items, actions, error, NULL, 0); +} + +/** + * Set a unique handle in a flow. + * + * The kernel supports TC rules with equal priority, as long as they use the + * same matching fields (e.g.: dst mac and ipv4) with different values (and + * full mask to ensure no collision is possible). + * In those rules, the handle (uint32_t) is the part that would identify + * specifically each rule. + * + * On 32-bit architectures, the handle can simply be the flow's pointer address. + * On 64-bit architectures, we rely on jhash(flow) to find a (sufficiently) + * unique handle. + * + * @param[in, out] flow + * The flow that needs its handle set. + */ +static void +tap_flow_set_handle(struct rte_flow *flow) +{ + uint32_t handle = 0; + + if (sizeof(flow) > 4) + handle = rte_jhash(&flow, sizeof(flow), 1); + else + handle = (uintptr_t)flow; + /* must be at least 1 to avoid letting the kernel choose one for us */ + if (!handle) + handle = 1; + flow->msg.t.tcm_handle = handle; +} + +/** + * Free the flow opened file descriptors and allocated memory + * + * @param[in] flow + * Pointer to the flow to free + * + */ +static void +tap_flow_free(struct pmd_internals *pmd, struct rte_flow *flow) +{ + int i; + + if (!flow) + return; + + if (pmd->rss_enabled) { + /* Close flow BPF file descriptors */ + for (i = 0; i < SEC_MAX; i++) + if (flow->bpf_fd[i] != 0) { + close(flow->bpf_fd[i]); + flow->bpf_fd[i] = 0; + } + + /* Release the map key for this RSS rule */ + bpf_rss_key(KEY_CMD_RELEASE, &flow->key_idx); + flow->key_idx = 0; + } + + /* Free flow allocated memory */ + rte_free(flow); +} + +/** + * Create a flow. + * + * @see rte_flow_create() + * @see rte_flow_ops + */ +static struct rte_flow * +tap_flow_create(struct rte_eth_dev *dev, + const struct rte_flow_attr *attr, + const struct rte_flow_item items[], + const struct rte_flow_action actions[], + struct rte_flow_error *error) +{ + struct pmd_internals *pmd = dev->data->dev_private; + struct rte_flow *remote_flow = NULL; + struct rte_flow *flow = NULL; + struct nlmsg *msg = NULL; + int err; + + if (!pmd->if_index) { + rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_HANDLE, + NULL, + "can't create rule, ifindex not found"); + goto fail; + } + /* + * No rules configured through standard rte_flow should be set on the + * priorities used by implicit rules. + */ + if ((attr->group == MAX_GROUP) && + attr->priority > (MAX_PRIORITY - TAP_REMOTE_MAX_IDX)) { + rte_flow_error_set( + error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY, + NULL, "priority value too big"); + goto fail; + } + flow = rte_malloc(__func__, sizeof(struct rte_flow), 0); + if (!flow) { + rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE, + NULL, "cannot allocate memory for rte_flow"); + goto fail; + } + msg = &flow->msg; + tc_init_msg(msg, pmd->if_index, RTM_NEWTFILTER, + NLM_F_REQUEST | NLM_F_ACK | NLM_F_EXCL | NLM_F_CREATE); + msg->t.tcm_info = TC_H_MAKE(0, htons(ETH_P_ALL)); + tap_flow_set_handle(flow); + if (priv_flow_process(pmd, attr, items, actions, error, flow, 0)) + goto fail; + err = tap_nl_send(pmd->nlsk_fd, &msg->nh); + if (err < 0) { + rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_HANDLE, + NULL, "couldn't send request to kernel"); + goto fail; + } + err = tap_nl_recv_ack(pmd->nlsk_fd); + if (err < 0) { + TAP_LOG(ERR, + "Kernel refused TC filter rule creation (%d): %s", + errno, strerror(errno)); + rte_flow_error_set(error, EEXIST, RTE_FLOW_ERROR_TYPE_HANDLE, + NULL, + "overlapping rules or Kernel too old for flower support"); + goto fail; + } + LIST_INSERT_HEAD(&pmd->flows, flow, next); + /** + * If a remote device is configured, a TC rule with identical items for + * matching must be set on that device, with a single action: redirect + * to the local pmd->if_index. + */ + if (pmd->remote_if_index) { + remote_flow = rte_malloc(__func__, sizeof(struct rte_flow), 0); + if (!remote_flow) { + rte_flow_error_set( + error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE, NULL, + "cannot allocate memory for rte_flow"); + goto fail; + } + msg = &remote_flow->msg; + /* set the rule if_index for the remote netdevice */ + tc_init_msg( + msg, pmd->remote_if_index, RTM_NEWTFILTER, + NLM_F_REQUEST | NLM_F_ACK | NLM_F_EXCL | NLM_F_CREATE); + msg->t.tcm_info = TC_H_MAKE(0, htons(ETH_P_ALL)); + tap_flow_set_handle(remote_flow); + if (priv_flow_process(pmd, attr, items, NULL, + error, remote_flow, TCA_EGRESS_REDIR)) { + rte_flow_error_set( + error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE, + NULL, "rte flow rule validation failed"); + goto fail; + } + err = tap_nl_send(pmd->nlsk_fd, &msg->nh); + if (err < 0) { + rte_flow_error_set( + error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE, + NULL, "Failure sending nl request"); + goto fail; + } + err = tap_nl_recv_ack(pmd->nlsk_fd); + if (err < 0) { + TAP_LOG(ERR, + "Kernel refused TC filter rule creation (%d): %s", + errno, strerror(errno)); + rte_flow_error_set( + error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE, + NULL, + "overlapping rules or Kernel too old for flower support"); + goto fail; + } + flow->remote_flow = remote_flow; + } + return flow; +fail: + if (remote_flow) + rte_free(remote_flow); + if (flow) + tap_flow_free(pmd, flow); + return NULL; +} + +/** + * Destroy a flow using pointer to pmd_internal. + * + * @param[in, out] pmd + * Pointer to private structure. + * @param[in] flow + * Pointer to the flow to destroy. + * @param[in, out] error + * Pointer to the flow error handler + * + * @return 0 if the flow could be destroyed, -1 otherwise. + */ +static int +tap_flow_destroy_pmd(struct pmd_internals *pmd, + struct rte_flow *flow, + struct rte_flow_error *error) +{ + struct rte_flow *remote_flow = flow->remote_flow; + int ret = 0; + + LIST_REMOVE(flow, next); + flow->msg.nh.nlmsg_flags = NLM_F_REQUEST | NLM_F_ACK; + flow->msg.nh.nlmsg_type = RTM_DELTFILTER; + + ret = tap_nl_send(pmd->nlsk_fd, &flow->msg.nh); + if (ret < 0) { + rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_HANDLE, + NULL, "couldn't send request to kernel"); + goto end; + } + ret = tap_nl_recv_ack(pmd->nlsk_fd); + /* If errno is ENOENT, the rule is already no longer in the kernel. */ + if (ret < 0 && errno == ENOENT) + ret = 0; + if (ret < 0) { + TAP_LOG(ERR, + "Kernel refused TC filter rule deletion (%d): %s", + errno, strerror(errno)); + rte_flow_error_set( + error, ENOTSUP, RTE_FLOW_ERROR_TYPE_HANDLE, NULL, + "couldn't receive kernel ack to our request"); + goto end; + } + + if (remote_flow) { + remote_flow->msg.nh.nlmsg_flags = NLM_F_REQUEST | NLM_F_ACK; + remote_flow->msg.nh.nlmsg_type = RTM_DELTFILTER; + + ret = tap_nl_send(pmd->nlsk_fd, &remote_flow->msg.nh); + if (ret < 0) { + rte_flow_error_set( + error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE, + NULL, "Failure sending nl request"); + goto end; + } + ret = tap_nl_recv_ack(pmd->nlsk_fd); + if (ret < 0 && errno == ENOENT) + ret = 0; + if (ret < 0) { + TAP_LOG(ERR, + "Kernel refused TC filter rule deletion (%d): %s", + errno, strerror(errno)); + rte_flow_error_set( + error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE, + NULL, "Failure trying to receive nl ack"); + goto end; + } + } +end: + if (remote_flow) + rte_free(remote_flow); + tap_flow_free(pmd, flow); + return ret; +} + +/** + * Destroy a flow. + * + * @see rte_flow_destroy() + * @see rte_flow_ops + */ +static int +tap_flow_destroy(struct rte_eth_dev *dev, + struct rte_flow *flow, + struct rte_flow_error *error) +{ + struct pmd_internals *pmd = dev->data->dev_private; + + return tap_flow_destroy_pmd(pmd, flow, error); +} + +/** + * Enable/disable flow isolation. + * + * @see rte_flow_isolate() + * @see rte_flow_ops + */ +static int +tap_flow_isolate(struct rte_eth_dev *dev, + int set, + struct rte_flow_error *error __rte_unused) +{ + struct pmd_internals *pmd = dev->data->dev_private; + + /* normalize 'set' variable to contain 0 or 1 values */ + if (set) + set = 1; + /* if already in the right isolation mode - nothing to do */ + if ((set ^ pmd->flow_isolate) == 0) + return 0; + /* mark the isolation mode for tap_flow_implicit_create() */ + pmd->flow_isolate = set; + /* + * If netdevice is there, setup appropriate flow rules immediately. + * Otherwise it will be set when bringing up the netdevice (tun_alloc). + */ + if (!pmd->rxq[0].fd) + return 0; + if (set) { + struct rte_flow *remote_flow; + + while (1) { + remote_flow = LIST_FIRST(&pmd->implicit_flows); + if (!remote_flow) + break; + /* + * Remove all implicit rules on the remote. + * Keep the local rule to redirect packets on TX. + * Keep also the last implicit local rule: ISOLATE. + */ + if (remote_flow->msg.t.tcm_ifindex == pmd->if_index) + break; + if (tap_flow_destroy_pmd(pmd, remote_flow, NULL) < 0) + goto error; + } + /* Switch the TC rule according to pmd->flow_isolate */ + if (tap_flow_implicit_create(pmd, TAP_ISOLATE) == -1) + goto error; + } else { + /* Switch the TC rule according to pmd->flow_isolate */ + if (tap_flow_implicit_create(pmd, TAP_ISOLATE) == -1) + goto error; + if (!pmd->remote_if_index) + return 0; + if (tap_flow_implicit_create(pmd, TAP_REMOTE_TX) < 0) + goto error; + if (tap_flow_implicit_create(pmd, TAP_REMOTE_LOCAL_MAC) < 0) + goto error; + if (tap_flow_implicit_create(pmd, TAP_REMOTE_BROADCAST) < 0) + goto error; + if (tap_flow_implicit_create(pmd, TAP_REMOTE_BROADCASTV6) < 0) + goto error; + if (dev->data->promiscuous && + tap_flow_implicit_create(pmd, TAP_REMOTE_PROMISC) < 0) + goto error; + if (dev->data->all_multicast && + tap_flow_implicit_create(pmd, TAP_REMOTE_ALLMULTI) < 0) + goto error; + } + return 0; +error: + pmd->flow_isolate = 0; + return rte_flow_error_set( + error, ENOTSUP, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, + "TC rule creation failed"); +} + +/** + * Destroy all flows. + * + * @see rte_flow_flush() + * @see rte_flow_ops + */ +int +tap_flow_flush(struct rte_eth_dev *dev, struct rte_flow_error *error) +{ + struct pmd_internals *pmd = dev->data->dev_private; + struct rte_flow *flow; + + while (!LIST_EMPTY(&pmd->flows)) { + flow = LIST_FIRST(&pmd->flows); + if (tap_flow_destroy(dev, flow, error) < 0) + return -1; + } + return 0; +} + +/** + * Add an implicit flow rule on the remote device to make sure traffic gets to + * the tap netdevice from there. + * + * @param pmd + * Pointer to private structure. + * @param[in] idx + * The idx in the implicit_rte_flows array specifying which rule to apply. + * + * @return -1 if the rule couldn't be applied, 0 otherwise. + */ +int tap_flow_implicit_create(struct pmd_internals *pmd, + enum implicit_rule_index idx) +{ + uint16_t flags = NLM_F_REQUEST | NLM_F_ACK | NLM_F_EXCL | NLM_F_CREATE; + struct rte_flow_action *actions = implicit_rte_flows[idx].actions; + struct rte_flow_action isolate_actions[2] = { + [1] = { + .type = RTE_FLOW_ACTION_TYPE_END, + }, + }; + struct rte_flow_item *items = implicit_rte_flows[idx].items; + struct rte_flow_attr *attr = &implicit_rte_flows[idx].attr; + struct rte_flow_item_eth eth_local = { .type = 0 }; + uint16_t if_index = pmd->remote_if_index; + struct rte_flow *remote_flow = NULL; + struct nlmsg *msg = NULL; + int err = 0; + struct rte_flow_item items_local[2] = { + [0] = { + .type = items[0].type, + .spec = ð_local, + .mask = items[0].mask, + }, + [1] = { + .type = items[1].type, + } + }; + + remote_flow = rte_malloc(__func__, sizeof(struct rte_flow), 0); + if (!remote_flow) { + TAP_LOG(ERR, "Cannot allocate memory for rte_flow"); + goto fail; + } + msg = &remote_flow->msg; + if (idx == TAP_REMOTE_TX) { + if_index = pmd->if_index; + } else if (idx == TAP_ISOLATE) { + if_index = pmd->if_index; + /* Don't be exclusive for this rule, it can be changed later. */ + flags = NLM_F_REQUEST | NLM_F_ACK | NLM_F_CREATE; + isolate_actions[0].type = pmd->flow_isolate ? + RTE_FLOW_ACTION_TYPE_DROP : + RTE_FLOW_ACTION_TYPE_PASSTHRU; + actions = isolate_actions; + } else if (idx == TAP_REMOTE_LOCAL_MAC) { + /* + * eth addr couldn't be set in implicit_rte_flows[] as it is not + * known at compile time. + */ + memcpy(ð_local.dst, &pmd->eth_addr, sizeof(pmd->eth_addr)); + items = items_local; + } + tc_init_msg(msg, if_index, RTM_NEWTFILTER, flags); + msg->t.tcm_info = TC_H_MAKE(0, htons(ETH_P_ALL)); + /* + * The ISOLATE rule is always present and must have a static handle, as + * the action is changed whether the feature is enabled (DROP) or + * disabled (PASSTHRU). + * There is just one REMOTE_PROMISCUOUS rule in all cases. It should + * have a static handle such that adding it twice will fail with EEXIST + * with any kernel version. Remark: old kernels may falsely accept the + * same REMOTE_PROMISCUOUS rules if they had different handles. + */ + if (idx == TAP_ISOLATE) + remote_flow->msg.t.tcm_handle = ISOLATE_HANDLE; + else if (idx == TAP_REMOTE_PROMISC) + remote_flow->msg.t.tcm_handle = REMOTE_PROMISCUOUS_HANDLE; + else + tap_flow_set_handle(remote_flow); + if (priv_flow_process(pmd, attr, items, actions, NULL, + remote_flow, implicit_rte_flows[idx].mirred)) { + TAP_LOG(ERR, "rte flow rule validation failed"); + goto fail; + } + err = tap_nl_send(pmd->nlsk_fd, &msg->nh); + if (err < 0) { + TAP_LOG(ERR, "Failure sending nl request"); + goto fail; + } + err = tap_nl_recv_ack(pmd->nlsk_fd); + if (err < 0) { + /* Silently ignore re-entering existing rule */ + if (errno == EEXIST) + goto success; + TAP_LOG(ERR, + "Kernel refused TC filter rule creation (%d): %s", + errno, strerror(errno)); + goto fail; + } + LIST_INSERT_HEAD(&pmd->implicit_flows, remote_flow, next); +success: + return 0; +fail: + if (remote_flow) + rte_free(remote_flow); + return -1; +} + +/** + * Remove specific implicit flow rule on the remote device. + * + * @param[in, out] pmd + * Pointer to private structure. + * @param[in] idx + * The idx in the implicit_rte_flows array specifying which rule to remove. + * + * @return -1 if one of the implicit rules couldn't be created, 0 otherwise. + */ +int tap_flow_implicit_destroy(struct pmd_internals *pmd, + enum implicit_rule_index idx) +{ + struct rte_flow *remote_flow; + int cur_prio = -1; + int idx_prio = implicit_rte_flows[idx].attr.priority + PRIORITY_OFFSET; + + for (remote_flow = LIST_FIRST(&pmd->implicit_flows); + remote_flow; + remote_flow = LIST_NEXT(remote_flow, next)) { + cur_prio = (remote_flow->msg.t.tcm_info >> 16) & PRIORITY_MASK; + if (cur_prio != idx_prio) + continue; + return tap_flow_destroy_pmd(pmd, remote_flow, NULL); + } + return 0; +} + +/** + * Destroy all implicit flows. + * + * @see rte_flow_flush() + */ +int +tap_flow_implicit_flush(struct pmd_internals *pmd, struct rte_flow_error *error) +{ + struct rte_flow *remote_flow; + + while (!LIST_EMPTY(&pmd->implicit_flows)) { + remote_flow = LIST_FIRST(&pmd->implicit_flows); + if (tap_flow_destroy_pmd(pmd, remote_flow, error) < 0) + return -1; + } + return 0; +} + +#define MAX_RSS_KEYS 256 +#define KEY_IDX_OFFSET (3 * MAX_RSS_KEYS) +#define SEC_NAME_CLS_Q "cls_q" + +const char *sec_name[SEC_MAX] = { + [SEC_L3_L4] = "l3_l4", +}; + +/** + * Enable RSS on tap: create TC rules for queuing. + * + * @param[in, out] pmd + * Pointer to private structure. + * + * @param[in] attr + * Pointer to rte_flow to get flow group + * + * @param[out] error + * Pointer to error reporting if not NULL. + * + * @return 0 on success, negative value on failure. + */ +static int rss_enable(struct pmd_internals *pmd, + const struct rte_flow_attr *attr, + struct rte_flow_error *error) +{ + struct rte_flow *rss_flow = NULL; + struct nlmsg *msg = NULL; + /* 4096 is the maximum number of instructions for a BPF program */ + char annotation[64]; + int i; + int err = 0; + + /* unlimit locked memory */ + struct rlimit memlock_limit = { + .rlim_cur = RLIM_INFINITY, + .rlim_max = RLIM_INFINITY, + }; + setrlimit(RLIMIT_MEMLOCK, &memlock_limit); + + /* Get a new map key for a new RSS rule */ + err = bpf_rss_key(KEY_CMD_INIT, NULL); + if (err < 0) { + rte_flow_error_set( + error, EINVAL, RTE_FLOW_ERROR_TYPE_HANDLE, NULL, + "Failed to initialize BPF RSS keys"); + + return -1; + } + + /* + * Create BPF RSS MAP + */ + pmd->map_fd = tap_flow_bpf_rss_map_create(sizeof(__u32), /* key size */ + sizeof(struct rss_key), + MAX_RSS_KEYS); + if (pmd->map_fd < 0) { + TAP_LOG(ERR, + "Failed to create BPF map (%d): %s", + errno, strerror(errno)); + rte_flow_error_set( + error, ENOTSUP, RTE_FLOW_ERROR_TYPE_HANDLE, NULL, + "Kernel too old or not configured " + "to support BPF maps"); + + return -ENOTSUP; + } + + /* + * Add a rule per queue to match reclassified packets and direct them to + * the correct queue. + */ + for (i = 0; i < pmd->dev->data->nb_rx_queues; i++) { + pmd->bpf_fd[i] = tap_flow_bpf_cls_q(i); + if (pmd->bpf_fd[i] < 0) { + TAP_LOG(ERR, + "Failed to load BPF section %s for queue %d", + SEC_NAME_CLS_Q, i); + rte_flow_error_set( + error, ENOTSUP, RTE_FLOW_ERROR_TYPE_HANDLE, + NULL, + "Kernel too old or not configured " + "to support BPF programs loading"); + + return -ENOTSUP; + } + + rss_flow = rte_malloc(__func__, sizeof(struct rte_flow), 0); + if (!rss_flow) { + TAP_LOG(ERR, + "Cannot allocate memory for rte_flow"); + return -1; + } + msg = &rss_flow->msg; + tc_init_msg(msg, pmd->if_index, RTM_NEWTFILTER, NLM_F_REQUEST | + NLM_F_ACK | NLM_F_EXCL | NLM_F_CREATE); + msg->t.tcm_info = TC_H_MAKE(0, htons(ETH_P_ALL)); + tap_flow_set_handle(rss_flow); + uint16_t group = attr->group << GROUP_SHIFT; + uint16_t prio = group | (i + PRIORITY_OFFSET); + msg->t.tcm_info = TC_H_MAKE(prio << 16, msg->t.tcm_info); + msg->t.tcm_parent = TC_H_MAKE(MULTIQ_MAJOR_HANDLE, 0); + + tap_nlattr_add(&msg->nh, TCA_KIND, sizeof("bpf"), "bpf"); + if (tap_nlattr_nested_start(msg, TCA_OPTIONS) < 0) + return -1; + tap_nlattr_add32(&msg->nh, TCA_BPF_FD, pmd->bpf_fd[i]); + snprintf(annotation, sizeof(annotation), "[%s%d]", + SEC_NAME_CLS_Q, i); + tap_nlattr_add(&msg->nh, TCA_BPF_NAME, strlen(annotation) + 1, + annotation); + /* Actions */ + { + struct action_data adata = { + .id = "skbedit", + .skbedit = { + .skbedit = { + .action = TC_ACT_PIPE, + }, + .queue = i, + }, + }; + if (add_actions(rss_flow, 1, &adata, TCA_BPF_ACT) < 0) + return -1; + } + tap_nlattr_nested_finish(msg); /* nested TCA_OPTIONS */ + + /* Netlink message is now ready to be sent */ + if (tap_nl_send(pmd->nlsk_fd, &msg->nh) < 0) + return -1; + err = tap_nl_recv_ack(pmd->nlsk_fd); + if (err < 0) { + TAP_LOG(ERR, + "Kernel refused TC filter rule creation (%d): %s", + errno, strerror(errno)); + return err; + } + LIST_INSERT_HEAD(&pmd->rss_flows, rss_flow, next); + } + + pmd->rss_enabled = 1; + return err; +} + +/** + * Manage bpf RSS keys repository with operations: init, get, release + * + * @param[in] cmd + * Command on RSS keys: init, get, release + * + * @param[in, out] key_idx + * Pointer to RSS Key index (out for get command, in for release command) + * + * @return -1 if couldn't get, release or init the RSS keys, 0 otherwise. + */ +static int bpf_rss_key(enum bpf_rss_key_e cmd, __u32 *key_idx) +{ + __u32 i; + int err = 0; + static __u32 num_used_keys; + static __u32 rss_keys[MAX_RSS_KEYS] = {KEY_STAT_UNSPEC}; + static __u32 rss_keys_initialized; + __u32 key; + + switch (cmd) { + case KEY_CMD_GET: + if (!rss_keys_initialized) { + err = -1; + break; + } + + if (num_used_keys == RTE_DIM(rss_keys)) { + err = -1; + break; + } + + *key_idx = num_used_keys % RTE_DIM(rss_keys); + while (rss_keys[*key_idx] == KEY_STAT_USED) + *key_idx = (*key_idx + 1) % RTE_DIM(rss_keys); + + rss_keys[*key_idx] = KEY_STAT_USED; + + /* + * Add an offset to key_idx in order to handle a case of + * RSS and non RSS flows mixture. + * If a non RSS flow is destroyed it has an eBPF map + * index 0 (initialized on flow creation) and might + * unintentionally remove RSS entry 0 from eBPF map. + * To avoid this issue, add an offset to the real index + * during a KEY_CMD_GET operation and subtract this offset + * during a KEY_CMD_RELEASE operation in order to restore + * the real index. + */ + *key_idx += KEY_IDX_OFFSET; + num_used_keys++; + break; + + case KEY_CMD_RELEASE: + if (!rss_keys_initialized) + break; + + /* + * Subtract offest to restore real key index + * If a non RSS flow is falsely trying to release map + * entry 0 - the offset subtraction will calculate the real + * map index as an out-of-range value and the release operation + * will be silently ignored. + */ + key = *key_idx - KEY_IDX_OFFSET; + if (key >= RTE_DIM(rss_keys)) + break; + + if (rss_keys[key] == KEY_STAT_USED) { + rss_keys[key] = KEY_STAT_AVAILABLE; + num_used_keys--; + } + break; + + case KEY_CMD_INIT: + for (i = 0; i < RTE_DIM(rss_keys); i++) + rss_keys[i] = KEY_STAT_AVAILABLE; + + rss_keys_initialized = 1; + num_used_keys = 0; + break; + + case KEY_CMD_DEINIT: + for (i = 0; i < RTE_DIM(rss_keys); i++) + rss_keys[i] = KEY_STAT_UNSPEC; + + rss_keys_initialized = 0; + num_used_keys = 0; + break; + + default: + break; + } + + return err; +} + +/** + * Add RSS hash calculations and queue selection + * + * @param[in, out] pmd + * Pointer to internal structure. Used to set/get RSS map fd + * + * @param[in] rss + * Pointer to RSS flow actions + * + * @param[out] error + * Pointer to error reporting if not NULL. + * + * @return 0 on success, negative value on failure + */ +static int rss_add_actions(struct rte_flow *flow, struct pmd_internals *pmd, + const struct rte_flow_action_rss *rss, + struct rte_flow_error *error) +{ + /* 4096 is the maximum number of instructions for a BPF program */ + unsigned int i; + int err; + struct rss_key rss_entry = { .hash_fields = 0, + .key_size = 0 }; + + /* Check supported RSS features */ + if (rss->func != RTE_ETH_HASH_FUNCTION_DEFAULT) + return rte_flow_error_set + (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, + "non-default RSS hash functions are not supported"); + if (rss->level) + return rte_flow_error_set + (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, + "a nonzero RSS encapsulation level is not supported"); + + /* Get a new map key for a new RSS rule */ + err = bpf_rss_key(KEY_CMD_GET, &flow->key_idx); + if (err < 0) { + rte_flow_error_set( + error, EINVAL, RTE_FLOW_ERROR_TYPE_HANDLE, NULL, + "Failed to get BPF RSS key"); + + return -1; + } + + /* Update RSS map entry with queues */ + rss_entry.nb_queues = rss->queue_num; + for (i = 0; i < rss->queue_num; i++) + rss_entry.queues[i] = rss->queue[i]; + rss_entry.hash_fields = + (1 << HASH_FIELD_IPV4_L3_L4) | (1 << HASH_FIELD_IPV6_L3_L4); + + /* Add this RSS entry to map */ + err = tap_flow_bpf_update_rss_elem(pmd->map_fd, + &flow->key_idx, &rss_entry); + + if (err) { + TAP_LOG(ERR, + "Failed to update BPF map entry #%u (%d): %s", + flow->key_idx, errno, strerror(errno)); + rte_flow_error_set( + error, ENOTSUP, RTE_FLOW_ERROR_TYPE_HANDLE, NULL, + "Kernel too old or not configured " + "to support BPF maps updates"); + + return -ENOTSUP; + } + + + /* + * Load bpf rules to calculate hash for this key_idx + */ + + flow->bpf_fd[SEC_L3_L4] = + tap_flow_bpf_calc_l3_l4_hash(flow->key_idx, pmd->map_fd); + if (flow->bpf_fd[SEC_L3_L4] < 0) { + TAP_LOG(ERR, + "Failed to load BPF section %s (%d): %s", + sec_name[SEC_L3_L4], errno, strerror(errno)); + rte_flow_error_set( + error, ENOTSUP, RTE_FLOW_ERROR_TYPE_HANDLE, NULL, + "Kernel too old or not configured " + "to support BPF program loading"); + + return -ENOTSUP; + } + + /* Actions */ + { + struct action_data adata[] = { + { + .id = "bpf", + .bpf = { + .bpf_fd = flow->bpf_fd[SEC_L3_L4], + .annotation = sec_name[SEC_L3_L4], + .bpf = { + .action = TC_ACT_PIPE, + }, + }, + }, + }; + + if (add_actions(flow, RTE_DIM(adata), adata, + TCA_FLOWER_ACT) < 0) + return -1; + } + + return 0; +} + +/** + * Manage filter operations. + * + * @param dev + * Pointer to Ethernet device structure. + * @param filter_type + * Filter type. + * @param filter_op + * Operation to perform. + * @param arg + * Pointer to operation-specific structure. + * + * @return + * 0 on success, negative errno value on failure. + */ +int +tap_dev_filter_ctrl(struct rte_eth_dev *dev, + enum rte_filter_type filter_type, + enum rte_filter_op filter_op, + void *arg) +{ + switch (filter_type) { + case RTE_ETH_FILTER_GENERIC: + if (filter_op != RTE_ETH_FILTER_GET) + return -EINVAL; + *(const void **)arg = &tap_flow_ops; + return 0; + default: + TAP_LOG(ERR, "%p: filter type (%d) not supported", + dev, filter_type); + } + return -EINVAL; +} diff --git a/src/spdk/dpdk/drivers/net/tap/tap_flow.h b/src/spdk/dpdk/drivers/net/tap/tap_flow.h new file mode 100644 index 00000000..ac60a9ae --- /dev/null +++ b/src/spdk/dpdk/drivers/net/tap/tap_flow.h @@ -0,0 +1,68 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright 2017 6WIND S.A. + * Copyright 2017 Mellanox Technologies, Ltd + */ + +#ifndef _TAP_FLOW_H_ +#define _TAP_FLOW_H_ + +#include <rte_flow.h> +#include <rte_flow_driver.h> +#include <rte_eth_tap.h> +#include <tap_autoconf.h> + +/** + * In TC, priority 0 means we require the kernel to allocate one for us. + * In rte_flow, however, we want the priority 0 to be the most important one. + * Use an offset to have the most important priority being 1 in TC. + */ +#define PRIORITY_OFFSET 1 +#define PRIORITY_MASK (0xfff) +#define MAX_PRIORITY (PRIORITY_MASK - PRIORITY_OFFSET) +#define GROUP_MASK (0xf) +#define GROUP_SHIFT 12 +#define MAX_GROUP GROUP_MASK +#define RSS_PRIORITY_OFFSET RTE_PMD_TAP_MAX_QUEUES + +/** + * These index are actually in reversed order: their priority is processed + * by subtracting their value to the lowest priority (PRIORITY_MASK). + * Thus the first one will have the lowest priority in the end + * (but biggest value). + */ +enum implicit_rule_index { + TAP_REMOTE_TX, + TAP_ISOLATE, + TAP_REMOTE_BROADCASTV6, + TAP_REMOTE_BROADCAST, + TAP_REMOTE_ALLMULTI, + TAP_REMOTE_PROMISC, + TAP_REMOTE_LOCAL_MAC, + TAP_REMOTE_MAX_IDX, +}; + +enum bpf_fd_idx { + SEC_L3_L4, + SEC_MAX, +}; + +int tap_dev_filter_ctrl(struct rte_eth_dev *dev, + enum rte_filter_type filter_type, + enum rte_filter_op filter_op, + void *arg); +int tap_flow_flush(struct rte_eth_dev *dev, struct rte_flow_error *error); + +int tap_flow_implicit_create(struct pmd_internals *pmd, + enum implicit_rule_index idx); +int tap_flow_implicit_destroy(struct pmd_internals *pmd, + enum implicit_rule_index idx); +int tap_flow_implicit_flush(struct pmd_internals *pmd, + struct rte_flow_error *error); + +int tap_flow_bpf_cls_q(__u32 queue_idx); +int tap_flow_bpf_calc_l3_l4_hash(__u32 key_idx, int map_fd); +int tap_flow_bpf_rss_map_create(unsigned int key_size, unsigned int value_size, + unsigned int max_entries); +int tap_flow_bpf_update_rss_elem(int fd, void *key, void *value); + +#endif /* _TAP_FLOW_H_ */ diff --git a/src/spdk/dpdk/drivers/net/tap/tap_intr.c b/src/spdk/dpdk/drivers/net/tap/tap_intr.c new file mode 100644 index 00000000..fc590181 --- /dev/null +++ b/src/spdk/dpdk/drivers/net/tap/tap_intr.c @@ -0,0 +1,110 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright 2018 Mellanox Technologies, Ltd + */ + +/** + * @file + * Interrupts handling for tap driver. + */ + +#include <assert.h> +#include <errno.h> +#include <fcntl.h> +#include <signal.h> +#include <stdint.h> +#include <stdlib.h> +#include <unistd.h> + +#include <rte_eth_tap.h> +#include <rte_errno.h> +#include <rte_interrupts.h> + + +/** + * Unregister Rx interrupts free the queue interrupt vector. + * + * @param dev + * Pointer to the tap rte_eth_dev structure. + */ +static void +tap_rx_intr_vec_uninstall(struct rte_eth_dev *dev) +{ + struct pmd_internals *pmd = dev->data->dev_private; + struct rte_intr_handle *intr_handle = &pmd->intr_handle; + + rte_intr_free_epoll_fd(intr_handle); + free(intr_handle->intr_vec); + intr_handle->intr_vec = NULL; + intr_handle->nb_efd = 0; +} + +/** + * Allocate Rx queue interrupt vector and register Rx interrupts. + * + * @param dev + * Pointer to the tap rte_eth_dev device structure. + * + * @return + * 0 on success, negative errno value otherwise and rte_errno is set. + */ +static int +tap_rx_intr_vec_install(struct rte_eth_dev *dev) +{ + struct pmd_internals *pmd = dev->data->dev_private; + unsigned int rxqs_n = pmd->dev->data->nb_rx_queues; + struct rte_intr_handle *intr_handle = &pmd->intr_handle; + unsigned int n = RTE_MIN(rxqs_n, (uint32_t)RTE_MAX_RXTX_INTR_VEC_ID); + unsigned int i; + unsigned int count = 0; + + if (!dev->data->dev_conf.intr_conf.rxq) + return 0; + intr_handle->intr_vec = malloc(sizeof(intr_handle->intr_vec[rxqs_n])); + if (intr_handle->intr_vec == NULL) { + rte_errno = ENOMEM; + TAP_LOG(ERR, + "failed to allocate memory for interrupt vector," + " Rx interrupts will not be supported"); + return -rte_errno; + } + for (i = 0; i < n; i++) { + struct rx_queue *rxq = pmd->dev->data->rx_queues[i]; + + /* Skip queues that cannot request interrupts. */ + if (!rxq || rxq->fd <= 0) { + /* Use invalid intr_vec[] index to disable entry. */ + intr_handle->intr_vec[i] = + RTE_INTR_VEC_RXTX_OFFSET + + RTE_MAX_RXTX_INTR_VEC_ID; + continue; + } + intr_handle->intr_vec[i] = RTE_INTR_VEC_RXTX_OFFSET + count; + intr_handle->efds[count] = rxq->fd; + count++; + } + if (!count) + tap_rx_intr_vec_uninstall(dev); + else + intr_handle->nb_efd = count; + return 0; +} + +/** + * Register or unregister the Rx interrupts. + * + * @param dev + * Pointer to the tap rte_eth_dev device structure. + * @param set + * should the operation be register or unregister the interrupts. + * + * @return + * 0 on success, negative errno value otherwise and rte_errno is set. + */ +int +tap_rx_intr_vec_set(struct rte_eth_dev *dev, int set) +{ + tap_rx_intr_vec_uninstall(dev); + if (set) + return tap_rx_intr_vec_install(dev); + return 0; +} diff --git a/src/spdk/dpdk/drivers/net/tap/tap_log.h b/src/spdk/dpdk/drivers/net/tap/tap_log.h new file mode 100644 index 00000000..fa06843a --- /dev/null +++ b/src/spdk/dpdk/drivers/net/tap/tap_log.h @@ -0,0 +1,10 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright 2017 6WIND S.A. + * Copyright 2017 Mellanox Technologies, Ltd + */ + +extern int tap_logtype; + +#define TAP_LOG(level, fmt, args...) \ + rte_log(RTE_LOG_ ## level, tap_logtype, "%s(): " fmt "\n", \ + __func__, ## args) diff --git a/src/spdk/dpdk/drivers/net/tap/tap_netlink.c b/src/spdk/dpdk/drivers/net/tap/tap_netlink.c new file mode 100644 index 00000000..6cb51009 --- /dev/null +++ b/src/spdk/dpdk/drivers/net/tap/tap_netlink.c @@ -0,0 +1,340 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright 2017 6WIND S.A. + * Copyright 2017 Mellanox Technologies, Ltd + */ + +#include <errno.h> +#include <inttypes.h> +#include <linux/netlink.h> +#include <string.h> +#include <sys/socket.h> +#include <unistd.h> + +#include <rte_malloc.h> +#include <tap_netlink.h> +#include <rte_random.h> +#include "tap_log.h" + +/* Must be quite large to support dumping a huge list of QDISC or filters. */ +#define BUF_SIZE (32 * 1024) /* Size of the buffer to receive kernel messages */ +#define SNDBUF_SIZE 32768 /* Send buffer size for the netlink socket */ +#define RCVBUF_SIZE 32768 /* Receive buffer size for the netlink socket */ + +struct nested_tail { + struct rtattr *tail; + struct nested_tail *prev; +}; + +/** + * Initialize a netlink socket for communicating with the kernel. + * + * @param nl_groups + * Set it to a netlink group value (e.g. RTMGRP_LINK) to receive messages for + * specific netlink multicast groups. Otherwise, no subscription will be made. + * + * @return + * netlink socket file descriptor on success, -1 otherwise. + */ +int +tap_nl_init(uint32_t nl_groups) +{ + int fd, sndbuf_size = SNDBUF_SIZE, rcvbuf_size = RCVBUF_SIZE; + struct sockaddr_nl local = { + .nl_family = AF_NETLINK, + .nl_groups = nl_groups, + }; + + fd = socket(AF_NETLINK, SOCK_RAW | SOCK_CLOEXEC, NETLINK_ROUTE); + if (fd < 0) { + TAP_LOG(ERR, "Unable to create a netlink socket"); + return -1; + } + if (setsockopt(fd, SOL_SOCKET, SO_SNDBUF, &sndbuf_size, sizeof(int))) { + TAP_LOG(ERR, "Unable to set socket buffer send size"); + return -1; + } + if (setsockopt(fd, SOL_SOCKET, SO_RCVBUF, &rcvbuf_size, sizeof(int))) { + TAP_LOG(ERR, "Unable to set socket buffer receive size"); + return -1; + } + if (bind(fd, (struct sockaddr *)&local, sizeof(local)) < 0) { + TAP_LOG(ERR, "Unable to bind to the netlink socket"); + return -1; + } + return fd; +} + +/** + * Clean up a netlink socket once all communicating with the kernel is finished. + * + * @param[in] nlsk_fd + * The netlink socket file descriptor used for communication. + * + * @return + * 0 on success, -1 otherwise. + */ +int +tap_nl_final(int nlsk_fd) +{ + if (close(nlsk_fd)) { + TAP_LOG(ERR, "Failed to close netlink socket: %s (%d)", + strerror(errno), errno); + return -1; + } + return 0; +} + +/** + * Send a message to the kernel on the netlink socket. + * + * @param[in] nlsk_fd + * The netlink socket file descriptor used for communication. + * @param[in] nh + * The netlink message send to the kernel. + * + * @return + * the number of sent bytes on success, -1 otherwise. + */ +int +tap_nl_send(int nlsk_fd, struct nlmsghdr *nh) +{ + /* man 7 netlink EXAMPLE */ + struct sockaddr_nl sa = { + .nl_family = AF_NETLINK, + }; + struct iovec iov = { + .iov_base = nh, + .iov_len = nh->nlmsg_len, + }; + struct msghdr msg = { + .msg_name = &sa, + .msg_namelen = sizeof(sa), + .msg_iov = &iov, + .msg_iovlen = 1, + }; + int send_bytes; + + nh->nlmsg_pid = 0; /* communication with the kernel uses pid 0 */ + nh->nlmsg_seq = (uint32_t)rte_rand(); + send_bytes = sendmsg(nlsk_fd, &msg, 0); + if (send_bytes < 0) { + TAP_LOG(ERR, "Failed to send netlink message: %s (%d)", + strerror(errno), errno); + return -1; + } + return send_bytes; +} + +/** + * Check that the kernel sends an appropriate ACK in response + * to an tap_nl_send(). + * + * @param[in] nlsk_fd + * The netlink socket file descriptor used for communication. + * + * @return + * 0 on success, -1 otherwise with errno set. + */ +int +tap_nl_recv_ack(int nlsk_fd) +{ + return tap_nl_recv(nlsk_fd, NULL, NULL); +} + +/** + * Receive a message from the kernel on the netlink socket, following an + * tap_nl_send(). + * + * @param[in] nlsk_fd + * The netlink socket file descriptor used for communication. + * @param[in] cb + * The callback function to call for each netlink message received. + * @param[in, out] arg + * Custom arguments for the callback. + * + * @return + * 0 on success, -1 otherwise with errno set. + */ +int +tap_nl_recv(int nlsk_fd, int (*cb)(struct nlmsghdr *, void *arg), void *arg) +{ + /* man 7 netlink EXAMPLE */ + struct sockaddr_nl sa; + char buf[BUF_SIZE]; + struct iovec iov = { + .iov_base = buf, + .iov_len = sizeof(buf), + }; + struct msghdr msg = { + .msg_name = &sa, + .msg_namelen = sizeof(sa), + .msg_iov = &iov, + /* One message at a time */ + .msg_iovlen = 1, + }; + int multipart = 0; + int ret = 0; + + do { + struct nlmsghdr *nh; + int recv_bytes = 0; + + recv_bytes = recvmsg(nlsk_fd, &msg, 0); + if (recv_bytes < 0) + return -1; + for (nh = (struct nlmsghdr *)buf; + NLMSG_OK(nh, (unsigned int)recv_bytes); + nh = NLMSG_NEXT(nh, recv_bytes)) { + if (nh->nlmsg_type == NLMSG_ERROR) { + struct nlmsgerr *err_data = NLMSG_DATA(nh); + + if (err_data->error < 0) { + errno = -err_data->error; + return -1; + } + /* Ack message. */ + return 0; + } + /* Multi-part msgs and their trailing DONE message. */ + if (nh->nlmsg_flags & NLM_F_MULTI) { + if (nh->nlmsg_type == NLMSG_DONE) + return 0; + multipart = 1; + } + if (cb) + ret = cb(nh, arg); + } + } while (multipart); + return ret; +} + +/** + * Append a netlink attribute to a message. + * + * @param[in, out] nh + * The netlink message to parse, received from the kernel. + * @param[in] type + * The type of attribute to append. + * @param[in] data_len + * The length of the data to append. + * @param[in] data + * The data to append. + */ +void +tap_nlattr_add(struct nlmsghdr *nh, unsigned short type, + unsigned int data_len, const void *data) +{ + /* see man 3 rtnetlink */ + struct rtattr *rta; + + rta = (struct rtattr *)NLMSG_TAIL(nh); + rta->rta_len = RTA_LENGTH(data_len); + rta->rta_type = type; + memcpy(RTA_DATA(rta), data, data_len); + nh->nlmsg_len = NLMSG_ALIGN(nh->nlmsg_len) + RTA_ALIGN(rta->rta_len); +} + +/** + * Append a uint8_t netlink attribute to a message. + * + * @param[in, out] nh + * The netlink message to parse, received from the kernel. + * @param[in] type + * The type of attribute to append. + * @param[in] data + * The data to append. + */ +void +tap_nlattr_add8(struct nlmsghdr *nh, unsigned short type, uint8_t data) +{ + tap_nlattr_add(nh, type, sizeof(uint8_t), &data); +} + +/** + * Append a uint16_t netlink attribute to a message. + * + * @param[in, out] nh + * The netlink message to parse, received from the kernel. + * @param[in] type + * The type of attribute to append. + * @param[in] data + * The data to append. + */ +void +tap_nlattr_add16(struct nlmsghdr *nh, unsigned short type, uint16_t data) +{ + tap_nlattr_add(nh, type, sizeof(uint16_t), &data); +} + +/** + * Append a uint16_t netlink attribute to a message. + * + * @param[in, out] nh + * The netlink message to parse, received from the kernel. + * @param[in] type + * The type of attribute to append. + * @param[in] data + * The data to append. + */ +void +tap_nlattr_add32(struct nlmsghdr *nh, unsigned short type, uint32_t data) +{ + tap_nlattr_add(nh, type, sizeof(uint32_t), &data); +} + +/** + * Start a nested netlink attribute. + * It must be followed later by a call to tap_nlattr_nested_finish(). + * + * @param[in, out] msg + * The netlink message where to edit the nested_tails metadata. + * @param[in] type + * The nested attribute type to append. + * + * @return + * -1 if adding a nested netlink attribute failed, 0 otherwise. + */ +int +tap_nlattr_nested_start(struct nlmsg *msg, uint16_t type) +{ + struct nested_tail *tail; + + tail = rte_zmalloc(NULL, sizeof(struct nested_tail), 0); + if (!tail) { + TAP_LOG(ERR, + "Couldn't allocate memory for nested netlink attribute"); + return -1; + } + + tail->tail = (struct rtattr *)NLMSG_TAIL(&msg->nh); + + tap_nlattr_add(&msg->nh, type, 0, NULL); + + tail->prev = msg->nested_tails; + + msg->nested_tails = tail; + + return 0; +} + +/** + * End a nested netlink attribute. + * It follows a call to tap_nlattr_nested_start(). + * In effect, it will modify the nested attribute length to include every bytes + * from the nested attribute start, up to here. + * + * @param[in, out] msg + * The netlink message where to edit the nested_tails metadata. + */ +void +tap_nlattr_nested_finish(struct nlmsg *msg) +{ + struct nested_tail *tail = msg->nested_tails; + + tail->tail->rta_len = (char *)NLMSG_TAIL(&msg->nh) - (char *)tail->tail; + + if (tail->prev) + msg->nested_tails = tail->prev; + + rte_free(tail); +} diff --git a/src/spdk/dpdk/drivers/net/tap/tap_netlink.h b/src/spdk/dpdk/drivers/net/tap/tap_netlink.h new file mode 100644 index 00000000..faa73ba1 --- /dev/null +++ b/src/spdk/dpdk/drivers/net/tap/tap_netlink.h @@ -0,0 +1,42 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright 2017 6WIND S.A. + * Copyright 2017 Mellanox Technologies, Ltd + */ + +#ifndef _TAP_NETLINK_H_ +#define _TAP_NETLINK_H_ + +#include <ctype.h> +#include <inttypes.h> +#include <linux/rtnetlink.h> +#include <linux/netlink.h> +#include <stdio.h> + +#include <rte_log.h> + +#define NLMSG_BUF 512 + +struct nlmsg { + struct nlmsghdr nh; + struct tcmsg t; + char buf[NLMSG_BUF]; + struct nested_tail *nested_tails; +}; + +#define NLMSG_TAIL(nlh) (void *)((char *)(nlh) + NLMSG_ALIGN((nlh)->nlmsg_len)) + +int tap_nl_init(uint32_t nl_groups); +int tap_nl_final(int nlsk_fd); +int tap_nl_send(int nlsk_fd, struct nlmsghdr *nh); +int tap_nl_recv(int nlsk_fd, int (*callback)(struct nlmsghdr *, void *), + void *arg); +int tap_nl_recv_ack(int nlsk_fd); +void tap_nlattr_add(struct nlmsghdr *nh, unsigned short type, + unsigned int data_len, const void *data); +void tap_nlattr_add8(struct nlmsghdr *nh, unsigned short type, uint8_t data); +void tap_nlattr_add16(struct nlmsghdr *nh, unsigned short type, uint16_t data); +void tap_nlattr_add32(struct nlmsghdr *nh, unsigned short type, uint32_t data); +int tap_nlattr_nested_start(struct nlmsg *msg, uint16_t type); +void tap_nlattr_nested_finish(struct nlmsg *msg); + +#endif /* _TAP_NETLINK_H_ */ diff --git a/src/spdk/dpdk/drivers/net/tap/tap_rss.h b/src/spdk/dpdk/drivers/net/tap/tap_rss.h new file mode 100644 index 00000000..17606b2d --- /dev/null +++ b/src/spdk/dpdk/drivers/net/tap/tap_rss.h @@ -0,0 +1,40 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright 2017 Mellanox Technologies, Ltd + */ + +#ifndef _TAP_RSS_H_ +#define _TAP_RSS_H_ + +#ifndef TAP_MAX_QUEUES +#define TAP_MAX_QUEUES 16 +#endif + +/* Fixed RSS hash key size in bytes. */ +#define TAP_RSS_HASH_KEY_SIZE 40 + +/* Supported RSS */ +#define TAP_RSS_HF_MASK (~(ETH_RSS_IP | ETH_RSS_UDP | ETH_RSS_TCP)) + +/* hashed fields for RSS */ +enum hash_field { + HASH_FIELD_IPV4_L3, /* IPv4 src/dst addr */ + HASH_FIELD_IPV4_L3_L4, /* IPv4 src/dst addr + L4 src/dst ports */ + HASH_FIELD_IPV6_L3, /* IPv6 src/dst addr */ + HASH_FIELD_IPV6_L3_L4, /* IPv6 src/dst addr + L4 src/dst ports */ + HASH_FIELD_L2_SRC, /* Ethernet src addr */ + HASH_FIELD_L2_DST, /* Ethernet dst addr */ + HASH_FIELD_L3_SRC, /* L3 src addr */ + HASH_FIELD_L3_DST, /* L3 dst addr */ + HASH_FIELD_L4_SRC, /* TCP/UDP src ports */ + HASH_FIELD_L4_DST, /* TCP/UDP dst ports */ +}; + +struct rss_key { + __u8 key[128]; + __u32 hash_fields; + __u32 key_size; + __u32 queues[TAP_MAX_QUEUES]; + __u32 nb_queues; +} __attribute__((packed)); + +#endif /* _TAP_RSS_H_ */ diff --git a/src/spdk/dpdk/drivers/net/tap/tap_tcmsgs.c b/src/spdk/dpdk/drivers/net/tap/tap_tcmsgs.c new file mode 100644 index 00000000..3c9d0366 --- /dev/null +++ b/src/spdk/dpdk/drivers/net/tap/tap_tcmsgs.c @@ -0,0 +1,296 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright 2017 6WIND S.A. + * Copyright 2017 Mellanox Technologies, Ltd + */ + +#include <inttypes.h> +#include <linux/netlink.h> +#include <net/if.h> +#include <string.h> + +#include <rte_log.h> +#include <tap_tcmsgs.h> +#include "tap_log.h" + +struct qdisc { + uint32_t handle; + uint32_t parent; +}; + +struct list_args { + int nlsk_fd; + uint16_t ifindex; + void *custom_arg; +}; + +struct qdisc_custom_arg { + uint32_t handle; + uint32_t parent; + uint8_t exists; +}; + +/** + * Initialize a netlink message with a TC header. + * + * @param[in, out] msg + * The netlink message to fill. + * @param[in] ifindex + * The netdevice ifindex where the rule will be applied. + * @param[in] type + * The type of TC message to create (RTM_NEWTFILTER, RTM_NEWQDISC, etc.). + * @param[in] flags + * Overrides the default netlink flags for this msg with those specified. + */ +void +tc_init_msg(struct nlmsg *msg, uint16_t ifindex, uint16_t type, uint16_t flags) +{ + struct nlmsghdr *n = &msg->nh; + + n->nlmsg_len = NLMSG_LENGTH(sizeof(struct tcmsg)); + n->nlmsg_type = type; + if (flags) + n->nlmsg_flags = flags; + else + n->nlmsg_flags = NLM_F_REQUEST | NLM_F_ACK; + msg->t.tcm_family = AF_UNSPEC; + msg->t.tcm_ifindex = ifindex; +} + +/** + * Delete a specific QDISC identified by its iface, and it's handle and parent. + * + * @param[in] nlsk_fd + * The netlink socket file descriptor used for communication. + * @param[in] ifindex + * The netdevice ifindex on whom the deletion will happen. + * @param[in] qinfo + * Additional info to identify the QDISC (handle and parent). + * + * @return + * 0 on success, -1 otherwise with errno set. + */ +static int +qdisc_del(int nlsk_fd, uint16_t ifindex, struct qdisc *qinfo) +{ + struct nlmsg msg; + int fd = 0; + + tc_init_msg(&msg, ifindex, RTM_DELQDISC, 0); + msg.t.tcm_handle = qinfo->handle; + msg.t.tcm_parent = qinfo->parent; + /* if no netlink socket is provided, create one */ + if (!nlsk_fd) { + fd = tap_nl_init(0); + if (fd < 0) { + TAP_LOG(ERR, + "Could not delete QDISC: null netlink socket"); + return -1; + } + } else { + fd = nlsk_fd; + } + if (tap_nl_send(fd, &msg.nh) < 0) + goto error; + if (tap_nl_recv_ack(fd) < 0) + goto error; + if (!nlsk_fd) + return tap_nl_final(fd); + return 0; +error: + if (!nlsk_fd) + tap_nl_final(fd); + return -1; +} + +/** + * Add the multiqueue QDISC with MULTIQ_MAJOR_HANDLE handle. + * + * @param[in] nlsk_fd + * The netlink socket file descriptor used for communication. + * @param[in] ifindex + * The netdevice ifindex where to add the multiqueue QDISC. + * + * @return + * 0 on success, -1 otherwise with errno set. + */ +int +qdisc_add_multiq(int nlsk_fd, uint16_t ifindex) +{ + struct tc_multiq_qopt opt; + struct nlmsg msg; + + tc_init_msg(&msg, ifindex, RTM_NEWQDISC, + NLM_F_REQUEST | NLM_F_ACK | NLM_F_EXCL | NLM_F_CREATE); + msg.t.tcm_handle = TC_H_MAKE(MULTIQ_MAJOR_HANDLE, 0); + msg.t.tcm_parent = TC_H_ROOT; + tap_nlattr_add(&msg.nh, TCA_KIND, sizeof("multiq"), "multiq"); + tap_nlattr_add(&msg.nh, TCA_OPTIONS, sizeof(opt), &opt); + if (tap_nl_send(nlsk_fd, &msg.nh) < 0) + return -1; + if (tap_nl_recv_ack(nlsk_fd) < 0) + return -1; + return 0; +} + +/** + * Add the ingress QDISC with default ffff: handle. + * + * @param[in] nlsk_fd + * The netlink socket file descriptor used for communication. + * @param[in] ifindex + * The netdevice ifindex where the QDISC will be added. + * + * @return + * 0 on success, -1 otherwise with errno set. + */ +int +qdisc_add_ingress(int nlsk_fd, uint16_t ifindex) +{ + struct nlmsg msg; + + tc_init_msg(&msg, ifindex, RTM_NEWQDISC, + NLM_F_REQUEST | NLM_F_ACK | NLM_F_EXCL | NLM_F_CREATE); + msg.t.tcm_handle = TC_H_MAKE(TC_H_INGRESS, 0); + msg.t.tcm_parent = TC_H_INGRESS; + tap_nlattr_add(&msg.nh, TCA_KIND, sizeof("ingress"), "ingress"); + if (tap_nl_send(nlsk_fd, &msg.nh) < 0) + return -1; + if (tap_nl_recv_ack(nlsk_fd) < 0) + return -1; + return 0; +} + +/** + * Callback function to delete a QDISC. + * + * @param[in] nh + * The netlink message to parse, received from the kernel. + * @param[in] arg + * Custom arguments for the callback. + * + * @return + * 0 on success, -1 otherwise with errno set. + */ +static int +qdisc_del_cb(struct nlmsghdr *nh, void *arg) +{ + struct tcmsg *t = NLMSG_DATA(nh); + struct list_args *args = arg; + + struct qdisc qinfo = { + .handle = t->tcm_handle, + .parent = t->tcm_parent, + }; + + /* filter out other ifaces' qdiscs */ + if (args->ifindex != (unsigned int)t->tcm_ifindex) + return 0; + /* + * Use another nlsk_fd (0) to avoid tampering with the current list + * iteration. + */ + return qdisc_del(0, args->ifindex, &qinfo); +} + +/** + * Iterate over all QDISC, and call the callback() function for each. + * + * @param[in] nlsk_fd + * The netlink socket file descriptor used for communication. + * @param[in] ifindex + * The netdevice ifindex where to find QDISCs. + * @param[in] callback + * The function to call for each QDISC. + * @param[in, out] arg + * The arguments to provide the callback function with. + * + * @return + * 0 on success, -1 otherwise with errno set. + */ +static int +qdisc_iterate(int nlsk_fd, uint16_t ifindex, + int (*callback)(struct nlmsghdr *, void *), void *arg) +{ + struct nlmsg msg; + struct list_args args = { + .nlsk_fd = nlsk_fd, + .ifindex = ifindex, + .custom_arg = arg, + }; + + tc_init_msg(&msg, ifindex, RTM_GETQDISC, NLM_F_REQUEST | NLM_F_DUMP); + if (tap_nl_send(nlsk_fd, &msg.nh) < 0) + return -1; + if (tap_nl_recv(nlsk_fd, callback, &args) < 0) + return -1; + return 0; +} + +/** + * Delete all QDISCs for a given netdevice. + * + * @param[in] nlsk_fd + * The netlink socket file descriptor used for communication. + * @param[in] ifindex + * The netdevice ifindex where to find QDISCs. + * + * @return + * 0 on success, -1 otherwise with errno set. + */ +int +qdisc_flush(int nlsk_fd, uint16_t ifindex) +{ + return qdisc_iterate(nlsk_fd, ifindex, qdisc_del_cb, NULL); +} + +/** + * Create the multiqueue QDISC, only if it does not exist already. + * + * @param[in] nlsk_fd + * The netlink socket file descriptor used for communication. + * @param[in] ifindex + * The netdevice ifindex where to add the multiqueue QDISC. + * + * @return + * 0 if the qdisc exists or if has been successfully added. + * Return -1 otherwise. + */ +int +qdisc_create_multiq(int nlsk_fd, uint16_t ifindex) +{ + int err = 0; + + err = qdisc_add_multiq(nlsk_fd, ifindex); + if (err < 0 && errno != -EEXIST) { + TAP_LOG(ERR, "Could not add multiq qdisc (%d): %s", + errno, strerror(errno)); + return -1; + } + return 0; +} + +/** + * Create the ingress QDISC, only if it does not exist already. + * + * @param[in] nlsk_fd + * The netlink socket file descriptor used for communication. + * @param[in] ifindex + * The netdevice ifindex where to add the ingress QDISC. + * + * @return + * 0 if the qdisc exists or if has been successfully added. + * Return -1 otherwise. + */ +int +qdisc_create_ingress(int nlsk_fd, uint16_t ifindex) +{ + int err = 0; + + err = qdisc_add_ingress(nlsk_fd, ifindex); + if (err < 0 && errno != -EEXIST) { + TAP_LOG(ERR, "Could not add ingress qdisc (%d): %s", + errno, strerror(errno)); + return -1; + } + return 0; +} diff --git a/src/spdk/dpdk/drivers/net/tap/tap_tcmsgs.h b/src/spdk/dpdk/drivers/net/tap/tap_tcmsgs.h new file mode 100644 index 00000000..8cedea84 --- /dev/null +++ b/src/spdk/dpdk/drivers/net/tap/tap_tcmsgs.h @@ -0,0 +1,37 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright 2017 6WIND S.A. + * Copyright 2017 Mellanox Technologies, Ltd + */ + +#ifndef _TAP_TCMSGS_H_ +#define _TAP_TCMSGS_H_ + +#include <tap_autoconf.h> +#include <linux/if_ether.h> +#include <linux/rtnetlink.h> +#include <linux/pkt_sched.h> +#include <linux/pkt_cls.h> +#include <linux/tc_act/tc_mirred.h> +#include <linux/tc_act/tc_gact.h> +#include <linux/tc_act/tc_skbedit.h> +#ifdef HAVE_TC_ACT_BPF +#include <linux/tc_act/tc_bpf.h> +#endif +#include <inttypes.h> + +#include <rte_ether.h> +#include <tap_netlink.h> + +#define MULTIQ_MAJOR_HANDLE (1 << 16) + +void tc_init_msg(struct nlmsg *msg, uint16_t ifindex, uint16_t type, + uint16_t flags); +int qdisc_list(int nlsk_fd, uint16_t ifindex); +int qdisc_flush(int nlsk_fd, uint16_t ifindex); +int qdisc_create_ingress(int nlsk_fd, uint16_t ifindex); +int qdisc_create_multiq(int nlsk_fd, uint16_t ifindex); +int qdisc_add_ingress(int nlsk_fd, uint16_t ifindex); +int qdisc_add_multiq(int nlsk_fd, uint16_t ifindex); +int filter_list_ingress(int nlsk_fd, uint16_t ifindex); + +#endif /* _TAP_TCMSGS_H_ */ |