summaryrefslogtreecommitdiffstats
path: root/src/spdk/dpdk/lib/librte_bpf
diff options
context:
space:
mode:
Diffstat (limited to 'src/spdk/dpdk/lib/librte_bpf')
-rw-r--r--src/spdk/dpdk/lib/librte_bpf/Makefile40
-rw-r--r--src/spdk/dpdk/lib/librte_bpf/bpf.c63
-rw-r--r--src/spdk/dpdk/lib/librte_bpf/bpf_def.h151
-rw-r--r--src/spdk/dpdk/lib/librte_bpf/bpf_exec.c453
-rw-r--r--src/spdk/dpdk/lib/librte_bpf/bpf_impl.h54
-rw-r--r--src/spdk/dpdk/lib/librte_bpf/bpf_jit_arm64.c1451
-rw-r--r--src/spdk/dpdk/lib/librte_bpf/bpf_jit_x86.c1368
-rw-r--r--src/spdk/dpdk/lib/librte_bpf/bpf_load.c150
-rw-r--r--src/spdk/dpdk/lib/librte_bpf/bpf_load_elf.c333
-rw-r--r--src/spdk/dpdk/lib/librte_bpf/bpf_pkt.c605
-rw-r--r--src/spdk/dpdk/lib/librte_bpf/bpf_validate.c2250
-rw-r--r--src/spdk/dpdk/lib/librte_bpf/meson.build29
-rw-r--r--src/spdk/dpdk/lib/librte_bpf/rte_bpf.h212
-rw-r--r--src/spdk/dpdk/lib/librte_bpf/rte_bpf_ethdev.h121
-rw-r--r--src/spdk/dpdk/lib/librte_bpf/rte_bpf_version.map16
15 files changed, 7296 insertions, 0 deletions
diff --git a/src/spdk/dpdk/lib/librte_bpf/Makefile b/src/spdk/dpdk/lib/librte_bpf/Makefile
new file mode 100644
index 000000000..3be675043
--- /dev/null
+++ b/src/spdk/dpdk/lib/librte_bpf/Makefile
@@ -0,0 +1,40 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2018 Intel Corporation
+
+include $(RTE_SDK)/mk/rte.vars.mk
+
+# library name
+LIB = librte_bpf.a
+
+CFLAGS += -O3
+CFLAGS += $(WERROR_FLAGS) -I$(SRCDIR)
+LDLIBS += -lrte_net -lrte_eal
+LDLIBS += -lrte_mempool -lrte_ring
+LDLIBS += -lrte_mbuf -lrte_ethdev
+ifeq ($(CONFIG_RTE_LIBRTE_BPF_ELF),y)
+LDLIBS += -lelf
+endif
+
+EXPORT_MAP := rte_bpf_version.map
+
+# all source are stored in SRCS-y
+SRCS-$(CONFIG_RTE_LIBRTE_BPF) += bpf.c
+SRCS-$(CONFIG_RTE_LIBRTE_BPF) += bpf_exec.c
+SRCS-$(CONFIG_RTE_LIBRTE_BPF) += bpf_load.c
+SRCS-$(CONFIG_RTE_LIBRTE_BPF) += bpf_pkt.c
+SRCS-$(CONFIG_RTE_LIBRTE_BPF) += bpf_validate.c
+ifeq ($(CONFIG_RTE_LIBRTE_BPF_ELF),y)
+SRCS-$(CONFIG_RTE_LIBRTE_BPF) += bpf_load_elf.c
+endif
+ifeq ($(CONFIG_RTE_ARCH_X86_64),y)
+SRCS-$(CONFIG_RTE_LIBRTE_BPF) += bpf_jit_x86.c
+else ifeq ($(CONFIG_RTE_ARCH_ARM64),y)
+SRCS-$(CONFIG_RTE_LIBRTE_BPF) += bpf_jit_arm64.c
+endif
+
+# install header files
+SYMLINK-$(CONFIG_RTE_LIBRTE_BPF)-include += bpf_def.h
+SYMLINK-$(CONFIG_RTE_LIBRTE_BPF)-include += rte_bpf.h
+SYMLINK-$(CONFIG_RTE_LIBRTE_BPF)-include += rte_bpf_ethdev.h
+
+include $(RTE_SDK)/mk/rte.lib.mk
diff --git a/src/spdk/dpdk/lib/librte_bpf/bpf.c b/src/spdk/dpdk/lib/librte_bpf/bpf.c
new file mode 100644
index 000000000..7e1879ffa
--- /dev/null
+++ b/src/spdk/dpdk/lib/librte_bpf/bpf.c
@@ -0,0 +1,63 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2018 Intel Corporation
+ */
+
+#include <stdarg.h>
+#include <stdio.h>
+#include <string.h>
+#include <errno.h>
+#include <stdint.h>
+#include <inttypes.h>
+
+#include <rte_common.h>
+#include <rte_eal.h>
+
+#include "bpf_impl.h"
+
+int rte_bpf_logtype;
+
+void
+rte_bpf_destroy(struct rte_bpf *bpf)
+{
+ if (bpf != NULL) {
+ if (bpf->jit.func != NULL)
+ munmap(bpf->jit.func, bpf->jit.sz);
+ munmap(bpf, bpf->sz);
+ }
+}
+
+int
+rte_bpf_get_jit(const struct rte_bpf *bpf, struct rte_bpf_jit *jit)
+{
+ if (bpf == NULL || jit == NULL)
+ return -EINVAL;
+
+ jit[0] = bpf->jit;
+ return 0;
+}
+
+int
+bpf_jit(struct rte_bpf *bpf)
+{
+ int32_t rc;
+
+#if defined(RTE_ARCH_X86_64)
+ rc = bpf_jit_x86(bpf);
+#elif defined(RTE_ARCH_ARM64)
+ rc = bpf_jit_arm64(bpf);
+#else
+ rc = -ENOTSUP;
+#endif
+
+ if (rc != 0)
+ RTE_BPF_LOG(WARNING, "%s(%p) failed, error code: %d;\n",
+ __func__, bpf, rc);
+ return rc;
+}
+
+RTE_INIT(rte_bpf_init_log)
+{
+ rte_bpf_logtype = rte_log_register("lib.bpf");
+ if (rte_bpf_logtype >= 0)
+ rte_log_set_level(rte_bpf_logtype, RTE_LOG_INFO);
+}
diff --git a/src/spdk/dpdk/lib/librte_bpf/bpf_def.h b/src/spdk/dpdk/lib/librte_bpf/bpf_def.h
new file mode 100644
index 000000000..d39992997
--- /dev/null
+++ b/src/spdk/dpdk/lib/librte_bpf/bpf_def.h
@@ -0,0 +1,151 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 1982, 1986, 1990, 1993
+ * The Regents of the University of California.
+ * Copyright(c) 2018 Intel Corporation.
+ */
+
+#ifndef _RTE_BPF_DEF_H_
+#define _RTE_BPF_DEF_H_
+
+/**
+ * @file
+ *
+ * classic BPF (cBPF) and extended BPF (eBPF) related defines.
+ * For more information regarding cBPF and eBPF ISA and their differences,
+ * please refer to:
+ * https://www.kernel.org/doc/Documentation/networking/filter.txt.
+ * As a rule of thumb for that file:
+ * all definitions used by both cBPF and eBPF start with bpf(BPF)_ prefix,
+ * while eBPF only ones start with ebpf(EBPF)) prefix.
+ */
+
+#include <stdint.h>
+
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/*
+ * The instruction encodings.
+ */
+
+/* Instruction classes */
+#define BPF_CLASS(code) ((code) & 0x07)
+#define BPF_LD 0x00
+#define BPF_LDX 0x01
+#define BPF_ST 0x02
+#define BPF_STX 0x03
+#define BPF_ALU 0x04
+#define BPF_JMP 0x05
+#define BPF_RET 0x06
+#define BPF_MISC 0x07
+
+#define EBPF_ALU64 0x07
+
+/* ld/ldx fields */
+#define BPF_SIZE(code) ((code) & 0x18)
+#define BPF_W 0x00
+#define BPF_H 0x08
+#define BPF_B 0x10
+#define EBPF_DW 0x18
+
+#define BPF_MODE(code) ((code) & 0xe0)
+#define BPF_IMM 0x00
+#define BPF_ABS 0x20
+#define BPF_IND 0x40
+#define BPF_MEM 0x60
+#define BPF_LEN 0x80
+#define BPF_MSH 0xa0
+
+#define EBPF_XADD 0xc0
+
+/* alu/jmp fields */
+#define BPF_OP(code) ((code) & 0xf0)
+#define BPF_ADD 0x00
+#define BPF_SUB 0x10
+#define BPF_MUL 0x20
+#define BPF_DIV 0x30
+#define BPF_OR 0x40
+#define BPF_AND 0x50
+#define BPF_LSH 0x60
+#define BPF_RSH 0x70
+#define BPF_NEG 0x80
+#define BPF_MOD 0x90
+#define BPF_XOR 0xa0
+
+#define EBPF_MOV 0xb0
+#define EBPF_ARSH 0xc0
+#define EBPF_END 0xd0
+
+#define BPF_JA 0x00
+#define BPF_JEQ 0x10
+#define BPF_JGT 0x20
+#define BPF_JGE 0x30
+#define BPF_JSET 0x40
+
+#define EBPF_JNE 0x50
+#define EBPF_JSGT 0x60
+#define EBPF_JSGE 0x70
+#define EBPF_CALL 0x80
+#define EBPF_EXIT 0x90
+#define EBPF_JLT 0xa0
+#define EBPF_JLE 0xb0
+#define EBPF_JSLT 0xc0
+#define EBPF_JSLE 0xd0
+
+#define BPF_SRC(code) ((code) & 0x08)
+#define BPF_K 0x00
+#define BPF_X 0x08
+
+/* if BPF_OP(code) == EBPF_END */
+#define EBPF_TO_LE 0x00 /* convert to little-endian */
+#define EBPF_TO_BE 0x08 /* convert to big-endian */
+
+/*
+ * eBPF registers
+ */
+enum {
+ EBPF_REG_0, /* return value from internal function/for eBPF program */
+ EBPF_REG_1, /* 0-th argument to internal function */
+ EBPF_REG_2, /* 1-th argument to internal function */
+ EBPF_REG_3, /* 2-th argument to internal function */
+ EBPF_REG_4, /* 3-th argument to internal function */
+ EBPF_REG_5, /* 4-th argument to internal function */
+ EBPF_REG_6, /* callee saved register */
+ EBPF_REG_7, /* callee saved register */
+ EBPF_REG_8, /* callee saved register */
+ EBPF_REG_9, /* callee saved register */
+ EBPF_REG_10, /* stack pointer (read-only) */
+ EBPF_REG_NUM,
+};
+
+/*
+ * When EBPF_CALL instruction has src_reg == EBPF_PSEUDO_CALL,
+ * it should be treated as pseudo-call instruction, where
+ * imm value contains pc-relative offset to another EBPF function.
+ * Right now DPDK EBPF library doesn't support it.
+ */
+#define EBPF_PSEUDO_CALL EBPF_REG_1
+
+/*
+ * eBPF instruction format
+ */
+struct ebpf_insn {
+ uint8_t code;
+ uint8_t dst_reg:4;
+ uint8_t src_reg:4;
+ int16_t off;
+ int32_t imm;
+};
+
+/*
+ * eBPF allows functions with R1-R5 as arguments.
+ */
+#define EBPF_FUNC_MAX_ARGS (EBPF_REG_6 - EBPF_REG_1)
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* RTE_BPF_DEF_H_ */
diff --git a/src/spdk/dpdk/lib/librte_bpf/bpf_exec.c b/src/spdk/dpdk/lib/librte_bpf/bpf_exec.c
new file mode 100644
index 000000000..1bb226643
--- /dev/null
+++ b/src/spdk/dpdk/lib/librte_bpf/bpf_exec.c
@@ -0,0 +1,453 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2018 Intel Corporation
+ */
+
+#include <stdarg.h>
+#include <stdio.h>
+#include <string.h>
+#include <errno.h>
+#include <stdint.h>
+#include <inttypes.h>
+
+#include <rte_common.h>
+#include <rte_log.h>
+#include <rte_debug.h>
+#include <rte_memory.h>
+#include <rte_eal.h>
+#include <rte_byteorder.h>
+
+#include "bpf_impl.h"
+
+#define BPF_JMP_UNC(ins) ((ins) += (ins)->off)
+
+#define BPF_JMP_CND_REG(reg, ins, op, type) \
+ ((ins) += \
+ ((type)(reg)[(ins)->dst_reg] op (type)(reg)[(ins)->src_reg]) ? \
+ (ins)->off : 0)
+
+#define BPF_JMP_CND_IMM(reg, ins, op, type) \
+ ((ins) += \
+ ((type)(reg)[(ins)->dst_reg] op (type)(ins)->imm) ? \
+ (ins)->off : 0)
+
+#define BPF_NEG_ALU(reg, ins, type) \
+ ((reg)[(ins)->dst_reg] = (type)(-(reg)[(ins)->dst_reg]))
+
+#define EBPF_MOV_ALU_REG(reg, ins, type) \
+ ((reg)[(ins)->dst_reg] = (type)(reg)[(ins)->src_reg])
+
+#define BPF_OP_ALU_REG(reg, ins, op, type) \
+ ((reg)[(ins)->dst_reg] = \
+ (type)(reg)[(ins)->dst_reg] op (type)(reg)[(ins)->src_reg])
+
+#define EBPF_MOV_ALU_IMM(reg, ins, type) \
+ ((reg)[(ins)->dst_reg] = (type)(ins)->imm)
+
+#define BPF_OP_ALU_IMM(reg, ins, op, type) \
+ ((reg)[(ins)->dst_reg] = \
+ (type)(reg)[(ins)->dst_reg] op (type)(ins)->imm)
+
+#define BPF_DIV_ZERO_CHECK(bpf, reg, ins, type) do { \
+ if ((type)(reg)[(ins)->src_reg] == 0) { \
+ RTE_BPF_LOG(ERR, \
+ "%s(%p): division by 0 at pc: %#zx;\n", \
+ __func__, bpf, \
+ (uintptr_t)(ins) - (uintptr_t)(bpf)->prm.ins); \
+ return 0; \
+ } \
+} while (0)
+
+#define BPF_LD_REG(reg, ins, type) \
+ ((reg)[(ins)->dst_reg] = \
+ *(type *)(uintptr_t)((reg)[(ins)->src_reg] + (ins)->off))
+
+#define BPF_ST_IMM(reg, ins, type) \
+ (*(type *)(uintptr_t)((reg)[(ins)->dst_reg] + (ins)->off) = \
+ (type)(ins)->imm)
+
+#define BPF_ST_REG(reg, ins, type) \
+ (*(type *)(uintptr_t)((reg)[(ins)->dst_reg] + (ins)->off) = \
+ (type)(reg)[(ins)->src_reg])
+
+#define BPF_ST_XADD_REG(reg, ins, tp) \
+ (rte_atomic##tp##_add((rte_atomic##tp##_t *) \
+ (uintptr_t)((reg)[(ins)->dst_reg] + (ins)->off), \
+ reg[ins->src_reg]))
+
+static inline void
+bpf_alu_be(uint64_t reg[EBPF_REG_NUM], const struct ebpf_insn *ins)
+{
+ uint64_t *v;
+
+ v = reg + ins->dst_reg;
+ switch (ins->imm) {
+ case 16:
+ *v = rte_cpu_to_be_16(*v);
+ break;
+ case 32:
+ *v = rte_cpu_to_be_32(*v);
+ break;
+ case 64:
+ *v = rte_cpu_to_be_64(*v);
+ break;
+ }
+}
+
+static inline void
+bpf_alu_le(uint64_t reg[EBPF_REG_NUM], const struct ebpf_insn *ins)
+{
+ uint64_t *v;
+
+ v = reg + ins->dst_reg;
+ switch (ins->imm) {
+ case 16:
+ *v = rte_cpu_to_le_16(*v);
+ break;
+ case 32:
+ *v = rte_cpu_to_le_32(*v);
+ break;
+ case 64:
+ *v = rte_cpu_to_le_64(*v);
+ break;
+ }
+}
+
+static inline uint64_t
+bpf_exec(const struct rte_bpf *bpf, uint64_t reg[EBPF_REG_NUM])
+{
+ const struct ebpf_insn *ins;
+
+ for (ins = bpf->prm.ins; ; ins++) {
+ switch (ins->code) {
+ /* 32 bit ALU IMM operations */
+ case (BPF_ALU | BPF_ADD | BPF_K):
+ BPF_OP_ALU_IMM(reg, ins, +, uint32_t);
+ break;
+ case (BPF_ALU | BPF_SUB | BPF_K):
+ BPF_OP_ALU_IMM(reg, ins, -, uint32_t);
+ break;
+ case (BPF_ALU | BPF_AND | BPF_K):
+ BPF_OP_ALU_IMM(reg, ins, &, uint32_t);
+ break;
+ case (BPF_ALU | BPF_OR | BPF_K):
+ BPF_OP_ALU_IMM(reg, ins, |, uint32_t);
+ break;
+ case (BPF_ALU | BPF_LSH | BPF_K):
+ BPF_OP_ALU_IMM(reg, ins, <<, uint32_t);
+ break;
+ case (BPF_ALU | BPF_RSH | BPF_K):
+ BPF_OP_ALU_IMM(reg, ins, >>, uint32_t);
+ break;
+ case (BPF_ALU | BPF_XOR | BPF_K):
+ BPF_OP_ALU_IMM(reg, ins, ^, uint32_t);
+ break;
+ case (BPF_ALU | BPF_MUL | BPF_K):
+ BPF_OP_ALU_IMM(reg, ins, *, uint32_t);
+ break;
+ case (BPF_ALU | BPF_DIV | BPF_K):
+ BPF_OP_ALU_IMM(reg, ins, /, uint32_t);
+ break;
+ case (BPF_ALU | BPF_MOD | BPF_K):
+ BPF_OP_ALU_IMM(reg, ins, %, uint32_t);
+ break;
+ case (BPF_ALU | EBPF_MOV | BPF_K):
+ EBPF_MOV_ALU_IMM(reg, ins, uint32_t);
+ break;
+ /* 32 bit ALU REG operations */
+ case (BPF_ALU | BPF_ADD | BPF_X):
+ BPF_OP_ALU_REG(reg, ins, +, uint32_t);
+ break;
+ case (BPF_ALU | BPF_SUB | BPF_X):
+ BPF_OP_ALU_REG(reg, ins, -, uint32_t);
+ break;
+ case (BPF_ALU | BPF_AND | BPF_X):
+ BPF_OP_ALU_REG(reg, ins, &, uint32_t);
+ break;
+ case (BPF_ALU | BPF_OR | BPF_X):
+ BPF_OP_ALU_REG(reg, ins, |, uint32_t);
+ break;
+ case (BPF_ALU | BPF_LSH | BPF_X):
+ BPF_OP_ALU_REG(reg, ins, <<, uint32_t);
+ break;
+ case (BPF_ALU | BPF_RSH | BPF_X):
+ BPF_OP_ALU_REG(reg, ins, >>, uint32_t);
+ break;
+ case (BPF_ALU | BPF_XOR | BPF_X):
+ BPF_OP_ALU_REG(reg, ins, ^, uint32_t);
+ break;
+ case (BPF_ALU | BPF_MUL | BPF_X):
+ BPF_OP_ALU_REG(reg, ins, *, uint32_t);
+ break;
+ case (BPF_ALU | BPF_DIV | BPF_X):
+ BPF_DIV_ZERO_CHECK(bpf, reg, ins, uint32_t);
+ BPF_OP_ALU_REG(reg, ins, /, uint32_t);
+ break;
+ case (BPF_ALU | BPF_MOD | BPF_X):
+ BPF_DIV_ZERO_CHECK(bpf, reg, ins, uint32_t);
+ BPF_OP_ALU_REG(reg, ins, %, uint32_t);
+ break;
+ case (BPF_ALU | EBPF_MOV | BPF_X):
+ EBPF_MOV_ALU_REG(reg, ins, uint32_t);
+ break;
+ case (BPF_ALU | BPF_NEG):
+ BPF_NEG_ALU(reg, ins, uint32_t);
+ break;
+ case (BPF_ALU | EBPF_END | EBPF_TO_BE):
+ bpf_alu_be(reg, ins);
+ break;
+ case (BPF_ALU | EBPF_END | EBPF_TO_LE):
+ bpf_alu_le(reg, ins);
+ break;
+ /* 64 bit ALU IMM operations */
+ case (EBPF_ALU64 | BPF_ADD | BPF_K):
+ BPF_OP_ALU_IMM(reg, ins, +, uint64_t);
+ break;
+ case (EBPF_ALU64 | BPF_SUB | BPF_K):
+ BPF_OP_ALU_IMM(reg, ins, -, uint64_t);
+ break;
+ case (EBPF_ALU64 | BPF_AND | BPF_K):
+ BPF_OP_ALU_IMM(reg, ins, &, uint64_t);
+ break;
+ case (EBPF_ALU64 | BPF_OR | BPF_K):
+ BPF_OP_ALU_IMM(reg, ins, |, uint64_t);
+ break;
+ case (EBPF_ALU64 | BPF_LSH | BPF_K):
+ BPF_OP_ALU_IMM(reg, ins, <<, uint64_t);
+ break;
+ case (EBPF_ALU64 | BPF_RSH | BPF_K):
+ BPF_OP_ALU_IMM(reg, ins, >>, uint64_t);
+ break;
+ case (EBPF_ALU64 | EBPF_ARSH | BPF_K):
+ BPF_OP_ALU_IMM(reg, ins, >>, int64_t);
+ break;
+ case (EBPF_ALU64 | BPF_XOR | BPF_K):
+ BPF_OP_ALU_IMM(reg, ins, ^, uint64_t);
+ break;
+ case (EBPF_ALU64 | BPF_MUL | BPF_K):
+ BPF_OP_ALU_IMM(reg, ins, *, uint64_t);
+ break;
+ case (EBPF_ALU64 | BPF_DIV | BPF_K):
+ BPF_OP_ALU_IMM(reg, ins, /, uint64_t);
+ break;
+ case (EBPF_ALU64 | BPF_MOD | BPF_K):
+ BPF_OP_ALU_IMM(reg, ins, %, uint64_t);
+ break;
+ case (EBPF_ALU64 | EBPF_MOV | BPF_K):
+ EBPF_MOV_ALU_IMM(reg, ins, uint64_t);
+ break;
+ /* 64 bit ALU REG operations */
+ case (EBPF_ALU64 | BPF_ADD | BPF_X):
+ BPF_OP_ALU_REG(reg, ins, +, uint64_t);
+ break;
+ case (EBPF_ALU64 | BPF_SUB | BPF_X):
+ BPF_OP_ALU_REG(reg, ins, -, uint64_t);
+ break;
+ case (EBPF_ALU64 | BPF_AND | BPF_X):
+ BPF_OP_ALU_REG(reg, ins, &, uint64_t);
+ break;
+ case (EBPF_ALU64 | BPF_OR | BPF_X):
+ BPF_OP_ALU_REG(reg, ins, |, uint64_t);
+ break;
+ case (EBPF_ALU64 | BPF_LSH | BPF_X):
+ BPF_OP_ALU_REG(reg, ins, <<, uint64_t);
+ break;
+ case (EBPF_ALU64 | BPF_RSH | BPF_X):
+ BPF_OP_ALU_REG(reg, ins, >>, uint64_t);
+ break;
+ case (EBPF_ALU64 | EBPF_ARSH | BPF_X):
+ BPF_OP_ALU_REG(reg, ins, >>, int64_t);
+ break;
+ case (EBPF_ALU64 | BPF_XOR | BPF_X):
+ BPF_OP_ALU_REG(reg, ins, ^, uint64_t);
+ break;
+ case (EBPF_ALU64 | BPF_MUL | BPF_X):
+ BPF_OP_ALU_REG(reg, ins, *, uint64_t);
+ break;
+ case (EBPF_ALU64 | BPF_DIV | BPF_X):
+ BPF_DIV_ZERO_CHECK(bpf, reg, ins, uint64_t);
+ BPF_OP_ALU_REG(reg, ins, /, uint64_t);
+ break;
+ case (EBPF_ALU64 | BPF_MOD | BPF_X):
+ BPF_DIV_ZERO_CHECK(bpf, reg, ins, uint64_t);
+ BPF_OP_ALU_REG(reg, ins, %, uint64_t);
+ break;
+ case (EBPF_ALU64 | EBPF_MOV | BPF_X):
+ EBPF_MOV_ALU_REG(reg, ins, uint64_t);
+ break;
+ case (EBPF_ALU64 | BPF_NEG):
+ BPF_NEG_ALU(reg, ins, uint64_t);
+ break;
+ /* load instructions */
+ case (BPF_LDX | BPF_MEM | BPF_B):
+ BPF_LD_REG(reg, ins, uint8_t);
+ break;
+ case (BPF_LDX | BPF_MEM | BPF_H):
+ BPF_LD_REG(reg, ins, uint16_t);
+ break;
+ case (BPF_LDX | BPF_MEM | BPF_W):
+ BPF_LD_REG(reg, ins, uint32_t);
+ break;
+ case (BPF_LDX | BPF_MEM | EBPF_DW):
+ BPF_LD_REG(reg, ins, uint64_t);
+ break;
+ /* load 64 bit immediate value */
+ case (BPF_LD | BPF_IMM | EBPF_DW):
+ reg[ins->dst_reg] = (uint32_t)ins[0].imm |
+ (uint64_t)(uint32_t)ins[1].imm << 32;
+ ins++;
+ break;
+ /* store instructions */
+ case (BPF_STX | BPF_MEM | BPF_B):
+ BPF_ST_REG(reg, ins, uint8_t);
+ break;
+ case (BPF_STX | BPF_MEM | BPF_H):
+ BPF_ST_REG(reg, ins, uint16_t);
+ break;
+ case (BPF_STX | BPF_MEM | BPF_W):
+ BPF_ST_REG(reg, ins, uint32_t);
+ break;
+ case (BPF_STX | BPF_MEM | EBPF_DW):
+ BPF_ST_REG(reg, ins, uint64_t);
+ break;
+ case (BPF_ST | BPF_MEM | BPF_B):
+ BPF_ST_IMM(reg, ins, uint8_t);
+ break;
+ case (BPF_ST | BPF_MEM | BPF_H):
+ BPF_ST_IMM(reg, ins, uint16_t);
+ break;
+ case (BPF_ST | BPF_MEM | BPF_W):
+ BPF_ST_IMM(reg, ins, uint32_t);
+ break;
+ case (BPF_ST | BPF_MEM | EBPF_DW):
+ BPF_ST_IMM(reg, ins, uint64_t);
+ break;
+ /* atomic add instructions */
+ case (BPF_STX | EBPF_XADD | BPF_W):
+ BPF_ST_XADD_REG(reg, ins, 32);
+ break;
+ case (BPF_STX | EBPF_XADD | EBPF_DW):
+ BPF_ST_XADD_REG(reg, ins, 64);
+ break;
+ /* jump instructions */
+ case (BPF_JMP | BPF_JA):
+ BPF_JMP_UNC(ins);
+ break;
+ /* jump IMM instructions */
+ case (BPF_JMP | BPF_JEQ | BPF_K):
+ BPF_JMP_CND_IMM(reg, ins, ==, uint64_t);
+ break;
+ case (BPF_JMP | EBPF_JNE | BPF_K):
+ BPF_JMP_CND_IMM(reg, ins, !=, uint64_t);
+ break;
+ case (BPF_JMP | BPF_JGT | BPF_K):
+ BPF_JMP_CND_IMM(reg, ins, >, uint64_t);
+ break;
+ case (BPF_JMP | EBPF_JLT | BPF_K):
+ BPF_JMP_CND_IMM(reg, ins, <, uint64_t);
+ break;
+ case (BPF_JMP | BPF_JGE | BPF_K):
+ BPF_JMP_CND_IMM(reg, ins, >=, uint64_t);
+ break;
+ case (BPF_JMP | EBPF_JLE | BPF_K):
+ BPF_JMP_CND_IMM(reg, ins, <=, uint64_t);
+ break;
+ case (BPF_JMP | EBPF_JSGT | BPF_K):
+ BPF_JMP_CND_IMM(reg, ins, >, int64_t);
+ break;
+ case (BPF_JMP | EBPF_JSLT | BPF_K):
+ BPF_JMP_CND_IMM(reg, ins, <, int64_t);
+ break;
+ case (BPF_JMP | EBPF_JSGE | BPF_K):
+ BPF_JMP_CND_IMM(reg, ins, >=, int64_t);
+ break;
+ case (BPF_JMP | EBPF_JSLE | BPF_K):
+ BPF_JMP_CND_IMM(reg, ins, <=, int64_t);
+ break;
+ case (BPF_JMP | BPF_JSET | BPF_K):
+ BPF_JMP_CND_IMM(reg, ins, &, uint64_t);
+ break;
+ /* jump REG instructions */
+ case (BPF_JMP | BPF_JEQ | BPF_X):
+ BPF_JMP_CND_REG(reg, ins, ==, uint64_t);
+ break;
+ case (BPF_JMP | EBPF_JNE | BPF_X):
+ BPF_JMP_CND_REG(reg, ins, !=, uint64_t);
+ break;
+ case (BPF_JMP | BPF_JGT | BPF_X):
+ BPF_JMP_CND_REG(reg, ins, >, uint64_t);
+ break;
+ case (BPF_JMP | EBPF_JLT | BPF_X):
+ BPF_JMP_CND_REG(reg, ins, <, uint64_t);
+ break;
+ case (BPF_JMP | BPF_JGE | BPF_X):
+ BPF_JMP_CND_REG(reg, ins, >=, uint64_t);
+ break;
+ case (BPF_JMP | EBPF_JLE | BPF_X):
+ BPF_JMP_CND_REG(reg, ins, <=, uint64_t);
+ break;
+ case (BPF_JMP | EBPF_JSGT | BPF_X):
+ BPF_JMP_CND_REG(reg, ins, >, int64_t);
+ break;
+ case (BPF_JMP | EBPF_JSLT | BPF_X):
+ BPF_JMP_CND_REG(reg, ins, <, int64_t);
+ break;
+ case (BPF_JMP | EBPF_JSGE | BPF_X):
+ BPF_JMP_CND_REG(reg, ins, >=, int64_t);
+ break;
+ case (BPF_JMP | EBPF_JSLE | BPF_X):
+ BPF_JMP_CND_REG(reg, ins, <=, int64_t);
+ break;
+ case (BPF_JMP | BPF_JSET | BPF_X):
+ BPF_JMP_CND_REG(reg, ins, &, uint64_t);
+ break;
+ /* call instructions */
+ case (BPF_JMP | EBPF_CALL):
+ reg[EBPF_REG_0] = bpf->prm.xsym[ins->imm].func.val(
+ reg[EBPF_REG_1], reg[EBPF_REG_2],
+ reg[EBPF_REG_3], reg[EBPF_REG_4],
+ reg[EBPF_REG_5]);
+ break;
+ /* return instruction */
+ case (BPF_JMP | EBPF_EXIT):
+ return reg[EBPF_REG_0];
+ default:
+ RTE_BPF_LOG(ERR,
+ "%s(%p): invalid opcode %#x at pc: %#zx;\n",
+ __func__, bpf, ins->code,
+ (uintptr_t)ins - (uintptr_t)bpf->prm.ins);
+ return 0;
+ }
+ }
+
+ /* should never be reached */
+ RTE_VERIFY(0);
+ return 0;
+}
+
+uint32_t
+rte_bpf_exec_burst(const struct rte_bpf *bpf, void *ctx[], uint64_t rc[],
+ uint32_t num)
+{
+ uint32_t i;
+ uint64_t reg[EBPF_REG_NUM];
+ uint64_t stack[MAX_BPF_STACK_SIZE / sizeof(uint64_t)];
+
+ for (i = 0; i != num; i++) {
+
+ reg[EBPF_REG_1] = (uintptr_t)ctx[i];
+ reg[EBPF_REG_10] = (uintptr_t)(stack + RTE_DIM(stack));
+
+ rc[i] = bpf_exec(bpf, reg);
+ }
+
+ return i;
+}
+
+uint64_t
+rte_bpf_exec(const struct rte_bpf *bpf, void *ctx)
+{
+ uint64_t rc;
+
+ rte_bpf_exec_burst(bpf, &ctx, &rc, 1);
+ return rc;
+}
diff --git a/src/spdk/dpdk/lib/librte_bpf/bpf_impl.h b/src/spdk/dpdk/lib/librte_bpf/bpf_impl.h
new file mode 100644
index 000000000..03ba0ae11
--- /dev/null
+++ b/src/spdk/dpdk/lib/librte_bpf/bpf_impl.h
@@ -0,0 +1,54 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2018 Intel Corporation
+ */
+
+#ifndef _BPF_H_
+#define _BPF_H_
+
+#include <rte_bpf.h>
+#include <sys/mman.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#define MAX_BPF_STACK_SIZE 0x200
+
+struct rte_bpf {
+ struct rte_bpf_prm prm;
+ struct rte_bpf_jit jit;
+ size_t sz;
+ uint32_t stack_sz;
+};
+
+extern int bpf_validate(struct rte_bpf *bpf);
+
+extern int bpf_jit(struct rte_bpf *bpf);
+
+extern int bpf_jit_x86(struct rte_bpf *);
+extern int bpf_jit_arm64(struct rte_bpf *);
+
+extern int rte_bpf_logtype;
+
+#define RTE_BPF_LOG(lvl, fmt, args...) \
+ rte_log(RTE_LOG_## lvl, rte_bpf_logtype, fmt, ##args)
+
+static inline size_t
+bpf_size(uint32_t bpf_op_sz)
+{
+ if (bpf_op_sz == BPF_B)
+ return sizeof(uint8_t);
+ else if (bpf_op_sz == BPF_H)
+ return sizeof(uint16_t);
+ else if (bpf_op_sz == BPF_W)
+ return sizeof(uint32_t);
+ else if (bpf_op_sz == EBPF_DW)
+ return sizeof(uint64_t);
+ return 0;
+}
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _BPF_H_ */
diff --git a/src/spdk/dpdk/lib/librte_bpf/bpf_jit_arm64.c b/src/spdk/dpdk/lib/librte_bpf/bpf_jit_arm64.c
new file mode 100644
index 000000000..a5a5d46f0
--- /dev/null
+++ b/src/spdk/dpdk/lib/librte_bpf/bpf_jit_arm64.c
@@ -0,0 +1,1451 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(C) 2019 Marvell International Ltd.
+ */
+
+#include <errno.h>
+#include <stdbool.h>
+
+#include <rte_common.h>
+#include <rte_byteorder.h>
+
+#include "bpf_impl.h"
+
+#define A64_REG_MASK(r) ((r) & 0x1f)
+#define A64_INVALID_OP_CODE (0xffffffff)
+
+#define TMP_REG_1 (EBPF_REG_10 + 1)
+#define TMP_REG_2 (EBPF_REG_10 + 2)
+#define TMP_REG_3 (EBPF_REG_10 + 3)
+
+#define EBPF_FP (EBPF_REG_10)
+#define EBPF_OP_GET(op) (BPF_OP(op) >> 4)
+
+#define A64_R(x) x
+#define A64_FP 29
+#define A64_LR 30
+#define A64_SP 31
+#define A64_ZR 31
+
+#define check_imm(n, val) (((val) >= 0) ? !!((val) >> (n)) : !!((~val) >> (n)))
+#define mask_imm(n, val) ((val) & ((1 << (n)) - 1))
+
+struct ebpf_a64_map {
+ uint32_t off; /* eBPF to arm64 insn offset mapping for jump */
+ uint8_t off_to_b; /* Offset to branch instruction delta */
+};
+
+struct a64_jit_ctx {
+ size_t stack_sz; /* Stack size */
+ uint32_t *ins; /* ARM64 instructions. NULL if first pass */
+ struct ebpf_a64_map *map; /* eBPF to arm64 insn mapping for jump */
+ uint32_t idx; /* Current instruction index */
+ uint32_t program_start; /* Program index, Just after prologue */
+ uint32_t program_sz; /* Program size. Found in first pass */
+ uint8_t foundcall; /* Found EBPF_CALL class code in eBPF pgm */
+};
+
+static int
+check_immr_imms(bool is64, uint8_t immr, uint8_t imms)
+{
+ const unsigned int width = is64 ? 64 : 32;
+
+ if (immr >= width || imms >= width)
+ return 1;
+
+ return 0;
+}
+
+static int
+check_mov_hw(bool is64, const uint8_t val)
+{
+ if (val == 16 || val == 0)
+ return 0;
+ else if (is64 && val != 64 && val != 48 && val != 32)
+ return 1;
+
+ return 0;
+}
+
+static int
+check_ls_sz(uint8_t sz)
+{
+ if (sz == BPF_B || sz == BPF_H || sz == BPF_W || sz == EBPF_DW)
+ return 0;
+
+ return 1;
+}
+
+static int
+check_reg(uint8_t r)
+{
+ return (r > 31) ? 1 : 0;
+}
+
+static int
+is_first_pass(struct a64_jit_ctx *ctx)
+{
+ return (ctx->ins == NULL);
+}
+
+static int
+check_invalid_args(struct a64_jit_ctx *ctx, uint32_t limit)
+{
+ uint32_t idx;
+
+ if (is_first_pass(ctx))
+ return 0;
+
+ for (idx = 0; idx < limit; idx++) {
+ if (rte_le_to_cpu_32(ctx->ins[idx]) == A64_INVALID_OP_CODE) {
+ RTE_BPF_LOG(ERR,
+ "%s: invalid opcode at %u;\n", __func__, idx);
+ return -EINVAL;
+ }
+ }
+ return 0;
+}
+
+static int
+jump_offset_init(struct a64_jit_ctx *ctx, struct rte_bpf *bpf)
+{
+ uint32_t i;
+
+ ctx->map = malloc(bpf->prm.nb_ins * sizeof(ctx->map[0]));
+ if (ctx->map == NULL)
+ return -ENOMEM;
+
+ /* Fill with fake offsets */
+ for (i = 0; i != bpf->prm.nb_ins; i++) {
+ ctx->map[i].off = INT32_MAX;
+ ctx->map[i].off_to_b = 0;
+ }
+ return 0;
+}
+
+static void
+jump_offset_fini(struct a64_jit_ctx *ctx)
+{
+ free(ctx->map);
+}
+
+static void
+jump_offset_update(struct a64_jit_ctx *ctx, uint32_t ebpf_idx)
+{
+ if (is_first_pass(ctx))
+ ctx->map[ebpf_idx].off = ctx->idx;
+}
+
+static void
+jump_offset_to_branch_update(struct a64_jit_ctx *ctx, uint32_t ebpf_idx)
+{
+ if (is_first_pass(ctx))
+ ctx->map[ebpf_idx].off_to_b = ctx->idx - ctx->map[ebpf_idx].off;
+
+}
+
+static int32_t
+jump_offset_get(struct a64_jit_ctx *ctx, uint32_t from, int16_t offset)
+{
+ int32_t a64_from, a64_to;
+
+ a64_from = ctx->map[from].off + ctx->map[from].off_to_b;
+ a64_to = ctx->map[from + offset + 1].off;
+
+ if (a64_to == INT32_MAX)
+ return a64_to;
+
+ return a64_to - a64_from;
+}
+
+enum a64_cond_e {
+ A64_EQ = 0x0, /* == */
+ A64_NE = 0x1, /* != */
+ A64_CS = 0x2, /* Unsigned >= */
+ A64_CC = 0x3, /* Unsigned < */
+ A64_MI = 0x4, /* < 0 */
+ A64_PL = 0x5, /* >= 0 */
+ A64_VS = 0x6, /* Overflow */
+ A64_VC = 0x7, /* No overflow */
+ A64_HI = 0x8, /* Unsigned > */
+ A64_LS = 0x9, /* Unsigned <= */
+ A64_GE = 0xa, /* Signed >= */
+ A64_LT = 0xb, /* Signed < */
+ A64_GT = 0xc, /* Signed > */
+ A64_LE = 0xd, /* Signed <= */
+ A64_AL = 0xe, /* Always */
+};
+
+static int
+check_cond(uint8_t cond)
+{
+ return (cond >= A64_AL) ? 1 : 0;
+}
+
+static uint8_t
+ebpf_to_a64_cond(uint8_t op)
+{
+ switch (BPF_OP(op)) {
+ case BPF_JEQ:
+ return A64_EQ;
+ case BPF_JGT:
+ return A64_HI;
+ case EBPF_JLT:
+ return A64_CC;
+ case BPF_JGE:
+ return A64_CS;
+ case EBPF_JLE:
+ return A64_LS;
+ case BPF_JSET:
+ case EBPF_JNE:
+ return A64_NE;
+ case EBPF_JSGT:
+ return A64_GT;
+ case EBPF_JSLT:
+ return A64_LT;
+ case EBPF_JSGE:
+ return A64_GE;
+ case EBPF_JSLE:
+ return A64_LE;
+ default:
+ return UINT8_MAX;
+ }
+}
+
+/* Emit an instruction */
+static inline void
+emit_insn(struct a64_jit_ctx *ctx, uint32_t insn, int error)
+{
+ if (error)
+ insn = A64_INVALID_OP_CODE;
+
+ if (ctx->ins)
+ ctx->ins[ctx->idx] = rte_cpu_to_le_32(insn);
+
+ ctx->idx++;
+}
+
+static void
+emit_ret(struct a64_jit_ctx *ctx)
+{
+ emit_insn(ctx, 0xd65f03c0, 0);
+}
+
+static void
+emit_add_sub_imm(struct a64_jit_ctx *ctx, bool is64, bool sub, uint8_t rd,
+ uint8_t rn, int16_t imm12)
+{
+ uint32_t insn, imm;
+
+ imm = mask_imm(12, imm12);
+ insn = (!!is64) << 31;
+ insn |= (!!sub) << 30;
+ insn |= 0x11000000;
+ insn |= rd;
+ insn |= rn << 5;
+ insn |= imm << 10;
+
+ emit_insn(ctx, insn,
+ check_reg(rd) || check_reg(rn) || check_imm(12, imm12));
+}
+
+static void
+emit_add_imm_64(struct a64_jit_ctx *ctx, uint8_t rd, uint8_t rn, uint16_t imm12)
+{
+ emit_add_sub_imm(ctx, 1, 0, rd, rn, imm12);
+}
+
+static void
+emit_sub_imm_64(struct a64_jit_ctx *ctx, uint8_t rd, uint8_t rn, uint16_t imm12)
+{
+ emit_add_sub_imm(ctx, 1, 1, rd, rn, imm12);
+}
+
+static void
+emit_mov(struct a64_jit_ctx *ctx, bool is64, uint8_t rd, uint8_t rn)
+{
+ emit_add_sub_imm(ctx, is64, 0, rd, rn, 0);
+}
+
+static void
+emit_mov_64(struct a64_jit_ctx *ctx, uint8_t rd, uint8_t rn)
+{
+ emit_mov(ctx, 1, rd, rn);
+}
+
+static void
+emit_ls_pair_64(struct a64_jit_ctx *ctx, uint8_t rt, uint8_t rt2, uint8_t rn,
+ bool push, bool load, bool pre_index)
+{
+ uint32_t insn;
+
+ insn = (!!load) << 22;
+ insn |= (!!pre_index) << 24;
+ insn |= 0xa8800000;
+ insn |= rt;
+ insn |= rn << 5;
+ insn |= rt2 << 10;
+ if (push)
+ insn |= 0x7e << 15; /* 0x7e means -2 with imm7 */
+ else
+ insn |= 0x2 << 15;
+
+ emit_insn(ctx, insn, check_reg(rn) || check_reg(rt) || check_reg(rt2));
+
+}
+
+/* Emit stp rt, rt2, [sp, #-16]! */
+static void
+emit_stack_push(struct a64_jit_ctx *ctx, uint8_t rt, uint8_t rt2)
+{
+ emit_ls_pair_64(ctx, rt, rt2, A64_SP, 1, 0, 1);
+}
+
+/* Emit ldp rt, rt2, [sp, #16] */
+static void
+emit_stack_pop(struct a64_jit_ctx *ctx, uint8_t rt, uint8_t rt2)
+{
+ emit_ls_pair_64(ctx, rt, rt2, A64_SP, 0, 1, 0);
+}
+
+#define A64_MOVN 0
+#define A64_MOVZ 2
+#define A64_MOVK 3
+static void
+mov_imm(struct a64_jit_ctx *ctx, bool is64, uint8_t rd, uint8_t type,
+ uint16_t imm16, uint8_t shift)
+{
+ uint32_t insn;
+
+ insn = (!!is64) << 31;
+ insn |= type << 29;
+ insn |= 0x25 << 23;
+ insn |= (shift/16) << 21;
+ insn |= imm16 << 5;
+ insn |= rd;
+
+ emit_insn(ctx, insn, check_reg(rd) || check_mov_hw(is64, shift));
+}
+
+static void
+emit_mov_imm32(struct a64_jit_ctx *ctx, bool is64, uint8_t rd, uint32_t val)
+{
+ uint16_t upper = val >> 16;
+ uint16_t lower = val & 0xffff;
+
+ /* Positive number */
+ if ((val & 1UL << 31) == 0) {
+ mov_imm(ctx, is64, rd, A64_MOVZ, lower, 0);
+ if (upper)
+ mov_imm(ctx, is64, rd, A64_MOVK, upper, 16);
+ } else { /* Negative number */
+ if (upper == 0xffff) {
+ mov_imm(ctx, is64, rd, A64_MOVN, ~lower, 0);
+ } else {
+ mov_imm(ctx, is64, rd, A64_MOVN, ~upper, 16);
+ if (lower != 0xffff)
+ mov_imm(ctx, is64, rd, A64_MOVK, lower, 0);
+ }
+ }
+}
+
+static int
+u16_blocks_weight(const uint64_t val, bool one)
+{
+ return (((val >> 0) & 0xffff) == (one ? 0xffff : 0x0000)) +
+ (((val >> 16) & 0xffff) == (one ? 0xffff : 0x0000)) +
+ (((val >> 32) & 0xffff) == (one ? 0xffff : 0x0000)) +
+ (((val >> 48) & 0xffff) == (one ? 0xffff : 0x0000));
+}
+
+static void
+emit_mov_imm(struct a64_jit_ctx *ctx, bool is64, uint8_t rd, uint64_t val)
+{
+ uint64_t nval = ~val;
+ int movn, sr;
+
+ if (is64 == 0)
+ return emit_mov_imm32(ctx, 0, rd, (uint32_t)(val & 0xffffffff));
+
+ /* Find MOVN or MOVZ first */
+ movn = u16_blocks_weight(val, true) > u16_blocks_weight(val, false);
+ /* Find shift right value */
+ sr = movn ? rte_fls_u64(nval) - 1 : rte_fls_u64(val) - 1;
+ sr = RTE_ALIGN_FLOOR(sr, 16);
+ sr = RTE_MAX(sr, 0);
+
+ if (movn)
+ mov_imm(ctx, 1, rd, A64_MOVN, (nval >> sr) & 0xffff, sr);
+ else
+ mov_imm(ctx, 1, rd, A64_MOVZ, (val >> sr) & 0xffff, sr);
+
+ sr -= 16;
+ while (sr >= 0) {
+ if (((val >> sr) & 0xffff) != (movn ? 0xffff : 0x0000))
+ mov_imm(ctx, 1, rd, A64_MOVK, (val >> sr) & 0xffff, sr);
+ sr -= 16;
+ }
+}
+
+static void
+emit_ls(struct a64_jit_ctx *ctx, uint8_t sz, uint8_t rt, uint8_t rn, uint8_t rm,
+ bool load)
+{
+ uint32_t insn;
+
+ insn = 0x1c1 << 21;
+ if (load)
+ insn |= 1 << 22;
+ if (sz == BPF_B)
+ insn |= 0 << 30;
+ else if (sz == BPF_H)
+ insn |= 1 << 30;
+ else if (sz == BPF_W)
+ insn |= 2 << 30;
+ else if (sz == EBPF_DW)
+ insn |= 3 << 30;
+
+ insn |= rm << 16;
+ insn |= 0x1a << 10; /* LSL and S = 0 */
+ insn |= rn << 5;
+ insn |= rt;
+
+ emit_insn(ctx, insn, check_reg(rt) || check_reg(rn) || check_reg(rm) ||
+ check_ls_sz(sz));
+}
+
+static void
+emit_str(struct a64_jit_ctx *ctx, uint8_t sz, uint8_t rt, uint8_t rn,
+ uint8_t rm)
+{
+ emit_ls(ctx, sz, rt, rn, rm, 0);
+}
+
+static void
+emit_ldr(struct a64_jit_ctx *ctx, uint8_t sz, uint8_t rt, uint8_t rn,
+ uint8_t rm)
+{
+ emit_ls(ctx, sz, rt, rn, rm, 1);
+}
+
+#define A64_ADD 0x58
+#define A64_SUB 0x258
+static void
+emit_add_sub(struct a64_jit_ctx *ctx, bool is64, uint8_t rd, uint8_t rn,
+ uint8_t rm, uint16_t op)
+{
+ uint32_t insn;
+
+ insn = (!!is64) << 31;
+ insn |= op << 21; /* shift == 0 */
+ insn |= rm << 16;
+ insn |= rn << 5;
+ insn |= rd;
+
+ emit_insn(ctx, insn, check_reg(rd) || check_reg(rm));
+}
+
+static void
+emit_add(struct a64_jit_ctx *ctx, bool is64, uint8_t rd, uint8_t rm)
+{
+ emit_add_sub(ctx, is64, rd, rd, rm, A64_ADD);
+}
+
+static void
+emit_sub(struct a64_jit_ctx *ctx, bool is64, uint8_t rd, uint8_t rm)
+{
+ emit_add_sub(ctx, is64, rd, rd, rm, A64_SUB);
+}
+
+static void
+emit_neg(struct a64_jit_ctx *ctx, bool is64, uint8_t rd)
+{
+ emit_add_sub(ctx, is64, rd, A64_ZR, rd, A64_SUB);
+}
+
+static void
+emit_mul(struct a64_jit_ctx *ctx, bool is64, uint8_t rd, uint8_t rm)
+{
+ uint32_t insn;
+
+ insn = (!!is64) << 31;
+ insn |= 0xd8 << 21;
+ insn |= rm << 16;
+ insn |= A64_ZR << 10;
+ insn |= rd << 5;
+ insn |= rd;
+
+ emit_insn(ctx, insn, check_reg(rd) || check_reg(rm));
+}
+
+#define A64_UDIV 0x2
+#define A64_LSLV 0x8
+#define A64_LSRV 0x9
+#define A64_ASRV 0xA
+static void
+emit_data_process_two_src(struct a64_jit_ctx *ctx, bool is64, uint8_t rd,
+ uint8_t rn, uint8_t rm, uint16_t op)
+
+{
+ uint32_t insn;
+
+ insn = (!!is64) << 31;
+ insn |= 0xd6 << 21;
+ insn |= rm << 16;
+ insn |= op << 10;
+ insn |= rn << 5;
+ insn |= rd;
+
+ emit_insn(ctx, insn, check_reg(rd) || check_reg(rm));
+}
+
+static void
+emit_div(struct a64_jit_ctx *ctx, bool is64, uint8_t rd, uint8_t rm)
+{
+ emit_data_process_two_src(ctx, is64, rd, rd, rm, A64_UDIV);
+}
+
+static void
+emit_lslv(struct a64_jit_ctx *ctx, bool is64, uint8_t rd, uint8_t rm)
+{
+ emit_data_process_two_src(ctx, is64, rd, rd, rm, A64_LSLV);
+}
+
+static void
+emit_lsrv(struct a64_jit_ctx *ctx, bool is64, uint8_t rd, uint8_t rm)
+{
+ emit_data_process_two_src(ctx, is64, rd, rd, rm, A64_LSRV);
+}
+
+static void
+emit_asrv(struct a64_jit_ctx *ctx, bool is64, uint8_t rd, uint8_t rm)
+{
+ emit_data_process_two_src(ctx, is64, rd, rd, rm, A64_ASRV);
+}
+
+#define A64_UBFM 0x2
+#define A64_SBFM 0x0
+static void
+emit_bitfield(struct a64_jit_ctx *ctx, bool is64, uint8_t rd, uint8_t rn,
+ uint8_t immr, uint8_t imms, uint16_t op)
+
+{
+ uint32_t insn;
+
+ insn = (!!is64) << 31;
+ if (insn)
+ insn |= 1 << 22; /* Set N bit when is64 is set */
+ insn |= op << 29;
+ insn |= 0x26 << 23;
+ insn |= immr << 16;
+ insn |= imms << 10;
+ insn |= rn << 5;
+ insn |= rd;
+
+ emit_insn(ctx, insn, check_reg(rd) || check_reg(rn) ||
+ check_immr_imms(is64, immr, imms));
+}
+static void
+emit_lsl(struct a64_jit_ctx *ctx, bool is64, uint8_t rd, uint8_t imm)
+{
+ const unsigned int width = is64 ? 64 : 32;
+ uint8_t imms, immr;
+
+ immr = (width - imm) & (width - 1);
+ imms = width - 1 - imm;
+
+ emit_bitfield(ctx, is64, rd, rd, immr, imms, A64_UBFM);
+}
+
+static void
+emit_lsr(struct a64_jit_ctx *ctx, bool is64, uint8_t rd, uint8_t imm)
+{
+ emit_bitfield(ctx, is64, rd, rd, imm, is64 ? 63 : 31, A64_UBFM);
+}
+
+static void
+emit_asr(struct a64_jit_ctx *ctx, bool is64, uint8_t rd, uint8_t imm)
+{
+ emit_bitfield(ctx, is64, rd, rd, imm, is64 ? 63 : 31, A64_SBFM);
+}
+
+#define A64_AND 0
+#define A64_OR 1
+#define A64_XOR 2
+static void
+emit_logical(struct a64_jit_ctx *ctx, bool is64, uint8_t rd,
+ uint8_t rm, uint16_t op)
+{
+ uint32_t insn;
+
+ insn = (!!is64) << 31;
+ insn |= op << 29;
+ insn |= 0x50 << 21;
+ insn |= rm << 16;
+ insn |= rd << 5;
+ insn |= rd;
+
+ emit_insn(ctx, insn, check_reg(rd) || check_reg(rm));
+}
+
+static void
+emit_or(struct a64_jit_ctx *ctx, bool is64, uint8_t rd, uint8_t rm)
+{
+ emit_logical(ctx, is64, rd, rm, A64_OR);
+}
+
+static void
+emit_and(struct a64_jit_ctx *ctx, bool is64, uint8_t rd, uint8_t rm)
+{
+ emit_logical(ctx, is64, rd, rm, A64_AND);
+}
+
+static void
+emit_xor(struct a64_jit_ctx *ctx, bool is64, uint8_t rd, uint8_t rm)
+{
+ emit_logical(ctx, is64, rd, rm, A64_XOR);
+}
+
+static void
+emit_msub(struct a64_jit_ctx *ctx, bool is64, uint8_t rd, uint8_t rn,
+ uint8_t rm, uint8_t ra)
+{
+ uint32_t insn;
+
+ insn = (!!is64) << 31;
+ insn |= 0xd8 << 21;
+ insn |= rm << 16;
+ insn |= 0x1 << 15;
+ insn |= ra << 10;
+ insn |= rn << 5;
+ insn |= rd;
+
+ emit_insn(ctx, insn, check_reg(rd) || check_reg(rn) || check_reg(rm) ||
+ check_reg(ra));
+}
+
+static void
+emit_mod(struct a64_jit_ctx *ctx, bool is64, uint8_t tmp, uint8_t rd,
+ uint8_t rm)
+{
+ emit_data_process_two_src(ctx, is64, tmp, rd, rm, A64_UDIV);
+ emit_msub(ctx, is64, rd, tmp, rm, rd);
+}
+
+static void
+emit_blr(struct a64_jit_ctx *ctx, uint8_t rn)
+{
+ uint32_t insn;
+
+ insn = 0xd63f0000;
+ insn |= rn << 5;
+
+ emit_insn(ctx, insn, check_reg(rn));
+}
+
+static void
+emit_zero_extend(struct a64_jit_ctx *ctx, uint8_t rd, int32_t imm)
+{
+ switch (imm) {
+ case 16:
+ /* Zero-extend 16 bits into 64 bits */
+ emit_bitfield(ctx, 1, rd, rd, 0, 15, A64_UBFM);
+ break;
+ case 32:
+ /* Zero-extend 32 bits into 64 bits */
+ emit_bitfield(ctx, 1, rd, rd, 0, 31, A64_UBFM);
+ break;
+ case 64:
+ break;
+ default:
+ /* Generate error */
+ emit_insn(ctx, 0, 1);
+ }
+}
+
+static void
+emit_rev(struct a64_jit_ctx *ctx, uint8_t rd, int32_t imm)
+{
+ uint32_t insn;
+
+ insn = 0xdac00000;
+ insn |= rd << 5;
+ insn |= rd;
+
+ switch (imm) {
+ case 16:
+ insn |= 1 << 10;
+ emit_insn(ctx, insn, check_reg(rd));
+ emit_zero_extend(ctx, rd, 16);
+ break;
+ case 32:
+ insn |= 2 << 10;
+ emit_insn(ctx, insn, check_reg(rd));
+ /* Upper 32 bits already cleared */
+ break;
+ case 64:
+ insn |= 3 << 10;
+ emit_insn(ctx, insn, check_reg(rd));
+ break;
+ default:
+ /* Generate error */
+ emit_insn(ctx, insn, 1);
+ }
+}
+
+static int
+is_be(void)
+{
+#if RTE_BYTE_ORDER == RTE_BIG_ENDIAN
+ return 1;
+#else
+ return 0;
+#endif
+}
+
+static void
+emit_be(struct a64_jit_ctx *ctx, uint8_t rd, int32_t imm)
+{
+ if (is_be())
+ emit_zero_extend(ctx, rd, imm);
+ else
+ emit_rev(ctx, rd, imm);
+}
+
+static void
+emit_le(struct a64_jit_ctx *ctx, uint8_t rd, int32_t imm)
+{
+ if (is_be())
+ emit_rev(ctx, rd, imm);
+ else
+ emit_zero_extend(ctx, rd, imm);
+}
+
+static uint8_t
+ebpf_to_a64_reg(struct a64_jit_ctx *ctx, uint8_t reg)
+{
+ const uint32_t ebpf2a64_has_call[] = {
+ /* Map A64 R7 register as EBPF return register */
+ [EBPF_REG_0] = A64_R(7),
+ /* Map A64 arguments register as EBPF arguments register */
+ [EBPF_REG_1] = A64_R(0),
+ [EBPF_REG_2] = A64_R(1),
+ [EBPF_REG_3] = A64_R(2),
+ [EBPF_REG_4] = A64_R(3),
+ [EBPF_REG_5] = A64_R(4),
+ /* Map A64 callee save register as EBPF callee save register */
+ [EBPF_REG_6] = A64_R(19),
+ [EBPF_REG_7] = A64_R(20),
+ [EBPF_REG_8] = A64_R(21),
+ [EBPF_REG_9] = A64_R(22),
+ [EBPF_FP] = A64_R(25),
+ /* Map A64 scratch registers as temporary storage */
+ [TMP_REG_1] = A64_R(9),
+ [TMP_REG_2] = A64_R(10),
+ [TMP_REG_3] = A64_R(11),
+ };
+
+ const uint32_t ebpf2a64_no_call[] = {
+ /* Map A64 R7 register as EBPF return register */
+ [EBPF_REG_0] = A64_R(7),
+ /* Map A64 arguments register as EBPF arguments register */
+ [EBPF_REG_1] = A64_R(0),
+ [EBPF_REG_2] = A64_R(1),
+ [EBPF_REG_3] = A64_R(2),
+ [EBPF_REG_4] = A64_R(3),
+ [EBPF_REG_5] = A64_R(4),
+ /*
+ * EBPF program does not have EBPF_CALL op code,
+ * Map A64 scratch registers as EBPF callee save registers.
+ */
+ [EBPF_REG_6] = A64_R(9),
+ [EBPF_REG_7] = A64_R(10),
+ [EBPF_REG_8] = A64_R(11),
+ [EBPF_REG_9] = A64_R(12),
+ /* Map A64 FP register as EBPF FP register */
+ [EBPF_FP] = A64_FP,
+ /* Map remaining A64 scratch registers as temporary storage */
+ [TMP_REG_1] = A64_R(13),
+ [TMP_REG_2] = A64_R(14),
+ [TMP_REG_3] = A64_R(15),
+ };
+
+ if (ctx->foundcall)
+ return ebpf2a64_has_call[reg];
+ else
+ return ebpf2a64_no_call[reg];
+}
+
+/*
+ * Procedure call standard for the arm64
+ * -------------------------------------
+ * R0..R7 - Parameter/result registers
+ * R8 - Indirect result location register
+ * R9..R15 - Scratch registers
+ * R15 - Platform Register
+ * R16 - First intra-procedure-call scratch register
+ * R17 - Second intra-procedure-call temporary register
+ * R19-R28 - Callee saved registers
+ * R29 - Frame pointer
+ * R30 - Link register
+ * R31 - Stack pointer
+ */
+static void
+emit_prologue_has_call(struct a64_jit_ctx *ctx)
+{
+ uint8_t r6, r7, r8, r9, fp;
+
+ r6 = ebpf_to_a64_reg(ctx, EBPF_REG_6);
+ r7 = ebpf_to_a64_reg(ctx, EBPF_REG_7);
+ r8 = ebpf_to_a64_reg(ctx, EBPF_REG_8);
+ r9 = ebpf_to_a64_reg(ctx, EBPF_REG_9);
+ fp = ebpf_to_a64_reg(ctx, EBPF_FP);
+
+ /*
+ * eBPF prog stack layout
+ *
+ * high
+ * eBPF prologue 0:+-----+ <= original A64_SP
+ * |FP/LR|
+ * -16:+-----+ <= current A64_FP
+ * Callee saved registers | ... |
+ * EBPF_FP => -64:+-----+
+ * | |
+ * eBPF prog stack | ... |
+ * | |
+ * (EBPF_FP - bpf->stack_sz)=> +-----+
+ * Pad for A64_SP 16B alignment| PAD |
+ * (EBPF_FP - ctx->stack_sz)=> +-----+ <= current A64_SP
+ * | |
+ * | ... | Function call stack
+ * | |
+ * +-----+
+ * low
+ */
+ emit_stack_push(ctx, A64_FP, A64_LR);
+ emit_mov_64(ctx, A64_FP, A64_SP);
+ emit_stack_push(ctx, r6, r7);
+ emit_stack_push(ctx, r8, r9);
+ /*
+ * There is no requirement to save A64_R(28) in stack. Doing it here,
+ * because, A64_SP needs be to 16B aligned and STR vs STP
+ * takes same number of cycles(typically).
+ */
+ emit_stack_push(ctx, fp, A64_R(28));
+ emit_mov_64(ctx, fp, A64_SP);
+ if (ctx->stack_sz)
+ emit_sub_imm_64(ctx, A64_SP, A64_SP, ctx->stack_sz);
+}
+
+static void
+emit_epilogue_has_call(struct a64_jit_ctx *ctx)
+{
+ uint8_t r6, r7, r8, r9, fp, r0;
+
+ r6 = ebpf_to_a64_reg(ctx, EBPF_REG_6);
+ r7 = ebpf_to_a64_reg(ctx, EBPF_REG_7);
+ r8 = ebpf_to_a64_reg(ctx, EBPF_REG_8);
+ r9 = ebpf_to_a64_reg(ctx, EBPF_REG_9);
+ fp = ebpf_to_a64_reg(ctx, EBPF_FP);
+ r0 = ebpf_to_a64_reg(ctx, EBPF_REG_0);
+
+ if (ctx->stack_sz)
+ emit_add_imm_64(ctx, A64_SP, A64_SP, ctx->stack_sz);
+ emit_stack_pop(ctx, fp, A64_R(28));
+ emit_stack_pop(ctx, r8, r9);
+ emit_stack_pop(ctx, r6, r7);
+ emit_stack_pop(ctx, A64_FP, A64_LR);
+ emit_mov_64(ctx, A64_R(0), r0);
+ emit_ret(ctx);
+}
+
+static void
+emit_prologue_no_call(struct a64_jit_ctx *ctx)
+{
+ /*
+ * eBPF prog stack layout without EBPF_CALL opcode
+ *
+ * high
+ * eBPF prologue(EBPF_FP) 0:+-----+ <= original A64_SP/current A64_FP
+ * | |
+ * | ... |
+ * eBPF prog stack | |
+ * | |
+ * (EBPF_FP - bpf->stack_sz)=> +-----+
+ * Pad for A64_SP 16B alignment| PAD |
+ * (EBPF_FP - ctx->stack_sz)=> +-----+ <= current A64_SP
+ * | |
+ * | ... | Function call stack
+ * | |
+ * +-----+
+ * low
+ */
+ if (ctx->stack_sz) {
+ emit_mov_64(ctx, A64_FP, A64_SP);
+ emit_sub_imm_64(ctx, A64_SP, A64_SP, ctx->stack_sz);
+ }
+}
+
+static void
+emit_epilogue_no_call(struct a64_jit_ctx *ctx)
+{
+ if (ctx->stack_sz)
+ emit_add_imm_64(ctx, A64_SP, A64_SP, ctx->stack_sz);
+ emit_mov_64(ctx, A64_R(0), ebpf_to_a64_reg(ctx, EBPF_REG_0));
+ emit_ret(ctx);
+}
+
+static void
+emit_prologue(struct a64_jit_ctx *ctx)
+{
+ if (ctx->foundcall)
+ emit_prologue_has_call(ctx);
+ else
+ emit_prologue_no_call(ctx);
+
+ ctx->program_start = ctx->idx;
+}
+
+static void
+emit_epilogue(struct a64_jit_ctx *ctx)
+{
+ ctx->program_sz = ctx->idx - ctx->program_start;
+
+ if (ctx->foundcall)
+ emit_epilogue_has_call(ctx);
+ else
+ emit_epilogue_no_call(ctx);
+}
+
+static void
+emit_call(struct a64_jit_ctx *ctx, uint8_t tmp, void *func)
+{
+ uint8_t r0 = ebpf_to_a64_reg(ctx, EBPF_REG_0);
+
+ emit_mov_imm(ctx, 1, tmp, (uint64_t)func);
+ emit_blr(ctx, tmp);
+ emit_mov_64(ctx, r0, A64_R(0));
+}
+
+static void
+emit_cbnz(struct a64_jit_ctx *ctx, bool is64, uint8_t rt, int32_t imm19)
+{
+ uint32_t insn, imm;
+
+ imm = mask_imm(19, imm19);
+ insn = (!!is64) << 31;
+ insn |= 0x35 << 24;
+ insn |= imm << 5;
+ insn |= rt;
+
+ emit_insn(ctx, insn, check_reg(rt) || check_imm(19, imm19));
+}
+
+static void
+emit_b(struct a64_jit_ctx *ctx, int32_t imm26)
+{
+ uint32_t insn, imm;
+
+ imm = mask_imm(26, imm26);
+ insn = 0x5 << 26;
+ insn |= imm;
+
+ emit_insn(ctx, insn, check_imm(26, imm26));
+}
+
+static void
+emit_return_zero_if_src_zero(struct a64_jit_ctx *ctx, bool is64, uint8_t src)
+{
+ uint8_t r0 = ebpf_to_a64_reg(ctx, EBPF_REG_0);
+ uint16_t jump_to_epilogue;
+
+ emit_cbnz(ctx, is64, src, 3);
+ emit_mov_imm(ctx, is64, r0, 0);
+ jump_to_epilogue = (ctx->program_start + ctx->program_sz) - ctx->idx;
+ emit_b(ctx, jump_to_epilogue);
+}
+
+static void
+emit_stadd(struct a64_jit_ctx *ctx, bool is64, uint8_t rs, uint8_t rn)
+{
+ uint32_t insn;
+
+ insn = 0xb820001f;
+ insn |= (!!is64) << 30;
+ insn |= rs << 16;
+ insn |= rn << 5;
+
+ emit_insn(ctx, insn, check_reg(rs) || check_reg(rn));
+}
+
+static void
+emit_ldxr(struct a64_jit_ctx *ctx, bool is64, uint8_t rt, uint8_t rn)
+{
+ uint32_t insn;
+
+ insn = 0x885f7c00;
+ insn |= (!!is64) << 30;
+ insn |= rn << 5;
+ insn |= rt;
+
+ emit_insn(ctx, insn, check_reg(rt) || check_reg(rn));
+}
+
+static void
+emit_stxr(struct a64_jit_ctx *ctx, bool is64, uint8_t rs, uint8_t rt,
+ uint8_t rn)
+{
+ uint32_t insn;
+
+ insn = 0x88007c00;
+ insn |= (!!is64) << 30;
+ insn |= rs << 16;
+ insn |= rn << 5;
+ insn |= rt;
+
+ emit_insn(ctx, insn, check_reg(rs) || check_reg(rt) || check_reg(rn));
+}
+
+static int
+has_atomics(void)
+{
+ int rc = 0;
+
+#if defined(__ARM_FEATURE_ATOMICS) || defined(RTE_ARM_FEATURE_ATOMICS)
+ rc = 1;
+#endif
+ return rc;
+}
+
+static void
+emit_xadd(struct a64_jit_ctx *ctx, uint8_t op, uint8_t tmp1, uint8_t tmp2,
+ uint8_t tmp3, uint8_t dst, int16_t off, uint8_t src)
+{
+ bool is64 = (BPF_SIZE(op) == EBPF_DW);
+ uint8_t rn;
+
+ if (off) {
+ emit_mov_imm(ctx, 1, tmp1, off);
+ emit_add(ctx, 1, tmp1, dst);
+ rn = tmp1;
+ } else {
+ rn = dst;
+ }
+
+ if (has_atomics()) {
+ emit_stadd(ctx, is64, src, rn);
+ } else {
+ emit_ldxr(ctx, is64, tmp2, rn);
+ emit_add(ctx, is64, tmp2, src);
+ emit_stxr(ctx, is64, tmp3, tmp2, rn);
+ emit_cbnz(ctx, is64, tmp3, -3);
+ }
+}
+
+#define A64_CMP 0x6b00000f
+#define A64_TST 0x6a00000f
+static void
+emit_cmp_tst(struct a64_jit_ctx *ctx, bool is64, uint8_t rn, uint8_t rm,
+ uint32_t opc)
+{
+ uint32_t insn;
+
+ insn = opc;
+ insn |= (!!is64) << 31;
+ insn |= rm << 16;
+ insn |= rn << 5;
+
+ emit_insn(ctx, insn, check_reg(rn) || check_reg(rm));
+}
+
+static void
+emit_cmp(struct a64_jit_ctx *ctx, bool is64, uint8_t rn, uint8_t rm)
+{
+ emit_cmp_tst(ctx, is64, rn, rm, A64_CMP);
+}
+
+static void
+emit_tst(struct a64_jit_ctx *ctx, bool is64, uint8_t rn, uint8_t rm)
+{
+ emit_cmp_tst(ctx, is64, rn, rm, A64_TST);
+}
+
+static void
+emit_b_cond(struct a64_jit_ctx *ctx, uint8_t cond, int32_t imm19)
+{
+ uint32_t insn, imm;
+
+ imm = mask_imm(19, imm19);
+ insn = 0x15 << 26;
+ insn |= imm << 5;
+ insn |= cond;
+
+ emit_insn(ctx, insn, check_cond(cond) || check_imm(19, imm19));
+}
+
+static void
+emit_branch(struct a64_jit_ctx *ctx, uint8_t op, uint32_t i, int16_t off)
+{
+ jump_offset_to_branch_update(ctx, i);
+ emit_b_cond(ctx, ebpf_to_a64_cond(op), jump_offset_get(ctx, i, off));
+}
+
+static void
+check_program_has_call(struct a64_jit_ctx *ctx, struct rte_bpf *bpf)
+{
+ const struct ebpf_insn *ins;
+ uint8_t op;
+ uint32_t i;
+
+ for (i = 0; i != bpf->prm.nb_ins; i++) {
+ ins = bpf->prm.ins + i;
+ op = ins->code;
+
+ switch (op) {
+ /* Call imm */
+ case (BPF_JMP | EBPF_CALL):
+ ctx->foundcall = 1;
+ return;
+ }
+ }
+}
+
+/*
+ * Walk through eBPF code and translate them to arm64 one.
+ */
+static int
+emit(struct a64_jit_ctx *ctx, struct rte_bpf *bpf)
+{
+ uint8_t op, dst, src, tmp1, tmp2, tmp3;
+ const struct ebpf_insn *ins;
+ uint64_t u64;
+ int16_t off;
+ int32_t imm;
+ uint32_t i;
+ bool is64;
+ int rc;
+
+ /* Reset context fields */
+ ctx->idx = 0;
+ /* arm64 SP must be aligned to 16 */
+ ctx->stack_sz = RTE_ALIGN_MUL_CEIL(bpf->stack_sz, 16);
+ tmp1 = ebpf_to_a64_reg(ctx, TMP_REG_1);
+ tmp2 = ebpf_to_a64_reg(ctx, TMP_REG_2);
+ tmp3 = ebpf_to_a64_reg(ctx, TMP_REG_3);
+
+ emit_prologue(ctx);
+
+ for (i = 0; i != bpf->prm.nb_ins; i++) {
+
+ jump_offset_update(ctx, i);
+ ins = bpf->prm.ins + i;
+ op = ins->code;
+ off = ins->off;
+ imm = ins->imm;
+
+ dst = ebpf_to_a64_reg(ctx, ins->dst_reg);
+ src = ebpf_to_a64_reg(ctx, ins->src_reg);
+ is64 = (BPF_CLASS(op) == EBPF_ALU64);
+
+ switch (op) {
+ /* dst = src */
+ case (BPF_ALU | EBPF_MOV | BPF_X):
+ case (EBPF_ALU64 | EBPF_MOV | BPF_X):
+ emit_mov(ctx, is64, dst, src);
+ break;
+ /* dst = imm */
+ case (BPF_ALU | EBPF_MOV | BPF_K):
+ case (EBPF_ALU64 | EBPF_MOV | BPF_K):
+ emit_mov_imm(ctx, is64, dst, imm);
+ break;
+ /* dst += src */
+ case (BPF_ALU | BPF_ADD | BPF_X):
+ case (EBPF_ALU64 | BPF_ADD | BPF_X):
+ emit_add(ctx, is64, dst, src);
+ break;
+ /* dst += imm */
+ case (BPF_ALU | BPF_ADD | BPF_K):
+ case (EBPF_ALU64 | BPF_ADD | BPF_K):
+ emit_mov_imm(ctx, is64, tmp1, imm);
+ emit_add(ctx, is64, dst, tmp1);
+ break;
+ /* dst -= src */
+ case (BPF_ALU | BPF_SUB | BPF_X):
+ case (EBPF_ALU64 | BPF_SUB | BPF_X):
+ emit_sub(ctx, is64, dst, src);
+ break;
+ /* dst -= imm */
+ case (BPF_ALU | BPF_SUB | BPF_K):
+ case (EBPF_ALU64 | BPF_SUB | BPF_K):
+ emit_mov_imm(ctx, is64, tmp1, imm);
+ emit_sub(ctx, is64, dst, tmp1);
+ break;
+ /* dst *= src */
+ case (BPF_ALU | BPF_MUL | BPF_X):
+ case (EBPF_ALU64 | BPF_MUL | BPF_X):
+ emit_mul(ctx, is64, dst, src);
+ break;
+ /* dst *= imm */
+ case (BPF_ALU | BPF_MUL | BPF_K):
+ case (EBPF_ALU64 | BPF_MUL | BPF_K):
+ emit_mov_imm(ctx, is64, tmp1, imm);
+ emit_mul(ctx, is64, dst, tmp1);
+ break;
+ /* dst /= src */
+ case (BPF_ALU | BPF_DIV | BPF_X):
+ case (EBPF_ALU64 | BPF_DIV | BPF_X):
+ emit_return_zero_if_src_zero(ctx, is64, src);
+ emit_div(ctx, is64, dst, src);
+ break;
+ /* dst /= imm */
+ case (BPF_ALU | BPF_DIV | BPF_K):
+ case (EBPF_ALU64 | BPF_DIV | BPF_K):
+ emit_mov_imm(ctx, is64, tmp1, imm);
+ emit_div(ctx, is64, dst, tmp1);
+ break;
+ /* dst %= src */
+ case (BPF_ALU | BPF_MOD | BPF_X):
+ case (EBPF_ALU64 | BPF_MOD | BPF_X):
+ emit_return_zero_if_src_zero(ctx, is64, src);
+ emit_mod(ctx, is64, tmp1, dst, src);
+ break;
+ /* dst %= imm */
+ case (BPF_ALU | BPF_MOD | BPF_K):
+ case (EBPF_ALU64 | BPF_MOD | BPF_K):
+ emit_mov_imm(ctx, is64, tmp1, imm);
+ emit_mod(ctx, is64, tmp2, dst, tmp1);
+ break;
+ /* dst |= src */
+ case (BPF_ALU | BPF_OR | BPF_X):
+ case (EBPF_ALU64 | BPF_OR | BPF_X):
+ emit_or(ctx, is64, dst, src);
+ break;
+ /* dst |= imm */
+ case (BPF_ALU | BPF_OR | BPF_K):
+ case (EBPF_ALU64 | BPF_OR | BPF_K):
+ emit_mov_imm(ctx, is64, tmp1, imm);
+ emit_or(ctx, is64, dst, tmp1);
+ break;
+ /* dst &= src */
+ case (BPF_ALU | BPF_AND | BPF_X):
+ case (EBPF_ALU64 | BPF_AND | BPF_X):
+ emit_and(ctx, is64, dst, src);
+ break;
+ /* dst &= imm */
+ case (BPF_ALU | BPF_AND | BPF_K):
+ case (EBPF_ALU64 | BPF_AND | BPF_K):
+ emit_mov_imm(ctx, is64, tmp1, imm);
+ emit_and(ctx, is64, dst, tmp1);
+ break;
+ /* dst ^= src */
+ case (BPF_ALU | BPF_XOR | BPF_X):
+ case (EBPF_ALU64 | BPF_XOR | BPF_X):
+ emit_xor(ctx, is64, dst, src);
+ break;
+ /* dst ^= imm */
+ case (BPF_ALU | BPF_XOR | BPF_K):
+ case (EBPF_ALU64 | BPF_XOR | BPF_K):
+ emit_mov_imm(ctx, is64, tmp1, imm);
+ emit_xor(ctx, is64, dst, tmp1);
+ break;
+ /* dst = -dst */
+ case (BPF_ALU | BPF_NEG):
+ case (EBPF_ALU64 | BPF_NEG):
+ emit_neg(ctx, is64, dst);
+ break;
+ /* dst <<= src */
+ case BPF_ALU | BPF_LSH | BPF_X:
+ case EBPF_ALU64 | BPF_LSH | BPF_X:
+ emit_lslv(ctx, is64, dst, src);
+ break;
+ /* dst <<= imm */
+ case BPF_ALU | BPF_LSH | BPF_K:
+ case EBPF_ALU64 | BPF_LSH | BPF_K:
+ emit_lsl(ctx, is64, dst, imm);
+ break;
+ /* dst >>= src */
+ case BPF_ALU | BPF_RSH | BPF_X:
+ case EBPF_ALU64 | BPF_RSH | BPF_X:
+ emit_lsrv(ctx, is64, dst, src);
+ break;
+ /* dst >>= imm */
+ case BPF_ALU | BPF_RSH | BPF_K:
+ case EBPF_ALU64 | BPF_RSH | BPF_K:
+ emit_lsr(ctx, is64, dst, imm);
+ break;
+ /* dst >>= src (arithmetic) */
+ case BPF_ALU | EBPF_ARSH | BPF_X:
+ case EBPF_ALU64 | EBPF_ARSH | BPF_X:
+ emit_asrv(ctx, is64, dst, src);
+ break;
+ /* dst >>= imm (arithmetic) */
+ case BPF_ALU | EBPF_ARSH | BPF_K:
+ case EBPF_ALU64 | EBPF_ARSH | BPF_K:
+ emit_asr(ctx, is64, dst, imm);
+ break;
+ /* dst = be##imm(dst) */
+ case (BPF_ALU | EBPF_END | EBPF_TO_BE):
+ emit_be(ctx, dst, imm);
+ break;
+ /* dst = le##imm(dst) */
+ case (BPF_ALU | EBPF_END | EBPF_TO_LE):
+ emit_le(ctx, dst, imm);
+ break;
+ /* dst = *(size *) (src + off) */
+ case (BPF_LDX | BPF_MEM | BPF_B):
+ case (BPF_LDX | BPF_MEM | BPF_H):
+ case (BPF_LDX | BPF_MEM | BPF_W):
+ case (BPF_LDX | BPF_MEM | EBPF_DW):
+ emit_mov_imm(ctx, 1, tmp1, off);
+ emit_ldr(ctx, BPF_SIZE(op), dst, src, tmp1);
+ break;
+ /* dst = imm64 */
+ case (BPF_LD | BPF_IMM | EBPF_DW):
+ u64 = ((uint64_t)ins[1].imm << 32) | (uint32_t)imm;
+ emit_mov_imm(ctx, 1, dst, u64);
+ i++;
+ break;
+ /* *(size *)(dst + off) = src */
+ case (BPF_STX | BPF_MEM | BPF_B):
+ case (BPF_STX | BPF_MEM | BPF_H):
+ case (BPF_STX | BPF_MEM | BPF_W):
+ case (BPF_STX | BPF_MEM | EBPF_DW):
+ emit_mov_imm(ctx, 1, tmp1, off);
+ emit_str(ctx, BPF_SIZE(op), src, dst, tmp1);
+ break;
+ /* *(size *)(dst + off) = imm */
+ case (BPF_ST | BPF_MEM | BPF_B):
+ case (BPF_ST | BPF_MEM | BPF_H):
+ case (BPF_ST | BPF_MEM | BPF_W):
+ case (BPF_ST | BPF_MEM | EBPF_DW):
+ emit_mov_imm(ctx, 1, tmp1, imm);
+ emit_mov_imm(ctx, 1, tmp2, off);
+ emit_str(ctx, BPF_SIZE(op), tmp1, dst, tmp2);
+ break;
+ /* STX XADD: lock *(size *)(dst + off) += src */
+ case (BPF_STX | EBPF_XADD | BPF_W):
+ case (BPF_STX | EBPF_XADD | EBPF_DW):
+ emit_xadd(ctx, op, tmp1, tmp2, tmp3, dst, off, src);
+ break;
+ /* PC += off */
+ case (BPF_JMP | BPF_JA):
+ emit_b(ctx, jump_offset_get(ctx, i, off));
+ break;
+ /* PC += off if dst COND imm */
+ case (BPF_JMP | BPF_JEQ | BPF_K):
+ case (BPF_JMP | EBPF_JNE | BPF_K):
+ case (BPF_JMP | BPF_JGT | BPF_K):
+ case (BPF_JMP | EBPF_JLT | BPF_K):
+ case (BPF_JMP | BPF_JGE | BPF_K):
+ case (BPF_JMP | EBPF_JLE | BPF_K):
+ case (BPF_JMP | EBPF_JSGT | BPF_K):
+ case (BPF_JMP | EBPF_JSLT | BPF_K):
+ case (BPF_JMP | EBPF_JSGE | BPF_K):
+ case (BPF_JMP | EBPF_JSLE | BPF_K):
+ emit_mov_imm(ctx, 1, tmp1, imm);
+ emit_cmp(ctx, 1, dst, tmp1);
+ emit_branch(ctx, op, i, off);
+ break;
+ case (BPF_JMP | BPF_JSET | BPF_K):
+ emit_mov_imm(ctx, 1, tmp1, imm);
+ emit_tst(ctx, 1, dst, tmp1);
+ emit_branch(ctx, op, i, off);
+ break;
+ /* PC += off if dst COND src */
+ case (BPF_JMP | BPF_JEQ | BPF_X):
+ case (BPF_JMP | EBPF_JNE | BPF_X):
+ case (BPF_JMP | BPF_JGT | BPF_X):
+ case (BPF_JMP | EBPF_JLT | BPF_X):
+ case (BPF_JMP | BPF_JGE | BPF_X):
+ case (BPF_JMP | EBPF_JLE | BPF_X):
+ case (BPF_JMP | EBPF_JSGT | BPF_X):
+ case (BPF_JMP | EBPF_JSLT | BPF_X):
+ case (BPF_JMP | EBPF_JSGE | BPF_X):
+ case (BPF_JMP | EBPF_JSLE | BPF_X):
+ emit_cmp(ctx, 1, dst, src);
+ emit_branch(ctx, op, i, off);
+ break;
+ case (BPF_JMP | BPF_JSET | BPF_X):
+ emit_tst(ctx, 1, dst, src);
+ emit_branch(ctx, op, i, off);
+ break;
+ /* Call imm */
+ case (BPF_JMP | EBPF_CALL):
+ emit_call(ctx, tmp1, bpf->prm.xsym[ins->imm].func.val);
+ break;
+ /* Return r0 */
+ case (BPF_JMP | EBPF_EXIT):
+ emit_epilogue(ctx);
+ break;
+ default:
+ RTE_BPF_LOG(ERR,
+ "%s(%p): invalid opcode %#x at pc: %u;\n",
+ __func__, bpf, ins->code, i);
+ return -EINVAL;
+ }
+ }
+ rc = check_invalid_args(ctx, ctx->idx);
+
+ return rc;
+}
+
+/*
+ * Produce a native ISA version of the given BPF code.
+ */
+int
+bpf_jit_arm64(struct rte_bpf *bpf)
+{
+ struct a64_jit_ctx ctx;
+ size_t size;
+ int rc;
+
+ /* Init JIT context */
+ memset(&ctx, 0, sizeof(ctx));
+
+ /* Initialize the memory for eBPF to a64 insn offset map for jump */
+ rc = jump_offset_init(&ctx, bpf);
+ if (rc)
+ goto error;
+
+ /* Find eBPF program has call class or not */
+ check_program_has_call(&ctx, bpf);
+
+ /* First pass to calculate total code size and valid jump offsets */
+ rc = emit(&ctx, bpf);
+ if (rc)
+ goto finish;
+
+ size = ctx.idx * sizeof(uint32_t);
+ /* Allocate JIT program memory */
+ ctx.ins = mmap(NULL, size, PROT_READ | PROT_WRITE,
+ MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+ if (ctx.ins == MAP_FAILED) {
+ rc = -ENOMEM;
+ goto finish;
+ }
+
+ /* Second pass to generate code */
+ rc = emit(&ctx, bpf);
+ if (rc)
+ goto munmap;
+
+ rc = mprotect(ctx.ins, size, PROT_READ | PROT_EXEC) != 0;
+ if (rc) {
+ rc = -errno;
+ goto munmap;
+ }
+
+ /* Flush the icache */
+ __builtin___clear_cache((char *)ctx.ins, (char *)(ctx.ins + ctx.idx));
+
+ bpf->jit.func = (void *)ctx.ins;
+ bpf->jit.sz = size;
+
+ goto finish;
+
+munmap:
+ munmap(ctx.ins, size);
+finish:
+ jump_offset_fini(&ctx);
+error:
+ return rc;
+}
diff --git a/src/spdk/dpdk/lib/librte_bpf/bpf_jit_x86.c b/src/spdk/dpdk/lib/librte_bpf/bpf_jit_x86.c
new file mode 100644
index 000000000..f70cd6be5
--- /dev/null
+++ b/src/spdk/dpdk/lib/librte_bpf/bpf_jit_x86.c
@@ -0,0 +1,1368 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2018 Intel Corporation
+ */
+
+#include <stdarg.h>
+#include <errno.h>
+#include <stdint.h>
+#include <inttypes.h>
+
+#include <rte_common.h>
+#include <rte_log.h>
+#include <rte_debug.h>
+#include <rte_memory.h>
+#include <rte_eal.h>
+#include <rte_byteorder.h>
+
+#include "bpf_impl.h"
+
+#define GET_BPF_OP(op) (BPF_OP(op) >> 4)
+
+enum {
+ RAX = 0, /* scratch, return value */
+ RCX = 1, /* scratch, 4th arg */
+ RDX = 2, /* scratch, 3rd arg */
+ RBX = 3, /* callee saved */
+ RSP = 4, /* stack pointer */
+ RBP = 5, /* frame pointer, callee saved */
+ RSI = 6, /* scratch, 2nd arg */
+ RDI = 7, /* scratch, 1st arg */
+ R8 = 8, /* scratch, 5th arg */
+ R9 = 9, /* scratch, 6th arg */
+ R10 = 10, /* scratch */
+ R11 = 11, /* scratch */
+ R12 = 12, /* callee saved */
+ R13 = 13, /* callee saved */
+ R14 = 14, /* callee saved */
+ R15 = 15, /* callee saved */
+};
+
+#define IS_EXT_REG(r) ((r) >= R8)
+
+enum {
+ REX_PREFIX = 0x40, /* fixed value 0100 */
+ REX_W = 0x8, /* 64bit operand size */
+ REX_R = 0x4, /* extension of the ModRM.reg field */
+ REX_X = 0x2, /* extension of the SIB.index field */
+ REX_B = 0x1, /* extension of the ModRM.rm field */
+};
+
+enum {
+ MOD_INDIRECT = 0,
+ MOD_IDISP8 = 1,
+ MOD_IDISP32 = 2,
+ MOD_DIRECT = 3,
+};
+
+enum {
+ SIB_SCALE_1 = 0,
+ SIB_SCALE_2 = 1,
+ SIB_SCALE_4 = 2,
+ SIB_SCALE_8 = 3,
+};
+
+/*
+ * eBPF to x86_64 register mappings.
+ */
+static const uint32_t ebpf2x86[] = {
+ [EBPF_REG_0] = RAX,
+ [EBPF_REG_1] = RDI,
+ [EBPF_REG_2] = RSI,
+ [EBPF_REG_3] = RDX,
+ [EBPF_REG_4] = RCX,
+ [EBPF_REG_5] = R8,
+ [EBPF_REG_6] = RBX,
+ [EBPF_REG_7] = R13,
+ [EBPF_REG_8] = R14,
+ [EBPF_REG_9] = R15,
+ [EBPF_REG_10] = RBP,
+};
+
+/*
+ * r10 and r11 are used as a scratch temporary registers.
+ */
+enum {
+ REG_DIV_IMM = R9,
+ REG_TMP0 = R11,
+ REG_TMP1 = R10,
+};
+
+/*
+ * callee saved registers list.
+ * keep RBP as the last one.
+ */
+static const uint32_t save_regs[] = {RBX, R12, R13, R14, R15, RBP};
+
+struct bpf_jit_state {
+ uint32_t idx;
+ size_t sz;
+ struct {
+ uint32_t num;
+ int32_t off;
+ } exit;
+ uint32_t reguse;
+ int32_t *off;
+ uint8_t *ins;
+};
+
+#define INUSE(v, r) (((v) >> (r)) & 1)
+#define USED(v, r) ((v) |= 1 << (r))
+
+union bpf_jit_imm {
+ uint32_t u32;
+ uint8_t u8[4];
+};
+
+/*
+ * In many cases for imm8 we can produce shorter code.
+ */
+static size_t
+imm_size(int32_t v)
+{
+ if (v == (int8_t)v)
+ return sizeof(int8_t);
+ return sizeof(int32_t);
+}
+
+static void
+emit_bytes(struct bpf_jit_state *st, const uint8_t ins[], uint32_t sz)
+{
+ uint32_t i;
+
+ if (st->ins != NULL) {
+ for (i = 0; i != sz; i++)
+ st->ins[st->sz + i] = ins[i];
+ }
+ st->sz += sz;
+}
+
+static void
+emit_imm(struct bpf_jit_state *st, const uint32_t imm, uint32_t sz)
+{
+ union bpf_jit_imm v;
+
+ v.u32 = imm;
+ emit_bytes(st, v.u8, sz);
+}
+
+/*
+ * emit REX byte
+ */
+static void
+emit_rex(struct bpf_jit_state *st, uint32_t op, uint32_t reg, uint32_t rm)
+{
+ uint8_t rex;
+
+ /* mark operand registers as used*/
+ USED(st->reguse, reg);
+ USED(st->reguse, rm);
+
+ rex = 0;
+ if (BPF_CLASS(op) == EBPF_ALU64 ||
+ op == (BPF_ST | BPF_MEM | EBPF_DW) ||
+ op == (BPF_STX | BPF_MEM | EBPF_DW) ||
+ op == (BPF_STX | EBPF_XADD | EBPF_DW) ||
+ op == (BPF_LD | BPF_IMM | EBPF_DW) ||
+ (BPF_CLASS(op) == BPF_LDX &&
+ BPF_MODE(op) == BPF_MEM &&
+ BPF_SIZE(op) != BPF_W))
+ rex |= REX_W;
+
+ if (IS_EXT_REG(reg))
+ rex |= REX_R;
+
+ if (IS_EXT_REG(rm))
+ rex |= REX_B;
+
+ /* store using SIL, DIL */
+ if (op == (BPF_STX | BPF_MEM | BPF_B) && (reg == RDI || reg == RSI))
+ rex |= REX_PREFIX;
+
+ if (rex != 0) {
+ rex |= REX_PREFIX;
+ emit_bytes(st, &rex, sizeof(rex));
+ }
+}
+
+/*
+ * emit MODRegRM byte
+ */
+static void
+emit_modregrm(struct bpf_jit_state *st, uint32_t mod, uint32_t reg, uint32_t rm)
+{
+ uint8_t v;
+
+ v = mod << 6 | (reg & 7) << 3 | (rm & 7);
+ emit_bytes(st, &v, sizeof(v));
+}
+
+/*
+ * emit SIB byte
+ */
+static void
+emit_sib(struct bpf_jit_state *st, uint32_t scale, uint32_t idx, uint32_t base)
+{
+ uint8_t v;
+
+ v = scale << 6 | (idx & 7) << 3 | (base & 7);
+ emit_bytes(st, &v, sizeof(v));
+}
+
+/*
+ * emit OPCODE+REGIDX byte
+ */
+static void
+emit_opcode(struct bpf_jit_state *st, uint8_t ops, uint32_t reg)
+{
+ uint8_t v;
+
+ v = ops | (reg & 7);
+ emit_bytes(st, &v, sizeof(v));
+}
+
+
+/*
+ * emit xchg %<sreg>, %<dreg>
+ */
+static void
+emit_xchg_reg(struct bpf_jit_state *st, uint32_t sreg, uint32_t dreg)
+{
+ const uint8_t ops = 0x87;
+
+ emit_rex(st, EBPF_ALU64, sreg, dreg);
+ emit_bytes(st, &ops, sizeof(ops));
+ emit_modregrm(st, MOD_DIRECT, sreg, dreg);
+}
+
+/*
+ * emit neg %<dreg>
+ */
+static void
+emit_neg(struct bpf_jit_state *st, uint32_t op, uint32_t dreg)
+{
+ const uint8_t ops = 0xF7;
+ const uint8_t mods = 3;
+
+ emit_rex(st, op, 0, dreg);
+ emit_bytes(st, &ops, sizeof(ops));
+ emit_modregrm(st, MOD_DIRECT, mods, dreg);
+}
+
+/*
+ * emit mov %<sreg>, %<dreg>
+ */
+static void
+emit_mov_reg(struct bpf_jit_state *st, uint32_t op, uint32_t sreg,
+ uint32_t dreg)
+{
+ const uint8_t ops = 0x89;
+
+ /* if operands are 32-bit, then it can be used to clear upper 32-bit */
+ if (sreg != dreg || BPF_CLASS(op) == BPF_ALU) {
+ emit_rex(st, op, sreg, dreg);
+ emit_bytes(st, &ops, sizeof(ops));
+ emit_modregrm(st, MOD_DIRECT, sreg, dreg);
+ }
+}
+
+/*
+ * emit movzwl %<sreg>, %<dreg>
+ */
+static void
+emit_movzwl(struct bpf_jit_state *st, uint32_t sreg, uint32_t dreg)
+{
+ static const uint8_t ops[] = {0x0F, 0xB7};
+
+ emit_rex(st, BPF_ALU, sreg, dreg);
+ emit_bytes(st, ops, sizeof(ops));
+ emit_modregrm(st, MOD_DIRECT, sreg, dreg);
+}
+
+/*
+ * emit ror <imm8>, %<dreg>
+ */
+static void
+emit_ror_imm(struct bpf_jit_state *st, uint32_t dreg, uint32_t imm)
+{
+ const uint8_t prfx = 0x66;
+ const uint8_t ops = 0xC1;
+ const uint8_t mods = 1;
+
+ emit_bytes(st, &prfx, sizeof(prfx));
+ emit_rex(st, BPF_ALU, 0, dreg);
+ emit_bytes(st, &ops, sizeof(ops));
+ emit_modregrm(st, MOD_DIRECT, mods, dreg);
+ emit_imm(st, imm, imm_size(imm));
+}
+
+/*
+ * emit bswap %<dreg>
+ */
+static void
+emit_be2le_48(struct bpf_jit_state *st, uint32_t dreg, uint32_t imm)
+{
+ uint32_t rop;
+
+ const uint8_t ops = 0x0F;
+ const uint8_t mods = 1;
+
+ rop = (imm == 64) ? EBPF_ALU64 : BPF_ALU;
+ emit_rex(st, rop, 0, dreg);
+ emit_bytes(st, &ops, sizeof(ops));
+ emit_modregrm(st, MOD_DIRECT, mods, dreg);
+}
+
+static void
+emit_be2le(struct bpf_jit_state *st, uint32_t dreg, uint32_t imm)
+{
+ if (imm == 16) {
+ emit_ror_imm(st, dreg, 8);
+ emit_movzwl(st, dreg, dreg);
+ } else
+ emit_be2le_48(st, dreg, imm);
+}
+
+/*
+ * In general it is NOP for x86.
+ * Just clear the upper bits.
+ */
+static void
+emit_le2be(struct bpf_jit_state *st, uint32_t dreg, uint32_t imm)
+{
+ if (imm == 16)
+ emit_movzwl(st, dreg, dreg);
+ else if (imm == 32)
+ emit_mov_reg(st, BPF_ALU | EBPF_MOV | BPF_X, dreg, dreg);
+}
+
+/*
+ * emit one of:
+ * add <imm>, %<dreg>
+ * and <imm>, %<dreg>
+ * or <imm>, %<dreg>
+ * sub <imm>, %<dreg>
+ * xor <imm>, %<dreg>
+ */
+static void
+emit_alu_imm(struct bpf_jit_state *st, uint32_t op, uint32_t dreg, uint32_t imm)
+{
+ uint8_t mod, opcode;
+ uint32_t bop, imsz;
+
+ const uint8_t op8 = 0x83;
+ const uint8_t op32 = 0x81;
+ static const uint8_t mods[] = {
+ [GET_BPF_OP(BPF_ADD)] = 0,
+ [GET_BPF_OP(BPF_AND)] = 4,
+ [GET_BPF_OP(BPF_OR)] = 1,
+ [GET_BPF_OP(BPF_SUB)] = 5,
+ [GET_BPF_OP(BPF_XOR)] = 6,
+ };
+
+ bop = GET_BPF_OP(op);
+ mod = mods[bop];
+
+ imsz = imm_size(imm);
+ opcode = (imsz == 1) ? op8 : op32;
+
+ emit_rex(st, op, 0, dreg);
+ emit_bytes(st, &opcode, sizeof(opcode));
+ emit_modregrm(st, MOD_DIRECT, mod, dreg);
+ emit_imm(st, imm, imsz);
+}
+
+/*
+ * emit one of:
+ * add %<sreg>, %<dreg>
+ * and %<sreg>, %<dreg>
+ * or %<sreg>, %<dreg>
+ * sub %<sreg>, %<dreg>
+ * xor %<sreg>, %<dreg>
+ */
+static void
+emit_alu_reg(struct bpf_jit_state *st, uint32_t op, uint32_t sreg,
+ uint32_t dreg)
+{
+ uint32_t bop;
+
+ static const uint8_t ops[] = {
+ [GET_BPF_OP(BPF_ADD)] = 0x01,
+ [GET_BPF_OP(BPF_AND)] = 0x21,
+ [GET_BPF_OP(BPF_OR)] = 0x09,
+ [GET_BPF_OP(BPF_SUB)] = 0x29,
+ [GET_BPF_OP(BPF_XOR)] = 0x31,
+ };
+
+ bop = GET_BPF_OP(op);
+
+ emit_rex(st, op, sreg, dreg);
+ emit_bytes(st, &ops[bop], sizeof(ops[bop]));
+ emit_modregrm(st, MOD_DIRECT, sreg, dreg);
+}
+
+static void
+emit_shift(struct bpf_jit_state *st, uint32_t op, uint32_t dreg)
+{
+ uint8_t mod;
+ uint32_t bop, opx;
+
+ static const uint8_t ops[] = {0xC1, 0xD3};
+ static const uint8_t mods[] = {
+ [GET_BPF_OP(BPF_LSH)] = 4,
+ [GET_BPF_OP(BPF_RSH)] = 5,
+ [GET_BPF_OP(EBPF_ARSH)] = 7,
+ };
+
+ bop = GET_BPF_OP(op);
+ mod = mods[bop];
+ opx = (BPF_SRC(op) == BPF_X);
+
+ emit_rex(st, op, 0, dreg);
+ emit_bytes(st, &ops[opx], sizeof(ops[opx]));
+ emit_modregrm(st, MOD_DIRECT, mod, dreg);
+}
+
+/*
+ * emit one of:
+ * shl <imm>, %<dreg>
+ * shr <imm>, %<dreg>
+ * sar <imm>, %<dreg>
+ */
+static void
+emit_shift_imm(struct bpf_jit_state *st, uint32_t op, uint32_t dreg,
+ uint32_t imm)
+{
+ emit_shift(st, op, dreg);
+ emit_imm(st, imm, imm_size(imm));
+}
+
+/*
+ * emit one of:
+ * shl %<dreg>
+ * shr %<dreg>
+ * sar %<dreg>
+ * note that rcx is implicitly used as a source register, so few extra
+ * instructions for register spillage might be necessary.
+ */
+static void
+emit_shift_reg(struct bpf_jit_state *st, uint32_t op, uint32_t sreg,
+ uint32_t dreg)
+{
+ if (sreg != RCX)
+ emit_xchg_reg(st, RCX, sreg);
+
+ emit_shift(st, op, (dreg == RCX) ? sreg : dreg);
+
+ if (sreg != RCX)
+ emit_xchg_reg(st, RCX, sreg);
+}
+
+/*
+ * emit mov <imm>, %<dreg>
+ */
+static void
+emit_mov_imm(struct bpf_jit_state *st, uint32_t op, uint32_t dreg, uint32_t imm)
+{
+ const uint8_t ops = 0xC7;
+
+ if (imm == 0) {
+ /* replace 'mov 0, %<dst>' with 'xor %<dst>, %<dst>' */
+ op = BPF_CLASS(op) | BPF_XOR | BPF_X;
+ emit_alu_reg(st, op, dreg, dreg);
+ return;
+ }
+
+ emit_rex(st, op, 0, dreg);
+ emit_bytes(st, &ops, sizeof(ops));
+ emit_modregrm(st, MOD_DIRECT, 0, dreg);
+ emit_imm(st, imm, sizeof(imm));
+}
+
+/*
+ * emit mov <imm64>, %<dreg>
+ */
+static void
+emit_ld_imm64(struct bpf_jit_state *st, uint32_t dreg, uint32_t imm0,
+ uint32_t imm1)
+{
+ uint32_t op;
+
+ const uint8_t ops = 0xB8;
+
+ op = (imm1 == 0) ? BPF_ALU : EBPF_ALU64;
+
+ emit_rex(st, op, 0, dreg);
+ emit_opcode(st, ops, dreg);
+
+ emit_imm(st, imm0, sizeof(imm0));
+ if (imm1 != 0)
+ emit_imm(st, imm1, sizeof(imm1));
+}
+
+/*
+ * note that rax:rdx are implicitly used as source/destination registers,
+ * so some reg spillage is necessary.
+ * emit:
+ * mov %rax, %r11
+ * mov %rdx, %r10
+ * mov %<dreg>, %rax
+ * either:
+ * mov %<sreg>, %rdx
+ * OR
+ * mov <imm>, %rdx
+ * mul %rdx
+ * mov %r10, %rdx
+ * mov %rax, %<dreg>
+ * mov %r11, %rax
+ */
+static void
+emit_mul(struct bpf_jit_state *st, uint32_t op, uint32_t sreg, uint32_t dreg,
+ uint32_t imm)
+{
+ const uint8_t ops = 0xF7;
+ const uint8_t mods = 4;
+
+ /* save rax & rdx */
+ emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, RAX, REG_TMP0);
+ emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, RDX, REG_TMP1);
+
+ /* rax = dreg */
+ emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, dreg, RAX);
+
+ if (BPF_SRC(op) == BPF_X)
+ /* rdx = sreg */
+ emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X,
+ sreg == RAX ? REG_TMP0 : sreg, RDX);
+ else
+ /* rdx = imm */
+ emit_mov_imm(st, EBPF_ALU64 | EBPF_MOV | BPF_K, RDX, imm);
+
+ emit_rex(st, op, RAX, RDX);
+ emit_bytes(st, &ops, sizeof(ops));
+ emit_modregrm(st, MOD_DIRECT, mods, RDX);
+
+ if (dreg != RDX)
+ /* restore rdx */
+ emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, REG_TMP1, RDX);
+
+ if (dreg != RAX) {
+ /* dreg = rax */
+ emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, RAX, dreg);
+ /* restore rax */
+ emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, REG_TMP0, RAX);
+ }
+}
+
+/*
+ * emit mov <ofs>(%<sreg>), %<dreg>
+ * note that for non 64-bit ops, higher bits have to be cleared.
+ */
+static void
+emit_ld_reg(struct bpf_jit_state *st, uint32_t op, uint32_t sreg, uint32_t dreg,
+ int32_t ofs)
+{
+ uint32_t mods, opsz;
+ const uint8_t op32 = 0x8B;
+ const uint8_t op16[] = {0x0F, 0xB7};
+ const uint8_t op8[] = {0x0F, 0xB6};
+
+ emit_rex(st, op, dreg, sreg);
+
+ opsz = BPF_SIZE(op);
+ if (opsz == BPF_B)
+ emit_bytes(st, op8, sizeof(op8));
+ else if (opsz == BPF_H)
+ emit_bytes(st, op16, sizeof(op16));
+ else
+ emit_bytes(st, &op32, sizeof(op32));
+
+ mods = (imm_size(ofs) == 1) ? MOD_IDISP8 : MOD_IDISP32;
+
+ emit_modregrm(st, mods, dreg, sreg);
+ if (sreg == RSP || sreg == R12)
+ emit_sib(st, SIB_SCALE_1, sreg, sreg);
+ emit_imm(st, ofs, imm_size(ofs));
+}
+
+/*
+ * emit one of:
+ * mov %<sreg>, <ofs>(%<dreg>)
+ * mov <imm>, <ofs>(%<dreg>)
+ */
+static void
+emit_st_common(struct bpf_jit_state *st, uint32_t op, uint32_t sreg,
+ uint32_t dreg, uint32_t imm, int32_t ofs)
+{
+ uint32_t mods, imsz, opsz, opx;
+ const uint8_t prfx16 = 0x66;
+
+ /* 8 bit instruction opcodes */
+ static const uint8_t op8[] = {0xC6, 0x88};
+
+ /* 16/32/64 bit instruction opcodes */
+ static const uint8_t ops[] = {0xC7, 0x89};
+
+ /* is the instruction has immediate value or src reg? */
+ opx = (BPF_CLASS(op) == BPF_STX);
+
+ opsz = BPF_SIZE(op);
+ if (opsz == BPF_H)
+ emit_bytes(st, &prfx16, sizeof(prfx16));
+
+ emit_rex(st, op, sreg, dreg);
+
+ if (opsz == BPF_B)
+ emit_bytes(st, &op8[opx], sizeof(op8[opx]));
+ else
+ emit_bytes(st, &ops[opx], sizeof(ops[opx]));
+
+ imsz = imm_size(ofs);
+ mods = (imsz == 1) ? MOD_IDISP8 : MOD_IDISP32;
+
+ emit_modregrm(st, mods, sreg, dreg);
+
+ if (dreg == RSP || dreg == R12)
+ emit_sib(st, SIB_SCALE_1, dreg, dreg);
+
+ emit_imm(st, ofs, imsz);
+
+ if (opx == 0) {
+ imsz = RTE_MIN(bpf_size(opsz), sizeof(imm));
+ emit_imm(st, imm, imsz);
+ }
+}
+
+static void
+emit_st_imm(struct bpf_jit_state *st, uint32_t op, uint32_t dreg, uint32_t imm,
+ int32_t ofs)
+{
+ emit_st_common(st, op, 0, dreg, imm, ofs);
+}
+
+static void
+emit_st_reg(struct bpf_jit_state *st, uint32_t op, uint32_t sreg, uint32_t dreg,
+ int32_t ofs)
+{
+ emit_st_common(st, op, sreg, dreg, 0, ofs);
+}
+
+/*
+ * emit lock add %<sreg>, <ofs>(%<dreg>)
+ */
+static void
+emit_st_xadd(struct bpf_jit_state *st, uint32_t op, uint32_t sreg,
+ uint32_t dreg, int32_t ofs)
+{
+ uint32_t imsz, mods;
+
+ const uint8_t lck = 0xF0; /* lock prefix */
+ const uint8_t ops = 0x01; /* add opcode */
+
+ imsz = imm_size(ofs);
+ mods = (imsz == 1) ? MOD_IDISP8 : MOD_IDISP32;
+
+ emit_bytes(st, &lck, sizeof(lck));
+ emit_rex(st, op, sreg, dreg);
+ emit_bytes(st, &ops, sizeof(ops));
+ emit_modregrm(st, mods, sreg, dreg);
+ emit_imm(st, ofs, imsz);
+}
+
+/*
+ * emit:
+ * mov <imm64>, (%rax)
+ * call *%rax
+ */
+static void
+emit_call(struct bpf_jit_state *st, uintptr_t trg)
+{
+ const uint8_t ops = 0xFF;
+ const uint8_t mods = 2;
+
+ emit_ld_imm64(st, RAX, trg, trg >> 32);
+ emit_bytes(st, &ops, sizeof(ops));
+ emit_modregrm(st, MOD_DIRECT, mods, RAX);
+}
+
+/*
+ * emit jmp <ofs>
+ * where 'ofs' is the target offset for the native code.
+ */
+static void
+emit_abs_jmp(struct bpf_jit_state *st, int32_t ofs)
+{
+ int32_t joff;
+ uint32_t imsz;
+
+ const uint8_t op8 = 0xEB;
+ const uint8_t op32 = 0xE9;
+
+ const int32_t sz8 = sizeof(op8) + sizeof(uint8_t);
+ const int32_t sz32 = sizeof(op32) + sizeof(uint32_t);
+
+ /* max possible jmp instruction size */
+ const int32_t iszm = RTE_MAX(sz8, sz32);
+
+ joff = ofs - st->sz;
+ imsz = RTE_MAX(imm_size(joff), imm_size(joff + iszm));
+
+ if (imsz == 1) {
+ emit_bytes(st, &op8, sizeof(op8));
+ joff -= sz8;
+ } else {
+ emit_bytes(st, &op32, sizeof(op32));
+ joff -= sz32;
+ }
+
+ emit_imm(st, joff, imsz);
+}
+
+/*
+ * emit jmp <ofs>
+ * where 'ofs' is the target offset for the BPF bytecode.
+ */
+static void
+emit_jmp(struct bpf_jit_state *st, int32_t ofs)
+{
+ emit_abs_jmp(st, st->off[st->idx + ofs]);
+}
+
+/*
+ * emit one of:
+ * cmovz %<sreg>, <%dreg>
+ * cmovne %<sreg>, <%dreg>
+ * cmova %<sreg>, <%dreg>
+ * cmovb %<sreg>, <%dreg>
+ * cmovae %<sreg>, <%dreg>
+ * cmovbe %<sreg>, <%dreg>
+ * cmovg %<sreg>, <%dreg>
+ * cmovl %<sreg>, <%dreg>
+ * cmovge %<sreg>, <%dreg>
+ * cmovle %<sreg>, <%dreg>
+ */
+static void
+emit_movcc_reg(struct bpf_jit_state *st, uint32_t op, uint32_t sreg,
+ uint32_t dreg)
+{
+ uint32_t bop;
+
+ static const uint8_t ops[][2] = {
+ [GET_BPF_OP(BPF_JEQ)] = {0x0F, 0x44}, /* CMOVZ */
+ [GET_BPF_OP(EBPF_JNE)] = {0x0F, 0x45}, /* CMOVNE */
+ [GET_BPF_OP(BPF_JGT)] = {0x0F, 0x47}, /* CMOVA */
+ [GET_BPF_OP(EBPF_JLT)] = {0x0F, 0x42}, /* CMOVB */
+ [GET_BPF_OP(BPF_JGE)] = {0x0F, 0x43}, /* CMOVAE */
+ [GET_BPF_OP(EBPF_JLE)] = {0x0F, 0x46}, /* CMOVBE */
+ [GET_BPF_OP(EBPF_JSGT)] = {0x0F, 0x4F}, /* CMOVG */
+ [GET_BPF_OP(EBPF_JSLT)] = {0x0F, 0x4C}, /* CMOVL */
+ [GET_BPF_OP(EBPF_JSGE)] = {0x0F, 0x4D}, /* CMOVGE */
+ [GET_BPF_OP(EBPF_JSLE)] = {0x0F, 0x4E}, /* CMOVLE */
+ [GET_BPF_OP(BPF_JSET)] = {0x0F, 0x45}, /* CMOVNE */
+ };
+
+ bop = GET_BPF_OP(op);
+
+ emit_rex(st, op, dreg, sreg);
+ emit_bytes(st, ops[bop], sizeof(ops[bop]));
+ emit_modregrm(st, MOD_DIRECT, dreg, sreg);
+}
+
+/*
+ * emit one of:
+ * je <ofs>
+ * jne <ofs>
+ * ja <ofs>
+ * jb <ofs>
+ * jae <ofs>
+ * jbe <ofs>
+ * jg <ofs>
+ * jl <ofs>
+ * jge <ofs>
+ * jle <ofs>
+ * where 'ofs' is the target offset for the native code.
+ */
+static void
+emit_abs_jcc(struct bpf_jit_state *st, uint32_t op, int32_t ofs)
+{
+ uint32_t bop, imsz;
+ int32_t joff;
+
+ static const uint8_t op8[] = {
+ [GET_BPF_OP(BPF_JEQ)] = 0x74, /* JE */
+ [GET_BPF_OP(EBPF_JNE)] = 0x75, /* JNE */
+ [GET_BPF_OP(BPF_JGT)] = 0x77, /* JA */
+ [GET_BPF_OP(EBPF_JLT)] = 0x72, /* JB */
+ [GET_BPF_OP(BPF_JGE)] = 0x73, /* JAE */
+ [GET_BPF_OP(EBPF_JLE)] = 0x76, /* JBE */
+ [GET_BPF_OP(EBPF_JSGT)] = 0x7F, /* JG */
+ [GET_BPF_OP(EBPF_JSLT)] = 0x7C, /* JL */
+ [GET_BPF_OP(EBPF_JSGE)] = 0x7D, /*JGE */
+ [GET_BPF_OP(EBPF_JSLE)] = 0x7E, /* JLE */
+ [GET_BPF_OP(BPF_JSET)] = 0x75, /*JNE */
+ };
+
+ static const uint8_t op32[][2] = {
+ [GET_BPF_OP(BPF_JEQ)] = {0x0F, 0x84}, /* JE */
+ [GET_BPF_OP(EBPF_JNE)] = {0x0F, 0x85}, /* JNE */
+ [GET_BPF_OP(BPF_JGT)] = {0x0F, 0x87}, /* JA */
+ [GET_BPF_OP(EBPF_JLT)] = {0x0F, 0x82}, /* JB */
+ [GET_BPF_OP(BPF_JGE)] = {0x0F, 0x83}, /* JAE */
+ [GET_BPF_OP(EBPF_JLE)] = {0x0F, 0x86}, /* JBE */
+ [GET_BPF_OP(EBPF_JSGT)] = {0x0F, 0x8F}, /* JG */
+ [GET_BPF_OP(EBPF_JSLT)] = {0x0F, 0x8C}, /* JL */
+ [GET_BPF_OP(EBPF_JSGE)] = {0x0F, 0x8D}, /*JGE */
+ [GET_BPF_OP(EBPF_JSLE)] = {0x0F, 0x8E}, /* JLE */
+ [GET_BPF_OP(BPF_JSET)] = {0x0F, 0x85}, /*JNE */
+ };
+
+ const int32_t sz8 = sizeof(op8[0]) + sizeof(uint8_t);
+ const int32_t sz32 = sizeof(op32[0]) + sizeof(uint32_t);
+
+ /* max possible jcc instruction size */
+ const int32_t iszm = RTE_MAX(sz8, sz32);
+
+ joff = ofs - st->sz;
+ imsz = RTE_MAX(imm_size(joff), imm_size(joff + iszm));
+
+ bop = GET_BPF_OP(op);
+
+ if (imsz == 1) {
+ emit_bytes(st, &op8[bop], sizeof(op8[bop]));
+ joff -= sz8;
+ } else {
+ emit_bytes(st, op32[bop], sizeof(op32[bop]));
+ joff -= sz32;
+ }
+
+ emit_imm(st, joff, imsz);
+}
+
+/*
+ * emit one of:
+ * je <ofs>
+ * jne <ofs>
+ * ja <ofs>
+ * jb <ofs>
+ * jae <ofs>
+ * jbe <ofs>
+ * jg <ofs>
+ * jl <ofs>
+ * jge <ofs>
+ * jle <ofs>
+ * where 'ofs' is the target offset for the BPF bytecode.
+ */
+static void
+emit_jcc(struct bpf_jit_state *st, uint32_t op, int32_t ofs)
+{
+ emit_abs_jcc(st, op, st->off[st->idx + ofs]);
+}
+
+
+/*
+ * emit cmp <imm>, %<dreg>
+ */
+static void
+emit_cmp_imm(struct bpf_jit_state *st, uint32_t op, uint32_t dreg, uint32_t imm)
+{
+ uint8_t ops;
+ uint32_t imsz;
+
+ const uint8_t op8 = 0x83;
+ const uint8_t op32 = 0x81;
+ const uint8_t mods = 7;
+
+ imsz = imm_size(imm);
+ ops = (imsz == 1) ? op8 : op32;
+
+ emit_rex(st, op, 0, dreg);
+ emit_bytes(st, &ops, sizeof(ops));
+ emit_modregrm(st, MOD_DIRECT, mods, dreg);
+ emit_imm(st, imm, imsz);
+}
+
+/*
+ * emit test <imm>, %<dreg>
+ */
+static void
+emit_tst_imm(struct bpf_jit_state *st, uint32_t op, uint32_t dreg, uint32_t imm)
+{
+ const uint8_t ops = 0xF7;
+ const uint8_t mods = 0;
+
+ emit_rex(st, op, 0, dreg);
+ emit_bytes(st, &ops, sizeof(ops));
+ emit_modregrm(st, MOD_DIRECT, mods, dreg);
+ emit_imm(st, imm, imm_size(imm));
+}
+
+static void
+emit_jcc_imm(struct bpf_jit_state *st, uint32_t op, uint32_t dreg,
+ uint32_t imm, int32_t ofs)
+{
+ if (BPF_OP(op) == BPF_JSET)
+ emit_tst_imm(st, EBPF_ALU64, dreg, imm);
+ else
+ emit_cmp_imm(st, EBPF_ALU64, dreg, imm);
+
+ emit_jcc(st, op, ofs);
+}
+
+/*
+ * emit test %<sreg>, %<dreg>
+ */
+static void
+emit_tst_reg(struct bpf_jit_state *st, uint32_t op, uint32_t sreg,
+ uint32_t dreg)
+{
+ const uint8_t ops = 0x85;
+
+ emit_rex(st, op, sreg, dreg);
+ emit_bytes(st, &ops, sizeof(ops));
+ emit_modregrm(st, MOD_DIRECT, sreg, dreg);
+}
+
+/*
+ * emit cmp %<sreg>, %<dreg>
+ */
+static void
+emit_cmp_reg(struct bpf_jit_state *st, uint32_t op, uint32_t sreg,
+ uint32_t dreg)
+{
+ const uint8_t ops = 0x39;
+
+ emit_rex(st, op, sreg, dreg);
+ emit_bytes(st, &ops, sizeof(ops));
+ emit_modregrm(st, MOD_DIRECT, sreg, dreg);
+
+}
+
+static void
+emit_jcc_reg(struct bpf_jit_state *st, uint32_t op, uint32_t sreg,
+ uint32_t dreg, int32_t ofs)
+{
+ if (BPF_OP(op) == BPF_JSET)
+ emit_tst_reg(st, EBPF_ALU64, sreg, dreg);
+ else
+ emit_cmp_reg(st, EBPF_ALU64, sreg, dreg);
+
+ emit_jcc(st, op, ofs);
+}
+
+/*
+ * note that rax:rdx are implicitly used as source/destination registers,
+ * so some reg spillage is necessary.
+ * emit:
+ * mov %rax, %r11
+ * mov %rdx, %r10
+ * mov %<dreg>, %rax
+ * xor %rdx, %rdx
+ * for divisor as immediate value:
+ * mov <imm>, %r9
+ * div %<divisor_reg>
+ * mov %r10, %rdx
+ * mov %rax, %<dreg>
+ * mov %r11, %rax
+ * either:
+ * mov %rax, %<dreg>
+ * OR
+ * mov %rdx, %<dreg>
+ * mov %r11, %rax
+ * mov %r10, %rdx
+ */
+static void
+emit_div(struct bpf_jit_state *st, uint32_t op, uint32_t sreg, uint32_t dreg,
+ uint32_t imm)
+{
+ uint32_t sr;
+
+ const uint8_t ops = 0xF7;
+ const uint8_t mods = 6;
+
+ if (BPF_SRC(op) == BPF_X) {
+
+ /* check that src divisor is not zero */
+ emit_tst_reg(st, BPF_CLASS(op), sreg, sreg);
+
+ /* exit with return value zero */
+ emit_movcc_reg(st, BPF_CLASS(op) | BPF_JEQ | BPF_X, sreg, RAX);
+ emit_abs_jcc(st, BPF_JMP | BPF_JEQ | BPF_K, st->exit.off);
+ }
+
+ /* save rax & rdx */
+ if (dreg != RAX)
+ emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, RAX, REG_TMP0);
+ if (dreg != RDX)
+ emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, RDX, REG_TMP1);
+
+ /* fill rax & rdx */
+ emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, dreg, RAX);
+ emit_mov_imm(st, EBPF_ALU64 | EBPF_MOV | BPF_K, RDX, 0);
+
+ if (BPF_SRC(op) == BPF_X) {
+ sr = sreg;
+ if (sr == RAX)
+ sr = REG_TMP0;
+ else if (sr == RDX)
+ sr = REG_TMP1;
+ } else {
+ sr = REG_DIV_IMM;
+ emit_mov_imm(st, EBPF_ALU64 | EBPF_MOV | BPF_K, sr, imm);
+ }
+
+ emit_rex(st, op, 0, sr);
+ emit_bytes(st, &ops, sizeof(ops));
+ emit_modregrm(st, MOD_DIRECT, mods, sr);
+
+ if (BPF_OP(op) == BPF_DIV)
+ emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, RAX, dreg);
+ else
+ emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, RDX, dreg);
+
+ if (dreg != RAX)
+ emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, REG_TMP0, RAX);
+ if (dreg != RDX)
+ emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, REG_TMP1, RDX);
+}
+
+static void
+emit_prolog(struct bpf_jit_state *st, int32_t stack_size)
+{
+ uint32_t i;
+ int32_t spil, ofs;
+
+ spil = 0;
+ for (i = 0; i != RTE_DIM(save_regs); i++)
+ spil += INUSE(st->reguse, save_regs[i]);
+
+ /* we can avoid touching the stack at all */
+ if (spil == 0)
+ return;
+
+
+ emit_alu_imm(st, EBPF_ALU64 | BPF_SUB | BPF_K, RSP,
+ spil * sizeof(uint64_t));
+
+ ofs = 0;
+ for (i = 0; i != RTE_DIM(save_regs); i++) {
+ if (INUSE(st->reguse, save_regs[i]) != 0) {
+ emit_st_reg(st, BPF_STX | BPF_MEM | EBPF_DW,
+ save_regs[i], RSP, ofs);
+ ofs += sizeof(uint64_t);
+ }
+ }
+
+ if (INUSE(st->reguse, RBP) != 0) {
+ emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, RSP, RBP);
+ emit_alu_imm(st, EBPF_ALU64 | BPF_SUB | BPF_K, RSP, stack_size);
+ }
+}
+
+/*
+ * emit ret
+ */
+static void
+emit_ret(struct bpf_jit_state *st)
+{
+ const uint8_t ops = 0xC3;
+
+ emit_bytes(st, &ops, sizeof(ops));
+}
+
+static void
+emit_epilog(struct bpf_jit_state *st)
+{
+ uint32_t i;
+ int32_t spil, ofs;
+
+ /* if we allready have an epilog generate a jump to it */
+ if (st->exit.num++ != 0) {
+ emit_abs_jmp(st, st->exit.off);
+ return;
+ }
+
+ /* store offset of epilog block */
+ st->exit.off = st->sz;
+
+ spil = 0;
+ for (i = 0; i != RTE_DIM(save_regs); i++)
+ spil += INUSE(st->reguse, save_regs[i]);
+
+ if (spil != 0) {
+
+ if (INUSE(st->reguse, RBP) != 0)
+ emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X,
+ RBP, RSP);
+
+ ofs = 0;
+ for (i = 0; i != RTE_DIM(save_regs); i++) {
+ if (INUSE(st->reguse, save_regs[i]) != 0) {
+ emit_ld_reg(st, BPF_LDX | BPF_MEM | EBPF_DW,
+ RSP, save_regs[i], ofs);
+ ofs += sizeof(uint64_t);
+ }
+ }
+
+ emit_alu_imm(st, EBPF_ALU64 | BPF_ADD | BPF_K, RSP,
+ spil * sizeof(uint64_t));
+ }
+
+ emit_ret(st);
+}
+
+/*
+ * walk through bpf code and translate them x86_64 one.
+ */
+static int
+emit(struct bpf_jit_state *st, const struct rte_bpf *bpf)
+{
+ uint32_t i, dr, op, sr;
+ const struct ebpf_insn *ins;
+
+ /* reset state fields */
+ st->sz = 0;
+ st->exit.num = 0;
+
+ emit_prolog(st, bpf->stack_sz);
+
+ for (i = 0; i != bpf->prm.nb_ins; i++) {
+
+ st->idx = i;
+ st->off[i] = st->sz;
+
+ ins = bpf->prm.ins + i;
+
+ dr = ebpf2x86[ins->dst_reg];
+ sr = ebpf2x86[ins->src_reg];
+ op = ins->code;
+
+ switch (op) {
+ /* 32 bit ALU IMM operations */
+ case (BPF_ALU | BPF_ADD | BPF_K):
+ case (BPF_ALU | BPF_SUB | BPF_K):
+ case (BPF_ALU | BPF_AND | BPF_K):
+ case (BPF_ALU | BPF_OR | BPF_K):
+ case (BPF_ALU | BPF_XOR | BPF_K):
+ emit_alu_imm(st, op, dr, ins->imm);
+ break;
+ case (BPF_ALU | BPF_LSH | BPF_K):
+ case (BPF_ALU | BPF_RSH | BPF_K):
+ emit_shift_imm(st, op, dr, ins->imm);
+ break;
+ case (BPF_ALU | EBPF_MOV | BPF_K):
+ emit_mov_imm(st, op, dr, ins->imm);
+ break;
+ /* 32 bit ALU REG operations */
+ case (BPF_ALU | BPF_ADD | BPF_X):
+ case (BPF_ALU | BPF_SUB | BPF_X):
+ case (BPF_ALU | BPF_AND | BPF_X):
+ case (BPF_ALU | BPF_OR | BPF_X):
+ case (BPF_ALU | BPF_XOR | BPF_X):
+ emit_alu_reg(st, op, sr, dr);
+ break;
+ case (BPF_ALU | BPF_LSH | BPF_X):
+ case (BPF_ALU | BPF_RSH | BPF_X):
+ emit_shift_reg(st, op, sr, dr);
+ break;
+ case (BPF_ALU | EBPF_MOV | BPF_X):
+ emit_mov_reg(st, op, sr, dr);
+ break;
+ case (BPF_ALU | BPF_NEG):
+ emit_neg(st, op, dr);
+ break;
+ case (BPF_ALU | EBPF_END | EBPF_TO_BE):
+ emit_be2le(st, dr, ins->imm);
+ break;
+ case (BPF_ALU | EBPF_END | EBPF_TO_LE):
+ emit_le2be(st, dr, ins->imm);
+ break;
+ /* 64 bit ALU IMM operations */
+ case (EBPF_ALU64 | BPF_ADD | BPF_K):
+ case (EBPF_ALU64 | BPF_SUB | BPF_K):
+ case (EBPF_ALU64 | BPF_AND | BPF_K):
+ case (EBPF_ALU64 | BPF_OR | BPF_K):
+ case (EBPF_ALU64 | BPF_XOR | BPF_K):
+ emit_alu_imm(st, op, dr, ins->imm);
+ break;
+ case (EBPF_ALU64 | BPF_LSH | BPF_K):
+ case (EBPF_ALU64 | BPF_RSH | BPF_K):
+ case (EBPF_ALU64 | EBPF_ARSH | BPF_K):
+ emit_shift_imm(st, op, dr, ins->imm);
+ break;
+ case (EBPF_ALU64 | EBPF_MOV | BPF_K):
+ emit_mov_imm(st, op, dr, ins->imm);
+ break;
+ /* 64 bit ALU REG operations */
+ case (EBPF_ALU64 | BPF_ADD | BPF_X):
+ case (EBPF_ALU64 | BPF_SUB | BPF_X):
+ case (EBPF_ALU64 | BPF_AND | BPF_X):
+ case (EBPF_ALU64 | BPF_OR | BPF_X):
+ case (EBPF_ALU64 | BPF_XOR | BPF_X):
+ emit_alu_reg(st, op, sr, dr);
+ break;
+ case (EBPF_ALU64 | BPF_LSH | BPF_X):
+ case (EBPF_ALU64 | BPF_RSH | BPF_X):
+ case (EBPF_ALU64 | EBPF_ARSH | BPF_X):
+ emit_shift_reg(st, op, sr, dr);
+ break;
+ case (EBPF_ALU64 | EBPF_MOV | BPF_X):
+ emit_mov_reg(st, op, sr, dr);
+ break;
+ case (EBPF_ALU64 | BPF_NEG):
+ emit_neg(st, op, dr);
+ break;
+ /* multiply instructions */
+ case (BPF_ALU | BPF_MUL | BPF_K):
+ case (BPF_ALU | BPF_MUL | BPF_X):
+ case (EBPF_ALU64 | BPF_MUL | BPF_K):
+ case (EBPF_ALU64 | BPF_MUL | BPF_X):
+ emit_mul(st, op, sr, dr, ins->imm);
+ break;
+ /* divide instructions */
+ case (BPF_ALU | BPF_DIV | BPF_K):
+ case (BPF_ALU | BPF_MOD | BPF_K):
+ case (BPF_ALU | BPF_DIV | BPF_X):
+ case (BPF_ALU | BPF_MOD | BPF_X):
+ case (EBPF_ALU64 | BPF_DIV | BPF_K):
+ case (EBPF_ALU64 | BPF_MOD | BPF_K):
+ case (EBPF_ALU64 | BPF_DIV | BPF_X):
+ case (EBPF_ALU64 | BPF_MOD | BPF_X):
+ emit_div(st, op, sr, dr, ins->imm);
+ break;
+ /* load instructions */
+ case (BPF_LDX | BPF_MEM | BPF_B):
+ case (BPF_LDX | BPF_MEM | BPF_H):
+ case (BPF_LDX | BPF_MEM | BPF_W):
+ case (BPF_LDX | BPF_MEM | EBPF_DW):
+ emit_ld_reg(st, op, sr, dr, ins->off);
+ break;
+ /* load 64 bit immediate value */
+ case (BPF_LD | BPF_IMM | EBPF_DW):
+ emit_ld_imm64(st, dr, ins[0].imm, ins[1].imm);
+ i++;
+ break;
+ /* store instructions */
+ case (BPF_STX | BPF_MEM | BPF_B):
+ case (BPF_STX | BPF_MEM | BPF_H):
+ case (BPF_STX | BPF_MEM | BPF_W):
+ case (BPF_STX | BPF_MEM | EBPF_DW):
+ emit_st_reg(st, op, sr, dr, ins->off);
+ break;
+ case (BPF_ST | BPF_MEM | BPF_B):
+ case (BPF_ST | BPF_MEM | BPF_H):
+ case (BPF_ST | BPF_MEM | BPF_W):
+ case (BPF_ST | BPF_MEM | EBPF_DW):
+ emit_st_imm(st, op, dr, ins->imm, ins->off);
+ break;
+ /* atomic add instructions */
+ case (BPF_STX | EBPF_XADD | BPF_W):
+ case (BPF_STX | EBPF_XADD | EBPF_DW):
+ emit_st_xadd(st, op, sr, dr, ins->off);
+ break;
+ /* jump instructions */
+ case (BPF_JMP | BPF_JA):
+ emit_jmp(st, ins->off + 1);
+ break;
+ /* jump IMM instructions */
+ case (BPF_JMP | BPF_JEQ | BPF_K):
+ case (BPF_JMP | EBPF_JNE | BPF_K):
+ case (BPF_JMP | BPF_JGT | BPF_K):
+ case (BPF_JMP | EBPF_JLT | BPF_K):
+ case (BPF_JMP | BPF_JGE | BPF_K):
+ case (BPF_JMP | EBPF_JLE | BPF_K):
+ case (BPF_JMP | EBPF_JSGT | BPF_K):
+ case (BPF_JMP | EBPF_JSLT | BPF_K):
+ case (BPF_JMP | EBPF_JSGE | BPF_K):
+ case (BPF_JMP | EBPF_JSLE | BPF_K):
+ case (BPF_JMP | BPF_JSET | BPF_K):
+ emit_jcc_imm(st, op, dr, ins->imm, ins->off + 1);
+ break;
+ /* jump REG instructions */
+ case (BPF_JMP | BPF_JEQ | BPF_X):
+ case (BPF_JMP | EBPF_JNE | BPF_X):
+ case (BPF_JMP | BPF_JGT | BPF_X):
+ case (BPF_JMP | EBPF_JLT | BPF_X):
+ case (BPF_JMP | BPF_JGE | BPF_X):
+ case (BPF_JMP | EBPF_JLE | BPF_X):
+ case (BPF_JMP | EBPF_JSGT | BPF_X):
+ case (BPF_JMP | EBPF_JSLT | BPF_X):
+ case (BPF_JMP | EBPF_JSGE | BPF_X):
+ case (BPF_JMP | EBPF_JSLE | BPF_X):
+ case (BPF_JMP | BPF_JSET | BPF_X):
+ emit_jcc_reg(st, op, sr, dr, ins->off + 1);
+ break;
+ /* call instructions */
+ case (BPF_JMP | EBPF_CALL):
+ emit_call(st,
+ (uintptr_t)bpf->prm.xsym[ins->imm].func.val);
+ break;
+ /* return instruction */
+ case (BPF_JMP | EBPF_EXIT):
+ emit_epilog(st);
+ break;
+ default:
+ RTE_BPF_LOG(ERR,
+ "%s(%p): invalid opcode %#x at pc: %u;\n",
+ __func__, bpf, ins->code, i);
+ return -EINVAL;
+ }
+ }
+
+ return 0;
+}
+
+/*
+ * produce a native ISA version of the given BPF code.
+ */
+int
+bpf_jit_x86(struct rte_bpf *bpf)
+{
+ int32_t rc;
+ uint32_t i;
+ size_t sz;
+ struct bpf_jit_state st;
+
+ /* init state */
+ memset(&st, 0, sizeof(st));
+ st.off = malloc(bpf->prm.nb_ins * sizeof(st.off[0]));
+ if (st.off == NULL)
+ return -ENOMEM;
+
+ /* fill with fake offsets */
+ st.exit.off = INT32_MAX;
+ for (i = 0; i != bpf->prm.nb_ins; i++)
+ st.off[i] = INT32_MAX;
+
+ /*
+ * dry runs, used to calculate total code size and valid jump offsets.
+ * stop when we get minimal possible size
+ */
+ do {
+ sz = st.sz;
+ rc = emit(&st, bpf);
+ } while (rc == 0 && sz != st.sz);
+
+ if (rc == 0) {
+
+ /* allocate memory needed */
+ st.ins = mmap(NULL, st.sz, PROT_READ | PROT_WRITE,
+ MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+ if (st.ins == MAP_FAILED)
+ rc = -ENOMEM;
+ else
+ /* generate code */
+ rc = emit(&st, bpf);
+ }
+
+ if (rc == 0 && mprotect(st.ins, st.sz, PROT_READ | PROT_EXEC) != 0)
+ rc = -ENOMEM;
+
+ if (rc != 0)
+ munmap(st.ins, st.sz);
+ else {
+ bpf->jit.func = (void *)st.ins;
+ bpf->jit.sz = st.sz;
+ }
+
+ free(st.off);
+ return rc;
+}
diff --git a/src/spdk/dpdk/lib/librte_bpf/bpf_load.c b/src/spdk/dpdk/lib/librte_bpf/bpf_load.c
new file mode 100644
index 000000000..2a3b901d7
--- /dev/null
+++ b/src/spdk/dpdk/lib/librte_bpf/bpf_load.c
@@ -0,0 +1,150 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2018 Intel Corporation
+ */
+
+#include <stdarg.h>
+#include <stdio.h>
+#include <string.h>
+#include <errno.h>
+#include <stdint.h>
+#include <unistd.h>
+#include <inttypes.h>
+
+#include <rte_common.h>
+#include <rte_log.h>
+#include <rte_debug.h>
+#include <rte_memory.h>
+#include <rte_eal.h>
+#include <rte_byteorder.h>
+#include <rte_errno.h>
+
+#include "bpf_impl.h"
+
+static struct rte_bpf *
+bpf_load(const struct rte_bpf_prm *prm)
+{
+ uint8_t *buf;
+ struct rte_bpf *bpf;
+ size_t sz, bsz, insz, xsz;
+
+ xsz = prm->nb_xsym * sizeof(prm->xsym[0]);
+ insz = prm->nb_ins * sizeof(prm->ins[0]);
+ bsz = sizeof(bpf[0]);
+ sz = insz + xsz + bsz;
+
+ buf = mmap(NULL, sz, PROT_READ | PROT_WRITE,
+ MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+ if (buf == MAP_FAILED)
+ return NULL;
+
+ bpf = (void *)buf;
+ bpf->sz = sz;
+
+ memcpy(&bpf->prm, prm, sizeof(bpf->prm));
+
+ memcpy(buf + bsz, prm->xsym, xsz);
+ memcpy(buf + bsz + xsz, prm->ins, insz);
+
+ bpf->prm.xsym = (void *)(buf + bsz);
+ bpf->prm.ins = (void *)(buf + bsz + xsz);
+
+ return bpf;
+}
+
+/*
+ * Check that user provided external symbol.
+ */
+static int
+bpf_check_xsym(const struct rte_bpf_xsym *xsym)
+{
+ uint32_t i;
+
+ if (xsym->name == NULL)
+ return -EINVAL;
+
+ if (xsym->type == RTE_BPF_XTYPE_VAR) {
+ if (xsym->var.desc.type == RTE_BPF_ARG_UNDEF)
+ return -EINVAL;
+ } else if (xsym->type == RTE_BPF_XTYPE_FUNC) {
+
+ if (xsym->func.nb_args > EBPF_FUNC_MAX_ARGS)
+ return -EINVAL;
+
+ /* check function arguments */
+ for (i = 0; i != xsym->func.nb_args; i++) {
+ if (xsym->func.args[i].type == RTE_BPF_ARG_UNDEF)
+ return -EINVAL;
+ }
+
+ /* check return value info */
+ if (xsym->func.ret.type != RTE_BPF_ARG_UNDEF &&
+ xsym->func.ret.size == 0)
+ return -EINVAL;
+ } else
+ return -EINVAL;
+
+ return 0;
+}
+
+struct rte_bpf *
+rte_bpf_load(const struct rte_bpf_prm *prm)
+{
+ struct rte_bpf *bpf;
+ int32_t rc;
+ uint32_t i;
+
+ if (prm == NULL || prm->ins == NULL ||
+ (prm->nb_xsym != 0 && prm->xsym == NULL)) {
+ rte_errno = EINVAL;
+ return NULL;
+ }
+
+ rc = 0;
+ for (i = 0; i != prm->nb_xsym && rc == 0; i++)
+ rc = bpf_check_xsym(prm->xsym + i);
+
+ if (rc != 0) {
+ rte_errno = -rc;
+ RTE_BPF_LOG(ERR, "%s: %d-th xsym is invalid\n", __func__, i);
+ return NULL;
+ }
+
+ bpf = bpf_load(prm);
+ if (bpf == NULL) {
+ rte_errno = ENOMEM;
+ return NULL;
+ }
+
+ rc = bpf_validate(bpf);
+ if (rc == 0) {
+ bpf_jit(bpf);
+ if (mprotect(bpf, bpf->sz, PROT_READ) != 0)
+ rc = -ENOMEM;
+ }
+
+ if (rc != 0) {
+ rte_bpf_destroy(bpf);
+ rte_errno = -rc;
+ return NULL;
+ }
+
+ return bpf;
+}
+
+#ifndef RTE_LIBRTE_BPF_ELF
+struct rte_bpf *
+rte_bpf_elf_load(const struct rte_bpf_prm *prm, const char *fname,
+ const char *sname)
+{
+ if (prm == NULL || fname == NULL || sname == NULL) {
+ rte_errno = EINVAL;
+ return NULL;
+ }
+
+ RTE_BPF_LOG(ERR, "%s() is not supported with current config\n"
+ "rebuild with libelf installed\n",
+ __func__);
+ rte_errno = ENOTSUP;
+ return NULL;
+}
+#endif
diff --git a/src/spdk/dpdk/lib/librte_bpf/bpf_load_elf.c b/src/spdk/dpdk/lib/librte_bpf/bpf_load_elf.c
new file mode 100644
index 000000000..2b11adeb5
--- /dev/null
+++ b/src/spdk/dpdk/lib/librte_bpf/bpf_load_elf.c
@@ -0,0 +1,333 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2018 Intel Corporation
+ */
+
+#include <stdarg.h>
+#include <stdio.h>
+#include <string.h>
+#include <errno.h>
+#include <stdint.h>
+#include <unistd.h>
+#include <inttypes.h>
+
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <sys/queue.h>
+#include <fcntl.h>
+
+#include <libelf.h>
+
+#include <rte_common.h>
+#include <rte_log.h>
+#include <rte_debug.h>
+#include <rte_memory.h>
+#include <rte_eal.h>
+#include <rte_byteorder.h>
+#include <rte_errno.h>
+
+#include "bpf_impl.h"
+
+/* To overcome compatibility issue */
+#ifndef EM_BPF
+#define EM_BPF 247
+#endif
+
+static uint32_t
+bpf_find_xsym(const char *sn, enum rte_bpf_xtype type,
+ const struct rte_bpf_xsym fp[], uint32_t fn)
+{
+ uint32_t i;
+
+ if (sn == NULL || fp == NULL)
+ return UINT32_MAX;
+
+ for (i = 0; i != fn; i++) {
+ if (fp[i].type == type && strcmp(sn, fp[i].name) == 0)
+ break;
+ }
+
+ return (i != fn) ? i : UINT32_MAX;
+}
+
+/*
+ * update BPF code at offset *ofs* with a proper address(index) for external
+ * symbol *sn*
+ */
+static int
+resolve_xsym(const char *sn, size_t ofs, struct ebpf_insn *ins, size_t ins_sz,
+ const struct rte_bpf_prm *prm)
+{
+ uint32_t idx, fidx;
+ enum rte_bpf_xtype type;
+
+ if (ofs % sizeof(ins[0]) != 0 || ofs >= ins_sz)
+ return -EINVAL;
+
+ idx = ofs / sizeof(ins[0]);
+ if (ins[idx].code == (BPF_JMP | EBPF_CALL))
+ type = RTE_BPF_XTYPE_FUNC;
+ else if (ins[idx].code == (BPF_LD | BPF_IMM | EBPF_DW) &&
+ ofs < ins_sz - sizeof(ins[idx]))
+ type = RTE_BPF_XTYPE_VAR;
+ else
+ return -EINVAL;
+
+ fidx = bpf_find_xsym(sn, type, prm->xsym, prm->nb_xsym);
+ if (fidx == UINT32_MAX)
+ return -ENOENT;
+
+ /* for function we just need an index in our xsym table */
+ if (type == RTE_BPF_XTYPE_FUNC) {
+
+ /* we don't support multiple functions per BPF module,
+ * so treat EBPF_PSEUDO_CALL to extrernal function
+ * as an ordinary EBPF_CALL.
+ */
+ if (ins[idx].src_reg == EBPF_PSEUDO_CALL) {
+ RTE_BPF_LOG(INFO, "%s(%u): "
+ "EBPF_PSEUDO_CALL to external function: %s\n",
+ __func__, idx, sn);
+ ins[idx].src_reg = EBPF_REG_0;
+ }
+ ins[idx].imm = fidx;
+ /* for variable we need to store its absolute address */
+ } else {
+ ins[idx].imm = (uintptr_t)prm->xsym[fidx].var.val;
+ ins[idx + 1].imm =
+ (uint64_t)(uintptr_t)prm->xsym[fidx].var.val >> 32;
+ }
+
+ return 0;
+}
+
+static int
+check_elf_header(const Elf64_Ehdr *eh)
+{
+ const char *err;
+
+ err = NULL;
+
+#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
+ if (eh->e_ident[EI_DATA] != ELFDATA2LSB)
+#else
+ if (eh->e_ident[EI_DATA] != ELFDATA2MSB)
+#endif
+ err = "not native byte order";
+ else if (eh->e_ident[EI_OSABI] != ELFOSABI_NONE)
+ err = "unexpected OS ABI";
+ else if (eh->e_type != ET_REL)
+ err = "unexpected ELF type";
+ else if (eh->e_machine != EM_NONE && eh->e_machine != EM_BPF)
+ err = "unexpected machine type";
+
+ if (err != NULL) {
+ RTE_BPF_LOG(ERR, "%s(): %s\n", __func__, err);
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
+/*
+ * helper function, find executable section by name.
+ */
+static int
+find_elf_code(Elf *elf, const char *section, Elf_Data **psd, size_t *pidx)
+{
+ Elf_Scn *sc;
+ const Elf64_Ehdr *eh;
+ const Elf64_Shdr *sh;
+ Elf_Data *sd;
+ const char *sn;
+ int32_t rc;
+
+ eh = elf64_getehdr(elf);
+ if (eh == NULL) {
+ rc = elf_errno();
+ RTE_BPF_LOG(ERR, "%s(%p, %s) error code: %d(%s)\n",
+ __func__, elf, section, rc, elf_errmsg(rc));
+ return -EINVAL;
+ }
+
+ if (check_elf_header(eh) != 0)
+ return -EINVAL;
+
+ /* find given section by name */
+ for (sc = elf_nextscn(elf, NULL); sc != NULL;
+ sc = elf_nextscn(elf, sc)) {
+ sh = elf64_getshdr(sc);
+ sn = elf_strptr(elf, eh->e_shstrndx, sh->sh_name);
+ if (sn != NULL && strcmp(section, sn) == 0 &&
+ sh->sh_type == SHT_PROGBITS &&
+ sh->sh_flags == (SHF_ALLOC | SHF_EXECINSTR))
+ break;
+ }
+
+ sd = elf_getdata(sc, NULL);
+ if (sd == NULL || sd->d_size == 0 ||
+ sd->d_size % sizeof(struct ebpf_insn) != 0) {
+ rc = elf_errno();
+ RTE_BPF_LOG(ERR, "%s(%p, %s) error code: %d(%s)\n",
+ __func__, elf, section, rc, elf_errmsg(rc));
+ return -EINVAL;
+ }
+
+ *psd = sd;
+ *pidx = elf_ndxscn(sc);
+ return 0;
+}
+
+/*
+ * helper function to process data from relocation table.
+ */
+static int
+process_reloc(Elf *elf, size_t sym_idx, Elf64_Rel *re, size_t re_sz,
+ struct ebpf_insn *ins, size_t ins_sz, const struct rte_bpf_prm *prm)
+{
+ int32_t rc;
+ uint32_t i, n;
+ size_t ofs, sym;
+ const char *sn;
+ const Elf64_Ehdr *eh;
+ Elf_Scn *sc;
+ const Elf_Data *sd;
+ Elf64_Sym *sm;
+
+ eh = elf64_getehdr(elf);
+
+ /* get symtable by section index */
+ sc = elf_getscn(elf, sym_idx);
+ sd = elf_getdata(sc, NULL);
+ if (sd == NULL)
+ return -EINVAL;
+ sm = sd->d_buf;
+
+ n = re_sz / sizeof(re[0]);
+ for (i = 0; i != n; i++) {
+
+ ofs = re[i].r_offset;
+
+ /* retrieve index in the symtable */
+ sym = ELF64_R_SYM(re[i].r_info);
+ if (sym * sizeof(sm[0]) >= sd->d_size)
+ return -EINVAL;
+
+ sn = elf_strptr(elf, eh->e_shstrndx, sm[sym].st_name);
+
+ rc = resolve_xsym(sn, ofs, ins, ins_sz, prm);
+ if (rc != 0) {
+ RTE_BPF_LOG(ERR,
+ "resolve_xsym(%s, %zu) error code: %d\n",
+ sn, ofs, rc);
+ return rc;
+ }
+ }
+
+ return 0;
+}
+
+/*
+ * helper function, find relocation information (if any)
+ * and update bpf code.
+ */
+static int
+elf_reloc_code(Elf *elf, Elf_Data *ed, size_t sidx,
+ const struct rte_bpf_prm *prm)
+{
+ Elf64_Rel *re;
+ Elf_Scn *sc;
+ const Elf64_Shdr *sh;
+ const Elf_Data *sd;
+ int32_t rc;
+
+ rc = 0;
+
+ /* walk through all sections */
+ for (sc = elf_nextscn(elf, NULL); sc != NULL && rc == 0;
+ sc = elf_nextscn(elf, sc)) {
+
+ sh = elf64_getshdr(sc);
+
+ /* relocation data for our code section */
+ if (sh->sh_type == SHT_REL && sh->sh_info == sidx) {
+ sd = elf_getdata(sc, NULL);
+ if (sd == NULL || sd->d_size == 0 ||
+ sd->d_size % sizeof(re[0]) != 0)
+ return -EINVAL;
+ rc = process_reloc(elf, sh->sh_link,
+ sd->d_buf, sd->d_size, ed->d_buf, ed->d_size,
+ prm);
+ }
+ }
+
+ return rc;
+}
+
+static struct rte_bpf *
+bpf_load_elf(const struct rte_bpf_prm *prm, int32_t fd, const char *section)
+{
+ Elf *elf;
+ Elf_Data *sd;
+ size_t sidx;
+ int32_t rc;
+ struct rte_bpf *bpf;
+ struct rte_bpf_prm np;
+
+ elf_version(EV_CURRENT);
+ elf = elf_begin(fd, ELF_C_READ, NULL);
+
+ rc = find_elf_code(elf, section, &sd, &sidx);
+ if (rc == 0)
+ rc = elf_reloc_code(elf, sd, sidx, prm);
+
+ if (rc == 0) {
+ np = prm[0];
+ np.ins = sd->d_buf;
+ np.nb_ins = sd->d_size / sizeof(struct ebpf_insn);
+ bpf = rte_bpf_load(&np);
+ } else {
+ bpf = NULL;
+ rte_errno = -rc;
+ }
+
+ elf_end(elf);
+ return bpf;
+}
+
+struct rte_bpf *
+rte_bpf_elf_load(const struct rte_bpf_prm *prm, const char *fname,
+ const char *sname)
+{
+ int32_t fd, rc;
+ struct rte_bpf *bpf;
+
+ if (prm == NULL || fname == NULL || sname == NULL) {
+ rte_errno = EINVAL;
+ return NULL;
+ }
+
+ fd = open(fname, O_RDONLY);
+ if (fd < 0) {
+ rc = errno;
+ RTE_BPF_LOG(ERR, "%s(%s) error code: %d(%s)\n",
+ __func__, fname, rc, strerror(rc));
+ rte_errno = EINVAL;
+ return NULL;
+ }
+
+ bpf = bpf_load_elf(prm, fd, sname);
+ close(fd);
+
+ if (bpf == NULL) {
+ RTE_BPF_LOG(ERR,
+ "%s(fname=\"%s\", sname=\"%s\") failed, "
+ "error code: %d\n",
+ __func__, fname, sname, rte_errno);
+ return NULL;
+ }
+
+ RTE_BPF_LOG(INFO, "%s(fname=\"%s\", sname=\"%s\") "
+ "successfully creates %p(jit={.func=%p,.sz=%zu});\n",
+ __func__, fname, sname, bpf, bpf->jit.func, bpf->jit.sz);
+ return bpf;
+}
diff --git a/src/spdk/dpdk/lib/librte_bpf/bpf_pkt.c b/src/spdk/dpdk/lib/librte_bpf/bpf_pkt.c
new file mode 100644
index 000000000..6e8248f0d
--- /dev/null
+++ b/src/spdk/dpdk/lib/librte_bpf/bpf_pkt.c
@@ -0,0 +1,605 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2018 Intel Corporation
+ */
+
+#include <stdarg.h>
+#include <stdio.h>
+#include <string.h>
+#include <errno.h>
+#include <stdint.h>
+#include <unistd.h>
+#include <inttypes.h>
+
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <sys/queue.h>
+
+#include <rte_common.h>
+#include <rte_byteorder.h>
+#include <rte_malloc.h>
+#include <rte_log.h>
+#include <rte_debug.h>
+#include <rte_cycles.h>
+#include <rte_eal.h>
+#include <rte_per_lcore.h>
+#include <rte_lcore.h>
+#include <rte_atomic.h>
+#include <rte_mbuf.h>
+#include <rte_ethdev.h>
+
+#include <rte_bpf_ethdev.h>
+#include "bpf_impl.h"
+
+/*
+ * information about installed BPF rx/tx callback
+ */
+
+struct bpf_eth_cbi {
+ /* used by both data & control path */
+ uint32_t use; /*usage counter */
+ const struct rte_eth_rxtx_callback *cb; /* callback handle */
+ struct rte_bpf *bpf;
+ struct rte_bpf_jit jit;
+ /* used by control path only */
+ LIST_ENTRY(bpf_eth_cbi) link;
+ uint16_t port;
+ uint16_t queue;
+} __rte_cache_aligned;
+
+/*
+ * Odd number means that callback is used by datapath.
+ * Even number means that callback is not used by datapath.
+ */
+#define BPF_ETH_CBI_INUSE 1
+
+/*
+ * List to manage RX/TX installed callbacks.
+ */
+LIST_HEAD(bpf_eth_cbi_list, bpf_eth_cbi);
+
+enum {
+ BPF_ETH_RX,
+ BPF_ETH_TX,
+ BPF_ETH_NUM,
+};
+
+/*
+ * information about all installed BPF rx/tx callbacks
+ */
+struct bpf_eth_cbh {
+ rte_spinlock_t lock;
+ struct bpf_eth_cbi_list list;
+ uint32_t type;
+};
+
+static struct bpf_eth_cbh rx_cbh = {
+ .lock = RTE_SPINLOCK_INITIALIZER,
+ .list = LIST_HEAD_INITIALIZER(list),
+ .type = BPF_ETH_RX,
+};
+
+static struct bpf_eth_cbh tx_cbh = {
+ .lock = RTE_SPINLOCK_INITIALIZER,
+ .list = LIST_HEAD_INITIALIZER(list),
+ .type = BPF_ETH_TX,
+};
+
+/*
+ * Marks given callback as used by datapath.
+ */
+static __rte_always_inline void
+bpf_eth_cbi_inuse(struct bpf_eth_cbi *cbi)
+{
+ cbi->use++;
+ /* make sure no store/load reordering could happen */
+ rte_smp_mb();
+}
+
+/*
+ * Marks given callback list as not used by datapath.
+ */
+static __rte_always_inline void
+bpf_eth_cbi_unuse(struct bpf_eth_cbi *cbi)
+{
+ /* make sure all previous loads are completed */
+ rte_smp_rmb();
+ cbi->use++;
+}
+
+/*
+ * Waits till datapath finished using given callback.
+ */
+static void
+bpf_eth_cbi_wait(const struct bpf_eth_cbi *cbi)
+{
+ uint32_t nuse, puse;
+
+ /* make sure all previous loads and stores are completed */
+ rte_smp_mb();
+
+ puse = cbi->use;
+
+ /* in use, busy wait till current RX/TX iteration is finished */
+ if ((puse & BPF_ETH_CBI_INUSE) != 0) {
+ do {
+ rte_pause();
+ rte_compiler_barrier();
+ nuse = cbi->use;
+ } while (nuse == puse);
+ }
+}
+
+static void
+bpf_eth_cbi_cleanup(struct bpf_eth_cbi *bc)
+{
+ bc->bpf = NULL;
+ memset(&bc->jit, 0, sizeof(bc->jit));
+}
+
+static struct bpf_eth_cbi *
+bpf_eth_cbh_find(struct bpf_eth_cbh *cbh, uint16_t port, uint16_t queue)
+{
+ struct bpf_eth_cbi *cbi;
+
+ LIST_FOREACH(cbi, &cbh->list, link) {
+ if (cbi->port == port && cbi->queue == queue)
+ break;
+ }
+ return cbi;
+}
+
+static struct bpf_eth_cbi *
+bpf_eth_cbh_add(struct bpf_eth_cbh *cbh, uint16_t port, uint16_t queue)
+{
+ struct bpf_eth_cbi *cbi;
+
+ /* return an existing one */
+ cbi = bpf_eth_cbh_find(cbh, port, queue);
+ if (cbi != NULL)
+ return cbi;
+
+ cbi = rte_zmalloc(NULL, sizeof(*cbi), RTE_CACHE_LINE_SIZE);
+ if (cbi != NULL) {
+ cbi->port = port;
+ cbi->queue = queue;
+ LIST_INSERT_HEAD(&cbh->list, cbi, link);
+ }
+ return cbi;
+}
+
+/*
+ * BPF packet processing routinies.
+ */
+
+static inline uint32_t
+apply_filter(struct rte_mbuf *mb[], const uint64_t rc[], uint32_t num,
+ uint32_t drop)
+{
+ uint32_t i, j, k;
+ struct rte_mbuf *dr[num];
+
+ for (i = 0, j = 0, k = 0; i != num; i++) {
+
+ /* filter matches */
+ if (rc[i] != 0)
+ mb[j++] = mb[i];
+ /* no match */
+ else
+ dr[k++] = mb[i];
+ }
+
+ if (drop != 0) {
+ /* free filtered out mbufs */
+ for (i = 0; i != k; i++)
+ rte_pktmbuf_free(dr[i]);
+ } else {
+ /* copy filtered out mbufs beyond good ones */
+ for (i = 0; i != k; i++)
+ mb[j + i] = dr[i];
+ }
+
+ return j;
+}
+
+static inline uint32_t
+pkt_filter_vm(const struct rte_bpf *bpf, struct rte_mbuf *mb[], uint32_t num,
+ uint32_t drop)
+{
+ uint32_t i;
+ void *dp[num];
+ uint64_t rc[num];
+
+ for (i = 0; i != num; i++)
+ dp[i] = rte_pktmbuf_mtod(mb[i], void *);
+
+ rte_bpf_exec_burst(bpf, dp, rc, num);
+ return apply_filter(mb, rc, num, drop);
+}
+
+static inline uint32_t
+pkt_filter_jit(const struct rte_bpf_jit *jit, struct rte_mbuf *mb[],
+ uint32_t num, uint32_t drop)
+{
+ uint32_t i, n;
+ void *dp;
+ uint64_t rc[num];
+
+ n = 0;
+ for (i = 0; i != num; i++) {
+ dp = rte_pktmbuf_mtod(mb[i], void *);
+ rc[i] = jit->func(dp);
+ n += (rc[i] == 0);
+ }
+
+ if (n != 0)
+ num = apply_filter(mb, rc, num, drop);
+
+ return num;
+}
+
+static inline uint32_t
+pkt_filter_mb_vm(const struct rte_bpf *bpf, struct rte_mbuf *mb[], uint32_t num,
+ uint32_t drop)
+{
+ uint64_t rc[num];
+
+ rte_bpf_exec_burst(bpf, (void **)mb, rc, num);
+ return apply_filter(mb, rc, num, drop);
+}
+
+static inline uint32_t
+pkt_filter_mb_jit(const struct rte_bpf_jit *jit, struct rte_mbuf *mb[],
+ uint32_t num, uint32_t drop)
+{
+ uint32_t i, n;
+ uint64_t rc[num];
+
+ n = 0;
+ for (i = 0; i != num; i++) {
+ rc[i] = jit->func(mb[i]);
+ n += (rc[i] == 0);
+ }
+
+ if (n != 0)
+ num = apply_filter(mb, rc, num, drop);
+
+ return num;
+}
+
+/*
+ * RX/TX callbacks for raw data bpf.
+ */
+
+static uint16_t
+bpf_rx_callback_vm(__rte_unused uint16_t port, __rte_unused uint16_t queue,
+ struct rte_mbuf *pkt[], uint16_t nb_pkts,
+ __rte_unused uint16_t max_pkts, void *user_param)
+{
+ struct bpf_eth_cbi *cbi;
+ uint16_t rc;
+
+ cbi = user_param;
+
+ bpf_eth_cbi_inuse(cbi);
+ rc = (cbi->cb != NULL) ?
+ pkt_filter_vm(cbi->bpf, pkt, nb_pkts, 1) :
+ nb_pkts;
+ bpf_eth_cbi_unuse(cbi);
+ return rc;
+}
+
+static uint16_t
+bpf_rx_callback_jit(__rte_unused uint16_t port, __rte_unused uint16_t queue,
+ struct rte_mbuf *pkt[], uint16_t nb_pkts,
+ __rte_unused uint16_t max_pkts, void *user_param)
+{
+ struct bpf_eth_cbi *cbi;
+ uint16_t rc;
+
+ cbi = user_param;
+ bpf_eth_cbi_inuse(cbi);
+ rc = (cbi->cb != NULL) ?
+ pkt_filter_jit(&cbi->jit, pkt, nb_pkts, 1) :
+ nb_pkts;
+ bpf_eth_cbi_unuse(cbi);
+ return rc;
+}
+
+static uint16_t
+bpf_tx_callback_vm(__rte_unused uint16_t port, __rte_unused uint16_t queue,
+ struct rte_mbuf *pkt[], uint16_t nb_pkts, void *user_param)
+{
+ struct bpf_eth_cbi *cbi;
+ uint16_t rc;
+
+ cbi = user_param;
+ bpf_eth_cbi_inuse(cbi);
+ rc = (cbi->cb != NULL) ?
+ pkt_filter_vm(cbi->bpf, pkt, nb_pkts, 0) :
+ nb_pkts;
+ bpf_eth_cbi_unuse(cbi);
+ return rc;
+}
+
+static uint16_t
+bpf_tx_callback_jit(__rte_unused uint16_t port, __rte_unused uint16_t queue,
+ struct rte_mbuf *pkt[], uint16_t nb_pkts, void *user_param)
+{
+ struct bpf_eth_cbi *cbi;
+ uint16_t rc;
+
+ cbi = user_param;
+ bpf_eth_cbi_inuse(cbi);
+ rc = (cbi->cb != NULL) ?
+ pkt_filter_jit(&cbi->jit, pkt, nb_pkts, 0) :
+ nb_pkts;
+ bpf_eth_cbi_unuse(cbi);
+ return rc;
+}
+
+/*
+ * RX/TX callbacks for mbuf.
+ */
+
+static uint16_t
+bpf_rx_callback_mb_vm(__rte_unused uint16_t port, __rte_unused uint16_t queue,
+ struct rte_mbuf *pkt[], uint16_t nb_pkts,
+ __rte_unused uint16_t max_pkts, void *user_param)
+{
+ struct bpf_eth_cbi *cbi;
+ uint16_t rc;
+
+ cbi = user_param;
+ bpf_eth_cbi_inuse(cbi);
+ rc = (cbi->cb != NULL) ?
+ pkt_filter_mb_vm(cbi->bpf, pkt, nb_pkts, 1) :
+ nb_pkts;
+ bpf_eth_cbi_unuse(cbi);
+ return rc;
+}
+
+static uint16_t
+bpf_rx_callback_mb_jit(__rte_unused uint16_t port, __rte_unused uint16_t queue,
+ struct rte_mbuf *pkt[], uint16_t nb_pkts,
+ __rte_unused uint16_t max_pkts, void *user_param)
+{
+ struct bpf_eth_cbi *cbi;
+ uint16_t rc;
+
+ cbi = user_param;
+ bpf_eth_cbi_inuse(cbi);
+ rc = (cbi->cb != NULL) ?
+ pkt_filter_mb_jit(&cbi->jit, pkt, nb_pkts, 1) :
+ nb_pkts;
+ bpf_eth_cbi_unuse(cbi);
+ return rc;
+}
+
+static uint16_t
+bpf_tx_callback_mb_vm(__rte_unused uint16_t port, __rte_unused uint16_t queue,
+ struct rte_mbuf *pkt[], uint16_t nb_pkts, void *user_param)
+{
+ struct bpf_eth_cbi *cbi;
+ uint16_t rc;
+
+ cbi = user_param;
+ bpf_eth_cbi_inuse(cbi);
+ rc = (cbi->cb != NULL) ?
+ pkt_filter_mb_vm(cbi->bpf, pkt, nb_pkts, 0) :
+ nb_pkts;
+ bpf_eth_cbi_unuse(cbi);
+ return rc;
+}
+
+static uint16_t
+bpf_tx_callback_mb_jit(__rte_unused uint16_t port, __rte_unused uint16_t queue,
+ struct rte_mbuf *pkt[], uint16_t nb_pkts, void *user_param)
+{
+ struct bpf_eth_cbi *cbi;
+ uint16_t rc;
+
+ cbi = user_param;
+ bpf_eth_cbi_inuse(cbi);
+ rc = (cbi->cb != NULL) ?
+ pkt_filter_mb_jit(&cbi->jit, pkt, nb_pkts, 0) :
+ nb_pkts;
+ bpf_eth_cbi_unuse(cbi);
+ return rc;
+}
+
+static rte_rx_callback_fn
+select_rx_callback(enum rte_bpf_arg_type type, uint32_t flags)
+{
+ if (flags & RTE_BPF_ETH_F_JIT) {
+ if (type == RTE_BPF_ARG_PTR)
+ return bpf_rx_callback_jit;
+ else if (type == RTE_BPF_ARG_PTR_MBUF)
+ return bpf_rx_callback_mb_jit;
+ } else if (type == RTE_BPF_ARG_PTR)
+ return bpf_rx_callback_vm;
+ else if (type == RTE_BPF_ARG_PTR_MBUF)
+ return bpf_rx_callback_mb_vm;
+
+ return NULL;
+}
+
+static rte_tx_callback_fn
+select_tx_callback(enum rte_bpf_arg_type type, uint32_t flags)
+{
+ if (flags & RTE_BPF_ETH_F_JIT) {
+ if (type == RTE_BPF_ARG_PTR)
+ return bpf_tx_callback_jit;
+ else if (type == RTE_BPF_ARG_PTR_MBUF)
+ return bpf_tx_callback_mb_jit;
+ } else if (type == RTE_BPF_ARG_PTR)
+ return bpf_tx_callback_vm;
+ else if (type == RTE_BPF_ARG_PTR_MBUF)
+ return bpf_tx_callback_mb_vm;
+
+ return NULL;
+}
+
+/*
+ * helper function to perform BPF unload for given port/queue.
+ * have to introduce extra complexity (and possible slowdown) here,
+ * as right now there is no safe generic way to remove RX/TX callback
+ * while IO is active.
+ * Still don't free memory allocated for callback handle itself,
+ * again right now there is no safe way to do that without stopping RX/TX
+ * on given port/queue first.
+ */
+static void
+bpf_eth_cbi_unload(struct bpf_eth_cbi *bc)
+{
+ /* mark this cbi as empty */
+ bc->cb = NULL;
+ rte_smp_mb();
+
+ /* make sure datapath doesn't use bpf anymore, then destroy bpf */
+ bpf_eth_cbi_wait(bc);
+ rte_bpf_destroy(bc->bpf);
+ bpf_eth_cbi_cleanup(bc);
+}
+
+static void
+bpf_eth_unload(struct bpf_eth_cbh *cbh, uint16_t port, uint16_t queue)
+{
+ struct bpf_eth_cbi *bc;
+
+ bc = bpf_eth_cbh_find(cbh, port, queue);
+ if (bc == NULL || bc->cb == NULL)
+ return;
+
+ if (cbh->type == BPF_ETH_RX)
+ rte_eth_remove_rx_callback(port, queue, bc->cb);
+ else
+ rte_eth_remove_tx_callback(port, queue, bc->cb);
+
+ bpf_eth_cbi_unload(bc);
+}
+
+
+void
+rte_bpf_eth_rx_unload(uint16_t port, uint16_t queue)
+{
+ struct bpf_eth_cbh *cbh;
+
+ cbh = &rx_cbh;
+ rte_spinlock_lock(&cbh->lock);
+ bpf_eth_unload(cbh, port, queue);
+ rte_spinlock_unlock(&cbh->lock);
+}
+
+void
+rte_bpf_eth_tx_unload(uint16_t port, uint16_t queue)
+{
+ struct bpf_eth_cbh *cbh;
+
+ cbh = &tx_cbh;
+ rte_spinlock_lock(&cbh->lock);
+ bpf_eth_unload(cbh, port, queue);
+ rte_spinlock_unlock(&cbh->lock);
+}
+
+static int
+bpf_eth_elf_load(struct bpf_eth_cbh *cbh, uint16_t port, uint16_t queue,
+ const struct rte_bpf_prm *prm, const char *fname, const char *sname,
+ uint32_t flags)
+{
+ int32_t rc;
+ struct bpf_eth_cbi *bc;
+ struct rte_bpf *bpf;
+ rte_rx_callback_fn frx;
+ rte_tx_callback_fn ftx;
+ struct rte_bpf_jit jit;
+
+ frx = NULL;
+ ftx = NULL;
+
+ if (prm == NULL || rte_eth_dev_is_valid_port(port) == 0 ||
+ queue >= RTE_MAX_QUEUES_PER_PORT)
+ return -EINVAL;
+
+ if (cbh->type == BPF_ETH_RX)
+ frx = select_rx_callback(prm->prog_arg.type, flags);
+ else
+ ftx = select_tx_callback(prm->prog_arg.type, flags);
+
+ if (frx == NULL && ftx == NULL) {
+ RTE_BPF_LOG(ERR, "%s(%u, %u): no callback selected;\n",
+ __func__, port, queue);
+ return -EINVAL;
+ }
+
+ bpf = rte_bpf_elf_load(prm, fname, sname);
+ if (bpf == NULL)
+ return -rte_errno;
+
+ rte_bpf_get_jit(bpf, &jit);
+
+ if ((flags & RTE_BPF_ETH_F_JIT) != 0 && jit.func == NULL) {
+ RTE_BPF_LOG(ERR, "%s(%u, %u): no JIT generated;\n",
+ __func__, port, queue);
+ rte_bpf_destroy(bpf);
+ return -ENOTSUP;
+ }
+
+ /* setup/update global callback info */
+ bc = bpf_eth_cbh_add(cbh, port, queue);
+ if (bc == NULL)
+ return -ENOMEM;
+
+ /* remove old one, if any */
+ if (bc->cb != NULL)
+ bpf_eth_unload(cbh, port, queue);
+
+ bc->bpf = bpf;
+ bc->jit = jit;
+
+ if (cbh->type == BPF_ETH_RX)
+ bc->cb = rte_eth_add_rx_callback(port, queue, frx, bc);
+ else
+ bc->cb = rte_eth_add_tx_callback(port, queue, ftx, bc);
+
+ if (bc->cb == NULL) {
+ rc = -rte_errno;
+ rte_bpf_destroy(bpf);
+ bpf_eth_cbi_cleanup(bc);
+ } else
+ rc = 0;
+
+ return rc;
+}
+
+int
+rte_bpf_eth_rx_elf_load(uint16_t port, uint16_t queue,
+ const struct rte_bpf_prm *prm, const char *fname, const char *sname,
+ uint32_t flags)
+{
+ int32_t rc;
+ struct bpf_eth_cbh *cbh;
+
+ cbh = &rx_cbh;
+ rte_spinlock_lock(&cbh->lock);
+ rc = bpf_eth_elf_load(cbh, port, queue, prm, fname, sname, flags);
+ rte_spinlock_unlock(&cbh->lock);
+
+ return rc;
+}
+
+int
+rte_bpf_eth_tx_elf_load(uint16_t port, uint16_t queue,
+ const struct rte_bpf_prm *prm, const char *fname, const char *sname,
+ uint32_t flags)
+{
+ int32_t rc;
+ struct bpf_eth_cbh *cbh;
+
+ cbh = &tx_cbh;
+ rte_spinlock_lock(&cbh->lock);
+ rc = bpf_eth_elf_load(cbh, port, queue, prm, fname, sname, flags);
+ rte_spinlock_unlock(&cbh->lock);
+
+ return rc;
+}
diff --git a/src/spdk/dpdk/lib/librte_bpf/bpf_validate.c b/src/spdk/dpdk/lib/librte_bpf/bpf_validate.c
new file mode 100644
index 000000000..6bd6f78e9
--- /dev/null
+++ b/src/spdk/dpdk/lib/librte_bpf/bpf_validate.c
@@ -0,0 +1,2250 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2018 Intel Corporation
+ */
+
+#include <stdarg.h>
+#include <stdio.h>
+#include <string.h>
+#include <errno.h>
+#include <stdint.h>
+#include <inttypes.h>
+
+#include <rte_common.h>
+#include <rte_eal.h>
+#include <rte_byteorder.h>
+
+#include "bpf_impl.h"
+
+#define BPF_ARG_PTR_STACK RTE_BPF_ARG_RESERVED
+
+struct bpf_reg_val {
+ struct rte_bpf_arg v;
+ uint64_t mask;
+ struct {
+ int64_t min;
+ int64_t max;
+ } s;
+ struct {
+ uint64_t min;
+ uint64_t max;
+ } u;
+};
+
+struct bpf_eval_state {
+ struct bpf_reg_val rv[EBPF_REG_NUM];
+ struct bpf_reg_val sv[MAX_BPF_STACK_SIZE / sizeof(uint64_t)];
+};
+
+/* possible instruction node colour */
+enum {
+ WHITE,
+ GREY,
+ BLACK,
+ MAX_NODE_COLOUR
+};
+
+/* possible edge types */
+enum {
+ UNKNOWN_EDGE,
+ TREE_EDGE,
+ BACK_EDGE,
+ CROSS_EDGE,
+ MAX_EDGE_TYPE
+};
+
+#define MAX_EDGES 2
+
+struct inst_node {
+ uint8_t colour;
+ uint8_t nb_edge:4;
+ uint8_t cur_edge:4;
+ uint8_t edge_type[MAX_EDGES];
+ uint32_t edge_dest[MAX_EDGES];
+ uint32_t prev_node;
+ struct bpf_eval_state *evst;
+};
+
+struct bpf_verifier {
+ const struct rte_bpf_prm *prm;
+ struct inst_node *in;
+ uint64_t stack_sz;
+ uint32_t nb_nodes;
+ uint32_t nb_jcc_nodes;
+ uint32_t node_colour[MAX_NODE_COLOUR];
+ uint32_t edge_type[MAX_EDGE_TYPE];
+ struct bpf_eval_state *evst;
+ struct inst_node *evin;
+ struct {
+ uint32_t num;
+ uint32_t cur;
+ struct bpf_eval_state *ent;
+ } evst_pool;
+};
+
+struct bpf_ins_check {
+ struct {
+ uint16_t dreg;
+ uint16_t sreg;
+ } mask;
+ struct {
+ uint16_t min;
+ uint16_t max;
+ } off;
+ struct {
+ uint32_t min;
+ uint32_t max;
+ } imm;
+ const char * (*check)(const struct ebpf_insn *);
+ const char * (*eval)(struct bpf_verifier *, const struct ebpf_insn *);
+};
+
+#define ALL_REGS RTE_LEN2MASK(EBPF_REG_NUM, uint16_t)
+#define WRT_REGS RTE_LEN2MASK(EBPF_REG_10, uint16_t)
+#define ZERO_REG RTE_LEN2MASK(EBPF_REG_1, uint16_t)
+
+/*
+ * check and evaluate functions for particular instruction types.
+ */
+
+static const char *
+check_alu_bele(const struct ebpf_insn *ins)
+{
+ if (ins->imm != 16 && ins->imm != 32 && ins->imm != 64)
+ return "invalid imm field";
+ return NULL;
+}
+
+static const char *
+eval_exit(struct bpf_verifier *bvf, const struct ebpf_insn *ins)
+{
+ RTE_SET_USED(ins);
+ if (bvf->evst->rv[EBPF_REG_0].v.type == RTE_BPF_ARG_UNDEF)
+ return "undefined return value";
+ return NULL;
+}
+
+/* setup max possible with this mask bounds */
+static void
+eval_umax_bound(struct bpf_reg_val *rv, uint64_t mask)
+{
+ rv->u.max = mask;
+ rv->u.min = 0;
+}
+
+static void
+eval_smax_bound(struct bpf_reg_val *rv, uint64_t mask)
+{
+ rv->s.max = mask >> 1;
+ rv->s.min = rv->s.max ^ UINT64_MAX;
+}
+
+static void
+eval_max_bound(struct bpf_reg_val *rv, uint64_t mask)
+{
+ eval_umax_bound(rv, mask);
+ eval_smax_bound(rv, mask);
+}
+
+static void
+eval_fill_max_bound(struct bpf_reg_val *rv, uint64_t mask)
+{
+ eval_max_bound(rv, mask);
+ rv->v.type = RTE_BPF_ARG_RAW;
+ rv->mask = mask;
+}
+
+static void
+eval_fill_imm64(struct bpf_reg_val *rv, uint64_t mask, uint64_t val)
+{
+ rv->mask = mask;
+ rv->s.min = val;
+ rv->s.max = val;
+ rv->u.min = val;
+ rv->u.max = val;
+}
+
+static void
+eval_fill_imm(struct bpf_reg_val *rv, uint64_t mask, int32_t imm)
+{
+ uint64_t v;
+
+ v = (uint64_t)imm & mask;
+
+ rv->v.type = RTE_BPF_ARG_RAW;
+ eval_fill_imm64(rv, mask, v);
+}
+
+static const char *
+eval_ld_imm64(struct bpf_verifier *bvf, const struct ebpf_insn *ins)
+{
+ uint32_t i;
+ uint64_t val;
+ struct bpf_reg_val *rd;
+
+ val = (uint32_t)ins[0].imm | (uint64_t)(uint32_t)ins[1].imm << 32;
+
+ rd = bvf->evst->rv + ins->dst_reg;
+ rd->v.type = RTE_BPF_ARG_RAW;
+ eval_fill_imm64(rd, UINT64_MAX, val);
+
+ for (i = 0; i != bvf->prm->nb_xsym; i++) {
+
+ /* load of external variable */
+ if (bvf->prm->xsym[i].type == RTE_BPF_XTYPE_VAR &&
+ (uintptr_t)bvf->prm->xsym[i].var.val == val) {
+ rd->v = bvf->prm->xsym[i].var.desc;
+ eval_fill_imm64(rd, UINT64_MAX, 0);
+ break;
+ }
+ }
+
+ return NULL;
+}
+
+static void
+eval_apply_mask(struct bpf_reg_val *rv, uint64_t mask)
+{
+ struct bpf_reg_val rt;
+
+ rt.u.min = rv->u.min & mask;
+ rt.u.max = rv->u.max & mask;
+ if (rt.u.min != rv->u.min || rt.u.max != rv->u.max) {
+ rv->u.max = RTE_MAX(rt.u.max, mask);
+ rv->u.min = 0;
+ }
+
+ eval_smax_bound(&rt, mask);
+ rv->s.max = RTE_MIN(rt.s.max, rv->s.max);
+ rv->s.min = RTE_MAX(rt.s.min, rv->s.min);
+
+ rv->mask = mask;
+}
+
+static void
+eval_add(struct bpf_reg_val *rd, const struct bpf_reg_val *rs, uint64_t msk)
+{
+ struct bpf_reg_val rv;
+
+ rv.u.min = (rd->u.min + rs->u.min) & msk;
+ rv.u.max = (rd->u.min + rs->u.max) & msk;
+ rv.s.min = (rd->s.min + rs->s.min) & msk;
+ rv.s.max = (rd->s.max + rs->s.max) & msk;
+
+ /*
+ * if at least one of the operands is not constant,
+ * then check for overflow
+ */
+ if ((rd->u.min != rd->u.max || rs->u.min != rs->u.max) &&
+ (rv.u.min < rd->u.min || rv.u.max < rd->u.max))
+ eval_umax_bound(&rv, msk);
+
+ if ((rd->s.min != rd->s.max || rs->s.min != rs->s.max) &&
+ (((rs->s.min < 0 && rv.s.min > rd->s.min) ||
+ rv.s.min < rd->s.min) ||
+ ((rs->s.max < 0 && rv.s.max > rd->s.max) ||
+ rv.s.max < rd->s.max)))
+ eval_smax_bound(&rv, msk);
+
+ rd->s = rv.s;
+ rd->u = rv.u;
+}
+
+static void
+eval_sub(struct bpf_reg_val *rd, const struct bpf_reg_val *rs, uint64_t msk)
+{
+ struct bpf_reg_val rv;
+
+ rv.u.min = (rd->u.min - rs->u.min) & msk;
+ rv.u.max = (rd->u.min - rs->u.max) & msk;
+ rv.s.min = (rd->s.min - rs->s.min) & msk;
+ rv.s.max = (rd->s.max - rs->s.max) & msk;
+
+ /*
+ * if at least one of the operands is not constant,
+ * then check for overflow
+ */
+ if ((rd->u.min != rd->u.max || rs->u.min != rs->u.max) &&
+ (rv.u.min > rd->u.min || rv.u.max > rd->u.max))
+ eval_umax_bound(&rv, msk);
+
+ if ((rd->s.min != rd->s.max || rs->s.min != rs->s.max) &&
+ (((rs->s.min < 0 && rv.s.min < rd->s.min) ||
+ rv.s.min > rd->s.min) ||
+ ((rs->s.max < 0 && rv.s.max < rd->s.max) ||
+ rv.s.max > rd->s.max)))
+ eval_smax_bound(&rv, msk);
+
+ rd->s = rv.s;
+ rd->u = rv.u;
+}
+
+static void
+eval_lsh(struct bpf_reg_val *rd, const struct bpf_reg_val *rs, size_t opsz,
+ uint64_t msk)
+{
+ /* check if shift value is less then max result bits */
+ if (rs->u.max >= opsz) {
+ eval_max_bound(rd, msk);
+ return;
+ }
+
+ /* check for overflow */
+ if (rd->u.max > RTE_LEN2MASK(opsz - rs->u.max, uint64_t))
+ eval_umax_bound(rd, msk);
+ else {
+ rd->u.max <<= rs->u.max;
+ rd->u.min <<= rs->u.min;
+ }
+
+ /* check that dreg values are and would remain always positive */
+ if ((uint64_t)rd->s.min >> (opsz - 1) != 0 || rd->s.max >=
+ RTE_LEN2MASK(opsz - rs->u.max - 1, int64_t))
+ eval_smax_bound(rd, msk);
+ else {
+ rd->s.max <<= rs->u.max;
+ rd->s.min <<= rs->u.min;
+ }
+}
+
+static void
+eval_rsh(struct bpf_reg_val *rd, const struct bpf_reg_val *rs, size_t opsz,
+ uint64_t msk)
+{
+ /* check if shift value is less then max result bits */
+ if (rs->u.max >= opsz) {
+ eval_max_bound(rd, msk);
+ return;
+ }
+
+ rd->u.max >>= rs->u.min;
+ rd->u.min >>= rs->u.max;
+
+ /* check that dreg values are always positive */
+ if ((uint64_t)rd->s.min >> (opsz - 1) != 0)
+ eval_smax_bound(rd, msk);
+ else {
+ rd->s.max >>= rs->u.min;
+ rd->s.min >>= rs->u.max;
+ }
+}
+
+static void
+eval_arsh(struct bpf_reg_val *rd, const struct bpf_reg_val *rs, size_t opsz,
+ uint64_t msk)
+{
+ uint32_t shv;
+
+ /* check if shift value is less then max result bits */
+ if (rs->u.max >= opsz) {
+ eval_max_bound(rd, msk);
+ return;
+ }
+
+ rd->u.max = (int64_t)rd->u.max >> rs->u.min;
+ rd->u.min = (int64_t)rd->u.min >> rs->u.max;
+
+ /* if we have 32-bit values - extend them to 64-bit */
+ if (opsz == sizeof(uint32_t) * CHAR_BIT) {
+ rd->s.min <<= opsz;
+ rd->s.max <<= opsz;
+ shv = opsz;
+ } else
+ shv = 0;
+
+ if (rd->s.min < 0)
+ rd->s.min = (rd->s.min >> (rs->u.min + shv)) & msk;
+ else
+ rd->s.min = (rd->s.min >> (rs->u.max + shv)) & msk;
+
+ if (rd->s.max < 0)
+ rd->s.max = (rd->s.max >> (rs->u.max + shv)) & msk;
+ else
+ rd->s.max = (rd->s.max >> (rs->u.min + shv)) & msk;
+}
+
+static uint64_t
+eval_umax_bits(uint64_t v, size_t opsz)
+{
+ if (v == 0)
+ return 0;
+
+ v = __builtin_clzll(v);
+ return RTE_LEN2MASK(opsz - v, uint64_t);
+}
+
+/* estimate max possible value for (v1 & v2) */
+static uint64_t
+eval_uand_max(uint64_t v1, uint64_t v2, size_t opsz)
+{
+ v1 = eval_umax_bits(v1, opsz);
+ v2 = eval_umax_bits(v2, opsz);
+ return (v1 & v2);
+}
+
+/* estimate max possible value for (v1 | v2) */
+static uint64_t
+eval_uor_max(uint64_t v1, uint64_t v2, size_t opsz)
+{
+ v1 = eval_umax_bits(v1, opsz);
+ v2 = eval_umax_bits(v2, opsz);
+ return (v1 | v2);
+}
+
+static void
+eval_and(struct bpf_reg_val *rd, const struct bpf_reg_val *rs, size_t opsz,
+ uint64_t msk)
+{
+ /* both operands are constants */
+ if (rd->u.min == rd->u.max && rs->u.min == rs->u.max) {
+ rd->u.min &= rs->u.min;
+ rd->u.max &= rs->u.max;
+ } else {
+ rd->u.max = eval_uand_max(rd->u.max, rs->u.max, opsz);
+ rd->u.min &= rs->u.min;
+ }
+
+ /* both operands are constants */
+ if (rd->s.min == rd->s.max && rs->s.min == rs->s.max) {
+ rd->s.min &= rs->s.min;
+ rd->s.max &= rs->s.max;
+ /* at least one of operand is non-negative */
+ } else if (rd->s.min >= 0 || rs->s.min >= 0) {
+ rd->s.max = eval_uand_max(rd->s.max & (msk >> 1),
+ rs->s.max & (msk >> 1), opsz);
+ rd->s.min &= rs->s.min;
+ } else
+ eval_smax_bound(rd, msk);
+}
+
+static void
+eval_or(struct bpf_reg_val *rd, const struct bpf_reg_val *rs, size_t opsz,
+ uint64_t msk)
+{
+ /* both operands are constants */
+ if (rd->u.min == rd->u.max && rs->u.min == rs->u.max) {
+ rd->u.min |= rs->u.min;
+ rd->u.max |= rs->u.max;
+ } else {
+ rd->u.max = eval_uor_max(rd->u.max, rs->u.max, opsz);
+ rd->u.min |= rs->u.min;
+ }
+
+ /* both operands are constants */
+ if (rd->s.min == rd->s.max && rs->s.min == rs->s.max) {
+ rd->s.min |= rs->s.min;
+ rd->s.max |= rs->s.max;
+
+ /* both operands are non-negative */
+ } else if (rd->s.min >= 0 || rs->s.min >= 0) {
+ rd->s.max = eval_uor_max(rd->s.max, rs->s.max, opsz);
+ rd->s.min |= rs->s.min;
+ } else
+ eval_smax_bound(rd, msk);
+}
+
+static void
+eval_xor(struct bpf_reg_val *rd, const struct bpf_reg_val *rs, size_t opsz,
+ uint64_t msk)
+{
+ /* both operands are constants */
+ if (rd->u.min == rd->u.max && rs->u.min == rs->u.max) {
+ rd->u.min ^= rs->u.min;
+ rd->u.max ^= rs->u.max;
+ } else {
+ rd->u.max = eval_uor_max(rd->u.max, rs->u.max, opsz);
+ rd->u.min = 0;
+ }
+
+ /* both operands are constants */
+ if (rd->s.min == rd->s.max && rs->s.min == rs->s.max) {
+ rd->s.min ^= rs->s.min;
+ rd->s.max ^= rs->s.max;
+
+ /* both operands are non-negative */
+ } else if (rd->s.min >= 0 || rs->s.min >= 0) {
+ rd->s.max = eval_uor_max(rd->s.max, rs->s.max, opsz);
+ rd->s.min = 0;
+ } else
+ eval_smax_bound(rd, msk);
+}
+
+static void
+eval_mul(struct bpf_reg_val *rd, const struct bpf_reg_val *rs, size_t opsz,
+ uint64_t msk)
+{
+ /* both operands are constants */
+ if (rd->u.min == rd->u.max && rs->u.min == rs->u.max) {
+ rd->u.min = (rd->u.min * rs->u.min) & msk;
+ rd->u.max = (rd->u.max * rs->u.max) & msk;
+ /* check for overflow */
+ } else if (rd->u.max <= msk >> opsz / 2 && rs->u.max <= msk >> opsz) {
+ rd->u.max *= rs->u.max;
+ rd->u.min *= rd->u.min;
+ } else
+ eval_umax_bound(rd, msk);
+
+ /* both operands are constants */
+ if (rd->s.min == rd->s.max && rs->s.min == rs->s.max) {
+ rd->s.min = (rd->s.min * rs->s.min) & msk;
+ rd->s.max = (rd->s.max * rs->s.max) & msk;
+ /* check that both operands are positive and no overflow */
+ } else if (rd->s.min >= 0 && rs->s.min >= 0) {
+ rd->s.max *= rs->s.max;
+ rd->s.min *= rd->s.min;
+ } else
+ eval_smax_bound(rd, msk);
+}
+
+static const char *
+eval_divmod(uint32_t op, struct bpf_reg_val *rd, struct bpf_reg_val *rs,
+ size_t opsz, uint64_t msk)
+{
+ /* both operands are constants */
+ if (rd->u.min == rd->u.max && rs->u.min == rs->u.max) {
+ if (rs->u.max == 0)
+ return "division by 0";
+ if (op == BPF_DIV) {
+ rd->u.min /= rs->u.min;
+ rd->u.max /= rs->u.max;
+ } else {
+ rd->u.min %= rs->u.min;
+ rd->u.max %= rs->u.max;
+ }
+ } else {
+ if (op == BPF_MOD)
+ rd->u.max = RTE_MIN(rd->u.max, rs->u.max - 1);
+ else
+ rd->u.max = rd->u.max;
+ rd->u.min = 0;
+ }
+
+ /* if we have 32-bit values - extend them to 64-bit */
+ if (opsz == sizeof(uint32_t) * CHAR_BIT) {
+ rd->s.min = (int32_t)rd->s.min;
+ rd->s.max = (int32_t)rd->s.max;
+ rs->s.min = (int32_t)rs->s.min;
+ rs->s.max = (int32_t)rs->s.max;
+ }
+
+ /* both operands are constants */
+ if (rd->s.min == rd->s.max && rs->s.min == rs->s.max) {
+ if (rs->s.max == 0)
+ return "division by 0";
+ if (op == BPF_DIV) {
+ rd->s.min /= rs->s.min;
+ rd->s.max /= rs->s.max;
+ } else {
+ rd->s.min %= rs->s.min;
+ rd->s.max %= rs->s.max;
+ }
+ } else if (op == BPF_MOD) {
+ rd->s.min = RTE_MAX(rd->s.max, 0);
+ rd->s.min = RTE_MIN(rd->s.min, 0);
+ } else
+ eval_smax_bound(rd, msk);
+
+ rd->s.max &= msk;
+ rd->s.min &= msk;
+
+ return NULL;
+}
+
+static void
+eval_neg(struct bpf_reg_val *rd, size_t opsz, uint64_t msk)
+{
+ uint64_t ux, uy;
+ int64_t sx, sy;
+
+ /* if we have 32-bit values - extend them to 64-bit */
+ if (opsz == sizeof(uint32_t) * CHAR_BIT) {
+ rd->u.min = (int32_t)rd->u.min;
+ rd->u.max = (int32_t)rd->u.max;
+ }
+
+ ux = -(int64_t)rd->u.min & msk;
+ uy = -(int64_t)rd->u.max & msk;
+
+ rd->u.max = RTE_MAX(ux, uy);
+ rd->u.min = RTE_MIN(ux, uy);
+
+ /* if we have 32-bit values - extend them to 64-bit */
+ if (opsz == sizeof(uint32_t) * CHAR_BIT) {
+ rd->s.min = (int32_t)rd->s.min;
+ rd->s.max = (int32_t)rd->s.max;
+ }
+
+ sx = -rd->s.min & msk;
+ sy = -rd->s.max & msk;
+
+ rd->s.max = RTE_MAX(sx, sy);
+ rd->s.min = RTE_MIN(sx, sy);
+}
+
+/*
+ * check that destination and source operand are in defined state.
+ */
+static const char *
+eval_defined(const struct bpf_reg_val *dst, const struct bpf_reg_val *src)
+{
+ if (dst != NULL && dst->v.type == RTE_BPF_ARG_UNDEF)
+ return "dest reg value is undefined";
+ if (src != NULL && src->v.type == RTE_BPF_ARG_UNDEF)
+ return "src reg value is undefined";
+ return NULL;
+}
+
+static const char *
+eval_alu(struct bpf_verifier *bvf, const struct ebpf_insn *ins)
+{
+ uint64_t msk;
+ uint32_t op;
+ size_t opsz;
+ const char *err;
+ struct bpf_eval_state *st;
+ struct bpf_reg_val *rd, rs;
+
+ opsz = (BPF_CLASS(ins->code) == BPF_ALU) ?
+ sizeof(uint32_t) : sizeof(uint64_t);
+ opsz = opsz * CHAR_BIT;
+ msk = RTE_LEN2MASK(opsz, uint64_t);
+
+ st = bvf->evst;
+ rd = st->rv + ins->dst_reg;
+
+ if (BPF_SRC(ins->code) == BPF_X) {
+ rs = st->rv[ins->src_reg];
+ eval_apply_mask(&rs, msk);
+ } else
+ eval_fill_imm(&rs, msk, ins->imm);
+
+ eval_apply_mask(rd, msk);
+
+ op = BPF_OP(ins->code);
+
+ err = eval_defined((op != EBPF_MOV) ? rd : NULL,
+ (op != BPF_NEG) ? &rs : NULL);
+ if (err != NULL)
+ return err;
+
+ if (op == BPF_ADD)
+ eval_add(rd, &rs, msk);
+ else if (op == BPF_SUB)
+ eval_sub(rd, &rs, msk);
+ else if (op == BPF_LSH)
+ eval_lsh(rd, &rs, opsz, msk);
+ else if (op == BPF_RSH)
+ eval_rsh(rd, &rs, opsz, msk);
+ else if (op == EBPF_ARSH)
+ eval_arsh(rd, &rs, opsz, msk);
+ else if (op == BPF_AND)
+ eval_and(rd, &rs, opsz, msk);
+ else if (op == BPF_OR)
+ eval_or(rd, &rs, opsz, msk);
+ else if (op == BPF_XOR)
+ eval_xor(rd, &rs, opsz, msk);
+ else if (op == BPF_MUL)
+ eval_mul(rd, &rs, opsz, msk);
+ else if (op == BPF_DIV || op == BPF_MOD)
+ err = eval_divmod(op, rd, &rs, opsz, msk);
+ else if (op == BPF_NEG)
+ eval_neg(rd, opsz, msk);
+ else if (op == EBPF_MOV)
+ *rd = rs;
+ else
+ eval_max_bound(rd, msk);
+
+ return err;
+}
+
+static const char *
+eval_bele(struct bpf_verifier *bvf, const struct ebpf_insn *ins)
+{
+ uint64_t msk;
+ struct bpf_eval_state *st;
+ struct bpf_reg_val *rd;
+ const char *err;
+
+ msk = RTE_LEN2MASK(ins->imm, uint64_t);
+
+ st = bvf->evst;
+ rd = st->rv + ins->dst_reg;
+
+ err = eval_defined(rd, NULL);
+ if (err != NULL)
+ return err;
+
+#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
+ if (ins->code == (BPF_ALU | EBPF_END | EBPF_TO_BE))
+ eval_max_bound(rd, msk);
+ else
+ eval_apply_mask(rd, msk);
+#else
+ if (ins->code == (BPF_ALU | EBPF_END | EBPF_TO_LE))
+ eval_max_bound(rd, msk);
+ else
+ eval_apply_mask(rd, msk);
+#endif
+
+ return NULL;
+}
+
+static const char *
+eval_ptr(struct bpf_verifier *bvf, struct bpf_reg_val *rm, uint32_t opsz,
+ uint32_t align, int16_t off)
+{
+ struct bpf_reg_val rv;
+
+ /* calculate reg + offset */
+ eval_fill_imm(&rv, rm->mask, off);
+ eval_add(rm, &rv, rm->mask);
+
+ if (RTE_BPF_ARG_PTR_TYPE(rm->v.type) == 0)
+ return "destination is not a pointer";
+
+ if (rm->mask != UINT64_MAX)
+ return "pointer truncation";
+
+ if (rm->u.max + opsz > rm->v.size ||
+ (uint64_t)rm->s.max + opsz > rm->v.size ||
+ rm->s.min < 0)
+ return "memory boundary violation";
+
+ if (rm->u.max % align != 0)
+ return "unaligned memory access";
+
+ if (rm->v.type == BPF_ARG_PTR_STACK) {
+
+ if (rm->u.max != rm->u.min || rm->s.max != rm->s.min ||
+ rm->u.max != (uint64_t)rm->s.max)
+ return "stack access with variable offset";
+
+ bvf->stack_sz = RTE_MAX(bvf->stack_sz, rm->v.size - rm->u.max);
+
+ /* pointer to mbuf */
+ } else if (rm->v.type == RTE_BPF_ARG_PTR_MBUF) {
+
+ if (rm->u.max != rm->u.min || rm->s.max != rm->s.min ||
+ rm->u.max != (uint64_t)rm->s.max)
+ return "mbuf access with variable offset";
+ }
+
+ return NULL;
+}
+
+static void
+eval_max_load(struct bpf_reg_val *rv, uint64_t mask)
+{
+ eval_umax_bound(rv, mask);
+
+ /* full 64-bit load */
+ if (mask == UINT64_MAX)
+ eval_smax_bound(rv, mask);
+
+ /* zero-extend load */
+ rv->s.min = rv->u.min;
+ rv->s.max = rv->u.max;
+}
+
+
+static const char *
+eval_load(struct bpf_verifier *bvf, const struct ebpf_insn *ins)
+{
+ uint32_t opsz;
+ uint64_t msk;
+ const char *err;
+ struct bpf_eval_state *st;
+ struct bpf_reg_val *rd, rs;
+ const struct bpf_reg_val *sv;
+
+ st = bvf->evst;
+ rd = st->rv + ins->dst_reg;
+ rs = st->rv[ins->src_reg];
+ opsz = bpf_size(BPF_SIZE(ins->code));
+ msk = RTE_LEN2MASK(opsz * CHAR_BIT, uint64_t);
+
+ err = eval_ptr(bvf, &rs, opsz, 1, ins->off);
+ if (err != NULL)
+ return err;
+
+ if (rs.v.type == BPF_ARG_PTR_STACK) {
+
+ sv = st->sv + rs.u.max / sizeof(uint64_t);
+ if (sv->v.type == RTE_BPF_ARG_UNDEF || sv->mask < msk)
+ return "undefined value on the stack";
+
+ *rd = *sv;
+
+ /* pointer to mbuf */
+ } else if (rs.v.type == RTE_BPF_ARG_PTR_MBUF) {
+
+ if (rs.u.max == offsetof(struct rte_mbuf, next)) {
+ eval_fill_imm(rd, msk, 0);
+ rd->v = rs.v;
+ } else if (rs.u.max == offsetof(struct rte_mbuf, buf_addr)) {
+ eval_fill_imm(rd, msk, 0);
+ rd->v.type = RTE_BPF_ARG_PTR;
+ rd->v.size = rs.v.buf_size;
+ } else if (rs.u.max == offsetof(struct rte_mbuf, data_off)) {
+ eval_fill_imm(rd, msk, RTE_PKTMBUF_HEADROOM);
+ rd->v.type = RTE_BPF_ARG_RAW;
+ } else {
+ eval_max_load(rd, msk);
+ rd->v.type = RTE_BPF_ARG_RAW;
+ }
+
+ /* pointer to raw data */
+ } else {
+ eval_max_load(rd, msk);
+ rd->v.type = RTE_BPF_ARG_RAW;
+ }
+
+ return NULL;
+}
+
+static const char *
+eval_mbuf_store(const struct bpf_reg_val *rv, uint32_t opsz)
+{
+ uint32_t i;
+
+ static const struct {
+ size_t off;
+ size_t sz;
+ } mbuf_ro_fileds[] = {
+ { .off = offsetof(struct rte_mbuf, buf_addr), },
+ { .off = offsetof(struct rte_mbuf, refcnt), },
+ { .off = offsetof(struct rte_mbuf, nb_segs), },
+ { .off = offsetof(struct rte_mbuf, buf_len), },
+ { .off = offsetof(struct rte_mbuf, pool), },
+ { .off = offsetof(struct rte_mbuf, next), },
+ { .off = offsetof(struct rte_mbuf, priv_size), },
+ };
+
+ for (i = 0; i != RTE_DIM(mbuf_ro_fileds) &&
+ (mbuf_ro_fileds[i].off + mbuf_ro_fileds[i].sz <=
+ rv->u.max || rv->u.max + opsz <= mbuf_ro_fileds[i].off);
+ i++)
+ ;
+
+ if (i != RTE_DIM(mbuf_ro_fileds))
+ return "store to the read-only mbuf field";
+
+ return NULL;
+
+}
+
+static const char *
+eval_store(struct bpf_verifier *bvf, const struct ebpf_insn *ins)
+{
+ uint32_t opsz;
+ uint64_t msk;
+ const char *err;
+ struct bpf_eval_state *st;
+ struct bpf_reg_val rd, rs, *sv;
+
+ opsz = bpf_size(BPF_SIZE(ins->code));
+ msk = RTE_LEN2MASK(opsz * CHAR_BIT, uint64_t);
+
+ st = bvf->evst;
+ rd = st->rv[ins->dst_reg];
+
+ if (BPF_CLASS(ins->code) == BPF_STX) {
+ rs = st->rv[ins->src_reg];
+ eval_apply_mask(&rs, msk);
+ } else
+ eval_fill_imm(&rs, msk, ins->imm);
+
+ err = eval_defined(NULL, &rs);
+ if (err != NULL)
+ return err;
+
+ err = eval_ptr(bvf, &rd, opsz, 1, ins->off);
+ if (err != NULL)
+ return err;
+
+ if (rd.v.type == BPF_ARG_PTR_STACK) {
+
+ sv = st->sv + rd.u.max / sizeof(uint64_t);
+ if (BPF_CLASS(ins->code) == BPF_STX &&
+ BPF_MODE(ins->code) == EBPF_XADD)
+ eval_max_bound(sv, msk);
+ else
+ *sv = rs;
+
+ /* pointer to mbuf */
+ } else if (rd.v.type == RTE_BPF_ARG_PTR_MBUF) {
+ err = eval_mbuf_store(&rd, opsz);
+ if (err != NULL)
+ return err;
+ }
+
+ return NULL;
+}
+
+static const char *
+eval_func_arg(struct bpf_verifier *bvf, const struct rte_bpf_arg *arg,
+ struct bpf_reg_val *rv)
+{
+ uint32_t i, n;
+ struct bpf_eval_state *st;
+ const char *err;
+
+ st = bvf->evst;
+
+ if (rv->v.type == RTE_BPF_ARG_UNDEF)
+ return "Undefined argument type";
+
+ if (arg->type != rv->v.type &&
+ arg->type != RTE_BPF_ARG_RAW &&
+ (arg->type != RTE_BPF_ARG_PTR ||
+ RTE_BPF_ARG_PTR_TYPE(rv->v.type) == 0))
+ return "Invalid argument type";
+
+ err = NULL;
+
+ /* argument is a pointer */
+ if (RTE_BPF_ARG_PTR_TYPE(arg->type) != 0) {
+
+ err = eval_ptr(bvf, rv, arg->size, 1, 0);
+
+ /*
+ * pointer to the variable on the stack is passed
+ * as an argument, mark stack space it occupies as initialized.
+ */
+ if (err == NULL && rv->v.type == BPF_ARG_PTR_STACK) {
+
+ i = rv->u.max / sizeof(uint64_t);
+ n = i + arg->size / sizeof(uint64_t);
+ while (i != n) {
+ eval_fill_max_bound(st->sv + i, UINT64_MAX);
+ i++;
+ };
+ }
+ }
+
+ return err;
+}
+
+static const char *
+eval_call(struct bpf_verifier *bvf, const struct ebpf_insn *ins)
+{
+ uint32_t i, idx;
+ struct bpf_reg_val *rv;
+ const struct rte_bpf_xsym *xsym;
+ const char *err;
+
+ idx = ins->imm;
+
+ if (idx >= bvf->prm->nb_xsym ||
+ bvf->prm->xsym[idx].type != RTE_BPF_XTYPE_FUNC)
+ return "invalid external function index";
+
+ /* for now don't support function calls on 32 bit platform */
+ if (sizeof(uint64_t) != sizeof(uintptr_t))
+ return "function calls are supported only for 64 bit apps";
+
+ xsym = bvf->prm->xsym + idx;
+
+ /* evaluate function arguments */
+ err = NULL;
+ for (i = 0; i != xsym->func.nb_args && err == NULL; i++) {
+ err = eval_func_arg(bvf, xsym->func.args + i,
+ bvf->evst->rv + EBPF_REG_1 + i);
+ }
+
+ /* R1-R5 argument/scratch registers */
+ for (i = EBPF_REG_1; i != EBPF_REG_6; i++)
+ bvf->evst->rv[i].v.type = RTE_BPF_ARG_UNDEF;
+
+ /* update return value */
+
+ rv = bvf->evst->rv + EBPF_REG_0;
+ rv->v = xsym->func.ret;
+ if (rv->v.type == RTE_BPF_ARG_RAW)
+ eval_fill_max_bound(rv,
+ RTE_LEN2MASK(rv->v.size * CHAR_BIT, uint64_t));
+ else if (RTE_BPF_ARG_PTR_TYPE(rv->v.type) != 0)
+ eval_fill_imm64(rv, UINTPTR_MAX, 0);
+
+ return err;
+}
+
+static void
+eval_jeq_jne(struct bpf_reg_val *trd, struct bpf_reg_val *trs)
+{
+ /* sreg is constant */
+ if (trs->u.min == trs->u.max) {
+ trd->u = trs->u;
+ /* dreg is constant */
+ } else if (trd->u.min == trd->u.max) {
+ trs->u = trd->u;
+ } else {
+ trd->u.max = RTE_MIN(trd->u.max, trs->u.max);
+ trd->u.min = RTE_MAX(trd->u.min, trs->u.min);
+ trs->u = trd->u;
+ }
+
+ /* sreg is constant */
+ if (trs->s.min == trs->s.max) {
+ trd->s = trs->s;
+ /* dreg is constant */
+ } else if (trd->s.min == trd->s.max) {
+ trs->s = trd->s;
+ } else {
+ trd->s.max = RTE_MIN(trd->s.max, trs->s.max);
+ trd->s.min = RTE_MAX(trd->s.min, trs->s.min);
+ trs->s = trd->s;
+ }
+}
+
+static void
+eval_jgt_jle(struct bpf_reg_val *trd, struct bpf_reg_val *trs,
+ struct bpf_reg_val *frd, struct bpf_reg_val *frs)
+{
+ frd->u.max = RTE_MIN(frd->u.max, frs->u.min);
+ trd->u.min = RTE_MAX(trd->u.min, trs->u.min + 1);
+}
+
+static void
+eval_jlt_jge(struct bpf_reg_val *trd, struct bpf_reg_val *trs,
+ struct bpf_reg_val *frd, struct bpf_reg_val *frs)
+{
+ frd->u.min = RTE_MAX(frd->u.min, frs->u.min);
+ trd->u.max = RTE_MIN(trd->u.max, trs->u.max - 1);
+}
+
+static void
+eval_jsgt_jsle(struct bpf_reg_val *trd, struct bpf_reg_val *trs,
+ struct bpf_reg_val *frd, struct bpf_reg_val *frs)
+{
+ frd->s.max = RTE_MIN(frd->s.max, frs->s.min);
+ trd->s.min = RTE_MAX(trd->s.min, trs->s.min + 1);
+}
+
+static void
+eval_jslt_jsge(struct bpf_reg_val *trd, struct bpf_reg_val *trs,
+ struct bpf_reg_val *frd, struct bpf_reg_val *frs)
+{
+ frd->s.min = RTE_MAX(frd->s.min, frs->s.min);
+ trd->s.max = RTE_MIN(trd->s.max, trs->s.max - 1);
+}
+
+static const char *
+eval_jcc(struct bpf_verifier *bvf, const struct ebpf_insn *ins)
+{
+ uint32_t op;
+ const char *err;
+ struct bpf_eval_state *fst, *tst;
+ struct bpf_reg_val *frd, *frs, *trd, *trs;
+ struct bpf_reg_val rvf, rvt;
+
+ tst = bvf->evst;
+ fst = bvf->evin->evst;
+
+ frd = fst->rv + ins->dst_reg;
+ trd = tst->rv + ins->dst_reg;
+
+ if (BPF_SRC(ins->code) == BPF_X) {
+ frs = fst->rv + ins->src_reg;
+ trs = tst->rv + ins->src_reg;
+ } else {
+ frs = &rvf;
+ trs = &rvt;
+ eval_fill_imm(frs, UINT64_MAX, ins->imm);
+ eval_fill_imm(trs, UINT64_MAX, ins->imm);
+ }
+
+ err = eval_defined(trd, trs);
+ if (err != NULL)
+ return err;
+
+ op = BPF_OP(ins->code);
+
+ if (op == BPF_JEQ)
+ eval_jeq_jne(trd, trs);
+ else if (op == EBPF_JNE)
+ eval_jeq_jne(frd, frs);
+ else if (op == BPF_JGT)
+ eval_jgt_jle(trd, trs, frd, frs);
+ else if (op == EBPF_JLE)
+ eval_jgt_jle(frd, frs, trd, trs);
+ else if (op == EBPF_JLT)
+ eval_jlt_jge(trd, trs, frd, frs);
+ else if (op == BPF_JGE)
+ eval_jlt_jge(frd, frs, trd, trs);
+ else if (op == EBPF_JSGT)
+ eval_jsgt_jsle(trd, trs, frd, frs);
+ else if (op == EBPF_JSLE)
+ eval_jsgt_jsle(frd, frs, trd, trs);
+ else if (op == EBPF_JLT)
+ eval_jslt_jsge(trd, trs, frd, frs);
+ else if (op == EBPF_JSGE)
+ eval_jslt_jsge(frd, frs, trd, trs);
+
+ return NULL;
+}
+
+/*
+ * validate parameters for each instruction type.
+ */
+static const struct bpf_ins_check ins_chk[UINT8_MAX + 1] = {
+ /* ALU IMM 32-bit instructions */
+ [(BPF_ALU | BPF_ADD | BPF_K)] = {
+ .mask = {.dreg = WRT_REGS, .sreg = ZERO_REG},
+ .off = { .min = 0, .max = 0},
+ .imm = { .min = 0, .max = UINT32_MAX,},
+ .eval = eval_alu,
+ },
+ [(BPF_ALU | BPF_SUB | BPF_K)] = {
+ .mask = {.dreg = WRT_REGS, .sreg = ZERO_REG},
+ .off = { .min = 0, .max = 0},
+ .imm = { .min = 0, .max = UINT32_MAX,},
+ .eval = eval_alu,
+ },
+ [(BPF_ALU | BPF_AND | BPF_K)] = {
+ .mask = {.dreg = WRT_REGS, .sreg = ZERO_REG},
+ .off = { .min = 0, .max = 0},
+ .imm = { .min = 0, .max = UINT32_MAX,},
+ .eval = eval_alu,
+ },
+ [(BPF_ALU | BPF_OR | BPF_K)] = {
+ .mask = {.dreg = WRT_REGS, .sreg = ZERO_REG},
+ .off = { .min = 0, .max = 0},
+ .imm = { .min = 0, .max = UINT32_MAX,},
+ .eval = eval_alu,
+ },
+ [(BPF_ALU | BPF_LSH | BPF_K)] = {
+ .mask = {.dreg = WRT_REGS, .sreg = ZERO_REG},
+ .off = { .min = 0, .max = 0},
+ .imm = { .min = 0, .max = UINT32_MAX,},
+ .eval = eval_alu,
+ },
+ [(BPF_ALU | BPF_RSH | BPF_K)] = {
+ .mask = {.dreg = WRT_REGS, .sreg = ZERO_REG},
+ .off = { .min = 0, .max = 0},
+ .imm = { .min = 0, .max = UINT32_MAX,},
+ .eval = eval_alu,
+ },
+ [(BPF_ALU | BPF_XOR | BPF_K)] = {
+ .mask = {.dreg = WRT_REGS, .sreg = ZERO_REG},
+ .off = { .min = 0, .max = 0},
+ .imm = { .min = 0, .max = UINT32_MAX,},
+ .eval = eval_alu,
+ },
+ [(BPF_ALU | BPF_MUL | BPF_K)] = {
+ .mask = {.dreg = WRT_REGS, .sreg = ZERO_REG},
+ .off = { .min = 0, .max = 0},
+ .imm = { .min = 0, .max = UINT32_MAX,},
+ .eval = eval_alu,
+ },
+ [(BPF_ALU | EBPF_MOV | BPF_K)] = {
+ .mask = {.dreg = WRT_REGS, .sreg = ZERO_REG},
+ .off = { .min = 0, .max = 0},
+ .imm = { .min = 0, .max = UINT32_MAX,},
+ .eval = eval_alu,
+ },
+ [(BPF_ALU | BPF_DIV | BPF_K)] = {
+ .mask = { .dreg = WRT_REGS, .sreg = ZERO_REG},
+ .off = { .min = 0, .max = 0},
+ .imm = { .min = 1, .max = UINT32_MAX},
+ .eval = eval_alu,
+ },
+ [(BPF_ALU | BPF_MOD | BPF_K)] = {
+ .mask = { .dreg = WRT_REGS, .sreg = ZERO_REG},
+ .off = { .min = 0, .max = 0},
+ .imm = { .min = 1, .max = UINT32_MAX},
+ .eval = eval_alu,
+ },
+ /* ALU IMM 64-bit instructions */
+ [(EBPF_ALU64 | BPF_ADD | BPF_K)] = {
+ .mask = {.dreg = WRT_REGS, .sreg = ZERO_REG},
+ .off = { .min = 0, .max = 0},
+ .imm = { .min = 0, .max = UINT32_MAX,},
+ .eval = eval_alu,
+ },
+ [(EBPF_ALU64 | BPF_SUB | BPF_K)] = {
+ .mask = {.dreg = WRT_REGS, .sreg = ZERO_REG},
+ .off = { .min = 0, .max = 0},
+ .imm = { .min = 0, .max = UINT32_MAX,},
+ .eval = eval_alu,
+ },
+ [(EBPF_ALU64 | BPF_AND | BPF_K)] = {
+ .mask = {.dreg = WRT_REGS, .sreg = ZERO_REG},
+ .off = { .min = 0, .max = 0},
+ .imm = { .min = 0, .max = UINT32_MAX,},
+ .eval = eval_alu,
+ },
+ [(EBPF_ALU64 | BPF_OR | BPF_K)] = {
+ .mask = {.dreg = WRT_REGS, .sreg = ZERO_REG},
+ .off = { .min = 0, .max = 0},
+ .imm = { .min = 0, .max = UINT32_MAX,},
+ .eval = eval_alu,
+ },
+ [(EBPF_ALU64 | BPF_LSH | BPF_K)] = {
+ .mask = {.dreg = WRT_REGS, .sreg = ZERO_REG},
+ .off = { .min = 0, .max = 0},
+ .imm = { .min = 0, .max = UINT32_MAX,},
+ .eval = eval_alu,
+ },
+ [(EBPF_ALU64 | BPF_RSH | BPF_K)] = {
+ .mask = {.dreg = WRT_REGS, .sreg = ZERO_REG},
+ .off = { .min = 0, .max = 0},
+ .imm = { .min = 0, .max = UINT32_MAX,},
+ .eval = eval_alu,
+ },
+ [(EBPF_ALU64 | EBPF_ARSH | BPF_K)] = {
+ .mask = {.dreg = WRT_REGS, .sreg = ZERO_REG},
+ .off = { .min = 0, .max = 0},
+ .imm = { .min = 0, .max = UINT32_MAX,},
+ .eval = eval_alu,
+ },
+ [(EBPF_ALU64 | BPF_XOR | BPF_K)] = {
+ .mask = {.dreg = WRT_REGS, .sreg = ZERO_REG},
+ .off = { .min = 0, .max = 0},
+ .imm = { .min = 0, .max = UINT32_MAX,},
+ .eval = eval_alu,
+ },
+ [(EBPF_ALU64 | BPF_MUL | BPF_K)] = {
+ .mask = {.dreg = WRT_REGS, .sreg = ZERO_REG},
+ .off = { .min = 0, .max = 0},
+ .imm = { .min = 0, .max = UINT32_MAX,},
+ .eval = eval_alu,
+ },
+ [(EBPF_ALU64 | EBPF_MOV | BPF_K)] = {
+ .mask = {.dreg = WRT_REGS, .sreg = ZERO_REG},
+ .off = { .min = 0, .max = 0},
+ .imm = { .min = 0, .max = UINT32_MAX,},
+ .eval = eval_alu,
+ },
+ [(EBPF_ALU64 | BPF_DIV | BPF_K)] = {
+ .mask = { .dreg = WRT_REGS, .sreg = ZERO_REG},
+ .off = { .min = 0, .max = 0},
+ .imm = { .min = 1, .max = UINT32_MAX},
+ .eval = eval_alu,
+ },
+ [(EBPF_ALU64 | BPF_MOD | BPF_K)] = {
+ .mask = { .dreg = WRT_REGS, .sreg = ZERO_REG},
+ .off = { .min = 0, .max = 0},
+ .imm = { .min = 1, .max = UINT32_MAX},
+ .eval = eval_alu,
+ },
+ /* ALU REG 32-bit instructions */
+ [(BPF_ALU | BPF_ADD | BPF_X)] = {
+ .mask = { .dreg = WRT_REGS, .sreg = ALL_REGS},
+ .off = { .min = 0, .max = 0},
+ .imm = { .min = 0, .max = 0},
+ .eval = eval_alu,
+ },
+ [(BPF_ALU | BPF_SUB | BPF_X)] = {
+ .mask = { .dreg = WRT_REGS, .sreg = ALL_REGS},
+ .off = { .min = 0, .max = 0},
+ .imm = { .min = 0, .max = 0},
+ .eval = eval_alu,
+ },
+ [(BPF_ALU | BPF_AND | BPF_X)] = {
+ .mask = { .dreg = WRT_REGS, .sreg = ALL_REGS},
+ .off = { .min = 0, .max = 0},
+ .imm = { .min = 0, .max = 0},
+ .eval = eval_alu,
+ },
+ [(BPF_ALU | BPF_OR | BPF_X)] = {
+ .mask = { .dreg = WRT_REGS, .sreg = ALL_REGS},
+ .off = { .min = 0, .max = 0},
+ .imm = { .min = 0, .max = 0},
+ .eval = eval_alu,
+ },
+ [(BPF_ALU | BPF_LSH | BPF_X)] = {
+ .mask = { .dreg = WRT_REGS, .sreg = ALL_REGS},
+ .off = { .min = 0, .max = 0},
+ .imm = { .min = 0, .max = 0},
+ .eval = eval_alu,
+ },
+ [(BPF_ALU | BPF_RSH | BPF_X)] = {
+ .mask = { .dreg = WRT_REGS, .sreg = ALL_REGS},
+ .off = { .min = 0, .max = 0},
+ .imm = { .min = 0, .max = 0},
+ .eval = eval_alu,
+ },
+ [(BPF_ALU | BPF_XOR | BPF_X)] = {
+ .mask = { .dreg = WRT_REGS, .sreg = ALL_REGS},
+ .off = { .min = 0, .max = 0},
+ .imm = { .min = 0, .max = 0},
+ .eval = eval_alu,
+ },
+ [(BPF_ALU | BPF_MUL | BPF_X)] = {
+ .mask = { .dreg = WRT_REGS, .sreg = ALL_REGS},
+ .off = { .min = 0, .max = 0},
+ .imm = { .min = 0, .max = 0},
+ .eval = eval_alu,
+ },
+ [(BPF_ALU | BPF_DIV | BPF_X)] = {
+ .mask = { .dreg = WRT_REGS, .sreg = ALL_REGS},
+ .off = { .min = 0, .max = 0},
+ .imm = { .min = 0, .max = 0},
+ .eval = eval_alu,
+ },
+ [(BPF_ALU | BPF_MOD | BPF_X)] = {
+ .mask = { .dreg = WRT_REGS, .sreg = ALL_REGS},
+ .off = { .min = 0, .max = 0},
+ .imm = { .min = 0, .max = 0},
+ .eval = eval_alu,
+ },
+ [(BPF_ALU | EBPF_MOV | BPF_X)] = {
+ .mask = { .dreg = WRT_REGS, .sreg = ALL_REGS},
+ .off = { .min = 0, .max = 0},
+ .imm = { .min = 0, .max = 0},
+ .eval = eval_alu,
+ },
+ [(BPF_ALU | BPF_NEG)] = {
+ .mask = { .dreg = WRT_REGS, .sreg = ZERO_REG},
+ .off = { .min = 0, .max = 0},
+ .imm = { .min = 0, .max = 0},
+ .eval = eval_alu,
+ },
+ [(BPF_ALU | EBPF_END | EBPF_TO_BE)] = {
+ .mask = { .dreg = WRT_REGS, .sreg = ZERO_REG},
+ .off = { .min = 0, .max = 0},
+ .imm = { .min = 16, .max = 64},
+ .check = check_alu_bele,
+ .eval = eval_bele,
+ },
+ [(BPF_ALU | EBPF_END | EBPF_TO_LE)] = {
+ .mask = { .dreg = WRT_REGS, .sreg = ZERO_REG},
+ .off = { .min = 0, .max = 0},
+ .imm = { .min = 16, .max = 64},
+ .check = check_alu_bele,
+ .eval = eval_bele,
+ },
+ /* ALU REG 64-bit instructions */
+ [(EBPF_ALU64 | BPF_ADD | BPF_X)] = {
+ .mask = { .dreg = WRT_REGS, .sreg = ALL_REGS},
+ .off = { .min = 0, .max = 0},
+ .imm = { .min = 0, .max = 0},
+ .eval = eval_alu,
+ },
+ [(EBPF_ALU64 | BPF_SUB | BPF_X)] = {
+ .mask = { .dreg = WRT_REGS, .sreg = ALL_REGS},
+ .off = { .min = 0, .max = 0},
+ .imm = { .min = 0, .max = 0},
+ .eval = eval_alu,
+ },
+ [(EBPF_ALU64 | BPF_AND | BPF_X)] = {
+ .mask = { .dreg = WRT_REGS, .sreg = ALL_REGS},
+ .off = { .min = 0, .max = 0},
+ .imm = { .min = 0, .max = 0},
+ .eval = eval_alu,
+ },
+ [(EBPF_ALU64 | BPF_OR | BPF_X)] = {
+ .mask = { .dreg = WRT_REGS, .sreg = ALL_REGS},
+ .off = { .min = 0, .max = 0},
+ .imm = { .min = 0, .max = 0},
+ .eval = eval_alu,
+ },
+ [(EBPF_ALU64 | BPF_LSH | BPF_X)] = {
+ .mask = { .dreg = WRT_REGS, .sreg = ALL_REGS},
+ .off = { .min = 0, .max = 0},
+ .imm = { .min = 0, .max = 0},
+ .eval = eval_alu,
+ },
+ [(EBPF_ALU64 | BPF_RSH | BPF_X)] = {
+ .mask = { .dreg = WRT_REGS, .sreg = ALL_REGS},
+ .off = { .min = 0, .max = 0},
+ .imm = { .min = 0, .max = 0},
+ .eval = eval_alu,
+ },
+ [(EBPF_ALU64 | EBPF_ARSH | BPF_X)] = {
+ .mask = { .dreg = WRT_REGS, .sreg = ALL_REGS},
+ .off = { .min = 0, .max = 0},
+ .imm = { .min = 0, .max = 0},
+ .eval = eval_alu,
+ },
+ [(EBPF_ALU64 | BPF_XOR | BPF_X)] = {
+ .mask = { .dreg = WRT_REGS, .sreg = ALL_REGS},
+ .off = { .min = 0, .max = 0},
+ .imm = { .min = 0, .max = 0},
+ .eval = eval_alu,
+ },
+ [(EBPF_ALU64 | BPF_MUL | BPF_X)] = {
+ .mask = { .dreg = WRT_REGS, .sreg = ALL_REGS},
+ .off = { .min = 0, .max = 0},
+ .imm = { .min = 0, .max = 0},
+ .eval = eval_alu,
+ },
+ [(EBPF_ALU64 | BPF_DIV | BPF_X)] = {
+ .mask = { .dreg = WRT_REGS, .sreg = ALL_REGS},
+ .off = { .min = 0, .max = 0},
+ .imm = { .min = 0, .max = 0},
+ .eval = eval_alu,
+ },
+ [(EBPF_ALU64 | BPF_MOD | BPF_X)] = {
+ .mask = { .dreg = WRT_REGS, .sreg = ALL_REGS},
+ .off = { .min = 0, .max = 0},
+ .imm = { .min = 0, .max = 0},
+ .eval = eval_alu,
+ },
+ [(EBPF_ALU64 | EBPF_MOV | BPF_X)] = {
+ .mask = { .dreg = WRT_REGS, .sreg = ALL_REGS},
+ .off = { .min = 0, .max = 0},
+ .imm = { .min = 0, .max = 0},
+ .eval = eval_alu,
+ },
+ [(EBPF_ALU64 | BPF_NEG)] = {
+ .mask = { .dreg = WRT_REGS, .sreg = ZERO_REG},
+ .off = { .min = 0, .max = 0},
+ .imm = { .min = 0, .max = 0},
+ .eval = eval_alu,
+ },
+ /* load instructions */
+ [(BPF_LDX | BPF_MEM | BPF_B)] = {
+ .mask = {. dreg = WRT_REGS, .sreg = ALL_REGS},
+ .off = { .min = 0, .max = UINT16_MAX},
+ .imm = { .min = 0, .max = 0},
+ .eval = eval_load,
+ },
+ [(BPF_LDX | BPF_MEM | BPF_H)] = {
+ .mask = {. dreg = WRT_REGS, .sreg = ALL_REGS},
+ .off = { .min = 0, .max = UINT16_MAX},
+ .imm = { .min = 0, .max = 0},
+ .eval = eval_load,
+ },
+ [(BPF_LDX | BPF_MEM | BPF_W)] = {
+ .mask = {. dreg = WRT_REGS, .sreg = ALL_REGS},
+ .off = { .min = 0, .max = UINT16_MAX},
+ .imm = { .min = 0, .max = 0},
+ .eval = eval_load,
+ },
+ [(BPF_LDX | BPF_MEM | EBPF_DW)] = {
+ .mask = {. dreg = WRT_REGS, .sreg = ALL_REGS},
+ .off = { .min = 0, .max = UINT16_MAX},
+ .imm = { .min = 0, .max = 0},
+ .eval = eval_load,
+ },
+ /* load 64 bit immediate value */
+ [(BPF_LD | BPF_IMM | EBPF_DW)] = {
+ .mask = { .dreg = WRT_REGS, .sreg = ZERO_REG},
+ .off = { .min = 0, .max = 0},
+ .imm = { .min = 0, .max = UINT32_MAX},
+ .eval = eval_ld_imm64,
+ },
+ /* store REG instructions */
+ [(BPF_STX | BPF_MEM | BPF_B)] = {
+ .mask = { .dreg = ALL_REGS, .sreg = ALL_REGS},
+ .off = { .min = 0, .max = UINT16_MAX},
+ .imm = { .min = 0, .max = 0},
+ .eval = eval_store,
+ },
+ [(BPF_STX | BPF_MEM | BPF_H)] = {
+ .mask = { .dreg = ALL_REGS, .sreg = ALL_REGS},
+ .off = { .min = 0, .max = UINT16_MAX},
+ .imm = { .min = 0, .max = 0},
+ .eval = eval_store,
+ },
+ [(BPF_STX | BPF_MEM | BPF_W)] = {
+ .mask = { .dreg = ALL_REGS, .sreg = ALL_REGS},
+ .off = { .min = 0, .max = UINT16_MAX},
+ .imm = { .min = 0, .max = 0},
+ .eval = eval_store,
+ },
+ [(BPF_STX | BPF_MEM | EBPF_DW)] = {
+ .mask = { .dreg = ALL_REGS, .sreg = ALL_REGS},
+ .off = { .min = 0, .max = UINT16_MAX},
+ .imm = { .min = 0, .max = 0},
+ .eval = eval_store,
+ },
+ /* atomic add instructions */
+ [(BPF_STX | EBPF_XADD | BPF_W)] = {
+ .mask = { .dreg = ALL_REGS, .sreg = ALL_REGS},
+ .off = { .min = 0, .max = UINT16_MAX},
+ .imm = { .min = 0, .max = 0},
+ .eval = eval_store,
+ },
+ [(BPF_STX | EBPF_XADD | EBPF_DW)] = {
+ .mask = { .dreg = ALL_REGS, .sreg = ALL_REGS},
+ .off = { .min = 0, .max = UINT16_MAX},
+ .imm = { .min = 0, .max = 0},
+ .eval = eval_store,
+ },
+ /* store IMM instructions */
+ [(BPF_ST | BPF_MEM | BPF_B)] = {
+ .mask = { .dreg = ALL_REGS, .sreg = ZERO_REG},
+ .off = { .min = 0, .max = UINT16_MAX},
+ .imm = { .min = 0, .max = UINT32_MAX},
+ .eval = eval_store,
+ },
+ [(BPF_ST | BPF_MEM | BPF_H)] = {
+ .mask = { .dreg = ALL_REGS, .sreg = ZERO_REG},
+ .off = { .min = 0, .max = UINT16_MAX},
+ .imm = { .min = 0, .max = UINT32_MAX},
+ .eval = eval_store,
+ },
+ [(BPF_ST | BPF_MEM | BPF_W)] = {
+ .mask = { .dreg = ALL_REGS, .sreg = ZERO_REG},
+ .off = { .min = 0, .max = UINT16_MAX},
+ .imm = { .min = 0, .max = UINT32_MAX},
+ .eval = eval_store,
+ },
+ [(BPF_ST | BPF_MEM | EBPF_DW)] = {
+ .mask = { .dreg = ALL_REGS, .sreg = ZERO_REG},
+ .off = { .min = 0, .max = UINT16_MAX},
+ .imm = { .min = 0, .max = UINT32_MAX},
+ .eval = eval_store,
+ },
+ /* jump instruction */
+ [(BPF_JMP | BPF_JA)] = {
+ .mask = { .dreg = ZERO_REG, .sreg = ZERO_REG},
+ .off = { .min = 0, .max = UINT16_MAX},
+ .imm = { .min = 0, .max = 0},
+ },
+ /* jcc IMM instructions */
+ [(BPF_JMP | BPF_JEQ | BPF_K)] = {
+ .mask = { .dreg = ALL_REGS, .sreg = ZERO_REG},
+ .off = { .min = 0, .max = UINT16_MAX},
+ .imm = { .min = 0, .max = UINT32_MAX},
+ .eval = eval_jcc,
+ },
+ [(BPF_JMP | EBPF_JNE | BPF_K)] = {
+ .mask = { .dreg = ALL_REGS, .sreg = ZERO_REG},
+ .off = { .min = 0, .max = UINT16_MAX},
+ .imm = { .min = 0, .max = UINT32_MAX},
+ .eval = eval_jcc,
+ },
+ [(BPF_JMP | BPF_JGT | BPF_K)] = {
+ .mask = { .dreg = ALL_REGS, .sreg = ZERO_REG},
+ .off = { .min = 0, .max = UINT16_MAX},
+ .imm = { .min = 0, .max = UINT32_MAX},
+ .eval = eval_jcc,
+ },
+ [(BPF_JMP | EBPF_JLT | BPF_K)] = {
+ .mask = { .dreg = ALL_REGS, .sreg = ZERO_REG},
+ .off = { .min = 0, .max = UINT16_MAX},
+ .imm = { .min = 0, .max = UINT32_MAX},
+ .eval = eval_jcc,
+ },
+ [(BPF_JMP | BPF_JGE | BPF_K)] = {
+ .mask = { .dreg = ALL_REGS, .sreg = ZERO_REG},
+ .off = { .min = 0, .max = UINT16_MAX},
+ .imm = { .min = 0, .max = UINT32_MAX},
+ .eval = eval_jcc,
+ },
+ [(BPF_JMP | EBPF_JLE | BPF_K)] = {
+ .mask = { .dreg = ALL_REGS, .sreg = ZERO_REG},
+ .off = { .min = 0, .max = UINT16_MAX},
+ .imm = { .min = 0, .max = UINT32_MAX},
+ .eval = eval_jcc,
+ },
+ [(BPF_JMP | EBPF_JSGT | BPF_K)] = {
+ .mask = { .dreg = ALL_REGS, .sreg = ZERO_REG},
+ .off = { .min = 0, .max = UINT16_MAX},
+ .imm = { .min = 0, .max = UINT32_MAX},
+ .eval = eval_jcc,
+ },
+ [(BPF_JMP | EBPF_JSLT | BPF_K)] = {
+ .mask = { .dreg = ALL_REGS, .sreg = ZERO_REG},
+ .off = { .min = 0, .max = UINT16_MAX},
+ .imm = { .min = 0, .max = UINT32_MAX},
+ .eval = eval_jcc,
+ },
+ [(BPF_JMP | EBPF_JSGE | BPF_K)] = {
+ .mask = { .dreg = ALL_REGS, .sreg = ZERO_REG},
+ .off = { .min = 0, .max = UINT16_MAX},
+ .imm = { .min = 0, .max = UINT32_MAX},
+ .eval = eval_jcc,
+ },
+ [(BPF_JMP | EBPF_JSLE | BPF_K)] = {
+ .mask = { .dreg = ALL_REGS, .sreg = ZERO_REG},
+ .off = { .min = 0, .max = UINT16_MAX},
+ .imm = { .min = 0, .max = UINT32_MAX},
+ .eval = eval_jcc,
+ },
+ [(BPF_JMP | BPF_JSET | BPF_K)] = {
+ .mask = { .dreg = ALL_REGS, .sreg = ZERO_REG},
+ .off = { .min = 0, .max = UINT16_MAX},
+ .imm = { .min = 0, .max = UINT32_MAX},
+ .eval = eval_jcc,
+ },
+ /* jcc REG instructions */
+ [(BPF_JMP | BPF_JEQ | BPF_X)] = {
+ .mask = { .dreg = ALL_REGS, .sreg = ALL_REGS},
+ .off = { .min = 0, .max = UINT16_MAX},
+ .imm = { .min = 0, .max = 0},
+ .eval = eval_jcc,
+ },
+ [(BPF_JMP | EBPF_JNE | BPF_X)] = {
+ .mask = { .dreg = ALL_REGS, .sreg = ALL_REGS},
+ .off = { .min = 0, .max = UINT16_MAX},
+ .imm = { .min = 0, .max = 0},
+ .eval = eval_jcc,
+ },
+ [(BPF_JMP | BPF_JGT | BPF_X)] = {
+ .mask = { .dreg = ALL_REGS, .sreg = ALL_REGS},
+ .off = { .min = 0, .max = UINT16_MAX},
+ .imm = { .min = 0, .max = 0},
+ .eval = eval_jcc,
+ },
+ [(BPF_JMP | EBPF_JLT | BPF_X)] = {
+ .mask = { .dreg = ALL_REGS, .sreg = ALL_REGS},
+ .off = { .min = 0, .max = UINT16_MAX},
+ .imm = { .min = 0, .max = 0},
+ .eval = eval_jcc,
+ },
+ [(BPF_JMP | BPF_JGE | BPF_X)] = {
+ .mask = { .dreg = ALL_REGS, .sreg = ALL_REGS},
+ .off = { .min = 0, .max = UINT16_MAX},
+ .imm = { .min = 0, .max = 0},
+ .eval = eval_jcc,
+ },
+ [(BPF_JMP | EBPF_JLE | BPF_X)] = {
+ .mask = { .dreg = ALL_REGS, .sreg = ALL_REGS},
+ .off = { .min = 0, .max = UINT16_MAX},
+ .imm = { .min = 0, .max = 0},
+ .eval = eval_jcc,
+ },
+ [(BPF_JMP | EBPF_JSGT | BPF_X)] = {
+ .mask = { .dreg = ALL_REGS, .sreg = ALL_REGS},
+ .off = { .min = 0, .max = UINT16_MAX},
+ .imm = { .min = 0, .max = 0},
+ .eval = eval_jcc,
+ },
+ [(BPF_JMP | EBPF_JSLT | BPF_X)] = {
+ .mask = { .dreg = ALL_REGS, .sreg = ALL_REGS},
+ .off = { .min = 0, .max = UINT16_MAX},
+ .imm = { .min = 0, .max = 0},
+ },
+ [(BPF_JMP | EBPF_JSGE | BPF_X)] = {
+ .mask = { .dreg = ALL_REGS, .sreg = ALL_REGS},
+ .off = { .min = 0, .max = UINT16_MAX},
+ .imm = { .min = 0, .max = 0},
+ .eval = eval_jcc,
+ },
+ [(BPF_JMP | EBPF_JSLE | BPF_X)] = {
+ .mask = { .dreg = ALL_REGS, .sreg = ALL_REGS},
+ .off = { .min = 0, .max = UINT16_MAX},
+ .imm = { .min = 0, .max = 0},
+ .eval = eval_jcc,
+ },
+ [(BPF_JMP | BPF_JSET | BPF_X)] = {
+ .mask = { .dreg = ALL_REGS, .sreg = ALL_REGS},
+ .off = { .min = 0, .max = UINT16_MAX},
+ .imm = { .min = 0, .max = 0},
+ .eval = eval_jcc,
+ },
+ /* call instruction */
+ [(BPF_JMP | EBPF_CALL)] = {
+ .mask = { .dreg = ZERO_REG, .sreg = ZERO_REG},
+ .off = { .min = 0, .max = 0},
+ .imm = { .min = 0, .max = UINT32_MAX},
+ .eval = eval_call,
+ },
+ /* ret instruction */
+ [(BPF_JMP | EBPF_EXIT)] = {
+ .mask = { .dreg = ZERO_REG, .sreg = ZERO_REG},
+ .off = { .min = 0, .max = 0},
+ .imm = { .min = 0, .max = 0},
+ .eval = eval_exit,
+ },
+};
+
+/*
+ * make sure that instruction syntax is valid,
+ * and it fields don't violate partciular instrcution type restrictions.
+ */
+static const char *
+check_syntax(const struct ebpf_insn *ins)
+{
+
+ uint8_t op;
+ uint16_t off;
+ uint32_t imm;
+
+ op = ins->code;
+
+ if (ins_chk[op].mask.dreg == 0)
+ return "invalid opcode";
+
+ if ((ins_chk[op].mask.dreg & 1 << ins->dst_reg) == 0)
+ return "invalid dst-reg field";
+
+ if ((ins_chk[op].mask.sreg & 1 << ins->src_reg) == 0)
+ return "invalid src-reg field";
+
+ off = ins->off;
+ if (ins_chk[op].off.min > off || ins_chk[op].off.max < off)
+ return "invalid off field";
+
+ imm = ins->imm;
+ if (ins_chk[op].imm.min > imm || ins_chk[op].imm.max < imm)
+ return "invalid imm field";
+
+ if (ins_chk[op].check != NULL)
+ return ins_chk[op].check(ins);
+
+ return NULL;
+}
+
+/*
+ * helper function, return instruction index for the given node.
+ */
+static uint32_t
+get_node_idx(const struct bpf_verifier *bvf, const struct inst_node *node)
+{
+ return node - bvf->in;
+}
+
+/*
+ * helper function, used to walk through constructed CFG.
+ */
+static struct inst_node *
+get_next_node(struct bpf_verifier *bvf, struct inst_node *node)
+{
+ uint32_t ce, ne, dst;
+
+ ne = node->nb_edge;
+ ce = node->cur_edge;
+ if (ce == ne)
+ return NULL;
+
+ node->cur_edge++;
+ dst = node->edge_dest[ce];
+ return bvf->in + dst;
+}
+
+static void
+set_node_colour(struct bpf_verifier *bvf, struct inst_node *node,
+ uint32_t new)
+{
+ uint32_t prev;
+
+ prev = node->colour;
+ node->colour = new;
+
+ bvf->node_colour[prev]--;
+ bvf->node_colour[new]++;
+}
+
+/*
+ * helper function, add new edge between two nodes.
+ */
+static int
+add_edge(struct bpf_verifier *bvf, struct inst_node *node, uint32_t nidx)
+{
+ uint32_t ne;
+
+ if (nidx > bvf->prm->nb_ins) {
+ RTE_BPF_LOG(ERR, "%s: program boundary violation at pc: %u, "
+ "next pc: %u\n",
+ __func__, get_node_idx(bvf, node), nidx);
+ return -EINVAL;
+ }
+
+ ne = node->nb_edge;
+ if (ne >= RTE_DIM(node->edge_dest)) {
+ RTE_BPF_LOG(ERR, "%s: internal error at pc: %u\n",
+ __func__, get_node_idx(bvf, node));
+ return -EINVAL;
+ }
+
+ node->edge_dest[ne] = nidx;
+ node->nb_edge = ne + 1;
+ return 0;
+}
+
+/*
+ * helper function, determine type of edge between two nodes.
+ */
+static void
+set_edge_type(struct bpf_verifier *bvf, struct inst_node *node,
+ const struct inst_node *next)
+{
+ uint32_t ce, clr, type;
+
+ ce = node->cur_edge - 1;
+ clr = next->colour;
+
+ type = UNKNOWN_EDGE;
+
+ if (clr == WHITE)
+ type = TREE_EDGE;
+ else if (clr == GREY)
+ type = BACK_EDGE;
+ else if (clr == BLACK)
+ /*
+ * in fact it could be either direct or cross edge,
+ * but for now, we don't need to distinguish between them.
+ */
+ type = CROSS_EDGE;
+
+ node->edge_type[ce] = type;
+ bvf->edge_type[type]++;
+}
+
+static struct inst_node *
+get_prev_node(struct bpf_verifier *bvf, struct inst_node *node)
+{
+ return bvf->in + node->prev_node;
+}
+
+/*
+ * Depth-First Search (DFS) through previously constructed
+ * Control Flow Graph (CFG).
+ * Information collected at this path would be used later
+ * to determine is there any loops, and/or unreachable instructions.
+ */
+static void
+dfs(struct bpf_verifier *bvf)
+{
+ struct inst_node *next, *node;
+
+ node = bvf->in;
+ while (node != NULL) {
+
+ if (node->colour == WHITE)
+ set_node_colour(bvf, node, GREY);
+
+ if (node->colour == GREY) {
+
+ /* find next unprocessed child node */
+ do {
+ next = get_next_node(bvf, node);
+ if (next == NULL)
+ break;
+ set_edge_type(bvf, node, next);
+ } while (next->colour != WHITE);
+
+ if (next != NULL) {
+ /* proceed with next child */
+ next->prev_node = get_node_idx(bvf, node);
+ node = next;
+ } else {
+ /*
+ * finished with current node and all it's kids,
+ * proceed with parent
+ */
+ set_node_colour(bvf, node, BLACK);
+ node->cur_edge = 0;
+ node = get_prev_node(bvf, node);
+ }
+ } else
+ node = NULL;
+ }
+}
+
+/*
+ * report unreachable instructions.
+ */
+static void
+log_unreachable(const struct bpf_verifier *bvf)
+{
+ uint32_t i;
+ struct inst_node *node;
+ const struct ebpf_insn *ins;
+
+ for (i = 0; i != bvf->prm->nb_ins; i++) {
+
+ node = bvf->in + i;
+ ins = bvf->prm->ins + i;
+
+ if (node->colour == WHITE &&
+ ins->code != (BPF_LD | BPF_IMM | EBPF_DW))
+ RTE_BPF_LOG(ERR, "unreachable code at pc: %u;\n", i);
+ }
+}
+
+/*
+ * report loops detected.
+ */
+static void
+log_loop(const struct bpf_verifier *bvf)
+{
+ uint32_t i, j;
+ struct inst_node *node;
+
+ for (i = 0; i != bvf->prm->nb_ins; i++) {
+
+ node = bvf->in + i;
+ if (node->colour != BLACK)
+ continue;
+
+ for (j = 0; j != node->nb_edge; j++) {
+ if (node->edge_type[j] == BACK_EDGE)
+ RTE_BPF_LOG(ERR,
+ "loop at pc:%u --> pc:%u;\n",
+ i, node->edge_dest[j]);
+ }
+ }
+}
+
+/*
+ * First pass goes though all instructions in the set, checks that each
+ * instruction is a valid one (correct syntax, valid field values, etc.)
+ * and constructs control flow graph (CFG).
+ * Then deapth-first search is performed over the constructed graph.
+ * Programs with unreachable instructions and/or loops will be rejected.
+ */
+static int
+validate(struct bpf_verifier *bvf)
+{
+ int32_t rc;
+ uint32_t i;
+ struct inst_node *node;
+ const struct ebpf_insn *ins;
+ const char *err;
+
+ rc = 0;
+ for (i = 0; i < bvf->prm->nb_ins; i++) {
+
+ ins = bvf->prm->ins + i;
+ node = bvf->in + i;
+
+ err = check_syntax(ins);
+ if (err != 0) {
+ RTE_BPF_LOG(ERR, "%s: %s at pc: %u\n",
+ __func__, err, i);
+ rc |= -EINVAL;
+ }
+
+ /*
+ * construct CFG, jcc nodes have to outgoing edges,
+ * 'exit' nodes - none, all others nodes have exaclty one
+ * outgoing edge.
+ */
+ switch (ins->code) {
+ case (BPF_JMP | EBPF_EXIT):
+ break;
+ case (BPF_JMP | BPF_JEQ | BPF_K):
+ case (BPF_JMP | EBPF_JNE | BPF_K):
+ case (BPF_JMP | BPF_JGT | BPF_K):
+ case (BPF_JMP | EBPF_JLT | BPF_K):
+ case (BPF_JMP | BPF_JGE | BPF_K):
+ case (BPF_JMP | EBPF_JLE | BPF_K):
+ case (BPF_JMP | EBPF_JSGT | BPF_K):
+ case (BPF_JMP | EBPF_JSLT | BPF_K):
+ case (BPF_JMP | EBPF_JSGE | BPF_K):
+ case (BPF_JMP | EBPF_JSLE | BPF_K):
+ case (BPF_JMP | BPF_JSET | BPF_K):
+ case (BPF_JMP | BPF_JEQ | BPF_X):
+ case (BPF_JMP | EBPF_JNE | BPF_X):
+ case (BPF_JMP | BPF_JGT | BPF_X):
+ case (BPF_JMP | EBPF_JLT | BPF_X):
+ case (BPF_JMP | BPF_JGE | BPF_X):
+ case (BPF_JMP | EBPF_JLE | BPF_X):
+ case (BPF_JMP | EBPF_JSGT | BPF_X):
+ case (BPF_JMP | EBPF_JSLT | BPF_X):
+ case (BPF_JMP | EBPF_JSGE | BPF_X):
+ case (BPF_JMP | EBPF_JSLE | BPF_X):
+ case (BPF_JMP | BPF_JSET | BPF_X):
+ rc |= add_edge(bvf, node, i + ins->off + 1);
+ rc |= add_edge(bvf, node, i + 1);
+ bvf->nb_jcc_nodes++;
+ break;
+ case (BPF_JMP | BPF_JA):
+ rc |= add_edge(bvf, node, i + ins->off + 1);
+ break;
+ /* load 64 bit immediate value */
+ case (BPF_LD | BPF_IMM | EBPF_DW):
+ rc |= add_edge(bvf, node, i + 2);
+ i++;
+ break;
+ default:
+ rc |= add_edge(bvf, node, i + 1);
+ break;
+ }
+
+ bvf->nb_nodes++;
+ bvf->node_colour[WHITE]++;
+ }
+
+ if (rc != 0)
+ return rc;
+
+ dfs(bvf);
+
+ RTE_BPF_LOG(DEBUG, "%s(%p) stats:\n"
+ "nb_nodes=%u;\n"
+ "nb_jcc_nodes=%u;\n"
+ "node_color={[WHITE]=%u, [GREY]=%u,, [BLACK]=%u};\n"
+ "edge_type={[UNKNOWN]=%u, [TREE]=%u, [BACK]=%u, [CROSS]=%u};\n",
+ __func__, bvf,
+ bvf->nb_nodes,
+ bvf->nb_jcc_nodes,
+ bvf->node_colour[WHITE], bvf->node_colour[GREY],
+ bvf->node_colour[BLACK],
+ bvf->edge_type[UNKNOWN_EDGE], bvf->edge_type[TREE_EDGE],
+ bvf->edge_type[BACK_EDGE], bvf->edge_type[CROSS_EDGE]);
+
+ if (bvf->node_colour[BLACK] != bvf->nb_nodes) {
+ RTE_BPF_LOG(ERR, "%s(%p) unreachable instructions;\n",
+ __func__, bvf);
+ log_unreachable(bvf);
+ return -EINVAL;
+ }
+
+ if (bvf->node_colour[GREY] != 0 || bvf->node_colour[WHITE] != 0 ||
+ bvf->edge_type[UNKNOWN_EDGE] != 0) {
+ RTE_BPF_LOG(ERR, "%s(%p) DFS internal error;\n",
+ __func__, bvf);
+ return -EINVAL;
+ }
+
+ if (bvf->edge_type[BACK_EDGE] != 0) {
+ RTE_BPF_LOG(ERR, "%s(%p) loops detected;\n",
+ __func__, bvf);
+ log_loop(bvf);
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
+/*
+ * helper functions get/free eval states.
+ */
+static struct bpf_eval_state *
+pull_eval_state(struct bpf_verifier *bvf)
+{
+ uint32_t n;
+
+ n = bvf->evst_pool.cur;
+ if (n == bvf->evst_pool.num)
+ return NULL;
+
+ bvf->evst_pool.cur = n + 1;
+ return bvf->evst_pool.ent + n;
+}
+
+static void
+push_eval_state(struct bpf_verifier *bvf)
+{
+ bvf->evst_pool.cur--;
+}
+
+static void
+evst_pool_fini(struct bpf_verifier *bvf)
+{
+ bvf->evst = NULL;
+ free(bvf->evst_pool.ent);
+ memset(&bvf->evst_pool, 0, sizeof(bvf->evst_pool));
+}
+
+static int
+evst_pool_init(struct bpf_verifier *bvf)
+{
+ uint32_t n;
+
+ n = bvf->nb_jcc_nodes + 1;
+
+ bvf->evst_pool.ent = calloc(n, sizeof(bvf->evst_pool.ent[0]));
+ if (bvf->evst_pool.ent == NULL)
+ return -ENOMEM;
+
+ bvf->evst_pool.num = n;
+ bvf->evst_pool.cur = 0;
+
+ bvf->evst = pull_eval_state(bvf);
+ return 0;
+}
+
+/*
+ * Save current eval state.
+ */
+static int
+save_eval_state(struct bpf_verifier *bvf, struct inst_node *node)
+{
+ struct bpf_eval_state *st;
+
+ /* get new eval_state for this node */
+ st = pull_eval_state(bvf);
+ if (st == NULL) {
+ RTE_BPF_LOG(ERR,
+ "%s: internal error (out of space) at pc: %u\n",
+ __func__, get_node_idx(bvf, node));
+ return -ENOMEM;
+ }
+
+ /* make a copy of current state */
+ memcpy(st, bvf->evst, sizeof(*st));
+
+ /* swap current state with new one */
+ node->evst = bvf->evst;
+ bvf->evst = st;
+
+ RTE_BPF_LOG(DEBUG, "%s(bvf=%p,node=%u) old/new states: %p/%p;\n",
+ __func__, bvf, get_node_idx(bvf, node), node->evst, bvf->evst);
+
+ return 0;
+}
+
+/*
+ * Restore previous eval state and mark current eval state as free.
+ */
+static void
+restore_eval_state(struct bpf_verifier *bvf, struct inst_node *node)
+{
+ RTE_BPF_LOG(DEBUG, "%s(bvf=%p,node=%u) old/new states: %p/%p;\n",
+ __func__, bvf, get_node_idx(bvf, node), bvf->evst, node->evst);
+
+ bvf->evst = node->evst;
+ node->evst = NULL;
+ push_eval_state(bvf);
+}
+
+static void
+log_eval_state(const struct bpf_verifier *bvf, const struct ebpf_insn *ins,
+ uint32_t pc, int32_t loglvl)
+{
+ const struct bpf_eval_state *st;
+ const struct bpf_reg_val *rv;
+
+ rte_log(loglvl, rte_bpf_logtype, "%s(pc=%u):\n", __func__, pc);
+
+ st = bvf->evst;
+ rv = st->rv + ins->dst_reg;
+
+ rte_log(loglvl, rte_bpf_logtype,
+ "r%u={\n"
+ "\tv={type=%u, size=%zu},\n"
+ "\tmask=0x%" PRIx64 ",\n"
+ "\tu={min=0x%" PRIx64 ", max=0x%" PRIx64 "},\n"
+ "\ts={min=%" PRId64 ", max=%" PRId64 "},\n"
+ "};\n",
+ ins->dst_reg,
+ rv->v.type, rv->v.size,
+ rv->mask,
+ rv->u.min, rv->u.max,
+ rv->s.min, rv->s.max);
+}
+
+/*
+ * Do second pass through CFG and try to evaluate instructions
+ * via each possible path.
+ * Right now evaluation functionality is quite limited.
+ * Still need to add extra checks for:
+ * - use/return uninitialized registers.
+ * - use uninitialized data from the stack.
+ * - memory boundaries violation.
+ */
+static int
+evaluate(struct bpf_verifier *bvf)
+{
+ int32_t rc;
+ uint32_t idx, op;
+ const char *err;
+ const struct ebpf_insn *ins;
+ struct inst_node *next, *node;
+
+ /* initial state of frame pointer */
+ static const struct bpf_reg_val rvfp = {
+ .v = {
+ .type = BPF_ARG_PTR_STACK,
+ .size = MAX_BPF_STACK_SIZE,
+ },
+ .mask = UINT64_MAX,
+ .u = {.min = MAX_BPF_STACK_SIZE, .max = MAX_BPF_STACK_SIZE},
+ .s = {.min = MAX_BPF_STACK_SIZE, .max = MAX_BPF_STACK_SIZE},
+ };
+
+ bvf->evst->rv[EBPF_REG_1].v = bvf->prm->prog_arg;
+ bvf->evst->rv[EBPF_REG_1].mask = UINT64_MAX;
+ if (bvf->prm->prog_arg.type == RTE_BPF_ARG_RAW)
+ eval_max_bound(bvf->evst->rv + EBPF_REG_1, UINT64_MAX);
+
+ bvf->evst->rv[EBPF_REG_10] = rvfp;
+
+ ins = bvf->prm->ins;
+ node = bvf->in;
+ next = node;
+ rc = 0;
+
+ while (node != NULL && rc == 0) {
+
+ /*
+ * current node evaluation, make sure we evaluate
+ * each node only once.
+ */
+ if (next != NULL) {
+
+ bvf->evin = node;
+ idx = get_node_idx(bvf, node);
+ op = ins[idx].code;
+
+ /* for jcc node make a copy of evaluatoion state */
+ if (node->nb_edge > 1)
+ rc |= save_eval_state(bvf, node);
+
+ if (ins_chk[op].eval != NULL && rc == 0) {
+ err = ins_chk[op].eval(bvf, ins + idx);
+ if (err != NULL) {
+ RTE_BPF_LOG(ERR, "%s: %s at pc: %u\n",
+ __func__, err, idx);
+ rc = -EINVAL;
+ }
+ }
+
+ log_eval_state(bvf, ins + idx, idx, RTE_LOG_DEBUG);
+ bvf->evin = NULL;
+ }
+
+ /* proceed through CFG */
+ next = get_next_node(bvf, node);
+ if (next != NULL) {
+
+ /* proceed with next child */
+ if (node->cur_edge == node->nb_edge &&
+ node->evst != NULL)
+ restore_eval_state(bvf, node);
+
+ next->prev_node = get_node_idx(bvf, node);
+ node = next;
+ } else {
+ /*
+ * finished with current node and all it's kids,
+ * proceed with parent
+ */
+ node->cur_edge = 0;
+ node = get_prev_node(bvf, node);
+
+ /* finished */
+ if (node == bvf->in)
+ node = NULL;
+ }
+ }
+
+ return rc;
+}
+
+int
+bpf_validate(struct rte_bpf *bpf)
+{
+ int32_t rc;
+ struct bpf_verifier bvf;
+
+ /* check input argument type, don't allow mbuf ptr on 32-bit */
+ if (bpf->prm.prog_arg.type != RTE_BPF_ARG_RAW &&
+ bpf->prm.prog_arg.type != RTE_BPF_ARG_PTR &&
+ (sizeof(uint64_t) != sizeof(uintptr_t) ||
+ bpf->prm.prog_arg.type != RTE_BPF_ARG_PTR_MBUF)) {
+ RTE_BPF_LOG(ERR, "%s: unsupported argument type\n", __func__);
+ return -ENOTSUP;
+ }
+
+ memset(&bvf, 0, sizeof(bvf));
+ bvf.prm = &bpf->prm;
+ bvf.in = calloc(bpf->prm.nb_ins, sizeof(bvf.in[0]));
+ if (bvf.in == NULL)
+ return -ENOMEM;
+
+ rc = validate(&bvf);
+
+ if (rc == 0) {
+ rc = evst_pool_init(&bvf);
+ if (rc == 0)
+ rc = evaluate(&bvf);
+ evst_pool_fini(&bvf);
+ }
+
+ free(bvf.in);
+
+ /* copy collected info */
+ if (rc == 0)
+ bpf->stack_sz = bvf.stack_sz;
+
+ return rc;
+}
diff --git a/src/spdk/dpdk/lib/librte_bpf/meson.build b/src/spdk/dpdk/lib/librte_bpf/meson.build
new file mode 100644
index 000000000..55d7a1560
--- /dev/null
+++ b/src/spdk/dpdk/lib/librte_bpf/meson.build
@@ -0,0 +1,29 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2018 Intel Corporation
+
+sources = files('bpf.c',
+ 'bpf_exec.c',
+ 'bpf_load.c',
+ 'bpf_pkt.c',
+ 'bpf_validate.c')
+
+if arch_subdir == 'x86' and dpdk_conf.get('RTE_ARCH_64')
+ sources += files('bpf_jit_x86.c')
+elif dpdk_conf.has('RTE_ARCH_ARM64')
+ sources += files('bpf_jit_arm64.c')
+endif
+
+install_headers('bpf_def.h',
+ 'rte_bpf.h',
+ 'rte_bpf_ethdev.h')
+
+deps += ['mbuf', 'net', 'ethdev']
+
+dep = dependency('libelf', required: false)
+if dep.found()
+ dpdk_conf.set('RTE_LIBRTE_BPF_ELF', 1)
+ sources += files('bpf_load_elf.c')
+ ext_deps += dep
+endif
+build = false
+reason = 'not needed by SPDK'
diff --git a/src/spdk/dpdk/lib/librte_bpf/rte_bpf.h b/src/spdk/dpdk/lib/librte_bpf/rte_bpf.h
new file mode 100644
index 000000000..cbf1cddac
--- /dev/null
+++ b/src/spdk/dpdk/lib/librte_bpf/rte_bpf.h
@@ -0,0 +1,212 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2018 Intel Corporation
+ */
+
+#ifndef _RTE_BPF_H_
+#define _RTE_BPF_H_
+
+/**
+ * @file rte_bpf.h
+ * @b EXPERIMENTAL: this API may change without prior notice
+ *
+ * RTE BPF support.
+ * librte_bpf provides a framework to load and execute eBPF bytecode
+ * inside user-space dpdk based applications.
+ * It supports basic set of features from eBPF spec
+ * (https://www.kernel.org/doc/Documentation/networking/filter.txt).
+ */
+
+#include <rte_common.h>
+#include <rte_mbuf.h>
+#include <bpf_def.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/**
+ * Possible types for function/BPF program arguments.
+ */
+enum rte_bpf_arg_type {
+ RTE_BPF_ARG_UNDEF, /**< undefined */
+ RTE_BPF_ARG_RAW, /**< scalar value */
+ RTE_BPF_ARG_PTR = 0x10, /**< pointer to data buffer */
+ RTE_BPF_ARG_PTR_MBUF, /**< pointer to rte_mbuf */
+ RTE_BPF_ARG_RESERVED, /**< reserved for internal use */
+};
+
+/**
+ * function argument information
+ */
+struct rte_bpf_arg {
+ enum rte_bpf_arg_type type;
+ /**
+ * for ptr type - max size of data buffer it points to
+ * for raw type - the size (in bytes) of the value
+ */
+ size_t size;
+ size_t buf_size;
+ /**< for mbuf ptr type, max size of rte_mbuf data buffer */
+};
+
+/**
+ * determine is argument a pointer
+ */
+#define RTE_BPF_ARG_PTR_TYPE(x) ((x) & RTE_BPF_ARG_PTR)
+
+/**
+ * Possible types for external symbols.
+ */
+enum rte_bpf_xtype {
+ RTE_BPF_XTYPE_FUNC, /**< function */
+ RTE_BPF_XTYPE_VAR, /**< variable */
+ RTE_BPF_XTYPE_NUM
+};
+
+/**
+ * Definition for external symbols available in the BPF program.
+ */
+struct rte_bpf_xsym {
+ const char *name; /**< name */
+ enum rte_bpf_xtype type; /**< type */
+ union {
+ struct {
+ uint64_t (*val)(uint64_t, uint64_t, uint64_t,
+ uint64_t, uint64_t);
+ uint32_t nb_args;
+ struct rte_bpf_arg args[EBPF_FUNC_MAX_ARGS];
+ /**< Function arguments descriptions. */
+ struct rte_bpf_arg ret; /**< function return value. */
+ } func;
+ struct {
+ void *val; /**< actual memory location */
+ struct rte_bpf_arg desc; /**< type, size, etc. */
+ } var; /**< external variable */
+ };
+};
+
+/**
+ * Input parameters for loading eBPF code.
+ */
+struct rte_bpf_prm {
+ const struct ebpf_insn *ins; /**< array of eBPF instructions */
+ uint32_t nb_ins; /**< number of instructions in ins */
+ const struct rte_bpf_xsym *xsym;
+ /**< array of external symbols that eBPF code is allowed to reference */
+ uint32_t nb_xsym; /**< number of elements in xsym */
+ struct rte_bpf_arg prog_arg; /**< eBPF program input arg description */
+};
+
+/**
+ * Information about compiled into native ISA eBPF code.
+ */
+struct rte_bpf_jit {
+ uint64_t (*func)(void *); /**< JIT-ed native code */
+ size_t sz; /**< size of JIT-ed code */
+};
+
+struct rte_bpf;
+
+/**
+ * De-allocate all memory used by this eBPF execution context.
+ *
+ * @param bpf
+ * BPF handle to destroy.
+ */
+__rte_experimental
+void
+rte_bpf_destroy(struct rte_bpf *bpf);
+
+/**
+ * Create a new eBPF execution context and load given BPF code into it.
+ *
+ * @param prm
+ * Parameters used to create and initialise the BPF execution context.
+ * @return
+ * BPF handle that is used in future BPF operations,
+ * or NULL on error, with error code set in rte_errno.
+ * Possible rte_errno errors include:
+ * - EINVAL - invalid parameter passed to function
+ * - ENOMEM - can't reserve enough memory
+ */
+__rte_experimental
+struct rte_bpf *
+rte_bpf_load(const struct rte_bpf_prm *prm);
+
+/**
+ * Create a new eBPF execution context and load BPF code from given ELF
+ * file into it.
+ * Note that if the function will encounter EBPF_PSEUDO_CALL instruction
+ * that references external symbol, it will treat is as standard BPF_CALL
+ * to the external helper function.
+ *
+ * @param prm
+ * Parameters used to create and initialise the BPF execution context.
+ * @param fname
+ * Pathname for a ELF file.
+ * @param sname
+ * Name of the executable section within the file to load.
+ * @return
+ * BPF handle that is used in future BPF operations,
+ * or NULL on error, with error code set in rte_errno.
+ * Possible rte_errno errors include:
+ * - EINVAL - invalid parameter passed to function
+ * - ENOMEM - can't reserve enough memory
+ */
+__rte_experimental
+struct rte_bpf *
+rte_bpf_elf_load(const struct rte_bpf_prm *prm, const char *fname,
+ const char *sname);
+/**
+ * Execute given BPF bytecode.
+ *
+ * @param bpf
+ * handle for the BPF code to execute.
+ * @param ctx
+ * pointer to input context.
+ * @return
+ * BPF execution return value.
+ */
+__rte_experimental
+uint64_t
+rte_bpf_exec(const struct rte_bpf *bpf, void *ctx);
+
+/**
+ * Execute given BPF bytecode over a set of input contexts.
+ *
+ * @param bpf
+ * handle for the BPF code to execute.
+ * @param ctx
+ * array of pointers to the input contexts.
+ * @param rc
+ * array of return values (one per input).
+ * @param num
+ * number of elements in ctx[] (and rc[]).
+ * @return
+ * number of successfully processed inputs.
+ */
+__rte_experimental
+uint32_t
+rte_bpf_exec_burst(const struct rte_bpf *bpf, void *ctx[], uint64_t rc[],
+ uint32_t num);
+
+/**
+ * Provide information about natively compiled code for given BPF handle.
+ *
+ * @param bpf
+ * handle for the BPF code.
+ * @param jit
+ * pointer to the rte_bpf_jit structure to be filled with related data.
+ * @return
+ * - -EINVAL if the parameters are invalid.
+ * - Zero if operation completed successfully.
+ */
+__rte_experimental
+int
+rte_bpf_get_jit(const struct rte_bpf *bpf, struct rte_bpf_jit *jit);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _RTE_BPF_H_ */
diff --git a/src/spdk/dpdk/lib/librte_bpf/rte_bpf_ethdev.h b/src/spdk/dpdk/lib/librte_bpf/rte_bpf_ethdev.h
new file mode 100644
index 000000000..0ce526e39
--- /dev/null
+++ b/src/spdk/dpdk/lib/librte_bpf/rte_bpf_ethdev.h
@@ -0,0 +1,121 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2018 Intel Corporation
+ */
+
+#ifndef _RTE_BPF_ETHDEV_H_
+#define _RTE_BPF_ETHDEV_H_
+
+/**
+ * @file rte_bpf_ethdev.h
+ * @b EXPERIMENTAL: this API may change without prior notice
+ *
+ * API to install BPF filter as RX/TX callbacks for eth devices.
+ * Note that right now:
+ * - it is not MT safe, i.e. it is not allowed to do load/unload for the
+ * same port/queue from different threads in parallel.
+ * - though it allows to do load/unload at runtime
+ * (while RX/TX is ongoing on given port/queue).
+ * - allows only one BPF program per port/queue,
+ * i.e. new load will replace previously loaded for that port/queue BPF program.
+ * Filter behaviour - if BPF program returns zero value for a given packet,
+ * then it will be dropped inside callback and no further processing
+ * on RX - it will be dropped inside callback and no further processing
+ * for that packet will happen.
+ * on TX - packet will remain unsent, and it is responsibility of the user
+ * to handle such situation (drop, try to send again, etc.).
+ */
+
+#include <rte_bpf.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+enum {
+ RTE_BPF_ETH_F_NONE = 0,
+ RTE_BPF_ETH_F_JIT = 0x1, /*< use compiled into native ISA code */
+};
+
+/**
+ * Unload previously loaded BPF program (if any) from given RX port/queue
+ * and remove appropriate RX port/queue callback.
+ *
+ * @param port
+ * The identifier of the ethernet port
+ * @param queue
+ * The identifier of the RX queue on the given port
+ */
+__rte_experimental
+void
+rte_bpf_eth_rx_unload(uint16_t port, uint16_t queue);
+
+/**
+ * Unload previously loaded BPF program (if any) from given TX port/queue
+ * and remove appropriate TX port/queue callback.
+ *
+ * @param port
+ * The identifier of the ethernet port
+ * @param queue
+ * The identifier of the TX queue on the given port
+ */
+__rte_experimental
+void
+rte_bpf_eth_tx_unload(uint16_t port, uint16_t queue);
+
+/**
+ * Load BPF program from the ELF file and install callback to execute it
+ * on given RX port/queue.
+ *
+ * @param port
+ * The identifier of the ethernet port
+ * @param queue
+ * The identifier of the RX queue on the given port
+ * @param fname
+ * Pathname for a ELF file.
+ * @param sname
+ * Name of the executable section within the file to load.
+ * @param prm
+ * Parameters used to create and initialise the BPF execution context.
+ * @param flags
+ * Flags that define expected behavior of the loaded filter
+ * (i.e. jited/non-jited version to use).
+ * @return
+ * Zero on successful completion or negative error code otherwise.
+ */
+__rte_experimental
+int
+rte_bpf_eth_rx_elf_load(uint16_t port, uint16_t queue,
+ const struct rte_bpf_prm *prm, const char *fname, const char *sname,
+ uint32_t flags);
+
+/**
+ * Load BPF program from the ELF file and install callback to execute it
+ * on given TX port/queue.
+ *
+ * @param port
+ * The identifier of the ethernet port
+ * @param queue
+ * The identifier of the TX queue on the given port
+ * @param fname
+ * Pathname for a ELF file.
+ * @param sname
+ * Name of the executable section within the file to load.
+ * @param prm
+ * Parameters used to create and initialise the BPF execution context.
+ * @param flags
+ * Flags that define expected expected behavior of the loaded filter
+ * (i.e. jited/non-jited version to use).
+ * @return
+ * Zero on successful completion or negative error code otherwise.
+ */
+__rte_experimental
+int
+rte_bpf_eth_tx_elf_load(uint16_t port, uint16_t queue,
+ const struct rte_bpf_prm *prm, const char *fname, const char *sname,
+ uint32_t flags);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _RTE_BPF_ETHDEV_H_ */
diff --git a/src/spdk/dpdk/lib/librte_bpf/rte_bpf_version.map b/src/spdk/dpdk/lib/librte_bpf/rte_bpf_version.map
new file mode 100644
index 000000000..a203e088e
--- /dev/null
+++ b/src/spdk/dpdk/lib/librte_bpf/rte_bpf_version.map
@@ -0,0 +1,16 @@
+EXPERIMENTAL {
+ global:
+
+ rte_bpf_destroy;
+ rte_bpf_elf_load;
+ rte_bpf_eth_rx_elf_load;
+ rte_bpf_eth_rx_unload;
+ rte_bpf_eth_tx_elf_load;
+ rte_bpf_eth_tx_unload;
+ rte_bpf_exec;
+ rte_bpf_exec_burst;
+ rte_bpf_get_jit;
+ rte_bpf_load;
+
+ local: *;
+};
generated by cgit v1.2.3 (git 2.39.1) at 2025-01-26 11:15:47 +0000