diff options
author | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-04-27 18:24:20 +0000 |
---|---|---|
committer | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-04-27 18:24:20 +0000 |
commit | 483eb2f56657e8e7f419ab1a4fab8dce9ade8609 (patch) | |
tree | e5d88d25d870d5dedacb6bbdbe2a966086a0a5cf /src/spdk/dpdk/lib/librte_pipeline | |
parent | Initial commit. (diff) | |
download | ceph-upstream.tar.xz ceph-upstream.zip |
Adding upstream version 14.2.21.upstream/14.2.21upstream
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'src/spdk/dpdk/lib/librte_pipeline')
-rw-r--r-- | src/spdk/dpdk/lib/librte_pipeline/Makefile | 31 | ||||
-rw-r--r-- | src/spdk/dpdk/lib/librte_pipeline/meson.build | 8 | ||||
-rw-r--r-- | src/spdk/dpdk/lib/librte_pipeline/rte_pipeline.c | 1619 | ||||
-rw-r--r-- | src/spdk/dpdk/lib/librte_pipeline/rte_pipeline.h | 848 | ||||
-rw-r--r-- | src/spdk/dpdk/lib/librte_pipeline/rte_pipeline_version.map | 75 | ||||
-rw-r--r-- | src/spdk/dpdk/lib/librte_pipeline/rte_port_in_action.c | 531 | ||||
-rw-r--r-- | src/spdk/dpdk/lib/librte_pipeline/rte_port_in_action.h | 301 | ||||
-rw-r--r-- | src/spdk/dpdk/lib/librte_pipeline/rte_table_action.c | 2386 | ||||
-rw-r--r-- | src/spdk/dpdk/lib/librte_pipeline/rte_table_action.h | 905 |
9 files changed, 6704 insertions, 0 deletions
diff --git a/src/spdk/dpdk/lib/librte_pipeline/Makefile b/src/spdk/dpdk/lib/librte_pipeline/Makefile new file mode 100644 index 00000000..84afe98c --- /dev/null +++ b/src/spdk/dpdk/lib/librte_pipeline/Makefile @@ -0,0 +1,31 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright(c) 2010-2016 Intel Corporation + +include $(RTE_SDK)/mk/rte.vars.mk + +# +# library name +# +LIB = librte_pipeline.a + +CFLAGS += -DALLOW_EXPERIMENTAL_API +CFLAGS += -O3 +CFLAGS += $(WERROR_FLAGS) +LDLIBS += -lrte_eal -lrte_mempool -lrte_mbuf -lrte_table +LDLIBS += -lrte_port -lrte_meter -lrte_sched + +EXPORT_MAP := rte_pipeline_version.map + +LIBABIVER := 3 + +# +# all source are stored in SRCS-y +# +SRCS-$(CONFIG_RTE_LIBRTE_PIPELINE) := rte_pipeline.c +SRCS-$(CONFIG_RTE_LIBRTE_PIPELINE) += rte_port_in_action.c +SRCS-$(CONFIG_RTE_LIBRTE_PIPELINE) += rte_table_action.c + +# install includes +SYMLINK-$(CONFIG_RTE_LIBRTE_PIPELINE)-include += rte_pipeline.h rte_port_in_action.h rte_table_action.h + +include $(RTE_SDK)/mk/rte.lib.mk diff --git a/src/spdk/dpdk/lib/librte_pipeline/meson.build b/src/spdk/dpdk/lib/librte_pipeline/meson.build new file mode 100644 index 00000000..dc16ab42 --- /dev/null +++ b/src/spdk/dpdk/lib/librte_pipeline/meson.build @@ -0,0 +1,8 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright(c) 2017 Intel Corporation + +version = 3 +allow_experimental_apis = true +sources = files('rte_pipeline.c', 'rte_port_in_action.c', 'rte_table_action.c') +headers = files('rte_pipeline.h', 'rte_port_in_action.h', 'rte_table_action.h') +deps += ['port', 'table', 'meter', 'sched'] diff --git a/src/spdk/dpdk/lib/librte_pipeline/rte_pipeline.c b/src/spdk/dpdk/lib/librte_pipeline/rte_pipeline.c new file mode 100644 index 00000000..0cb8b804 --- /dev/null +++ b/src/spdk/dpdk/lib/librte_pipeline/rte_pipeline.c @@ -0,0 +1,1619 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2010-2016 Intel Corporation + */ + +#include <string.h> +#include <stdio.h> + +#include <rte_common.h> +#include <rte_memory.h> +#include <rte_cycles.h> +#include <rte_prefetch.h> +#include <rte_branch_prediction.h> +#include <rte_mbuf.h> +#include <rte_malloc.h> +#include <rte_string_fns.h> + +#include "rte_pipeline.h" + +#define RTE_TABLE_INVALID UINT32_MAX + +#ifdef RTE_PIPELINE_STATS_COLLECT + +#define RTE_PIPELINE_STATS_AH_DROP_WRITE(p, mask) \ + ({ (p)->n_pkts_ah_drop = __builtin_popcountll(mask); }) + +#define RTE_PIPELINE_STATS_AH_DROP_READ(p, counter) \ + ({ (counter) += (p)->n_pkts_ah_drop; (p)->n_pkts_ah_drop = 0; }) + +#define RTE_PIPELINE_STATS_TABLE_DROP0(p) \ + ({ (p)->pkts_drop_mask = (p)->action_mask0[RTE_PIPELINE_ACTION_DROP]; }) + +#define RTE_PIPELINE_STATS_TABLE_DROP1(p, counter) \ +({ \ + uint64_t mask = (p)->action_mask0[RTE_PIPELINE_ACTION_DROP]; \ + mask ^= (p)->pkts_drop_mask; \ + (counter) += __builtin_popcountll(mask); \ +}) + +#else + +#define RTE_PIPELINE_STATS_AH_DROP_WRITE(p, mask) +#define RTE_PIPELINE_STATS_AH_DROP_READ(p, counter) +#define RTE_PIPELINE_STATS_TABLE_DROP0(p) +#define RTE_PIPELINE_STATS_TABLE_DROP1(p, counter) + +#endif + +struct rte_port_in { + /* Input parameters */ + struct rte_port_in_ops ops; + rte_pipeline_port_in_action_handler f_action; + void *arg_ah; + uint32_t burst_size; + + /* The table to which this port is connected */ + uint32_t table_id; + + /* Handle to low-level port */ + void *h_port; + + /* List of enabled ports */ + struct rte_port_in *next; + + /* Statistics */ + uint64_t n_pkts_dropped_by_ah; +}; + +struct rte_port_out { + /* Input parameters */ + struct rte_port_out_ops ops; + rte_pipeline_port_out_action_handler f_action; + void *arg_ah; + + /* Handle to low-level port */ + void *h_port; + + /* Statistics */ + uint64_t n_pkts_dropped_by_ah; +}; + +struct rte_table { + /* Input parameters */ + struct rte_table_ops ops; + rte_pipeline_table_action_handler_hit f_action_hit; + rte_pipeline_table_action_handler_miss f_action_miss; + void *arg_ah; + struct rte_pipeline_table_entry *default_entry; + uint32_t entry_size; + + uint32_t table_next_id; + uint32_t table_next_id_valid; + + /* Handle to the low-level table object */ + void *h_table; + + /* Statistics */ + uint64_t n_pkts_dropped_by_lkp_hit_ah; + uint64_t n_pkts_dropped_by_lkp_miss_ah; + uint64_t n_pkts_dropped_lkp_hit; + uint64_t n_pkts_dropped_lkp_miss; +}; + +#define RTE_PIPELINE_MAX_NAME_SZ 124 + +struct rte_pipeline { + /* Input parameters */ + char name[RTE_PIPELINE_MAX_NAME_SZ]; + int socket_id; + uint32_t offset_port_id; + + /* Internal tables */ + struct rte_port_in ports_in[RTE_PIPELINE_PORT_IN_MAX]; + struct rte_port_out ports_out[RTE_PIPELINE_PORT_OUT_MAX]; + struct rte_table tables[RTE_PIPELINE_TABLE_MAX]; + + /* Occupancy of internal tables */ + uint32_t num_ports_in; + uint32_t num_ports_out; + uint32_t num_tables; + + /* List of enabled ports */ + uint64_t enabled_port_in_mask; + struct rte_port_in *port_in_next; + + /* Pipeline run structures */ + struct rte_mbuf *pkts[RTE_PORT_IN_BURST_SIZE_MAX]; + struct rte_pipeline_table_entry *entries[RTE_PORT_IN_BURST_SIZE_MAX]; + uint64_t action_mask0[RTE_PIPELINE_ACTIONS]; + uint64_t action_mask1[RTE_PIPELINE_ACTIONS]; + uint64_t pkts_mask; + uint64_t n_pkts_ah_drop; + uint64_t pkts_drop_mask; +} __rte_cache_aligned; + +static inline uint32_t +rte_mask_get_next(uint64_t mask, uint32_t pos) +{ + uint64_t mask_rot = (mask << ((63 - pos) & 0x3F)) | + (mask >> ((pos + 1) & 0x3F)); + return (__builtin_ctzll(mask_rot) - (63 - pos)) & 0x3F; +} + +static inline uint32_t +rte_mask_get_prev(uint64_t mask, uint32_t pos) +{ + uint64_t mask_rot = (mask >> (pos & 0x3F)) | + (mask << ((64 - pos) & 0x3F)); + return ((63 - __builtin_clzll(mask_rot)) + pos) & 0x3F; +} + +static void +rte_pipeline_table_free(struct rte_table *table); + +static void +rte_pipeline_port_in_free(struct rte_port_in *port); + +static void +rte_pipeline_port_out_free(struct rte_port_out *port); + +/* + * Pipeline + * + */ +static int +rte_pipeline_check_params(struct rte_pipeline_params *params) +{ + if (params == NULL) { + RTE_LOG(ERR, PIPELINE, + "%s: Incorrect value for parameter params\n", __func__); + return -EINVAL; + } + + /* name */ + if (params->name == NULL) { + RTE_LOG(ERR, PIPELINE, + "%s: Incorrect value for parameter name\n", __func__); + return -EINVAL; + } + + /* socket */ + if ((params->socket_id < 0) || + (params->socket_id >= RTE_MAX_NUMA_NODES)) { + RTE_LOG(ERR, PIPELINE, + "%s: Incorrect value for parameter socket_id\n", + __func__); + return -EINVAL; + } + + return 0; +} + +struct rte_pipeline * +rte_pipeline_create(struct rte_pipeline_params *params) +{ + struct rte_pipeline *p; + int status; + + /* Check input parameters */ + status = rte_pipeline_check_params(params); + if (status != 0) { + RTE_LOG(ERR, PIPELINE, + "%s: Pipeline params check failed (%d)\n", + __func__, status); + return NULL; + } + + /* Allocate memory for the pipeline on requested socket */ + p = rte_zmalloc_socket("PIPELINE", sizeof(struct rte_pipeline), + RTE_CACHE_LINE_SIZE, params->socket_id); + + if (p == NULL) { + RTE_LOG(ERR, PIPELINE, + "%s: Pipeline memory allocation failed\n", __func__); + return NULL; + } + + /* Save input parameters */ + snprintf(p->name, RTE_PIPELINE_MAX_NAME_SZ, "%s", params->name); + p->socket_id = params->socket_id; + p->offset_port_id = params->offset_port_id; + + /* Initialize pipeline internal data structure */ + p->num_ports_in = 0; + p->num_ports_out = 0; + p->num_tables = 0; + p->enabled_port_in_mask = 0; + p->port_in_next = NULL; + p->pkts_mask = 0; + p->n_pkts_ah_drop = 0; + + return p; +} + +int +rte_pipeline_free(struct rte_pipeline *p) +{ + uint32_t i; + + /* Check input parameters */ + if (p == NULL) { + RTE_LOG(ERR, PIPELINE, + "%s: rte_pipeline parameter is NULL\n", __func__); + return -EINVAL; + } + + /* Free input ports */ + for (i = 0; i < p->num_ports_in; i++) { + struct rte_port_in *port = &p->ports_in[i]; + + rte_pipeline_port_in_free(port); + } + + /* Free tables */ + for (i = 0; i < p->num_tables; i++) { + struct rte_table *table = &p->tables[i]; + + rte_pipeline_table_free(table); + } + + /* Free output ports */ + for (i = 0; i < p->num_ports_out; i++) { + struct rte_port_out *port = &p->ports_out[i]; + + rte_pipeline_port_out_free(port); + } + + /* Free pipeline memory */ + rte_free(p); + + return 0; +} + +/* + * Table + * + */ +static int +rte_table_check_params(struct rte_pipeline *p, + struct rte_pipeline_table_params *params, + uint32_t *table_id) +{ + if (p == NULL) { + RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter is NULL\n", + __func__); + return -EINVAL; + } + if (params == NULL) { + RTE_LOG(ERR, PIPELINE, "%s: params parameter is NULL\n", + __func__); + return -EINVAL; + } + if (table_id == NULL) { + RTE_LOG(ERR, PIPELINE, "%s: table_id parameter is NULL\n", + __func__); + return -EINVAL; + } + + /* ops */ + if (params->ops == NULL) { + RTE_LOG(ERR, PIPELINE, "%s: params->ops is NULL\n", + __func__); + return -EINVAL; + } + + if (params->ops->f_create == NULL) { + RTE_LOG(ERR, PIPELINE, + "%s: f_create function pointer is NULL\n", __func__); + return -EINVAL; + } + + if (params->ops->f_lookup == NULL) { + RTE_LOG(ERR, PIPELINE, + "%s: f_lookup function pointer is NULL\n", __func__); + return -EINVAL; + } + + /* De we have room for one more table? */ + if (p->num_tables == RTE_PIPELINE_TABLE_MAX) { + RTE_LOG(ERR, PIPELINE, + "%s: Incorrect value for num_tables parameter\n", + __func__); + return -EINVAL; + } + + return 0; +} + +int +rte_pipeline_table_create(struct rte_pipeline *p, + struct rte_pipeline_table_params *params, + uint32_t *table_id) +{ + struct rte_table *table; + struct rte_pipeline_table_entry *default_entry; + void *h_table; + uint32_t entry_size, id; + int status; + + /* Check input arguments */ + status = rte_table_check_params(p, params, table_id); + if (status != 0) + return status; + + id = p->num_tables; + table = &p->tables[id]; + + /* Allocate space for the default table entry */ + entry_size = sizeof(struct rte_pipeline_table_entry) + + params->action_data_size; + default_entry = rte_zmalloc_socket( + "PIPELINE", entry_size, RTE_CACHE_LINE_SIZE, p->socket_id); + if (default_entry == NULL) { + RTE_LOG(ERR, PIPELINE, + "%s: Failed to allocate default entry\n", __func__); + return -EINVAL; + } + + /* Create the table */ + h_table = params->ops->f_create(params->arg_create, p->socket_id, + entry_size); + if (h_table == NULL) { + rte_free(default_entry); + RTE_LOG(ERR, PIPELINE, "%s: Table creation failed\n", __func__); + return -EINVAL; + } + + /* Commit current table to the pipeline */ + p->num_tables++; + *table_id = id; + + /* Save input parameters */ + memcpy(&table->ops, params->ops, sizeof(struct rte_table_ops)); + table->f_action_hit = params->f_action_hit; + table->f_action_miss = params->f_action_miss; + table->arg_ah = params->arg_ah; + table->entry_size = entry_size; + + /* Clear the lookup miss actions (to be set later through API) */ + table->default_entry = default_entry; + table->default_entry->action = RTE_PIPELINE_ACTION_DROP; + + /* Initialize table internal data structure */ + table->h_table = h_table; + table->table_next_id = 0; + table->table_next_id_valid = 0; + + return 0; +} + +void +rte_pipeline_table_free(struct rte_table *table) +{ + if (table->ops.f_free != NULL) + table->ops.f_free(table->h_table); + + rte_free(table->default_entry); +} + +int +rte_pipeline_table_default_entry_add(struct rte_pipeline *p, + uint32_t table_id, + struct rte_pipeline_table_entry *default_entry, + struct rte_pipeline_table_entry **default_entry_ptr) +{ + struct rte_table *table; + + /* Check input arguments */ + if (p == NULL) { + RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter is NULL\n", + __func__); + return -EINVAL; + } + + if (default_entry == NULL) { + RTE_LOG(ERR, PIPELINE, + "%s: default_entry parameter is NULL\n", __func__); + return -EINVAL; + } + + if (table_id >= p->num_tables) { + RTE_LOG(ERR, PIPELINE, + "%s: table_id %d out of range\n", __func__, table_id); + return -EINVAL; + } + + table = &p->tables[table_id]; + + if ((default_entry->action == RTE_PIPELINE_ACTION_TABLE) && + table->table_next_id_valid && + (default_entry->table_id != table->table_next_id)) { + RTE_LOG(ERR, PIPELINE, + "%s: Tree-like topologies not allowed\n", __func__); + return -EINVAL; + } + + /* Set the lookup miss actions */ + if ((default_entry->action == RTE_PIPELINE_ACTION_TABLE) && + (table->table_next_id_valid == 0)) { + table->table_next_id = default_entry->table_id; + table->table_next_id_valid = 1; + } + + memcpy(table->default_entry, default_entry, table->entry_size); + + *default_entry_ptr = table->default_entry; + return 0; +} + +int +rte_pipeline_table_default_entry_delete(struct rte_pipeline *p, + uint32_t table_id, + struct rte_pipeline_table_entry *entry) +{ + struct rte_table *table; + + /* Check input arguments */ + if (p == NULL) { + RTE_LOG(ERR, PIPELINE, + "%s: pipeline parameter is NULL\n", __func__); + return -EINVAL; + } + + if (table_id >= p->num_tables) { + RTE_LOG(ERR, PIPELINE, + "%s: table_id %d out of range\n", __func__, table_id); + return -EINVAL; + } + + table = &p->tables[table_id]; + + /* Save the current contents of the default entry */ + if (entry) + memcpy(entry, table->default_entry, table->entry_size); + + /* Clear the lookup miss actions */ + memset(table->default_entry, 0, table->entry_size); + table->default_entry->action = RTE_PIPELINE_ACTION_DROP; + + return 0; +} + +int +rte_pipeline_table_entry_add(struct rte_pipeline *p, + uint32_t table_id, + void *key, + struct rte_pipeline_table_entry *entry, + int *key_found, + struct rte_pipeline_table_entry **entry_ptr) +{ + struct rte_table *table; + + /* Check input arguments */ + if (p == NULL) { + RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter is NULL\n", + __func__); + return -EINVAL; + } + + if (key == NULL) { + RTE_LOG(ERR, PIPELINE, "%s: key parameter is NULL\n", __func__); + return -EINVAL; + } + + if (entry == NULL) { + RTE_LOG(ERR, PIPELINE, "%s: entry parameter is NULL\n", + __func__); + return -EINVAL; + } + + if (table_id >= p->num_tables) { + RTE_LOG(ERR, PIPELINE, + "%s: table_id %d out of range\n", __func__, table_id); + return -EINVAL; + } + + table = &p->tables[table_id]; + + if (table->ops.f_add == NULL) { + RTE_LOG(ERR, PIPELINE, "%s: f_add function pointer NULL\n", + __func__); + return -EINVAL; + } + + if ((entry->action == RTE_PIPELINE_ACTION_TABLE) && + table->table_next_id_valid && + (entry->table_id != table->table_next_id)) { + RTE_LOG(ERR, PIPELINE, + "%s: Tree-like topologies not allowed\n", __func__); + return -EINVAL; + } + + /* Add entry */ + if ((entry->action == RTE_PIPELINE_ACTION_TABLE) && + (table->table_next_id_valid == 0)) { + table->table_next_id = entry->table_id; + table->table_next_id_valid = 1; + } + + return (table->ops.f_add)(table->h_table, key, (void *) entry, + key_found, (void **) entry_ptr); +} + +int +rte_pipeline_table_entry_delete(struct rte_pipeline *p, + uint32_t table_id, + void *key, + int *key_found, + struct rte_pipeline_table_entry *entry) +{ + struct rte_table *table; + + /* Check input arguments */ + if (p == NULL) { + RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n", + __func__); + return -EINVAL; + } + + if (key == NULL) { + RTE_LOG(ERR, PIPELINE, "%s: key parameter is NULL\n", + __func__); + return -EINVAL; + } + + if (table_id >= p->num_tables) { + RTE_LOG(ERR, PIPELINE, + "%s: table_id %d out of range\n", __func__, table_id); + return -EINVAL; + } + + table = &p->tables[table_id]; + + if (table->ops.f_delete == NULL) { + RTE_LOG(ERR, PIPELINE, + "%s: f_delete function pointer NULL\n", __func__); + return -EINVAL; + } + + return (table->ops.f_delete)(table->h_table, key, key_found, entry); +} + +int rte_pipeline_table_entry_add_bulk(struct rte_pipeline *p, + uint32_t table_id, + void **keys, + struct rte_pipeline_table_entry **entries, + uint32_t n_keys, + int *key_found, + struct rte_pipeline_table_entry **entries_ptr) +{ + struct rte_table *table; + uint32_t i; + + /* Check input arguments */ + if (p == NULL) { + RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter is NULL\n", + __func__); + return -EINVAL; + } + + if (keys == NULL) { + RTE_LOG(ERR, PIPELINE, "%s: keys parameter is NULL\n", __func__); + return -EINVAL; + } + + if (entries == NULL) { + RTE_LOG(ERR, PIPELINE, "%s: entries parameter is NULL\n", + __func__); + return -EINVAL; + } + + if (table_id >= p->num_tables) { + RTE_LOG(ERR, PIPELINE, + "%s: table_id %d out of range\n", __func__, table_id); + return -EINVAL; + } + + table = &p->tables[table_id]; + + if (table->ops.f_add_bulk == NULL) { + RTE_LOG(ERR, PIPELINE, "%s: f_add_bulk function pointer NULL\n", + __func__); + return -EINVAL; + } + + for (i = 0; i < n_keys; i++) { + if ((entries[i]->action == RTE_PIPELINE_ACTION_TABLE) && + table->table_next_id_valid && + (entries[i]->table_id != table->table_next_id)) { + RTE_LOG(ERR, PIPELINE, + "%s: Tree-like topologies not allowed\n", __func__); + return -EINVAL; + } + } + + /* Add entry */ + for (i = 0; i < n_keys; i++) { + if ((entries[i]->action == RTE_PIPELINE_ACTION_TABLE) && + (table->table_next_id_valid == 0)) { + table->table_next_id = entries[i]->table_id; + table->table_next_id_valid = 1; + } + } + + return (table->ops.f_add_bulk)(table->h_table, keys, (void **) entries, + n_keys, key_found, (void **) entries_ptr); +} + +int rte_pipeline_table_entry_delete_bulk(struct rte_pipeline *p, + uint32_t table_id, + void **keys, + uint32_t n_keys, + int *key_found, + struct rte_pipeline_table_entry **entries) +{ + struct rte_table *table; + + /* Check input arguments */ + if (p == NULL) { + RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n", + __func__); + return -EINVAL; + } + + if (keys == NULL) { + RTE_LOG(ERR, PIPELINE, "%s: key parameter is NULL\n", + __func__); + return -EINVAL; + } + + if (table_id >= p->num_tables) { + RTE_LOG(ERR, PIPELINE, + "%s: table_id %d out of range\n", __func__, table_id); + return -EINVAL; + } + + table = &p->tables[table_id]; + + if (table->ops.f_delete_bulk == NULL) { + RTE_LOG(ERR, PIPELINE, + "%s: f_delete function pointer NULL\n", __func__); + return -EINVAL; + } + + return (table->ops.f_delete_bulk)(table->h_table, keys, n_keys, key_found, + (void **) entries); +} + +/* + * Port + * + */ +static int +rte_pipeline_port_in_check_params(struct rte_pipeline *p, + struct rte_pipeline_port_in_params *params, + uint32_t *port_id) +{ + if (p == NULL) { + RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n", + __func__); + return -EINVAL; + } + if (params == NULL) { + RTE_LOG(ERR, PIPELINE, "%s: params parameter NULL\n", __func__); + return -EINVAL; + } + if (port_id == NULL) { + RTE_LOG(ERR, PIPELINE, "%s: port_id parameter NULL\n", + __func__); + return -EINVAL; + } + + /* ops */ + if (params->ops == NULL) { + RTE_LOG(ERR, PIPELINE, "%s: params->ops parameter NULL\n", + __func__); + return -EINVAL; + } + + if (params->ops->f_create == NULL) { + RTE_LOG(ERR, PIPELINE, + "%s: f_create function pointer NULL\n", __func__); + return -EINVAL; + } + + if (params->ops->f_rx == NULL) { + RTE_LOG(ERR, PIPELINE, "%s: f_rx function pointer NULL\n", + __func__); + return -EINVAL; + } + + /* burst_size */ + if ((params->burst_size == 0) || + (params->burst_size > RTE_PORT_IN_BURST_SIZE_MAX)) { + RTE_LOG(ERR, PIPELINE, "%s: invalid value for burst_size\n", + __func__); + return -EINVAL; + } + + /* Do we have room for one more port? */ + if (p->num_ports_in == RTE_PIPELINE_PORT_IN_MAX) { + RTE_LOG(ERR, PIPELINE, + "%s: invalid value for num_ports_in\n", __func__); + return -EINVAL; + } + + return 0; +} + +static int +rte_pipeline_port_out_check_params(struct rte_pipeline *p, + struct rte_pipeline_port_out_params *params, + uint32_t *port_id) +{ + if (p == NULL) { + RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n", + __func__); + return -EINVAL; + } + + if (params == NULL) { + RTE_LOG(ERR, PIPELINE, "%s: params parameter NULL\n", __func__); + return -EINVAL; + } + + if (port_id == NULL) { + RTE_LOG(ERR, PIPELINE, "%s: port_id parameter NULL\n", + __func__); + return -EINVAL; + } + + /* ops */ + if (params->ops == NULL) { + RTE_LOG(ERR, PIPELINE, "%s: params->ops parameter NULL\n", + __func__); + return -EINVAL; + } + + if (params->ops->f_create == NULL) { + RTE_LOG(ERR, PIPELINE, + "%s: f_create function pointer NULL\n", __func__); + return -EINVAL; + } + + if (params->ops->f_tx == NULL) { + RTE_LOG(ERR, PIPELINE, + "%s: f_tx function pointer NULL\n", __func__); + return -EINVAL; + } + + if (params->ops->f_tx_bulk == NULL) { + RTE_LOG(ERR, PIPELINE, + "%s: f_tx_bulk function pointer NULL\n", __func__); + return -EINVAL; + } + + /* Do we have room for one more port? */ + if (p->num_ports_out == RTE_PIPELINE_PORT_OUT_MAX) { + RTE_LOG(ERR, PIPELINE, + "%s: invalid value for num_ports_out\n", __func__); + return -EINVAL; + } + + return 0; +} + +int +rte_pipeline_port_in_create(struct rte_pipeline *p, + struct rte_pipeline_port_in_params *params, + uint32_t *port_id) +{ + struct rte_port_in *port; + void *h_port; + uint32_t id; + int status; + + /* Check input arguments */ + status = rte_pipeline_port_in_check_params(p, params, port_id); + if (status != 0) + return status; + + id = p->num_ports_in; + port = &p->ports_in[id]; + + /* Create the port */ + h_port = params->ops->f_create(params->arg_create, p->socket_id); + if (h_port == NULL) { + RTE_LOG(ERR, PIPELINE, "%s: Port creation failed\n", __func__); + return -EINVAL; + } + + /* Commit current table to the pipeline */ + p->num_ports_in++; + *port_id = id; + + /* Save input parameters */ + memcpy(&port->ops, params->ops, sizeof(struct rte_port_in_ops)); + port->f_action = params->f_action; + port->arg_ah = params->arg_ah; + port->burst_size = params->burst_size; + + /* Initialize port internal data structure */ + port->table_id = RTE_TABLE_INVALID; + port->h_port = h_port; + port->next = NULL; + + return 0; +} + +void +rte_pipeline_port_in_free(struct rte_port_in *port) +{ + if (port->ops.f_free != NULL) + port->ops.f_free(port->h_port); +} + +int +rte_pipeline_port_out_create(struct rte_pipeline *p, + struct rte_pipeline_port_out_params *params, + uint32_t *port_id) +{ + struct rte_port_out *port; + void *h_port; + uint32_t id; + int status; + + /* Check input arguments */ + status = rte_pipeline_port_out_check_params(p, params, port_id); + if (status != 0) + return status; + + id = p->num_ports_out; + port = &p->ports_out[id]; + + /* Create the port */ + h_port = params->ops->f_create(params->arg_create, p->socket_id); + if (h_port == NULL) { + RTE_LOG(ERR, PIPELINE, "%s: Port creation failed\n", __func__); + return -EINVAL; + } + + /* Commit current table to the pipeline */ + p->num_ports_out++; + *port_id = id; + + /* Save input parameters */ + memcpy(&port->ops, params->ops, sizeof(struct rte_port_out_ops)); + port->f_action = params->f_action; + port->arg_ah = params->arg_ah; + + /* Initialize port internal data structure */ + port->h_port = h_port; + + return 0; +} + +void +rte_pipeline_port_out_free(struct rte_port_out *port) +{ + if (port->ops.f_free != NULL) + port->ops.f_free(port->h_port); +} + +int +rte_pipeline_port_in_connect_to_table(struct rte_pipeline *p, + uint32_t port_id, + uint32_t table_id) +{ + struct rte_port_in *port; + + /* Check input arguments */ + if (p == NULL) { + RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n", + __func__); + return -EINVAL; + } + + if (port_id >= p->num_ports_in) { + RTE_LOG(ERR, PIPELINE, + "%s: port IN ID %u is out of range\n", + __func__, port_id); + return -EINVAL; + } + + if (table_id >= p->num_tables) { + RTE_LOG(ERR, PIPELINE, + "%s: Table ID %u is out of range\n", + __func__, table_id); + return -EINVAL; + } + + port = &p->ports_in[port_id]; + port->table_id = table_id; + + return 0; +} + +int +rte_pipeline_port_in_enable(struct rte_pipeline *p, uint32_t port_id) +{ + struct rte_port_in *port, *port_prev, *port_next; + uint64_t port_mask; + uint32_t port_prev_id, port_next_id; + + /* Check input arguments */ + if (p == NULL) { + RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n", + __func__); + return -EINVAL; + } + + if (port_id >= p->num_ports_in) { + RTE_LOG(ERR, PIPELINE, + "%s: port IN ID %u is out of range\n", + __func__, port_id); + return -EINVAL; + } + + port = &p->ports_in[port_id]; + + /* Return if current input port is already enabled */ + port_mask = 1LLU << port_id; + if (p->enabled_port_in_mask & port_mask) + return 0; + + p->enabled_port_in_mask |= port_mask; + + /* Add current input port to the pipeline chain of enabled ports */ + port_prev_id = rte_mask_get_prev(p->enabled_port_in_mask, port_id); + port_next_id = rte_mask_get_next(p->enabled_port_in_mask, port_id); + + port_prev = &p->ports_in[port_prev_id]; + port_next = &p->ports_in[port_next_id]; + + port_prev->next = port; + port->next = port_next; + + /* Check if list of enabled ports was previously empty */ + if (p->enabled_port_in_mask == port_mask) + p->port_in_next = port; + + return 0; +} + +int +rte_pipeline_port_in_disable(struct rte_pipeline *p, uint32_t port_id) +{ + struct rte_port_in *port, *port_prev, *port_next; + uint64_t port_mask; + uint32_t port_prev_id, port_next_id; + + /* Check input arguments */ + if (p == NULL) { + RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n", + __func__); + return -EINVAL; + } + + if (port_id >= p->num_ports_in) { + RTE_LOG(ERR, PIPELINE, "%s: port IN ID %u is out of range\n", + __func__, port_id); + return -EINVAL; + } + + port = &p->ports_in[port_id]; + + /* Return if current input port is already disabled */ + port_mask = 1LLU << port_id; + if ((p->enabled_port_in_mask & port_mask) == 0) + return 0; + + p->enabled_port_in_mask &= ~port_mask; + + /* Return if no other enabled ports */ + if (p->enabled_port_in_mask == 0) { + p->port_in_next = NULL; + + return 0; + } + + /* Add current input port to the pipeline chain of enabled ports */ + port_prev_id = rte_mask_get_prev(p->enabled_port_in_mask, port_id); + port_next_id = rte_mask_get_next(p->enabled_port_in_mask, port_id); + + port_prev = &p->ports_in[port_prev_id]; + port_next = &p->ports_in[port_next_id]; + + port_prev->next = port_next; + + /* Check if the port which has just been disabled is next to serve */ + if (port == p->port_in_next) + p->port_in_next = port_next; + + return 0; +} + +/* + * Pipeline run-time + * + */ +int +rte_pipeline_check(struct rte_pipeline *p) +{ + uint32_t port_in_id; + + /* Check input arguments */ + if (p == NULL) { + RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n", + __func__); + return -EINVAL; + } + + /* Check that pipeline has at least one input port, one table and one + output port */ + if (p->num_ports_in == 0) { + RTE_LOG(ERR, PIPELINE, "%s: must have at least 1 input port\n", + __func__); + return -EINVAL; + } + if (p->num_tables == 0) { + RTE_LOG(ERR, PIPELINE, "%s: must have at least 1 table\n", + __func__); + return -EINVAL; + } + if (p->num_ports_out == 0) { + RTE_LOG(ERR, PIPELINE, "%s: must have at least 1 output port\n", + __func__); + return -EINVAL; + } + + /* Check that all input ports are connected */ + for (port_in_id = 0; port_in_id < p->num_ports_in; port_in_id++) { + struct rte_port_in *port_in = &p->ports_in[port_in_id]; + + if (port_in->table_id == RTE_TABLE_INVALID) { + RTE_LOG(ERR, PIPELINE, + "%s: Port IN ID %u is not connected\n", + __func__, port_in_id); + return -EINVAL; + } + } + + return 0; +} + +static inline void +rte_pipeline_compute_masks(struct rte_pipeline *p, uint64_t pkts_mask) +{ + p->action_mask1[RTE_PIPELINE_ACTION_DROP] = 0; + p->action_mask1[RTE_PIPELINE_ACTION_PORT] = 0; + p->action_mask1[RTE_PIPELINE_ACTION_PORT_META] = 0; + p->action_mask1[RTE_PIPELINE_ACTION_TABLE] = 0; + + if ((pkts_mask & (pkts_mask + 1)) == 0) { + uint64_t n_pkts = __builtin_popcountll(pkts_mask); + uint32_t i; + + for (i = 0; i < n_pkts; i++) { + uint64_t pkt_mask = 1LLU << i; + uint32_t pos = p->entries[i]->action; + + p->action_mask1[pos] |= pkt_mask; + } + } else { + uint32_t i; + + for (i = 0; i < RTE_PORT_IN_BURST_SIZE_MAX; i++) { + uint64_t pkt_mask = 1LLU << i; + uint32_t pos; + + if ((pkt_mask & pkts_mask) == 0) + continue; + + pos = p->entries[i]->action; + p->action_mask1[pos] |= pkt_mask; + } + } +} + +static inline void +rte_pipeline_action_handler_port_bulk(struct rte_pipeline *p, + uint64_t pkts_mask, uint32_t port_id) +{ + struct rte_port_out *port_out = &p->ports_out[port_id]; + + p->pkts_mask = pkts_mask; + + /* Output port user actions */ + if (port_out->f_action != NULL) { + port_out->f_action(p, p->pkts, pkts_mask, port_out->arg_ah); + + RTE_PIPELINE_STATS_AH_DROP_READ(p, + port_out->n_pkts_dropped_by_ah); + } + + /* Output port TX */ + if (p->pkts_mask != 0) + port_out->ops.f_tx_bulk(port_out->h_port, + p->pkts, + p->pkts_mask); +} + +static inline void +rte_pipeline_action_handler_port(struct rte_pipeline *p, uint64_t pkts_mask) +{ + p->pkts_mask = pkts_mask; + + if ((pkts_mask & (pkts_mask + 1)) == 0) { + uint64_t n_pkts = __builtin_popcountll(pkts_mask); + uint32_t i; + + for (i = 0; i < n_pkts; i++) { + struct rte_mbuf *pkt = p->pkts[i]; + uint32_t port_out_id = p->entries[i]->port_id; + struct rte_port_out *port_out = + &p->ports_out[port_out_id]; + + /* Output port user actions */ + if (port_out->f_action == NULL) /* Output port TX */ + port_out->ops.f_tx(port_out->h_port, pkt); + else { + uint64_t pkt_mask = 1LLU << i; + + port_out->f_action(p, + p->pkts, + pkt_mask, + port_out->arg_ah); + + RTE_PIPELINE_STATS_AH_DROP_READ(p, + port_out->n_pkts_dropped_by_ah); + + /* Output port TX */ + if (pkt_mask & p->pkts_mask) + port_out->ops.f_tx(port_out->h_port, + pkt); + } + } + } else { + uint32_t i; + + for (i = 0; i < RTE_PORT_IN_BURST_SIZE_MAX; i++) { + uint64_t pkt_mask = 1LLU << i; + struct rte_mbuf *pkt; + struct rte_port_out *port_out; + uint32_t port_out_id; + + if ((pkt_mask & pkts_mask) == 0) + continue; + + pkt = p->pkts[i]; + port_out_id = p->entries[i]->port_id; + port_out = &p->ports_out[port_out_id]; + + /* Output port user actions */ + if (port_out->f_action == NULL) /* Output port TX */ + port_out->ops.f_tx(port_out->h_port, pkt); + else { + port_out->f_action(p, + p->pkts, + pkt_mask, + port_out->arg_ah); + + RTE_PIPELINE_STATS_AH_DROP_READ(p, + port_out->n_pkts_dropped_by_ah); + + /* Output port TX */ + if (pkt_mask & p->pkts_mask) + port_out->ops.f_tx(port_out->h_port, + pkt); + } + } + } +} + +static inline void +rte_pipeline_action_handler_port_meta(struct rte_pipeline *p, + uint64_t pkts_mask) +{ + p->pkts_mask = pkts_mask; + + if ((pkts_mask & (pkts_mask + 1)) == 0) { + uint64_t n_pkts = __builtin_popcountll(pkts_mask); + uint32_t i; + + for (i = 0; i < n_pkts; i++) { + struct rte_mbuf *pkt = p->pkts[i]; + uint32_t port_out_id = + RTE_MBUF_METADATA_UINT32(pkt, + p->offset_port_id); + struct rte_port_out *port_out = &p->ports_out[ + port_out_id]; + + /* Output port user actions */ + if (port_out->f_action == NULL) /* Output port TX */ + port_out->ops.f_tx(port_out->h_port, pkt); + else { + uint64_t pkt_mask = 1LLU << i; + + port_out->f_action(p, + p->pkts, + pkt_mask, + port_out->arg_ah); + + RTE_PIPELINE_STATS_AH_DROP_READ(p, + port_out->n_pkts_dropped_by_ah); + + /* Output port TX */ + if (pkt_mask & p->pkts_mask) + port_out->ops.f_tx(port_out->h_port, + pkt); + } + } + } else { + uint32_t i; + + for (i = 0; i < RTE_PORT_IN_BURST_SIZE_MAX; i++) { + uint64_t pkt_mask = 1LLU << i; + struct rte_mbuf *pkt; + struct rte_port_out *port_out; + uint32_t port_out_id; + + if ((pkt_mask & pkts_mask) == 0) + continue; + + pkt = p->pkts[i]; + port_out_id = RTE_MBUF_METADATA_UINT32(pkt, + p->offset_port_id); + port_out = &p->ports_out[port_out_id]; + + /* Output port user actions */ + if (port_out->f_action == NULL) /* Output port TX */ + port_out->ops.f_tx(port_out->h_port, pkt); + else { + port_out->f_action(p, + p->pkts, + pkt_mask, + port_out->arg_ah); + + RTE_PIPELINE_STATS_AH_DROP_READ(p, + port_out->n_pkts_dropped_by_ah); + + /* Output port TX */ + if (pkt_mask & p->pkts_mask) + port_out->ops.f_tx(port_out->h_port, + pkt); + } + } + } +} + +static inline void +rte_pipeline_action_handler_drop(struct rte_pipeline *p, uint64_t pkts_mask) +{ + if ((pkts_mask & (pkts_mask + 1)) == 0) { + uint64_t n_pkts = __builtin_popcountll(pkts_mask); + uint32_t i; + + for (i = 0; i < n_pkts; i++) + rte_pktmbuf_free(p->pkts[i]); + } else { + uint32_t i; + + for (i = 0; i < RTE_PORT_IN_BURST_SIZE_MAX; i++) { + uint64_t pkt_mask = 1LLU << i; + + if ((pkt_mask & pkts_mask) == 0) + continue; + + rte_pktmbuf_free(p->pkts[i]); + } + } +} + +int +rte_pipeline_run(struct rte_pipeline *p) +{ + struct rte_port_in *port_in = p->port_in_next; + uint32_t n_pkts, table_id; + + if (port_in == NULL) + return 0; + + /* Input port RX */ + n_pkts = port_in->ops.f_rx(port_in->h_port, p->pkts, + port_in->burst_size); + if (n_pkts == 0) { + p->port_in_next = port_in->next; + return 0; + } + + p->pkts_mask = RTE_LEN2MASK(n_pkts, uint64_t); + p->action_mask0[RTE_PIPELINE_ACTION_DROP] = 0; + p->action_mask0[RTE_PIPELINE_ACTION_PORT] = 0; + p->action_mask0[RTE_PIPELINE_ACTION_PORT_META] = 0; + p->action_mask0[RTE_PIPELINE_ACTION_TABLE] = 0; + + /* Input port user actions */ + if (port_in->f_action != NULL) { + port_in->f_action(p, p->pkts, n_pkts, port_in->arg_ah); + + RTE_PIPELINE_STATS_AH_DROP_READ(p, + port_in->n_pkts_dropped_by_ah); + } + + /* Table */ + for (table_id = port_in->table_id; p->pkts_mask != 0; ) { + struct rte_table *table; + uint64_t lookup_hit_mask, lookup_miss_mask; + + /* Lookup */ + table = &p->tables[table_id]; + table->ops.f_lookup(table->h_table, p->pkts, p->pkts_mask, + &lookup_hit_mask, (void **) p->entries); + lookup_miss_mask = p->pkts_mask & (~lookup_hit_mask); + + /* Lookup miss */ + if (lookup_miss_mask != 0) { + struct rte_pipeline_table_entry *default_entry = + table->default_entry; + + p->pkts_mask = lookup_miss_mask; + + /* Table user actions */ + if (table->f_action_miss != NULL) { + table->f_action_miss(p, + p->pkts, + lookup_miss_mask, + default_entry, + table->arg_ah); + + RTE_PIPELINE_STATS_AH_DROP_READ(p, + table->n_pkts_dropped_by_lkp_miss_ah); + } + + /* Table reserved actions */ + if ((default_entry->action == RTE_PIPELINE_ACTION_PORT) && + (p->pkts_mask != 0)) + rte_pipeline_action_handler_port_bulk(p, + p->pkts_mask, + default_entry->port_id); + else { + uint32_t pos = default_entry->action; + + RTE_PIPELINE_STATS_TABLE_DROP0(p); + + p->action_mask0[pos] |= p->pkts_mask; + + RTE_PIPELINE_STATS_TABLE_DROP1(p, + table->n_pkts_dropped_lkp_miss); + } + } + + /* Lookup hit */ + if (lookup_hit_mask != 0) { + p->pkts_mask = lookup_hit_mask; + + /* Table user actions */ + if (table->f_action_hit != NULL) { + table->f_action_hit(p, + p->pkts, + lookup_hit_mask, + p->entries, + table->arg_ah); + + RTE_PIPELINE_STATS_AH_DROP_READ(p, + table->n_pkts_dropped_by_lkp_hit_ah); + } + + /* Table reserved actions */ + RTE_PIPELINE_STATS_TABLE_DROP0(p); + rte_pipeline_compute_masks(p, p->pkts_mask); + p->action_mask0[RTE_PIPELINE_ACTION_DROP] |= + p->action_mask1[ + RTE_PIPELINE_ACTION_DROP]; + p->action_mask0[RTE_PIPELINE_ACTION_PORT] |= + p->action_mask1[ + RTE_PIPELINE_ACTION_PORT]; + p->action_mask0[RTE_PIPELINE_ACTION_PORT_META] |= + p->action_mask1[ + RTE_PIPELINE_ACTION_PORT_META]; + p->action_mask0[RTE_PIPELINE_ACTION_TABLE] |= + p->action_mask1[ + RTE_PIPELINE_ACTION_TABLE]; + + RTE_PIPELINE_STATS_TABLE_DROP1(p, + table->n_pkts_dropped_lkp_hit); + } + + /* Prepare for next iteration */ + p->pkts_mask = p->action_mask0[RTE_PIPELINE_ACTION_TABLE]; + table_id = table->table_next_id; + p->action_mask0[RTE_PIPELINE_ACTION_TABLE] = 0; + } + + /* Table reserved action PORT */ + rte_pipeline_action_handler_port(p, + p->action_mask0[RTE_PIPELINE_ACTION_PORT]); + + /* Table reserved action PORT META */ + rte_pipeline_action_handler_port_meta(p, + p->action_mask0[RTE_PIPELINE_ACTION_PORT_META]); + + /* Table reserved action DROP */ + rte_pipeline_action_handler_drop(p, + p->action_mask0[RTE_PIPELINE_ACTION_DROP]); + + /* Pick candidate for next port IN to serve */ + p->port_in_next = port_in->next; + + return (int) n_pkts; +} + +int +rte_pipeline_flush(struct rte_pipeline *p) +{ + uint32_t port_id; + + /* Check input arguments */ + if (p == NULL) { + RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n", + __func__); + return -EINVAL; + } + + for (port_id = 0; port_id < p->num_ports_out; port_id++) { + struct rte_port_out *port = &p->ports_out[port_id]; + + if (port->ops.f_flush != NULL) + port->ops.f_flush(port->h_port); + } + + return 0; +} + +int +rte_pipeline_port_out_packet_insert(struct rte_pipeline *p, + uint32_t port_id, struct rte_mbuf *pkt) +{ + struct rte_port_out *port_out = &p->ports_out[port_id]; + + port_out->ops.f_tx(port_out->h_port, pkt); /* Output port TX */ + + return 0; +} + +int rte_pipeline_ah_packet_hijack(struct rte_pipeline *p, + uint64_t pkts_mask) +{ + pkts_mask &= p->pkts_mask; + p->pkts_mask &= ~pkts_mask; + + return 0; +} + +int rte_pipeline_ah_packet_drop(struct rte_pipeline *p, + uint64_t pkts_mask) +{ + pkts_mask &= p->pkts_mask; + p->pkts_mask &= ~pkts_mask; + p->action_mask0[RTE_PIPELINE_ACTION_DROP] |= pkts_mask; + + RTE_PIPELINE_STATS_AH_DROP_WRITE(p, pkts_mask); + return 0; +} + +int rte_pipeline_port_in_stats_read(struct rte_pipeline *p, uint32_t port_id, + struct rte_pipeline_port_in_stats *stats, int clear) +{ + struct rte_port_in *port; + int retval; + + if (p == NULL) { + RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n", + __func__); + return -EINVAL; + } + + if (port_id >= p->num_ports_in) { + RTE_LOG(ERR, PIPELINE, + "%s: port IN ID %u is out of range\n", + __func__, port_id); + return -EINVAL; + } + + port = &p->ports_in[port_id]; + + if (port->ops.f_stats != NULL) { + retval = port->ops.f_stats(port->h_port, &stats->stats, clear); + if (retval) + return retval; + } else if (stats != NULL) + memset(&stats->stats, 0, sizeof(stats->stats)); + + if (stats != NULL) + stats->n_pkts_dropped_by_ah = port->n_pkts_dropped_by_ah; + + if (clear != 0) + port->n_pkts_dropped_by_ah = 0; + + return 0; +} + +int rte_pipeline_port_out_stats_read(struct rte_pipeline *p, uint32_t port_id, + struct rte_pipeline_port_out_stats *stats, int clear) +{ + struct rte_port_out *port; + int retval; + + if (p == NULL) { + RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n", __func__); + return -EINVAL; + } + + if (port_id >= p->num_ports_out) { + RTE_LOG(ERR, PIPELINE, + "%s: port OUT ID %u is out of range\n", __func__, port_id); + return -EINVAL; + } + + port = &p->ports_out[port_id]; + if (port->ops.f_stats != NULL) { + retval = port->ops.f_stats(port->h_port, &stats->stats, clear); + if (retval != 0) + return retval; + } else if (stats != NULL) + memset(&stats->stats, 0, sizeof(stats->stats)); + + if (stats != NULL) + stats->n_pkts_dropped_by_ah = port->n_pkts_dropped_by_ah; + + if (clear != 0) + port->n_pkts_dropped_by_ah = 0; + + return 0; +} + +int rte_pipeline_table_stats_read(struct rte_pipeline *p, uint32_t table_id, + struct rte_pipeline_table_stats *stats, int clear) +{ + struct rte_table *table; + int retval; + + if (p == NULL) { + RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n", + __func__); + return -EINVAL; + } + + if (table_id >= p->num_tables) { + RTE_LOG(ERR, PIPELINE, + "%s: table %u is out of range\n", __func__, table_id); + return -EINVAL; + } + + table = &p->tables[table_id]; + if (table->ops.f_stats != NULL) { + retval = table->ops.f_stats(table->h_table, &stats->stats, clear); + if (retval != 0) + return retval; + } else if (stats != NULL) + memset(&stats->stats, 0, sizeof(stats->stats)); + + if (stats != NULL) { + stats->n_pkts_dropped_by_lkp_hit_ah = + table->n_pkts_dropped_by_lkp_hit_ah; + stats->n_pkts_dropped_by_lkp_miss_ah = + table->n_pkts_dropped_by_lkp_miss_ah; + stats->n_pkts_dropped_lkp_hit = table->n_pkts_dropped_lkp_hit; + stats->n_pkts_dropped_lkp_miss = table->n_pkts_dropped_lkp_miss; + } + + if (clear != 0) { + table->n_pkts_dropped_by_lkp_hit_ah = 0; + table->n_pkts_dropped_by_lkp_miss_ah = 0; + table->n_pkts_dropped_lkp_hit = 0; + table->n_pkts_dropped_lkp_miss = 0; + } + + return 0; +} diff --git a/src/spdk/dpdk/lib/librte_pipeline/rte_pipeline.h b/src/spdk/dpdk/lib/librte_pipeline/rte_pipeline.h new file mode 100644 index 00000000..3cfb6868 --- /dev/null +++ b/src/spdk/dpdk/lib/librte_pipeline/rte_pipeline.h @@ -0,0 +1,848 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2010-2016 Intel Corporation + */ + +#ifndef __INCLUDE_RTE_PIPELINE_H__ +#define __INCLUDE_RTE_PIPELINE_H__ + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * @file + * RTE Pipeline + * + * This tool is part of the DPDK Packet Framework tool suite and provides + * a standard methodology (logically similar to OpenFlow) for rapid development + * of complex packet processing pipelines out of ports, tables and actions. + * + * <B>Basic operation.</B> A pipeline is constructed by connecting its input + * ports to its output ports through a chain of lookup tables. As result of + * lookup operation into the current table, one of the table entries (or the + * default table entry, in case of lookup miss) is identified to provide the + * actions to be executed on the current packet and the associated action + * meta-data. The behavior of user actions is defined through the configurable + * table action handler, while the reserved actions define the next hop for the + * current packet (either another table, an output port or packet drop) and are + * handled transparently by the framework. + * + * <B>Initialization and run-time flows.</B> Once all the pipeline elements + * (input ports, tables, output ports) have been created, input ports connected + * to tables, table action handlers configured, tables populated with the + * initial set of entries (actions and action meta-data) and input ports + * enabled, the pipeline runs automatically, pushing packets from input ports + * to tables and output ports. At each table, the identified user actions are + * being executed, resulting in action meta-data (stored in the table entry) + * and packet meta-data (stored with the packet descriptor) being updated. The + * pipeline tables can have further updates and input ports can be disabled or + * enabled later on as required. + * + * <B>Multi-core scaling.</B> Typically, each CPU core will run its own + * pipeline instance. Complex application-level pipelines can be implemented by + * interconnecting multiple CPU core-level pipelines in tree-like topologies, + * as the same port devices (e.g. SW rings) can serve as output ports for the + * pipeline running on CPU core A, as well as input ports for the pipeline + * running on CPU core B. This approach enables the application development + * using the pipeline (CPU cores connected serially), cluster/run-to-completion + * (CPU cores connected in parallel) or mixed (pipeline of CPU core clusters) + * programming models. + * + * <B>Thread safety.</B> It is possible to have multiple pipelines running on + * the same CPU core, but it is not allowed (for thread safety reasons) to have + * multiple CPU cores running the same pipeline instance. + * + ***/ + +#include <stdint.h> + +#include <rte_port.h> +#include <rte_table.h> +#include <rte_common.h> + +struct rte_mbuf; + +/* + * Pipeline + * + */ +/** Opaque data type for pipeline */ +struct rte_pipeline; + +/** Parameters for pipeline creation */ +struct rte_pipeline_params { + /** Pipeline name */ + const char *name; + + /** CPU socket ID where memory for the pipeline and its elements (ports + and tables) should be allocated */ + int socket_id; + + /** Offset within packet meta-data to port_id to be used by action + "Send packet to output port read from packet meta-data". Has to be + 4-byte aligned. */ + uint32_t offset_port_id; +}; + +/** Pipeline port in stats. */ +struct rte_pipeline_port_in_stats { + /** Port in stats. */ + struct rte_port_in_stats stats; + + /** Number of packets dropped by action handler. */ + uint64_t n_pkts_dropped_by_ah; + +}; + +/** Pipeline port out stats. */ +struct rte_pipeline_port_out_stats { + /** Port out stats. */ + struct rte_port_out_stats stats; + + /** Number of packets dropped by action handler. */ + uint64_t n_pkts_dropped_by_ah; +}; + +/** Pipeline table stats. */ +struct rte_pipeline_table_stats { + /** Table stats. */ + struct rte_table_stats stats; + + /** Number of packets dropped by lookup hit action handler. */ + uint64_t n_pkts_dropped_by_lkp_hit_ah; + + /** Number of packets dropped by lookup miss action handler. */ + uint64_t n_pkts_dropped_by_lkp_miss_ah; + + /** Number of packets dropped by pipeline in behalf of this + * table based on action specified in table entry. */ + uint64_t n_pkts_dropped_lkp_hit; + + /** Number of packets dropped by pipeline in behalf of this + * table based on action specified in table entry. */ + uint64_t n_pkts_dropped_lkp_miss; +}; + +/** + * Pipeline create + * + * @param params + * Parameters for pipeline creation + * @return + * Handle to pipeline instance on success or NULL otherwise + */ +struct rte_pipeline *rte_pipeline_create(struct rte_pipeline_params *params); + +/** + * Pipeline free + * + * @param p + * Handle to pipeline instance + * @return + * 0 on success, error code otherwise + */ +int rte_pipeline_free(struct rte_pipeline *p); + +/** + * Pipeline consistency check + * + * @param p + * Handle to pipeline instance + * @return + * 0 on success, error code otherwise + */ +int rte_pipeline_check(struct rte_pipeline *p); + +/** + * Pipeline run + * + * @param p + * Handle to pipeline instance + * @return + * Number of packets read and processed + */ +int rte_pipeline_run(struct rte_pipeline *p); + +/** + * Pipeline flush + * + * @param p + * Handle to pipeline instance + * @return + * 0 on success, error code otherwise + */ +int rte_pipeline_flush(struct rte_pipeline *p); + +/* + * Actions + * + */ +/** Reserved actions */ +enum rte_pipeline_action { + /** Drop the packet */ + RTE_PIPELINE_ACTION_DROP = 0, + + /** Send packet to output port */ + RTE_PIPELINE_ACTION_PORT, + + /** Send packet to output port read from packet meta-data */ + RTE_PIPELINE_ACTION_PORT_META, + + /** Send packet to table */ + RTE_PIPELINE_ACTION_TABLE, + + /** Number of reserved actions */ + RTE_PIPELINE_ACTIONS +}; + +/* + * Table + * + */ +/** Maximum number of tables allowed for any given pipeline instance. The + value of this parameter cannot be changed. */ +#define RTE_PIPELINE_TABLE_MAX 64 + +/** + * Head format for the table entry of any pipeline table. For any given + * pipeline table, all table entries should have the same size and format. For + * any given pipeline table, the table entry has to start with a head of this + * structure, which contains the reserved actions and their associated + * meta-data, and then optionally continues with user actions and their + * associated meta-data. As all the currently defined reserved actions are + * mutually exclusive, only one reserved action can be set per table entry. + */ +struct rte_pipeline_table_entry { + /** Reserved action */ + enum rte_pipeline_action action; + + RTE_STD_C11 + union { + /** Output port ID (meta-data for "Send packet to output port" + action) */ + uint32_t port_id; + /** Table ID (meta-data for "Send packet to table" action) */ + uint32_t table_id; + }; + /** Start of table entry area for user defined actions and meta-data */ + __extension__ uint8_t action_data[0]; +}; + +/** + * Pipeline table action handler on lookup hit + * + * The action handler can decide to drop packets by resetting the associated + * packet bit in the pkts_mask parameter. In this case, the action handler is + * required not to free the packet buffer, which will be freed eventually by + * the pipeline. + * + * @param p + * Handle to pipeline instance + * @param pkts + * Burst of input packets specified as array of up to 64 pointers to struct + * rte_mbuf + * @param pkts_mask + * 64-bit bitmask specifying which packets in the input burst are valid. When + * pkts_mask bit n is set, then element n of pkts array is pointing to a + * valid packet and element n of entries array is pointing to a valid table + * entry associated with the packet, with the association typically done by + * the table lookup operation. Otherwise, element n of pkts array and element + * n of entries array will not be accessed. + * @param entries + * Set of table entries specified as array of up to 64 pointers to struct + * rte_pipeline_table_entry + * @param arg + * Opaque parameter registered by the user at the pipeline table creation + * time + * @return + * 0 on success, error code otherwise + */ +typedef int (*rte_pipeline_table_action_handler_hit)( + struct rte_pipeline *p, + struct rte_mbuf **pkts, + uint64_t pkts_mask, + struct rte_pipeline_table_entry **entries, + void *arg); + +/** + * Pipeline table action handler on lookup miss + * + * The action handler can decide to drop packets by resetting the associated + * packet bit in the pkts_mask parameter. In this case, the action handler is + * required not to free the packet buffer, which will be freed eventually by + * the pipeline. + * + * @param p + * Handle to pipeline instance + * @param pkts + * Burst of input packets specified as array of up to 64 pointers to struct + * rte_mbuf + * @param pkts_mask + * 64-bit bitmask specifying which packets in the input burst are valid. When + * pkts_mask bit n is set, then element n of pkts array is pointing to a + * valid packet. Otherwise, element n of pkts array will not be accessed. + * @param entry + * Single table entry associated with all the valid packets from the input + * burst, specified as pointer to struct rte_pipeline_table_entry. + * This entry is the pipeline table default entry that is associated by the + * table lookup operation with the input packets that have resulted in lookup + * miss. + * @param arg + * Opaque parameter registered by the user at the pipeline table creation + * time + * @return + * 0 on success, error code otherwise + */ +typedef int (*rte_pipeline_table_action_handler_miss)( + struct rte_pipeline *p, + struct rte_mbuf **pkts, + uint64_t pkts_mask, + struct rte_pipeline_table_entry *entry, + void *arg); + +/** Parameters for pipeline table creation. Action handlers have to be either + both enabled or both disabled (they can be disabled by setting them to + NULL). */ +struct rte_pipeline_table_params { + /** Table operations (specific to each table type) */ + struct rte_table_ops *ops; + /** Opaque param to be passed to the table create operation when + invoked */ + void *arg_create; + /** Callback function to execute the user actions on input packets in + case of lookup hit */ + rte_pipeline_table_action_handler_hit f_action_hit; + /** Callback function to execute the user actions on input packets in + case of lookup miss */ + rte_pipeline_table_action_handler_miss f_action_miss; + + /** Opaque parameter to be passed to lookup hit and/or lookup miss + action handlers when invoked */ + void *arg_ah; + /** Memory size to be reserved per table entry for storing the user + actions and their meta-data */ + uint32_t action_data_size; +}; + +/** + * Pipeline table create + * + * @param p + * Handle to pipeline instance + * @param params + * Parameters for pipeline table creation + * @param table_id + * Table ID. Valid only within the scope of table IDs of the current + * pipeline. Only returned after a successful invocation. + * @return + * 0 on success, error code otherwise + */ +int rte_pipeline_table_create(struct rte_pipeline *p, + struct rte_pipeline_table_params *params, + uint32_t *table_id); + +/** + * Pipeline table default entry add + * + * The contents of the table default entry is updated with the provided actions + * and meta-data. When the default entry is not configured (by using this + * function), the built-in default entry has the action "Drop" and meta-data + * set to all-zeros. + * + * @param p + * Handle to pipeline instance + * @param table_id + * Table ID (returned by previous invocation of pipeline table create) + * @param default_entry + * New contents for the table default entry + * @param default_entry_ptr + * On successful invocation, pointer to the default table entry which can be + * used for further read-write accesses to this table entry. This pointer + * is valid until the default entry is deleted or re-added. + * @return + * 0 on success, error code otherwise + */ +int rte_pipeline_table_default_entry_add(struct rte_pipeline *p, + uint32_t table_id, + struct rte_pipeline_table_entry *default_entry, + struct rte_pipeline_table_entry **default_entry_ptr); + +/** + * Pipeline table default entry delete + * + * The new contents of the table default entry is set to reserved action "Drop + * the packet" with meta-data cleared (i.e. set to all-zeros). + * + * @param p + * Handle to pipeline instance + * @param table_id + * Table ID (returned by previous invocation of pipeline table create) + * @param entry + * On successful invocation, when entry points to a valid buffer, the + * previous contents of the table default entry (as it was just before the + * delete operation) is copied to this buffer + * @return + * 0 on success, error code otherwise + */ +int rte_pipeline_table_default_entry_delete(struct rte_pipeline *p, + uint32_t table_id, + struct rte_pipeline_table_entry *entry); + +/** + * Pipeline table entry add + * + * @param p + * Handle to pipeline instance + * @param table_id + * Table ID (returned by previous invocation of pipeline table create) + * @param key + * Table entry key + * @param entry + * New contents for the table entry identified by key + * @param key_found + * On successful invocation, set to TRUE (value different than 0) if key was + * already present in the table before the add operation and to FALSE (value + * 0) if not + * @param entry_ptr + * On successful invocation, pointer to the table entry associated with key. + * This can be used for further read-write accesses to this table entry and + * is valid until the key is deleted from the table or re-added (usually for + * associating different actions and/or action meta-data to the current key) + * @return + * 0 on success, error code otherwise + */ +int rte_pipeline_table_entry_add(struct rte_pipeline *p, + uint32_t table_id, + void *key, + struct rte_pipeline_table_entry *entry, + int *key_found, + struct rte_pipeline_table_entry **entry_ptr); + +/** + * Pipeline table entry delete + * + * @param p + * Handle to pipeline instance + * @param table_id + * Table ID (returned by previous invocation of pipeline table create) + * @param key + * Table entry key + * @param key_found + * On successful invocation, set to TRUE (value different than 0) if key was + * found in the table before the delete operation and to FALSE (value 0) if + * not + * @param entry + * On successful invocation, when key is found in the table and entry points + * to a valid buffer, the table entry contents (as it was before the delete + * was performed) is copied to this buffer + * @return + * 0 on success, error code otherwise + */ +int rte_pipeline_table_entry_delete(struct rte_pipeline *p, + uint32_t table_id, + void *key, + int *key_found, + struct rte_pipeline_table_entry *entry); + +/** + * Pipeline table entry add bulk + * + * @param p + * Handle to pipeline instance + * @param table_id + * Table ID (returned by previous invocation of pipeline table create) + * @param keys + * Array containing table entry keys + * @param entries + * Array containing new contents for every table entry identified by key + * @param n_keys + * Number of keys to add + * @param key_found + * On successful invocation, key_found for every item in the array is set to + * TRUE (value different than 0) if key was already present in the table + * before the add operation and to FALSE (value 0) if not + * @param entries_ptr + * On successful invocation, array *entries_ptr stores pointer to every table + * entry associated with key. This can be used for further read-write accesses + * to this table entry and is valid until the key is deleted from the table or + * re-added (usually for associating different actions and/or action meta-data + * to the current key) + * @return + * 0 on success, error code otherwise + */ +int rte_pipeline_table_entry_add_bulk(struct rte_pipeline *p, + uint32_t table_id, + void **keys, + struct rte_pipeline_table_entry **entries, + uint32_t n_keys, + int *key_found, + struct rte_pipeline_table_entry **entries_ptr); + +/** + * Pipeline table entry delete bulk + * + * @param p + * Handle to pipeline instance + * @param table_id + * Table ID (returned by previous invocation of pipeline table create) + * @param keys + * Array containing table entry keys + * @param n_keys + * Number of keys to delete + * @param key_found + * On successful invocation, key_found for every item in the array is set to + * TRUE (value different than 0) if key was found in the table before the + * delete operation and to FALSE (value 0) if not + * @param entries + * If entries pointer is NULL, this pointer is ignored for every entry found. + * Else, after successful invocation, if specific key is found in the table + * and entry points to a valid buffer, the table entry contents (as it was + * before the delete was performed) is copied to this buffer. + * @return + * 0 on success, error code otherwise + */ +int rte_pipeline_table_entry_delete_bulk(struct rte_pipeline *p, + uint32_t table_id, + void **keys, + uint32_t n_keys, + int *key_found, + struct rte_pipeline_table_entry **entries); + +/** + * Read pipeline table stats. + * + * This function reads table statistics identified by *table_id* of given + * pipeline *p*. + * + * @param p + * Handle to pipeline instance. + * @param table_id + * Port ID what stats will be returned. + * @param stats + * Statistics buffer. + * @param clear + * If not 0 clear stats after reading. + * @return + * 0 on success, error code otherwise + */ +int rte_pipeline_table_stats_read(struct rte_pipeline *p, uint32_t table_id, + struct rte_pipeline_table_stats *stats, int clear); + +/* + * Port IN + * + */ +/** Maximum number of input ports allowed for any given pipeline instance. The + value of this parameter cannot be changed. */ +#define RTE_PIPELINE_PORT_IN_MAX 64 + +/** + * Pipeline input port action handler + * + * The action handler can decide to drop packets by resetting the associated + * packet bit in the pkts_mask parameter. In this case, the action handler is + * required not to free the packet buffer, which will be freed eventually by + * the pipeline. + * + * @param p + * Handle to pipeline instance + * @param pkts + * Burst of input packets specified as array of up to 64 pointers to struct + * rte_mbuf + * @param n + * Number of packets in the input burst. This parameter specifies that + * elements 0 to (n-1) of pkts array are valid. + * @param arg + * Opaque parameter registered by the user at the pipeline table creation + * time + * @return + * 0 on success, error code otherwise + */ +typedef int (*rte_pipeline_port_in_action_handler)( + struct rte_pipeline *p, + struct rte_mbuf **pkts, + uint32_t n, + void *arg); + +/** Parameters for pipeline input port creation */ +struct rte_pipeline_port_in_params { + /** Input port operations (specific to each table type) */ + struct rte_port_in_ops *ops; + /** Opaque parameter to be passed to create operation when invoked */ + void *arg_create; + + /** Callback function to execute the user actions on input packets. + Disabled if set to NULL. */ + rte_pipeline_port_in_action_handler f_action; + /** Opaque parameter to be passed to the action handler when invoked */ + void *arg_ah; + + /** Recommended burst size for the RX operation(in number of pkts) */ + uint32_t burst_size; +}; + +/** + * Pipeline input port create + * + * @param p + * Handle to pipeline instance + * @param params + * Parameters for pipeline input port creation + * @param port_id + * Input port ID. Valid only within the scope of input port IDs of the + * current pipeline. Only returned after a successful invocation. + * @return + * 0 on success, error code otherwise + */ +int rte_pipeline_port_in_create(struct rte_pipeline *p, + struct rte_pipeline_port_in_params *params, + uint32_t *port_id); + +/** + * Pipeline input port connect to table + * + * @param p + * Handle to pipeline instance + * @param port_id + * Port ID (returned by previous invocation of pipeline input port create) + * @param table_id + * Table ID (returned by previous invocation of pipeline table create) + * @return + * 0 on success, error code otherwise + */ +int rte_pipeline_port_in_connect_to_table(struct rte_pipeline *p, + uint32_t port_id, + uint32_t table_id); + +/** + * Pipeline input port enable + * + * @param p + * Handle to pipeline instance + * @param port_id + * Port ID (returned by previous invocation of pipeline input port create) + * @return + * 0 on success, error code otherwise + */ +int rte_pipeline_port_in_enable(struct rte_pipeline *p, + uint32_t port_id); + +/** + * Pipeline input port disable + * + * @param p + * Handle to pipeline instance + * @param port_id + * Port ID (returned by previous invocation of pipeline input port create) + * @return + * 0 on success, error code otherwise + */ +int rte_pipeline_port_in_disable(struct rte_pipeline *p, + uint32_t port_id); + +/** + * Read pipeline port in stats. + * + * This function reads port in statistics identified by *port_id* of given + * pipeline *p*. + * + * @param p + * Handle to pipeline instance. + * @param port_id + * Port ID what stats will be returned. + * @param stats + * Statistics buffer. + * @param clear + * If not 0 clear stats after reading. + * @return + * 0 on success, error code otherwise + */ +int rte_pipeline_port_in_stats_read(struct rte_pipeline *p, uint32_t port_id, + struct rte_pipeline_port_in_stats *stats, int clear); + +/* + * Port OUT + * + */ +/** Maximum number of output ports allowed for any given pipeline instance. The + value of this parameter cannot be changed. */ +#define RTE_PIPELINE_PORT_OUT_MAX 64 + +/** + * Pipeline output port action handler + * + * The action handler can decide to drop packets by resetting the associated + * packet bit in the pkts_mask parameter. In this case, the action handler is + * required not to free the packet buffer, which will be freed eventually by + * the pipeline. + * + * @param p + * Handle to pipeline instance + * @param pkts + * Burst of input packets specified as array of up to 64 pointers to struct + * rte_mbuf + * @param pkts_mask + * 64-bit bitmask specifying which packets in the input burst are valid. When + * pkts_mask bit n is set, then element n of pkts array is pointing to a + * valid packet. Otherwise, element n of pkts array will not be accessed. + * @param arg + * Opaque parameter registered by the user at the pipeline table creation + * time + * @return + * 0 on success, error code otherwise + */ +typedef int (*rte_pipeline_port_out_action_handler)( + struct rte_pipeline *p, + struct rte_mbuf **pkts, + uint64_t pkts_mask, + void *arg); + +/** Parameters for pipeline output port creation. The action handlers have to +be either both enabled or both disabled (by setting them to NULL). When +enabled, the pipeline selects between them at different moments, based on the +number of packets that have to be sent to the same output port. */ +struct rte_pipeline_port_out_params { + /** Output port operations (specific to each table type) */ + struct rte_port_out_ops *ops; + /** Opaque parameter to be passed to create operation when invoked */ + void *arg_create; + + /** Callback function executing the user actions on bust of input + packets */ + rte_pipeline_port_out_action_handler f_action; + /** Opaque parameter to be passed to the action handler when invoked */ + void *arg_ah; +}; + +/** + * Pipeline output port create + * + * @param p + * Handle to pipeline instance + * @param params + * Parameters for pipeline output port creation + * @param port_id + * Output port ID. Valid only within the scope of output port IDs of the + * current pipeline. Only returned after a successful invocation. + * @return + * 0 on success, error code otherwise + */ +int rte_pipeline_port_out_create(struct rte_pipeline *p, + struct rte_pipeline_port_out_params *params, + uint32_t *port_id); + +/** + * Read pipeline port out stats. + * + * This function reads port out statistics identified by *port_id* of given + * pipeline *p*. + * + * @param p + * Handle to pipeline instance. + * @param port_id + * Port ID what stats will be returned. + * @param stats + * Statistics buffer. + * @param clear + * If not 0 clear stats after reading. + * @return + * 0 on success, error code otherwise + */ +int rte_pipeline_port_out_stats_read(struct rte_pipeline *p, uint32_t port_id, + struct rte_pipeline_port_out_stats *stats, int clear); + +/* + * Functions to be called as part of the port IN/OUT or table action handlers + * + */ +/** + * Action handler packet insert to output port + * + * This function can be called by any input/output port or table action handler + * to send a packet out through one of the pipeline output ports. This packet is + * generated by the action handler, i.e. this packet is not part of the burst of + * packets read from one of the pipeline input ports and currently processed by + * the pipeline (this packet is not an element of the pkts array input parameter + * of the action handler). + * + * @param p + * Handle to pipeline instance + * @param port_id + * Output port ID (returned by previous invocation of pipeline output port + * create) to send the packet specified by pkt + * @param pkt + * New packet generated by the action handler + * @return + * 0 on success, error code otherwise + */ +int rte_pipeline_port_out_packet_insert(struct rte_pipeline *p, + uint32_t port_id, + struct rte_mbuf *pkt); + +#define rte_pipeline_ah_port_out_packet_insert \ + rte_pipeline_port_out_packet_insert + +/** + * Action handler packet hijack + * + * This function can be called by any input/output port or table action handler + * to hijack selected packets from the burst of packets read from one of the + * pipeline input ports and currently processed by the pipeline. The hijacked + * packets are removed from any further pipeline processing, with the action + * handler now having the full ownership for these packets. + * + * The action handler can further send the hijacked packets out through any + * pipeline output port by calling the rte_pipeline_ah_port_out_packet_insert() + * function. The action handler can also drop these packets by calling the + * rte_pktmbuf_free() function, although a better alternative is provided by + * the action handler using the rte_pipeline_ah_packet_drop() function. + * + * @param p + * Handle to pipeline instance + * @param pkts_mask + * 64-bit bitmask specifying which of the packets handed over for processing + * to the action handler is to be hijacked by the action handler. When + * pkts_mask bit n is set, then element n of the pkts array (input argument to + * the action handler) is hijacked. + * @return + * 0 on success, error code otherwise + */ +int rte_pipeline_ah_packet_hijack(struct rte_pipeline *p, + uint64_t pkts_mask); + +/** + * Action handler packet drop + * + * This function is called by the pipeline action handlers (port in/out, table) + * to drop the packets selected using packet mask. + * + * This function can be called by any input/output port or table action handler + * to drop selected packets from the burst of packets read from one of the + * pipeline input ports and currently processed by the pipeline. The dropped + * packets are removed from any further pipeline processing and the packet + * buffers are eventually freed to their buffer pool. + * + * This function updates the drop statistics counters correctly, therefore the + * recommended approach for dropping packets by the action handlers is to call + * this function as opposed to the action handler hijacking the packets first + * and then dropping them invisibly to the pipeline (by using the + * rte_pktmbuf_free() function). + * + * @param p + * Handle to pipeline instance + * @param pkts_mask + * 64-bit bitmask specifying which of the packets handed over for processing + * to the action handler is to be dropped by the action handler. When + * pkts_mask bit n is set, then element n of the pkts array (input argument to + * the action handler) is dropped. + * @return + * 0 on success, error code otherwise + */ +int rte_pipeline_ah_packet_drop(struct rte_pipeline *p, + uint64_t pkts_mask); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/src/spdk/dpdk/lib/librte_pipeline/rte_pipeline_version.map b/src/spdk/dpdk/lib/librte_pipeline/rte_pipeline_version.map new file mode 100644 index 00000000..d820b22f --- /dev/null +++ b/src/spdk/dpdk/lib/librte_pipeline/rte_pipeline_version.map @@ -0,0 +1,75 @@ +DPDK_2.0 { + global: + + rte_pipeline_check; + rte_pipeline_create; + rte_pipeline_flush; + rte_pipeline_free; + rte_pipeline_port_in_connect_to_table; + rte_pipeline_port_in_create; + rte_pipeline_port_in_disable; + rte_pipeline_port_in_enable; + rte_pipeline_port_out_create; + rte_pipeline_port_out_packet_insert; + rte_pipeline_run; + rte_pipeline_table_create; + rte_pipeline_table_default_entry_add; + rte_pipeline_table_default_entry_delete; + rte_pipeline_table_entry_add; + rte_pipeline_table_entry_delete; + + local: *; +}; + +DPDK_2.1 { + global: + + rte_pipeline_port_in_stats_read; + rte_pipeline_port_out_stats_read; + rte_pipeline_table_stats_read; + +} DPDK_2.0; + +DPDK_2.2 { + global: + + rte_pipeline_table_entry_add_bulk; + rte_pipeline_table_entry_delete_bulk; + +} DPDK_2.1; + +DPDK_16.04 { + global: + + rte_pipeline_ah_packet_hijack; + rte_pipeline_ah_packet_drop; + +} DPDK_2.2; + +EXPERIMENTAL { + global: + + rte_port_in_action_apply; + rte_port_in_action_create; + rte_port_in_action_free; + rte_port_in_action_params_get; + rte_port_in_action_profile_action_register; + rte_port_in_action_profile_create; + rte_port_in_action_profile_free; + rte_port_in_action_profile_freeze; + rte_table_action_apply; + rte_table_action_create; + rte_table_action_dscp_table_update; + rte_table_action_free; + rte_table_action_meter_profile_add; + rte_table_action_meter_profile_delete; + rte_table_action_meter_read; + rte_table_action_profile_action_register; + rte_table_action_profile_create; + rte_table_action_profile_free; + rte_table_action_profile_freeze; + rte_table_action_table_params_get; + rte_table_action_stats_read; + rte_table_action_time_read; + rte_table_action_ttl_read; +}; diff --git a/src/spdk/dpdk/lib/librte_pipeline/rte_port_in_action.c b/src/spdk/dpdk/lib/librte_pipeline/rte_port_in_action.c new file mode 100644 index 00000000..e3b00df8 --- /dev/null +++ b/src/spdk/dpdk/lib/librte_pipeline/rte_port_in_action.c @@ -0,0 +1,531 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2010-2018 Intel Corporation + */ + +#include <stdlib.h> +#include <string.h> + +#include <rte_common.h> +#include <rte_byteorder.h> +#include <rte_malloc.h> +#include <rte_memcpy.h> + +#include "rte_port_in_action.h" + +/** + * RTE_PORT_IN_ACTION_FLTR + */ +static int +fltr_cfg_check(struct rte_port_in_action_fltr_config *cfg) +{ + if (cfg == NULL) + return -1; + + return 0; +} + +struct fltr_data { + uint32_t port_id; +}; + +static void +fltr_init(struct fltr_data *data, + struct rte_port_in_action_fltr_config *cfg) +{ + data->port_id = cfg->port_id; +} + +static int +fltr_apply(struct fltr_data *data, + struct rte_port_in_action_fltr_params *p) +{ + /* Check input arguments */ + if (p == NULL) + return -1; + + data->port_id = p->port_id; + + return 0; +} + +/** + * RTE_PORT_IN_ACTION_LB + */ +static int +lb_cfg_check(struct rte_port_in_action_lb_config *cfg) +{ + if ((cfg == NULL) || + (cfg->key_size < RTE_PORT_IN_ACTION_LB_KEY_SIZE_MIN) || + (cfg->key_size > RTE_PORT_IN_ACTION_LB_KEY_SIZE_MAX) || + (!rte_is_power_of_2(cfg->key_size)) || + (cfg->f_hash == NULL)) + return -1; + + return 0; +} + +struct lb_data { + uint32_t port_id[RTE_PORT_IN_ACTION_LB_TABLE_SIZE]; +}; + +static void +lb_init(struct lb_data *data, + struct rte_port_in_action_lb_config *cfg) +{ + memcpy(data->port_id, cfg->port_id, sizeof(cfg->port_id)); +} + +static int +lb_apply(struct lb_data *data, + struct rte_port_in_action_lb_params *p) +{ + /* Check input arguments */ + if (p == NULL) + return -1; + + memcpy(data->port_id, p->port_id, sizeof(p->port_id)); + + return 0; +} + +/** + * Action profile + */ +static int +action_valid(enum rte_port_in_action_type action) +{ + switch (action) { + case RTE_PORT_IN_ACTION_FLTR: + case RTE_PORT_IN_ACTION_LB: + return 1; + default: + return 0; + } +} + +#define RTE_PORT_IN_ACTION_MAX 64 + +struct ap_config { + uint64_t action_mask; + struct rte_port_in_action_fltr_config fltr; + struct rte_port_in_action_lb_config lb; +}; + +static size_t +action_cfg_size(enum rte_port_in_action_type action) +{ + switch (action) { + case RTE_PORT_IN_ACTION_FLTR: + return sizeof(struct rte_port_in_action_fltr_config); + case RTE_PORT_IN_ACTION_LB: + return sizeof(struct rte_port_in_action_lb_config); + default: + return 0; + } +} + +static void* +action_cfg_get(struct ap_config *ap_config, + enum rte_port_in_action_type type) +{ + switch (type) { + case RTE_PORT_IN_ACTION_FLTR: + return &ap_config->fltr; + + case RTE_PORT_IN_ACTION_LB: + return &ap_config->lb; + + default: + return NULL; + } +} + +static void +action_cfg_set(struct ap_config *ap_config, + enum rte_port_in_action_type type, + void *action_cfg) +{ + void *dst = action_cfg_get(ap_config, type); + + if (dst) + memcpy(dst, action_cfg, action_cfg_size(type)); + + ap_config->action_mask |= 1LLU << type; +} + +struct ap_data { + size_t offset[RTE_PORT_IN_ACTION_MAX]; + size_t total_size; +}; + +static size_t +action_data_size(enum rte_port_in_action_type action, + struct ap_config *ap_config __rte_unused) +{ + switch (action) { + case RTE_PORT_IN_ACTION_FLTR: + return sizeof(struct fltr_data); + + case RTE_PORT_IN_ACTION_LB: + return sizeof(struct lb_data); + + default: + return 0; + } +} + +static void +action_data_offset_set(struct ap_data *ap_data, + struct ap_config *ap_config) +{ + uint64_t action_mask = ap_config->action_mask; + size_t offset; + uint32_t action; + + memset(ap_data->offset, 0, sizeof(ap_data->offset)); + + offset = 0; + for (action = 0; action < RTE_PORT_IN_ACTION_MAX; action++) + if (action_mask & (1LLU << action)) { + ap_data->offset[action] = offset; + offset += action_data_size((enum rte_port_in_action_type)action, + ap_config); + } + + ap_data->total_size = offset; +} + +struct rte_port_in_action_profile { + struct ap_config cfg; + struct ap_data data; + int frozen; +}; + +struct rte_port_in_action_profile * +rte_port_in_action_profile_create(uint32_t socket_id) +{ + struct rte_port_in_action_profile *ap; + + /* Memory allocation */ + ap = rte_zmalloc_socket(NULL, + sizeof(struct rte_port_in_action_profile), + RTE_CACHE_LINE_SIZE, + socket_id); + if (ap == NULL) + return NULL; + + return ap; +} + +int +rte_port_in_action_profile_action_register(struct rte_port_in_action_profile *profile, + enum rte_port_in_action_type type, + void *action_config) +{ + int status; + + /* Check input arguments */ + if ((profile == NULL) || + profile->frozen || + (action_valid(type) == 0) || + (profile->cfg.action_mask & (1LLU << type)) || + ((action_cfg_size(type) == 0) && action_config) || + (action_cfg_size(type) && (action_config == NULL))) + return -EINVAL; + + switch (type) { + case RTE_PORT_IN_ACTION_FLTR: + status = fltr_cfg_check(action_config); + break; + + case RTE_PORT_IN_ACTION_LB: + status = lb_cfg_check(action_config); + break; + + default: + status = 0; + break; + } + + if (status) + return status; + + /* Action enable */ + action_cfg_set(&profile->cfg, type, action_config); + + return 0; +} + +int +rte_port_in_action_profile_freeze(struct rte_port_in_action_profile *profile) +{ + if (profile->frozen) + return -EBUSY; + + action_data_offset_set(&profile->data, &profile->cfg); + profile->frozen = 1; + + return 0; +} + +int +rte_port_in_action_profile_free(struct rte_port_in_action_profile *profile) +{ + if (profile == NULL) + return 0; + + free(profile); + return 0; +} + +/** + * Action + */ +struct rte_port_in_action { + struct ap_config cfg; + struct ap_data data; + uint8_t memory[0] __rte_cache_aligned; +}; + +static __rte_always_inline void * +action_data_get(struct rte_port_in_action *action, + enum rte_port_in_action_type type) +{ + size_t offset = action->data.offset[type]; + + return &action->memory[offset]; +} + +static void +action_data_init(struct rte_port_in_action *action, + enum rte_port_in_action_type type) +{ + void *data = action_data_get(action, type); + + switch (type) { + case RTE_PORT_IN_ACTION_FLTR: + fltr_init(data, &action->cfg.fltr); + return; + + case RTE_PORT_IN_ACTION_LB: + lb_init(data, &action->cfg.lb); + return; + + default: + return; + } +} + +struct rte_port_in_action * +rte_port_in_action_create(struct rte_port_in_action_profile *profile, + uint32_t socket_id) +{ + struct rte_port_in_action *action; + size_t size; + uint32_t i; + + /* Check input arguments */ + if ((profile == NULL) || + (profile->frozen == 0)) + return NULL; + + /* Memory allocation */ + size = sizeof(struct rte_port_in_action) + profile->data.total_size; + size = RTE_CACHE_LINE_ROUNDUP(size); + + action = rte_zmalloc_socket(NULL, + size, + RTE_CACHE_LINE_SIZE, + socket_id); + if (action == NULL) + return NULL; + + /* Initialization */ + memcpy(&action->cfg, &profile->cfg, sizeof(profile->cfg)); + memcpy(&action->data, &profile->data, sizeof(profile->data)); + + for (i = 0; i < RTE_PORT_IN_ACTION_MAX; i++) + if (action->cfg.action_mask & (1LLU << i)) + action_data_init(action, + (enum rte_port_in_action_type)i); + + return action; +} + +int +rte_port_in_action_apply(struct rte_port_in_action *action, + enum rte_port_in_action_type type, + void *action_params) +{ + void *action_data; + + /* Check input arguments */ + if ((action == NULL) || + (action_valid(type) == 0) || + ((action->cfg.action_mask & (1LLU << type)) == 0) || + (action_params == NULL)) + return -EINVAL; + + /* Data update */ + action_data = action_data_get(action, type); + + switch (type) { + case RTE_PORT_IN_ACTION_FLTR: + return fltr_apply(action_data, + action_params); + + case RTE_PORT_IN_ACTION_LB: + return lb_apply(action_data, + action_params); + + default: + return -EINVAL; + } +} + +static int +ah_filter_on_match(struct rte_pipeline *p, + struct rte_mbuf **pkts, + uint32_t n_pkts, + void *arg) +{ + struct rte_port_in_action *action = arg; + struct rte_port_in_action_fltr_config *cfg = &action->cfg.fltr; + uint64_t *key_mask = (uint64_t *) cfg->key_mask; + uint64_t *key = (uint64_t *) cfg->key; + uint32_t key_offset = cfg->key_offset; + struct fltr_data *data = action_data_get(action, + RTE_PORT_IN_ACTION_FLTR); + uint32_t i; + + for (i = 0; i < n_pkts; i++) { + struct rte_mbuf *pkt = pkts[i]; + uint64_t *pkt_key = RTE_MBUF_METADATA_UINT64_PTR(pkt, + key_offset); + + uint64_t xor0 = (pkt_key[0] & key_mask[0]) ^ key[0]; + uint64_t xor1 = (pkt_key[1] & key_mask[1]) ^ key[1]; + uint64_t or = xor0 | xor1; + + if (or == 0) { + rte_pipeline_ah_packet_hijack(p, 1LLU << i); + rte_pipeline_port_out_packet_insert(p, + data->port_id, pkt); + } + } + + return 0; +} + +static int +ah_filter_on_mismatch(struct rte_pipeline *p, + struct rte_mbuf **pkts, + uint32_t n_pkts, + void *arg) +{ + struct rte_port_in_action *action = arg; + struct rte_port_in_action_fltr_config *cfg = &action->cfg.fltr; + uint64_t *key_mask = (uint64_t *) cfg->key_mask; + uint64_t *key = (uint64_t *) cfg->key; + uint32_t key_offset = cfg->key_offset; + struct fltr_data *data = action_data_get(action, + RTE_PORT_IN_ACTION_FLTR); + uint32_t i; + + for (i = 0; i < n_pkts; i++) { + struct rte_mbuf *pkt = pkts[i]; + uint64_t *pkt_key = RTE_MBUF_METADATA_UINT64_PTR(pkt, + key_offset); + + uint64_t xor0 = (pkt_key[0] & key_mask[0]) ^ key[0]; + uint64_t xor1 = (pkt_key[1] & key_mask[1]) ^ key[1]; + uint64_t or = xor0 | xor1; + + if (or) { + rte_pipeline_ah_packet_hijack(p, 1LLU << i); + rte_pipeline_port_out_packet_insert(p, + data->port_id, pkt); + } + } + + return 0; +} + +static int +ah_lb(struct rte_pipeline *p, + struct rte_mbuf **pkts, + uint32_t n_pkts, + void *arg) +{ + struct rte_port_in_action *action = arg; + struct rte_port_in_action_lb_config *cfg = &action->cfg.lb; + struct lb_data *data = action_data_get(action, RTE_PORT_IN_ACTION_LB); + uint64_t pkt_mask = RTE_LEN2MASK(n_pkts, uint64_t); + uint32_t i; + + rte_pipeline_ah_packet_hijack(p, pkt_mask); + + for (i = 0; i < n_pkts; i++) { + struct rte_mbuf *pkt = pkts[i]; + uint8_t *pkt_key = RTE_MBUF_METADATA_UINT8_PTR(pkt, + cfg->key_offset); + + uint64_t digest = cfg->f_hash(pkt_key, + cfg->key_mask, + cfg->key_size, + cfg->seed); + uint64_t pos = digest & (RTE_PORT_IN_ACTION_LB_TABLE_SIZE - 1); + uint32_t port_id = data->port_id[pos]; + + rte_pipeline_port_out_packet_insert(p, port_id, pkt); + } + + return 0; +} + +static rte_pipeline_port_in_action_handler +ah_selector(struct rte_port_in_action *action) +{ + if (action->cfg.action_mask == 0) + return NULL; + + if (action->cfg.action_mask == 1LLU << RTE_PORT_IN_ACTION_FLTR) + return (action->cfg.fltr.filter_on_match) ? + ah_filter_on_match : ah_filter_on_mismatch; + + if (action->cfg.action_mask == 1LLU << RTE_PORT_IN_ACTION_LB) + return ah_lb; + + return NULL; +} + +int +rte_port_in_action_params_get(struct rte_port_in_action *action, + struct rte_pipeline_port_in_params *params) +{ + rte_pipeline_port_in_action_handler f_action; + + /* Check input arguments */ + if ((action == NULL) || + (params == NULL)) + return -EINVAL; + + f_action = ah_selector(action); + + /* Fill in params */ + params->f_action = f_action; + params->arg_ah = (f_action) ? action : NULL; + + return 0; +} + +int +rte_port_in_action_free(struct rte_port_in_action *action) +{ + if (action == NULL) + return 0; + + rte_free(action); + + return 0; +} diff --git a/src/spdk/dpdk/lib/librte_pipeline/rte_port_in_action.h b/src/spdk/dpdk/lib/librte_pipeline/rte_port_in_action.h new file mode 100644 index 00000000..0a85e4e0 --- /dev/null +++ b/src/spdk/dpdk/lib/librte_pipeline/rte_port_in_action.h @@ -0,0 +1,301 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2018 Intel Corporation + */ + +#ifndef __INCLUDE_RTE_PORT_IN_ACTION_H__ +#define __INCLUDE_RTE_PORT_IN_ACTION_H__ + +/** + * @file + * RTE Pipeline Input Port Actions + * + * This API provides a common set of actions for pipeline input ports to speed + * up application development. + * + * Each pipeline input port can be assigned an action handler to be executed + * on every input packet during the pipeline execution. The pipeline library + * allows the user to define his own input port actions by providing customized + * input port action handler. While the user can still follow this process, this + * API is intended to provide a quicker development alternative for a set of + * predefined actions. + * + * The typical steps to use this API are: + * - Define an input port action profile. This is a configuration template that + * can potentially be shared by multiple input ports from the same or + * different pipelines, with different input ports from the same pipeline + * able to use different action profiles. For every input port using a given + * action profile, the profile defines the set of actions and the action + * configuration to be executed by the input port. API functions: + * rte_port_in_action_profile_create(), + * rte_port_in_action_profile_action_register(), + * rte_port_in_action_profile_freeze(). + * + * - Instantiate the input port action profile to create input port action + * objects. Each pipeline input port has its own action object. + * API functions: rte_port_in_action_create(). + * + * - Use the input port action object to generate the input port action handler + * invoked by the pipeline. API functions: + * rte_port_in_action_params_get(). + * + * - Use the input port action object to generate the internal data structures + * used by the input port action handler based on given action parameters. + * API functions: rte_port_in_action_apply(). + * + * @warning + * @b EXPERIMENTAL: this API may change without prior notice + */ + +#ifdef __cplusplus +extern "C" { +#endif + +#include <stdint.h> + +#include <rte_compat.h> +#include <rte_table_hash.h> + +#include "rte_pipeline.h" + +/** Input port actions. */ +enum rte_port_in_action_type { + /** Filter selected input packets. */ + RTE_PORT_IN_ACTION_FLTR = 0, + + /** Load balance. */ + RTE_PORT_IN_ACTION_LB, +}; + +/** + * RTE_PORT_IN_ACTION_FLTR + */ +/** Filter key size (number of bytes) */ +#define RTE_PORT_IN_ACTION_FLTR_KEY_SIZE 16 + +/** Filter action configuration (per action profile). */ +struct rte_port_in_action_fltr_config { + /** Key offset within the input packet buffer. Offset 0 points to the + * first byte of the MBUF structure. + */ + uint32_t key_offset; + + /** Key mask. */ + uint8_t key_mask[RTE_PORT_IN_ACTION_FLTR_KEY_SIZE]; + + /** Key value. */ + uint8_t key[RTE_PORT_IN_ACTION_FLTR_KEY_SIZE]; + + /** When non-zero, all the input packets that match the *key* (with the + * *key_mask* applied) are sent to the pipeline output port *port_id*. + * When zero, all the input packets that do NOT match the *key* (with + * *key_mask* applied) are sent to the pipeline output port *port_id*. + */ + int filter_on_match; + + /** Pipeline output port ID to send the filtered input packets to. + * Can be updated later. + * + * @see struct rte_port_in_action_fltr_params + */ + uint32_t port_id; +}; + +/** Filter action parameters (per action). */ +struct rte_port_in_action_fltr_params { + /** Pipeline output port ID to send the filtered input packets to. */ + uint32_t port_id; +}; + +/** + * RTE_PORT_IN_ACTION_LB + */ +/** Load balance key size min (number of bytes). */ +#define RTE_PORT_IN_ACTION_LB_KEY_SIZE_MIN 8 + +/** Load balance key size max (number of bytes). */ +#define RTE_PORT_IN_ACTION_LB_KEY_SIZE_MAX 64 + +/** Load balance table size. */ +#define RTE_PORT_IN_ACTION_LB_TABLE_SIZE 16 + +/** Load balance action configuration (per action profile). */ +struct rte_port_in_action_lb_config { + /** Key size (number of bytes). */ + uint32_t key_size; + + /** Key offset within the input packet buffer. Offset 0 points to the + * first byte of the MBUF structure. + */ + uint32_t key_offset; + + /** Key mask(*key_size* bytes are valid). */ + uint8_t key_mask[RTE_PORT_IN_ACTION_LB_KEY_SIZE_MAX]; + + /** Hash function. */ + rte_table_hash_op_hash f_hash; + + /** Seed value for *f_hash*. */ + uint64_t seed; + + /** Table defining the weight of each pipeline output port. The weights + * are set in 1/RTE_PORT_IN_ACTION_LB_TABLE_SIZE increments. To assign a + * weight of N/RTE_PORT_IN_ACTION_LB_TABLE_SIZE to a given output port + * (0 <= N <= RTE_PORT_IN_ACTION_LB_TABLE_SIZE), the output port needs + * to show up exactly N times in this table. Can be updated later. + * + * @see struct rte_port_in_action_lb_params + */ + uint32_t port_id[RTE_PORT_IN_ACTION_LB_TABLE_SIZE]; +}; + +/** Load balance action parameters (per action). */ +struct rte_port_in_action_lb_params { + /** Table defining the weight of each pipeline output port. The weights + * are set in 1/RTE_PORT_IN_ACTION_LB_TABLE_SIZE increments. To assign a + * weight of N/RTE_PORT_IN_ACTION_LB_TABLE_SIZE to a given output port + * (0 <= N <= RTE_PORT_IN_ACTION_LB_TABLE_SIZE), the output port needs + * to show up exactly N times in this table. + */ + uint32_t port_id[RTE_PORT_IN_ACTION_LB_TABLE_SIZE]; +}; + +/** + * Input port action profile. + */ +struct rte_port_in_action_profile; + +/** + * Input port action profile create. + * + * @param[in] socket_id + * CPU socket ID for the internal data structures memory allocation. + * @return + * Input port action profile handle on success, NULL otherwise. + */ +struct rte_port_in_action_profile * __rte_experimental +rte_port_in_action_profile_create(uint32_t socket_id); + +/** + * Input port action profile free. + * + * @param[in] profile + * Input port action profile handle (needs to be valid). + * @return + * Zero on success, non-zero error code otherwise. + */ +int __rte_experimental +rte_port_in_action_profile_free(struct rte_port_in_action_profile *profile); + +/** + * Input port action profile action register. + * + * @param[in] profile + * Input port action profile handle (needs to be valid and not in frozen + * state). + * @param[in] type + * Specific input port action to be registered for *profile*. + * @param[in] action_config + * Configuration for the *type* action. + * If struct rte_port_in_action_*type*_config is defined, it needs to point to + * a valid instance of this structure, otherwise it needs to be set to NULL. + * @return + * Zero on success, non-zero error code otherwise. + */ +int __rte_experimental +rte_port_in_action_profile_action_register( + struct rte_port_in_action_profile *profile, + enum rte_port_in_action_type type, + void *action_config); + +/** + * Input port action profile freeze. + * + * Once this function is called successfully, the given profile enters the + * frozen state with the following immediate effects: no more actions can be + * registered for this profile, so the profile can be instantiated to create + * input port action objects. + * + * @param[in] profile + * Input port profile action handle (needs to be valid and not in frozen + * state). + * @return + * Zero on success, non-zero error code otherwise. + * + * @see rte_port_in_action_create() + */ +int __rte_experimental +rte_port_in_action_profile_freeze(struct rte_port_in_action_profile *profile); + +/** + * Input port action. + */ +struct rte_port_in_action; + +/** + * Input port action create. + * + * Instantiates the given input port action profile to create an input port + * action object. + * + * @param[in] profile + * Input port profile action handle (needs to be valid and in frozen state). + * @param[in] socket_id + * CPU socket ID where the internal data structures required by the new input + * port action object should be allocated. + * @return + * Handle to input port action object on success, NULL on error. + */ +struct rte_port_in_action * __rte_experimental +rte_port_in_action_create(struct rte_port_in_action_profile *profile, + uint32_t socket_id); + +/** + * Input port action free. + * + * @param[in] action + * Handle to input port action object (needs to be valid). + * @return + * Zero on success, non-zero error code otherwise. + */ +int __rte_experimental +rte_port_in_action_free(struct rte_port_in_action *action); + +/** + * Input port params get. + * + * @param[in] action + * Handle to input port action object (needs to be valid). + * @param[inout] params + * Pipeline input port parameters (needs to be pre-allocated). + * @return + * Zero on success, non-zero error code otherwise. + */ +int __rte_experimental +rte_port_in_action_params_get(struct rte_port_in_action *action, + struct rte_pipeline_port_in_params *params); + +/** + * Input port action apply. + * + * @param[in] action + * Handle to input port action object (needs to be valid). + * @param[in] type + * Specific input port action previously registered for the input port action + * profile of the *action* object. + * @param[in] action_params + * Parameters for the *type* action. + * If struct rte_port_in_action_*type*_params is defined, it needs to point to + * a valid instance of this structure, otherwise it needs to be set to NULL. + * @return + * Zero on success, non-zero error code otherwise. + */ +int __rte_experimental +rte_port_in_action_apply(struct rte_port_in_action *action, + enum rte_port_in_action_type type, + void *action_params); + +#ifdef __cplusplus +} +#endif + +#endif /* __INCLUDE_RTE_PORT_IN_ACTION_H__ */ diff --git a/src/spdk/dpdk/lib/librte_pipeline/rte_table_action.c b/src/spdk/dpdk/lib/librte_pipeline/rte_table_action.c new file mode 100644 index 00000000..83ffa5de --- /dev/null +++ b/src/spdk/dpdk/lib/librte_pipeline/rte_table_action.c @@ -0,0 +1,2386 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2010-2018 Intel Corporation + */ + +#include <stdlib.h> +#include <string.h> + +#include <rte_common.h> +#include <rte_byteorder.h> +#include <rte_cycles.h> +#include <rte_malloc.h> +#include <rte_memcpy.h> +#include <rte_ether.h> +#include <rte_ip.h> +#include <rte_esp.h> +#include <rte_tcp.h> +#include <rte_udp.h> + +#include "rte_table_action.h" + +#define rte_htons rte_cpu_to_be_16 +#define rte_htonl rte_cpu_to_be_32 + +#define rte_ntohs rte_be_to_cpu_16 +#define rte_ntohl rte_be_to_cpu_32 + +/** + * RTE_TABLE_ACTION_FWD + */ +#define fwd_data rte_pipeline_table_entry + +static int +fwd_apply(struct fwd_data *data, + struct rte_table_action_fwd_params *p) +{ + data->action = p->action; + + if (p->action == RTE_PIPELINE_ACTION_PORT) + data->port_id = p->id; + + if (p->action == RTE_PIPELINE_ACTION_TABLE) + data->table_id = p->id; + + return 0; +} + +/** + * RTE_TABLE_ACTION_LB + */ +static int +lb_cfg_check(struct rte_table_action_lb_config *cfg) +{ + if ((cfg == NULL) || + (cfg->key_size < RTE_TABLE_ACTION_LB_KEY_SIZE_MIN) || + (cfg->key_size > RTE_TABLE_ACTION_LB_KEY_SIZE_MAX) || + (!rte_is_power_of_2(cfg->key_size)) || + (cfg->f_hash == NULL)) + return -1; + + return 0; +} + +struct lb_data { + uint32_t out[RTE_TABLE_ACTION_LB_TABLE_SIZE]; +} __attribute__((__packed__)); + +static int +lb_apply(struct lb_data *data, + struct rte_table_action_lb_params *p) +{ + memcpy(data->out, p->out, sizeof(data->out)); + + return 0; +} + +static __rte_always_inline void +pkt_work_lb(struct rte_mbuf *mbuf, + struct lb_data *data, + struct rte_table_action_lb_config *cfg) +{ + uint8_t *pkt_key = RTE_MBUF_METADATA_UINT8_PTR(mbuf, cfg->key_offset); + uint32_t *out = RTE_MBUF_METADATA_UINT32_PTR(mbuf, cfg->out_offset); + uint64_t digest, pos; + uint32_t out_val; + + digest = cfg->f_hash(pkt_key, + cfg->key_mask, + cfg->key_size, + cfg->seed); + pos = digest & (RTE_TABLE_ACTION_LB_TABLE_SIZE - 1); + out_val = data->out[pos]; + + *out = out_val; +} + +/** + * RTE_TABLE_ACTION_MTR + */ +static int +mtr_cfg_check(struct rte_table_action_mtr_config *mtr) +{ + if ((mtr->alg == RTE_TABLE_ACTION_METER_SRTCM) || + ((mtr->n_tc != 1) && (mtr->n_tc != 4)) || + (mtr->n_bytes_enabled != 0)) + return -ENOTSUP; + return 0; +} + +#define MBUF_SCHED_QUEUE_TC_COLOR(queue, tc, color) \ + ((uint16_t)((((uint64_t)(queue)) & 0x3) | \ + ((((uint64_t)(tc)) & 0x3) << 2) | \ + ((((uint64_t)(color)) & 0x3) << 4))) + +#define MBUF_SCHED_COLOR(sched, color) \ + (((sched) & (~0x30LLU)) | ((color) << 4)) + +struct mtr_trtcm_data { + struct rte_meter_trtcm trtcm; + uint64_t stats[e_RTE_METER_COLORS]; +} __attribute__((__packed__)); + +#define MTR_TRTCM_DATA_METER_PROFILE_ID_GET(data) \ + (((data)->stats[e_RTE_METER_GREEN] & 0xF8LLU) >> 3) + +static void +mtr_trtcm_data_meter_profile_id_set(struct mtr_trtcm_data *data, + uint32_t profile_id) +{ + data->stats[e_RTE_METER_GREEN] &= ~0xF8LLU; + data->stats[e_RTE_METER_GREEN] |= (profile_id % 32) << 3; +} + +#define MTR_TRTCM_DATA_POLICER_ACTION_DROP_GET(data, color)\ + (((data)->stats[(color)] & 4LLU) >> 2) + +#define MTR_TRTCM_DATA_POLICER_ACTION_COLOR_GET(data, color)\ + ((enum rte_meter_color)((data)->stats[(color)] & 3LLU)) + +static void +mtr_trtcm_data_policer_action_set(struct mtr_trtcm_data *data, + enum rte_meter_color color, + enum rte_table_action_policer action) +{ + if (action == RTE_TABLE_ACTION_POLICER_DROP) { + data->stats[color] |= 4LLU; + } else { + data->stats[color] &= ~7LLU; + data->stats[color] |= color & 3LLU; + } +} + +static uint64_t +mtr_trtcm_data_stats_get(struct mtr_trtcm_data *data, + enum rte_meter_color color) +{ + return data->stats[color] >> 8; +} + +static void +mtr_trtcm_data_stats_reset(struct mtr_trtcm_data *data, + enum rte_meter_color color) +{ + data->stats[color] &= 0xFFLU; +} + +#define MTR_TRTCM_DATA_STATS_INC(data, color) \ + ((data)->stats[(color)] += (1LLU << 8)) + +static size_t +mtr_data_size(struct rte_table_action_mtr_config *mtr) +{ + return mtr->n_tc * sizeof(struct mtr_trtcm_data); +} + +struct dscp_table_entry_data { + enum rte_meter_color color; + uint16_t tc; + uint16_t queue_tc_color; +}; + +struct dscp_table_data { + struct dscp_table_entry_data entry[64]; +}; + +struct meter_profile_data { + struct rte_meter_trtcm_profile profile; + uint32_t profile_id; + int valid; +}; + +static struct meter_profile_data * +meter_profile_data_find(struct meter_profile_data *mp, + uint32_t mp_size, + uint32_t profile_id) +{ + uint32_t i; + + for (i = 0; i < mp_size; i++) { + struct meter_profile_data *mp_data = &mp[i]; + + if (mp_data->valid && (mp_data->profile_id == profile_id)) + return mp_data; + } + + return NULL; +} + +static struct meter_profile_data * +meter_profile_data_find_unused(struct meter_profile_data *mp, + uint32_t mp_size) +{ + uint32_t i; + + for (i = 0; i < mp_size; i++) { + struct meter_profile_data *mp_data = &mp[i]; + + if (!mp_data->valid) + return mp_data; + } + + return NULL; +} + +static int +mtr_apply_check(struct rte_table_action_mtr_params *p, + struct rte_table_action_mtr_config *cfg, + struct meter_profile_data *mp, + uint32_t mp_size) +{ + uint32_t i; + + if (p->tc_mask > RTE_LEN2MASK(cfg->n_tc, uint32_t)) + return -EINVAL; + + for (i = 0; i < RTE_TABLE_ACTION_TC_MAX; i++) { + struct rte_table_action_mtr_tc_params *p_tc = &p->mtr[i]; + struct meter_profile_data *mp_data; + + if ((p->tc_mask & (1LLU << i)) == 0) + continue; + + mp_data = meter_profile_data_find(mp, + mp_size, + p_tc->meter_profile_id); + if (!mp_data) + return -EINVAL; + } + + return 0; +} + +static int +mtr_apply(struct mtr_trtcm_data *data, + struct rte_table_action_mtr_params *p, + struct rte_table_action_mtr_config *cfg, + struct meter_profile_data *mp, + uint32_t mp_size) +{ + uint32_t i; + int status; + + /* Check input arguments */ + status = mtr_apply_check(p, cfg, mp, mp_size); + if (status) + return status; + + /* Apply */ + for (i = 0; i < RTE_TABLE_ACTION_TC_MAX; i++) { + struct rte_table_action_mtr_tc_params *p_tc = &p->mtr[i]; + struct mtr_trtcm_data *data_tc = &data[i]; + struct meter_profile_data *mp_data; + + if ((p->tc_mask & (1LLU << i)) == 0) + continue; + + /* Find profile */ + mp_data = meter_profile_data_find(mp, + mp_size, + p_tc->meter_profile_id); + if (!mp_data) + return -EINVAL; + + memset(data_tc, 0, sizeof(*data_tc)); + + /* Meter object */ + status = rte_meter_trtcm_config(&data_tc->trtcm, + &mp_data->profile); + if (status) + return status; + + /* Meter profile */ + mtr_trtcm_data_meter_profile_id_set(data_tc, + mp_data - mp); + + /* Policer actions */ + mtr_trtcm_data_policer_action_set(data_tc, + e_RTE_METER_GREEN, + p_tc->policer[e_RTE_METER_GREEN]); + + mtr_trtcm_data_policer_action_set(data_tc, + e_RTE_METER_YELLOW, + p_tc->policer[e_RTE_METER_YELLOW]); + + mtr_trtcm_data_policer_action_set(data_tc, + e_RTE_METER_RED, + p_tc->policer[e_RTE_METER_RED]); + } + + return 0; +} + +static __rte_always_inline uint64_t +pkt_work_mtr(struct rte_mbuf *mbuf, + struct mtr_trtcm_data *data, + struct dscp_table_data *dscp_table, + struct meter_profile_data *mp, + uint64_t time, + uint32_t dscp, + uint16_t total_length) +{ + uint64_t drop_mask, sched; + uint64_t *sched_ptr = (uint64_t *) &mbuf->hash.sched; + struct dscp_table_entry_data *dscp_entry = &dscp_table->entry[dscp]; + enum rte_meter_color color_in, color_meter, color_policer; + uint32_t tc, mp_id; + + tc = dscp_entry->tc; + color_in = dscp_entry->color; + data += tc; + mp_id = MTR_TRTCM_DATA_METER_PROFILE_ID_GET(data); + sched = *sched_ptr; + + /* Meter */ + color_meter = rte_meter_trtcm_color_aware_check( + &data->trtcm, + &mp[mp_id].profile, + time, + total_length, + color_in); + + /* Stats */ + MTR_TRTCM_DATA_STATS_INC(data, color_meter); + + /* Police */ + drop_mask = MTR_TRTCM_DATA_POLICER_ACTION_DROP_GET(data, color_meter); + color_policer = + MTR_TRTCM_DATA_POLICER_ACTION_COLOR_GET(data, color_meter); + *sched_ptr = MBUF_SCHED_COLOR(sched, color_policer); + + return drop_mask; +} + +/** + * RTE_TABLE_ACTION_TM + */ +static int +tm_cfg_check(struct rte_table_action_tm_config *tm) +{ + if ((tm->n_subports_per_port == 0) || + (rte_is_power_of_2(tm->n_subports_per_port) == 0) || + (tm->n_subports_per_port > UINT16_MAX) || + (tm->n_pipes_per_subport == 0) || + (rte_is_power_of_2(tm->n_pipes_per_subport) == 0)) + return -ENOTSUP; + + return 0; +} + +struct tm_data { + uint16_t queue_tc_color; + uint16_t subport; + uint32_t pipe; +} __attribute__((__packed__)); + +static int +tm_apply_check(struct rte_table_action_tm_params *p, + struct rte_table_action_tm_config *cfg) +{ + if ((p->subport_id >= cfg->n_subports_per_port) || + (p->pipe_id >= cfg->n_pipes_per_subport)) + return -EINVAL; + + return 0; +} + +static int +tm_apply(struct tm_data *data, + struct rte_table_action_tm_params *p, + struct rte_table_action_tm_config *cfg) +{ + int status; + + /* Check input arguments */ + status = tm_apply_check(p, cfg); + if (status) + return status; + + /* Apply */ + data->queue_tc_color = 0; + data->subport = (uint16_t) p->subport_id; + data->pipe = p->pipe_id; + + return 0; +} + +static __rte_always_inline void +pkt_work_tm(struct rte_mbuf *mbuf, + struct tm_data *data, + struct dscp_table_data *dscp_table, + uint32_t dscp) +{ + struct dscp_table_entry_data *dscp_entry = &dscp_table->entry[dscp]; + struct tm_data *sched_ptr = (struct tm_data *) &mbuf->hash.sched; + struct tm_data sched; + + sched = *data; + sched.queue_tc_color = dscp_entry->queue_tc_color; + *sched_ptr = sched; +} + +/** + * RTE_TABLE_ACTION_ENCAP + */ +static int +encap_valid(enum rte_table_action_encap_type encap) +{ + switch (encap) { + case RTE_TABLE_ACTION_ENCAP_ETHER: + case RTE_TABLE_ACTION_ENCAP_VLAN: + case RTE_TABLE_ACTION_ENCAP_QINQ: + case RTE_TABLE_ACTION_ENCAP_MPLS: + case RTE_TABLE_ACTION_ENCAP_PPPOE: + return 1; + default: + return 0; + } +} + +static int +encap_cfg_check(struct rte_table_action_encap_config *encap) +{ + if ((encap->encap_mask == 0) || + (__builtin_popcountll(encap->encap_mask) != 1)) + return -ENOTSUP; + + return 0; +} + +struct encap_ether_data { + struct ether_hdr ether; +} __attribute__((__packed__)); + +#define VLAN(pcp, dei, vid) \ + ((uint16_t)((((uint64_t)(pcp)) & 0x7LLU) << 13) | \ + ((((uint64_t)(dei)) & 0x1LLU) << 12) | \ + (((uint64_t)(vid)) & 0xFFFLLU)) \ + +struct encap_vlan_data { + struct ether_hdr ether; + struct vlan_hdr vlan; +} __attribute__((__packed__)); + +struct encap_qinq_data { + struct ether_hdr ether; + struct vlan_hdr svlan; + struct vlan_hdr cvlan; +} __attribute__((__packed__)); + +#define ETHER_TYPE_MPLS_UNICAST 0x8847 + +#define ETHER_TYPE_MPLS_MULTICAST 0x8848 + +#define MPLS(label, tc, s, ttl) \ + ((uint32_t)(((((uint64_t)(label)) & 0xFFFFFLLU) << 12) |\ + ((((uint64_t)(tc)) & 0x7LLU) << 9) | \ + ((((uint64_t)(s)) & 0x1LLU) << 8) | \ + (((uint64_t)(ttl)) & 0xFFLLU))) + +struct encap_mpls_data { + struct ether_hdr ether; + uint32_t mpls[RTE_TABLE_ACTION_MPLS_LABELS_MAX]; + uint32_t mpls_count; +} __attribute__((__packed__)); + +#define ETHER_TYPE_PPPOE_SESSION 0x8864 + +#define PPP_PROTOCOL_IP 0x0021 + +struct pppoe_ppp_hdr { + uint16_t ver_type_code; + uint16_t session_id; + uint16_t length; + uint16_t protocol; +} __attribute__((__packed__)); + +struct encap_pppoe_data { + struct ether_hdr ether; + struct pppoe_ppp_hdr pppoe_ppp; +} __attribute__((__packed__)); + +static size_t +encap_data_size(struct rte_table_action_encap_config *encap) +{ + switch (encap->encap_mask) { + case 1LLU << RTE_TABLE_ACTION_ENCAP_ETHER: + return sizeof(struct encap_ether_data); + + case 1LLU << RTE_TABLE_ACTION_ENCAP_VLAN: + return sizeof(struct encap_vlan_data); + + case 1LLU << RTE_TABLE_ACTION_ENCAP_QINQ: + return sizeof(struct encap_qinq_data); + + case 1LLU << RTE_TABLE_ACTION_ENCAP_MPLS: + return sizeof(struct encap_mpls_data); + + case 1LLU << RTE_TABLE_ACTION_ENCAP_PPPOE: + return sizeof(struct encap_pppoe_data); + + default: + return 0; + } +} + +static int +encap_apply_check(struct rte_table_action_encap_params *p, + struct rte_table_action_encap_config *cfg) +{ + if ((encap_valid(p->type) == 0) || + ((cfg->encap_mask & (1LLU << p->type)) == 0)) + return -EINVAL; + + switch (p->type) { + case RTE_TABLE_ACTION_ENCAP_ETHER: + return 0; + + case RTE_TABLE_ACTION_ENCAP_VLAN: + return 0; + + case RTE_TABLE_ACTION_ENCAP_QINQ: + return 0; + + case RTE_TABLE_ACTION_ENCAP_MPLS: + if ((p->mpls.mpls_count == 0) || + (p->mpls.mpls_count > RTE_TABLE_ACTION_MPLS_LABELS_MAX)) + return -EINVAL; + + return 0; + + case RTE_TABLE_ACTION_ENCAP_PPPOE: + return 0; + + default: + return -EINVAL; + } +} + +static int +encap_ether_apply(void *data, + struct rte_table_action_encap_params *p, + struct rte_table_action_common_config *common_cfg) +{ + struct encap_ether_data *d = data; + uint16_t ethertype = (common_cfg->ip_version) ? + ETHER_TYPE_IPv4 : + ETHER_TYPE_IPv6; + + /* Ethernet */ + ether_addr_copy(&p->ether.ether.da, &d->ether.d_addr); + ether_addr_copy(&p->ether.ether.sa, &d->ether.s_addr); + d->ether.ether_type = rte_htons(ethertype); + + return 0; +} + +static int +encap_vlan_apply(void *data, + struct rte_table_action_encap_params *p, + struct rte_table_action_common_config *common_cfg) +{ + struct encap_vlan_data *d = data; + uint16_t ethertype = (common_cfg->ip_version) ? + ETHER_TYPE_IPv4 : + ETHER_TYPE_IPv6; + + /* Ethernet */ + ether_addr_copy(&p->vlan.ether.da, &d->ether.d_addr); + ether_addr_copy(&p->vlan.ether.sa, &d->ether.s_addr); + d->ether.ether_type = rte_htons(ETHER_TYPE_VLAN); + + /* VLAN */ + d->vlan.vlan_tci = rte_htons(VLAN(p->vlan.vlan.pcp, + p->vlan.vlan.dei, + p->vlan.vlan.vid)); + d->vlan.eth_proto = rte_htons(ethertype); + + return 0; +} + +static int +encap_qinq_apply(void *data, + struct rte_table_action_encap_params *p, + struct rte_table_action_common_config *common_cfg) +{ + struct encap_qinq_data *d = data; + uint16_t ethertype = (common_cfg->ip_version) ? + ETHER_TYPE_IPv4 : + ETHER_TYPE_IPv6; + + /* Ethernet */ + ether_addr_copy(&p->qinq.ether.da, &d->ether.d_addr); + ether_addr_copy(&p->qinq.ether.sa, &d->ether.s_addr); + d->ether.ether_type = rte_htons(ETHER_TYPE_QINQ); + + /* SVLAN */ + d->svlan.vlan_tci = rte_htons(VLAN(p->qinq.svlan.pcp, + p->qinq.svlan.dei, + p->qinq.svlan.vid)); + d->svlan.eth_proto = rte_htons(ETHER_TYPE_VLAN); + + /* CVLAN */ + d->cvlan.vlan_tci = rte_htons(VLAN(p->qinq.cvlan.pcp, + p->qinq.cvlan.dei, + p->qinq.cvlan.vid)); + d->cvlan.eth_proto = rte_htons(ethertype); + + return 0; +} + +static int +encap_mpls_apply(void *data, + struct rte_table_action_encap_params *p) +{ + struct encap_mpls_data *d = data; + uint16_t ethertype = (p->mpls.unicast) ? + ETHER_TYPE_MPLS_UNICAST : + ETHER_TYPE_MPLS_MULTICAST; + uint32_t i; + + /* Ethernet */ + ether_addr_copy(&p->mpls.ether.da, &d->ether.d_addr); + ether_addr_copy(&p->mpls.ether.sa, &d->ether.s_addr); + d->ether.ether_type = rte_htons(ethertype); + + /* MPLS */ + for (i = 0; i < p->mpls.mpls_count - 1; i++) + d->mpls[i] = rte_htonl(MPLS(p->mpls.mpls[i].label, + p->mpls.mpls[i].tc, + 0, + p->mpls.mpls[i].ttl)); + + d->mpls[i] = rte_htonl(MPLS(p->mpls.mpls[i].label, + p->mpls.mpls[i].tc, + 1, + p->mpls.mpls[i].ttl)); + + d->mpls_count = p->mpls.mpls_count; + return 0; +} + +static int +encap_pppoe_apply(void *data, + struct rte_table_action_encap_params *p) +{ + struct encap_pppoe_data *d = data; + + /* Ethernet */ + ether_addr_copy(&p->pppoe.ether.da, &d->ether.d_addr); + ether_addr_copy(&p->pppoe.ether.sa, &d->ether.s_addr); + d->ether.ether_type = rte_htons(ETHER_TYPE_PPPOE_SESSION); + + /* PPPoE and PPP*/ + d->pppoe_ppp.ver_type_code = rte_htons(0x1100); + d->pppoe_ppp.session_id = rte_htons(p->pppoe.pppoe.session_id); + d->pppoe_ppp.length = 0; /* not pre-computed */ + d->pppoe_ppp.protocol = rte_htons(PPP_PROTOCOL_IP); + + return 0; +} + +static int +encap_apply(void *data, + struct rte_table_action_encap_params *p, + struct rte_table_action_encap_config *cfg, + struct rte_table_action_common_config *common_cfg) +{ + int status; + + /* Check input arguments */ + status = encap_apply_check(p, cfg); + if (status) + return status; + + switch (p->type) { + case RTE_TABLE_ACTION_ENCAP_ETHER: + return encap_ether_apply(data, p, common_cfg); + + case RTE_TABLE_ACTION_ENCAP_VLAN: + return encap_vlan_apply(data, p, common_cfg); + + case RTE_TABLE_ACTION_ENCAP_QINQ: + return encap_qinq_apply(data, p, common_cfg); + + case RTE_TABLE_ACTION_ENCAP_MPLS: + return encap_mpls_apply(data, p); + + case RTE_TABLE_ACTION_ENCAP_PPPOE: + return encap_pppoe_apply(data, p); + + default: + return -EINVAL; + } +} + +static __rte_always_inline void * +encap(void *dst, const void *src, size_t n) +{ + dst = ((uint8_t *) dst) - n; + return rte_memcpy(dst, src, n); +} + +static __rte_always_inline void +pkt_work_encap(struct rte_mbuf *mbuf, + void *data, + struct rte_table_action_encap_config *cfg, + void *ip, + uint16_t total_length, + uint32_t ip_offset) +{ + switch (cfg->encap_mask) { + case 1LLU << RTE_TABLE_ACTION_ENCAP_ETHER: + encap(ip, data, sizeof(struct encap_ether_data)); + mbuf->data_off = ip_offset - (sizeof(struct rte_mbuf) + + sizeof(struct encap_ether_data)); + mbuf->pkt_len = mbuf->data_len = total_length + + sizeof(struct encap_ether_data); + break; + + case 1LLU << RTE_TABLE_ACTION_ENCAP_VLAN: + encap(ip, data, sizeof(struct encap_vlan_data)); + mbuf->data_off = ip_offset - (sizeof(struct rte_mbuf) + + sizeof(struct encap_vlan_data)); + mbuf->pkt_len = mbuf->data_len = total_length + + sizeof(struct encap_vlan_data); + break; + + case 1LLU << RTE_TABLE_ACTION_ENCAP_QINQ: + encap(ip, data, sizeof(struct encap_qinq_data)); + mbuf->data_off = ip_offset - (sizeof(struct rte_mbuf) + + sizeof(struct encap_qinq_data)); + mbuf->pkt_len = mbuf->data_len = total_length + + sizeof(struct encap_qinq_data); + break; + + case 1LLU << RTE_TABLE_ACTION_ENCAP_MPLS: + { + struct encap_mpls_data *mpls = data; + size_t size = sizeof(struct ether_hdr) + + mpls->mpls_count * 4; + + encap(ip, data, size); + mbuf->data_off = ip_offset - (sizeof(struct rte_mbuf) + size); + mbuf->pkt_len = mbuf->data_len = total_length + size; + break; + } + + case 1LLU << RTE_TABLE_ACTION_ENCAP_PPPOE: + { + struct encap_pppoe_data *pppoe = + encap(ip, data, sizeof(struct encap_pppoe_data)); + pppoe->pppoe_ppp.length = rte_htons(total_length + 2); + mbuf->data_off = ip_offset - (sizeof(struct rte_mbuf) + + sizeof(struct encap_pppoe_data)); + mbuf->pkt_len = mbuf->data_len = total_length + + sizeof(struct encap_pppoe_data); + break; + } + + default: + break; + } +} + +/** + * RTE_TABLE_ACTION_NAT + */ +static int +nat_cfg_check(struct rte_table_action_nat_config *nat) +{ + if ((nat->proto != 0x06) && + (nat->proto != 0x11)) + return -ENOTSUP; + + return 0; +} + +struct nat_ipv4_data { + uint32_t addr; + uint16_t port; +} __attribute__((__packed__)); + +struct nat_ipv6_data { + uint8_t addr[16]; + uint16_t port; +} __attribute__((__packed__)); + +static size_t +nat_data_size(struct rte_table_action_nat_config *nat __rte_unused, + struct rte_table_action_common_config *common) +{ + int ip_version = common->ip_version; + + return (ip_version) ? + sizeof(struct nat_ipv4_data) : + sizeof(struct nat_ipv6_data); +} + +static int +nat_apply_check(struct rte_table_action_nat_params *p, + struct rte_table_action_common_config *cfg) +{ + if ((p->ip_version && (cfg->ip_version == 0)) || + ((p->ip_version == 0) && cfg->ip_version)) + return -EINVAL; + + return 0; +} + +static int +nat_apply(void *data, + struct rte_table_action_nat_params *p, + struct rte_table_action_common_config *cfg) +{ + int status; + + /* Check input arguments */ + status = nat_apply_check(p, cfg); + if (status) + return status; + + /* Apply */ + if (p->ip_version) { + struct nat_ipv4_data *d = data; + + d->addr = rte_htonl(p->addr.ipv4); + d->port = rte_htons(p->port); + } else { + struct nat_ipv6_data *d = data; + + memcpy(d->addr, p->addr.ipv6, sizeof(d->addr)); + d->port = rte_htons(p->port); + } + + return 0; +} + +static __rte_always_inline uint16_t +nat_ipv4_checksum_update(uint16_t cksum0, + uint32_t ip0, + uint32_t ip1) +{ + int32_t cksum1; + + cksum1 = cksum0; + cksum1 = ~cksum1 & 0xFFFF; + + /* Subtract ip0 (one's complement logic) */ + cksum1 -= (ip0 >> 16) + (ip0 & 0xFFFF); + cksum1 = (cksum1 & 0xFFFF) + (cksum1 >> 16); + cksum1 = (cksum1 & 0xFFFF) + (cksum1 >> 16); + + /* Add ip1 (one's complement logic) */ + cksum1 += (ip1 >> 16) + (ip1 & 0xFFFF); + cksum1 = (cksum1 & 0xFFFF) + (cksum1 >> 16); + cksum1 = (cksum1 & 0xFFFF) + (cksum1 >> 16); + + return (uint16_t)(~cksum1); +} + +static __rte_always_inline uint16_t +nat_ipv4_tcp_udp_checksum_update(uint16_t cksum0, + uint32_t ip0, + uint32_t ip1, + uint16_t port0, + uint16_t port1) +{ + int32_t cksum1; + + cksum1 = cksum0; + cksum1 = ~cksum1 & 0xFFFF; + + /* Subtract ip0 and port 0 (one's complement logic) */ + cksum1 -= (ip0 >> 16) + (ip0 & 0xFFFF) + port0; + cksum1 = (cksum1 & 0xFFFF) + (cksum1 >> 16); + cksum1 = (cksum1 & 0xFFFF) + (cksum1 >> 16); + + /* Add ip1 and port1 (one's complement logic) */ + cksum1 += (ip1 >> 16) + (ip1 & 0xFFFF) + port1; + cksum1 = (cksum1 & 0xFFFF) + (cksum1 >> 16); + cksum1 = (cksum1 & 0xFFFF) + (cksum1 >> 16); + + return (uint16_t)(~cksum1); +} + +static __rte_always_inline uint16_t +nat_ipv6_tcp_udp_checksum_update(uint16_t cksum0, + uint16_t *ip0, + uint16_t *ip1, + uint16_t port0, + uint16_t port1) +{ + int32_t cksum1; + + cksum1 = cksum0; + cksum1 = ~cksum1 & 0xFFFF; + + /* Subtract ip0 and port 0 (one's complement logic) */ + cksum1 -= ip0[0] + ip0[1] + ip0[2] + ip0[3] + + ip0[4] + ip0[5] + ip0[6] + ip0[7] + port0; + cksum1 = (cksum1 & 0xFFFF) + (cksum1 >> 16); + cksum1 = (cksum1 & 0xFFFF) + (cksum1 >> 16); + + /* Add ip1 and port1 (one's complement logic) */ + cksum1 += ip1[0] + ip1[1] + ip1[2] + ip1[3] + + ip1[4] + ip1[5] + ip1[6] + ip1[7] + port1; + cksum1 = (cksum1 & 0xFFFF) + (cksum1 >> 16); + cksum1 = (cksum1 & 0xFFFF) + (cksum1 >> 16); + + return (uint16_t)(~cksum1); +} + +static __rte_always_inline void +pkt_ipv4_work_nat(struct ipv4_hdr *ip, + struct nat_ipv4_data *data, + struct rte_table_action_nat_config *cfg) +{ + if (cfg->source_nat) { + if (cfg->proto == 0x6) { + struct tcp_hdr *tcp = (struct tcp_hdr *) &ip[1]; + uint16_t ip_cksum, tcp_cksum; + + ip_cksum = nat_ipv4_checksum_update(ip->hdr_checksum, + ip->src_addr, + data->addr); + + tcp_cksum = nat_ipv4_tcp_udp_checksum_update(tcp->cksum, + ip->src_addr, + data->addr, + tcp->src_port, + data->port); + + ip->src_addr = data->addr; + ip->hdr_checksum = ip_cksum; + tcp->src_port = data->port; + tcp->cksum = tcp_cksum; + } else { + struct udp_hdr *udp = (struct udp_hdr *) &ip[1]; + uint16_t ip_cksum, udp_cksum; + + ip_cksum = nat_ipv4_checksum_update(ip->hdr_checksum, + ip->src_addr, + data->addr); + + udp_cksum = nat_ipv4_tcp_udp_checksum_update(udp->dgram_cksum, + ip->src_addr, + data->addr, + udp->src_port, + data->port); + + ip->src_addr = data->addr; + ip->hdr_checksum = ip_cksum; + udp->src_port = data->port; + if (udp->dgram_cksum) + udp->dgram_cksum = udp_cksum; + } + } else { + if (cfg->proto == 0x6) { + struct tcp_hdr *tcp = (struct tcp_hdr *) &ip[1]; + uint16_t ip_cksum, tcp_cksum; + + ip_cksum = nat_ipv4_checksum_update(ip->hdr_checksum, + ip->dst_addr, + data->addr); + + tcp_cksum = nat_ipv4_tcp_udp_checksum_update(tcp->cksum, + ip->dst_addr, + data->addr, + tcp->dst_port, + data->port); + + ip->dst_addr = data->addr; + ip->hdr_checksum = ip_cksum; + tcp->dst_port = data->port; + tcp->cksum = tcp_cksum; + } else { + struct udp_hdr *udp = (struct udp_hdr *) &ip[1]; + uint16_t ip_cksum, udp_cksum; + + ip_cksum = nat_ipv4_checksum_update(ip->hdr_checksum, + ip->dst_addr, + data->addr); + + udp_cksum = nat_ipv4_tcp_udp_checksum_update(udp->dgram_cksum, + ip->dst_addr, + data->addr, + udp->dst_port, + data->port); + + ip->dst_addr = data->addr; + ip->hdr_checksum = ip_cksum; + udp->dst_port = data->port; + if (udp->dgram_cksum) + udp->dgram_cksum = udp_cksum; + } + } +} + +static __rte_always_inline void +pkt_ipv6_work_nat(struct ipv6_hdr *ip, + struct nat_ipv6_data *data, + struct rte_table_action_nat_config *cfg) +{ + if (cfg->source_nat) { + if (cfg->proto == 0x6) { + struct tcp_hdr *tcp = (struct tcp_hdr *) &ip[1]; + uint16_t tcp_cksum; + + tcp_cksum = nat_ipv6_tcp_udp_checksum_update(tcp->cksum, + (uint16_t *)ip->src_addr, + (uint16_t *)data->addr, + tcp->src_port, + data->port); + + rte_memcpy(ip->src_addr, data->addr, 16); + tcp->src_port = data->port; + tcp->cksum = tcp_cksum; + } else { + struct udp_hdr *udp = (struct udp_hdr *) &ip[1]; + uint16_t udp_cksum; + + udp_cksum = nat_ipv6_tcp_udp_checksum_update(udp->dgram_cksum, + (uint16_t *)ip->src_addr, + (uint16_t *)data->addr, + udp->src_port, + data->port); + + rte_memcpy(ip->src_addr, data->addr, 16); + udp->src_port = data->port; + udp->dgram_cksum = udp_cksum; + } + } else { + if (cfg->proto == 0x6) { + struct tcp_hdr *tcp = (struct tcp_hdr *) &ip[1]; + uint16_t tcp_cksum; + + tcp_cksum = nat_ipv6_tcp_udp_checksum_update(tcp->cksum, + (uint16_t *)ip->dst_addr, + (uint16_t *)data->addr, + tcp->dst_port, + data->port); + + rte_memcpy(ip->dst_addr, data->addr, 16); + tcp->dst_port = data->port; + tcp->cksum = tcp_cksum; + } else { + struct udp_hdr *udp = (struct udp_hdr *) &ip[1]; + uint16_t udp_cksum; + + udp_cksum = nat_ipv6_tcp_udp_checksum_update(udp->dgram_cksum, + (uint16_t *)ip->dst_addr, + (uint16_t *)data->addr, + udp->dst_port, + data->port); + + rte_memcpy(ip->dst_addr, data->addr, 16); + udp->dst_port = data->port; + udp->dgram_cksum = udp_cksum; + } + } +} + +/** + * RTE_TABLE_ACTION_TTL + */ +static int +ttl_cfg_check(struct rte_table_action_ttl_config *ttl) +{ + if (ttl->drop == 0) + return -ENOTSUP; + + return 0; +} + +struct ttl_data { + uint32_t n_packets; +} __attribute__((__packed__)); + +#define TTL_INIT(data, decrement) \ + ((data)->n_packets = (decrement) ? 1 : 0) + +#define TTL_DEC_GET(data) \ + ((uint8_t)((data)->n_packets & 1)) + +#define TTL_STATS_RESET(data) \ + ((data)->n_packets = ((data)->n_packets & 1)) + +#define TTL_STATS_READ(data) \ + ((data)->n_packets >> 1) + +#define TTL_STATS_ADD(data, value) \ + ((data)->n_packets = \ + (((((data)->n_packets >> 1) + (value)) << 1) | \ + ((data)->n_packets & 1))) + +static int +ttl_apply(void *data, + struct rte_table_action_ttl_params *p) +{ + struct ttl_data *d = data; + + TTL_INIT(d, p->decrement); + + return 0; +} + +static __rte_always_inline uint64_t +pkt_ipv4_work_ttl(struct ipv4_hdr *ip, + struct ttl_data *data) +{ + uint32_t drop; + uint16_t cksum = ip->hdr_checksum; + uint8_t ttl = ip->time_to_live; + uint8_t ttl_diff = TTL_DEC_GET(data); + + cksum += ttl_diff; + ttl -= ttl_diff; + + ip->hdr_checksum = cksum; + ip->time_to_live = ttl; + + drop = (ttl == 0) ? 1 : 0; + TTL_STATS_ADD(data, drop); + + return drop; +} + +static __rte_always_inline uint64_t +pkt_ipv6_work_ttl(struct ipv6_hdr *ip, + struct ttl_data *data) +{ + uint32_t drop; + uint8_t ttl = ip->hop_limits; + uint8_t ttl_diff = TTL_DEC_GET(data); + + ttl -= ttl_diff; + + ip->hop_limits = ttl; + + drop = (ttl == 0) ? 1 : 0; + TTL_STATS_ADD(data, drop); + + return drop; +} + +/** + * RTE_TABLE_ACTION_STATS + */ +static int +stats_cfg_check(struct rte_table_action_stats_config *stats) +{ + if ((stats->n_packets_enabled == 0) && (stats->n_bytes_enabled == 0)) + return -EINVAL; + + return 0; +} + +struct stats_data { + uint64_t n_packets; + uint64_t n_bytes; +} __attribute__((__packed__)); + +static int +stats_apply(struct stats_data *data, + struct rte_table_action_stats_params *p) +{ + data->n_packets = p->n_packets; + data->n_bytes = p->n_bytes; + + return 0; +} + +static __rte_always_inline void +pkt_work_stats(struct stats_data *data, + uint16_t total_length) +{ + data->n_packets++; + data->n_bytes += total_length; +} + +/** + * RTE_TABLE_ACTION_TIME + */ +struct time_data { + uint64_t time; +} __attribute__((__packed__)); + +static int +time_apply(struct time_data *data, + struct rte_table_action_time_params *p) +{ + data->time = p->time; + return 0; +} + +static __rte_always_inline void +pkt_work_time(struct time_data *data, + uint64_t time) +{ + data->time = time; +} + +/** + * Action profile + */ +static int +action_valid(enum rte_table_action_type action) +{ + switch (action) { + case RTE_TABLE_ACTION_FWD: + case RTE_TABLE_ACTION_LB: + case RTE_TABLE_ACTION_MTR: + case RTE_TABLE_ACTION_TM: + case RTE_TABLE_ACTION_ENCAP: + case RTE_TABLE_ACTION_NAT: + case RTE_TABLE_ACTION_TTL: + case RTE_TABLE_ACTION_STATS: + case RTE_TABLE_ACTION_TIME: + return 1; + default: + return 0; + } +} + + +#define RTE_TABLE_ACTION_MAX 64 + +struct ap_config { + uint64_t action_mask; + struct rte_table_action_common_config common; + struct rte_table_action_lb_config lb; + struct rte_table_action_mtr_config mtr; + struct rte_table_action_tm_config tm; + struct rte_table_action_encap_config encap; + struct rte_table_action_nat_config nat; + struct rte_table_action_ttl_config ttl; + struct rte_table_action_stats_config stats; +}; + +static size_t +action_cfg_size(enum rte_table_action_type action) +{ + switch (action) { + case RTE_TABLE_ACTION_LB: + return sizeof(struct rte_table_action_lb_config); + case RTE_TABLE_ACTION_MTR: + return sizeof(struct rte_table_action_mtr_config); + case RTE_TABLE_ACTION_TM: + return sizeof(struct rte_table_action_tm_config); + case RTE_TABLE_ACTION_ENCAP: + return sizeof(struct rte_table_action_encap_config); + case RTE_TABLE_ACTION_NAT: + return sizeof(struct rte_table_action_nat_config); + case RTE_TABLE_ACTION_TTL: + return sizeof(struct rte_table_action_ttl_config); + case RTE_TABLE_ACTION_STATS: + return sizeof(struct rte_table_action_stats_config); + default: + return 0; + } +} + +static void* +action_cfg_get(struct ap_config *ap_config, + enum rte_table_action_type type) +{ + switch (type) { + case RTE_TABLE_ACTION_LB: + return &ap_config->lb; + + case RTE_TABLE_ACTION_MTR: + return &ap_config->mtr; + + case RTE_TABLE_ACTION_TM: + return &ap_config->tm; + + case RTE_TABLE_ACTION_ENCAP: + return &ap_config->encap; + + case RTE_TABLE_ACTION_NAT: + return &ap_config->nat; + + case RTE_TABLE_ACTION_TTL: + return &ap_config->ttl; + + case RTE_TABLE_ACTION_STATS: + return &ap_config->stats; + + default: + return NULL; + } +} + +static void +action_cfg_set(struct ap_config *ap_config, + enum rte_table_action_type type, + void *action_cfg) +{ + void *dst = action_cfg_get(ap_config, type); + + if (dst) + memcpy(dst, action_cfg, action_cfg_size(type)); + + ap_config->action_mask |= 1LLU << type; +} + +struct ap_data { + size_t offset[RTE_TABLE_ACTION_MAX]; + size_t total_size; +}; + +static size_t +action_data_size(enum rte_table_action_type action, + struct ap_config *ap_config) +{ + switch (action) { + case RTE_TABLE_ACTION_FWD: + return sizeof(struct fwd_data); + + case RTE_TABLE_ACTION_LB: + return sizeof(struct lb_data); + + case RTE_TABLE_ACTION_MTR: + return mtr_data_size(&ap_config->mtr); + + case RTE_TABLE_ACTION_TM: + return sizeof(struct tm_data); + + case RTE_TABLE_ACTION_ENCAP: + return encap_data_size(&ap_config->encap); + + case RTE_TABLE_ACTION_NAT: + return nat_data_size(&ap_config->nat, + &ap_config->common); + + case RTE_TABLE_ACTION_TTL: + return sizeof(struct ttl_data); + + case RTE_TABLE_ACTION_STATS: + return sizeof(struct stats_data); + + case RTE_TABLE_ACTION_TIME: + return sizeof(struct time_data); + + default: + return 0; + } +} + + +static void +action_data_offset_set(struct ap_data *ap_data, + struct ap_config *ap_config) +{ + uint64_t action_mask = ap_config->action_mask; + size_t offset; + uint32_t action; + + memset(ap_data->offset, 0, sizeof(ap_data->offset)); + + offset = 0; + for (action = 0; action < RTE_TABLE_ACTION_MAX; action++) + if (action_mask & (1LLU << action)) { + ap_data->offset[action] = offset; + offset += action_data_size((enum rte_table_action_type)action, + ap_config); + } + + ap_data->total_size = offset; +} + +struct rte_table_action_profile { + struct ap_config cfg; + struct ap_data data; + int frozen; +}; + +struct rte_table_action_profile * +rte_table_action_profile_create(struct rte_table_action_common_config *common) +{ + struct rte_table_action_profile *ap; + + /* Check input arguments */ + if (common == NULL) + return NULL; + + /* Memory allocation */ + ap = calloc(1, sizeof(struct rte_table_action_profile)); + if (ap == NULL) + return NULL; + + /* Initialization */ + memcpy(&ap->cfg.common, common, sizeof(*common)); + + return ap; +} + + +int +rte_table_action_profile_action_register(struct rte_table_action_profile *profile, + enum rte_table_action_type type, + void *action_config) +{ + int status; + + /* Check input arguments */ + if ((profile == NULL) || + profile->frozen || + (action_valid(type) == 0) || + (profile->cfg.action_mask & (1LLU << type)) || + ((action_cfg_size(type) == 0) && action_config) || + (action_cfg_size(type) && (action_config == NULL))) + return -EINVAL; + + switch (type) { + case RTE_TABLE_ACTION_LB: + status = lb_cfg_check(action_config); + break; + + case RTE_TABLE_ACTION_MTR: + status = mtr_cfg_check(action_config); + break; + + case RTE_TABLE_ACTION_TM: + status = tm_cfg_check(action_config); + break; + + case RTE_TABLE_ACTION_ENCAP: + status = encap_cfg_check(action_config); + break; + + case RTE_TABLE_ACTION_NAT: + status = nat_cfg_check(action_config); + break; + + case RTE_TABLE_ACTION_TTL: + status = ttl_cfg_check(action_config); + break; + + case RTE_TABLE_ACTION_STATS: + status = stats_cfg_check(action_config); + break; + + default: + status = 0; + break; + } + + if (status) + return status; + + /* Action enable */ + action_cfg_set(&profile->cfg, type, action_config); + + return 0; +} + +int +rte_table_action_profile_freeze(struct rte_table_action_profile *profile) +{ + if (profile->frozen) + return -EBUSY; + + profile->cfg.action_mask |= 1LLU << RTE_TABLE_ACTION_FWD; + action_data_offset_set(&profile->data, &profile->cfg); + profile->frozen = 1; + + return 0; +} + +int +rte_table_action_profile_free(struct rte_table_action_profile *profile) +{ + if (profile == NULL) + return 0; + + free(profile); + return 0; +} + +/** + * Action + */ +#define METER_PROFILES_MAX 32 + +struct rte_table_action { + struct ap_config cfg; + struct ap_data data; + struct dscp_table_data dscp_table; + struct meter_profile_data mp[METER_PROFILES_MAX]; +}; + +struct rte_table_action * +rte_table_action_create(struct rte_table_action_profile *profile, + uint32_t socket_id) +{ + struct rte_table_action *action; + + /* Check input arguments */ + if ((profile == NULL) || + (profile->frozen == 0)) + return NULL; + + /* Memory allocation */ + action = rte_zmalloc_socket(NULL, + sizeof(struct rte_table_action), + RTE_CACHE_LINE_SIZE, + socket_id); + if (action == NULL) + return NULL; + + /* Initialization */ + memcpy(&action->cfg, &profile->cfg, sizeof(profile->cfg)); + memcpy(&action->data, &profile->data, sizeof(profile->data)); + + return action; +} + +static __rte_always_inline void * +action_data_get(void *data, + struct rte_table_action *action, + enum rte_table_action_type type) +{ + size_t offset = action->data.offset[type]; + uint8_t *data_bytes = data; + + return &data_bytes[offset]; +} + +int +rte_table_action_apply(struct rte_table_action *action, + void *data, + enum rte_table_action_type type, + void *action_params) +{ + void *action_data; + + /* Check input arguments */ + if ((action == NULL) || + (data == NULL) || + (action_valid(type) == 0) || + ((action->cfg.action_mask & (1LLU << type)) == 0) || + (action_params == NULL)) + return -EINVAL; + + /* Data update */ + action_data = action_data_get(data, action, type); + + switch (type) { + case RTE_TABLE_ACTION_FWD: + return fwd_apply(action_data, + action_params); + + case RTE_TABLE_ACTION_LB: + return lb_apply(action_data, + action_params); + + case RTE_TABLE_ACTION_MTR: + return mtr_apply(action_data, + action_params, + &action->cfg.mtr, + action->mp, + RTE_DIM(action->mp)); + + case RTE_TABLE_ACTION_TM: + return tm_apply(action_data, + action_params, + &action->cfg.tm); + + case RTE_TABLE_ACTION_ENCAP: + return encap_apply(action_data, + action_params, + &action->cfg.encap, + &action->cfg.common); + + case RTE_TABLE_ACTION_NAT: + return nat_apply(action_data, + action_params, + &action->cfg.common); + + case RTE_TABLE_ACTION_TTL: + return ttl_apply(action_data, + action_params); + + case RTE_TABLE_ACTION_STATS: + return stats_apply(action_data, + action_params); + + case RTE_TABLE_ACTION_TIME: + return time_apply(action_data, + action_params); + + default: + return -EINVAL; + } +} + +int +rte_table_action_dscp_table_update(struct rte_table_action *action, + uint64_t dscp_mask, + struct rte_table_action_dscp_table *table) +{ + uint32_t i; + + /* Check input arguments */ + if ((action == NULL) || + ((action->cfg.action_mask & ((1LLU << RTE_TABLE_ACTION_MTR) | + (1LLU << RTE_TABLE_ACTION_TM))) == 0) || + (dscp_mask == 0) || + (table == NULL)) + return -EINVAL; + + for (i = 0; i < RTE_DIM(table->entry); i++) { + struct dscp_table_entry_data *data = + &action->dscp_table.entry[i]; + struct rte_table_action_dscp_table_entry *entry = + &table->entry[i]; + uint16_t queue_tc_color = + MBUF_SCHED_QUEUE_TC_COLOR(entry->tc_queue_id, + entry->tc_id, + entry->color); + + if ((dscp_mask & (1LLU << i)) == 0) + continue; + + data->color = entry->color; + data->tc = entry->tc_id; + data->queue_tc_color = queue_tc_color; + } + + return 0; +} + +int +rte_table_action_meter_profile_add(struct rte_table_action *action, + uint32_t meter_profile_id, + struct rte_table_action_meter_profile *profile) +{ + struct meter_profile_data *mp_data; + uint32_t status; + + /* Check input arguments */ + if ((action == NULL) || + ((action->cfg.action_mask & (1LLU << RTE_TABLE_ACTION_MTR)) == 0) || + (profile == NULL)) + return -EINVAL; + + if (profile->alg != RTE_TABLE_ACTION_METER_TRTCM) + return -ENOTSUP; + + mp_data = meter_profile_data_find(action->mp, + RTE_DIM(action->mp), + meter_profile_id); + if (mp_data) + return -EEXIST; + + mp_data = meter_profile_data_find_unused(action->mp, + RTE_DIM(action->mp)); + if (!mp_data) + return -ENOSPC; + + /* Install new profile */ + status = rte_meter_trtcm_profile_config(&mp_data->profile, + &profile->trtcm); + if (status) + return status; + + mp_data->profile_id = meter_profile_id; + mp_data->valid = 1; + + return 0; +} + +int +rte_table_action_meter_profile_delete(struct rte_table_action *action, + uint32_t meter_profile_id) +{ + struct meter_profile_data *mp_data; + + /* Check input arguments */ + if ((action == NULL) || + ((action->cfg.action_mask & (1LLU << RTE_TABLE_ACTION_MTR)) == 0)) + return -EINVAL; + + mp_data = meter_profile_data_find(action->mp, + RTE_DIM(action->mp), + meter_profile_id); + if (!mp_data) + return 0; + + /* Uninstall profile */ + mp_data->valid = 0; + + return 0; +} + +int +rte_table_action_meter_read(struct rte_table_action *action, + void *data, + uint32_t tc_mask, + struct rte_table_action_mtr_counters *stats, + int clear) +{ + struct mtr_trtcm_data *mtr_data; + uint32_t i; + + /* Check input arguments */ + if ((action == NULL) || + ((action->cfg.action_mask & (1LLU << RTE_TABLE_ACTION_MTR)) == 0) || + (data == NULL) || + (tc_mask > RTE_LEN2MASK(action->cfg.mtr.n_tc, uint32_t))) + return -EINVAL; + + mtr_data = action_data_get(data, action, RTE_TABLE_ACTION_MTR); + + /* Read */ + if (stats) { + for (i = 0; i < RTE_TABLE_ACTION_TC_MAX; i++) { + struct rte_table_action_mtr_counters_tc *dst = + &stats->stats[i]; + struct mtr_trtcm_data *src = &mtr_data[i]; + + if ((tc_mask & (1 << i)) == 0) + continue; + + dst->n_packets[e_RTE_METER_GREEN] = + mtr_trtcm_data_stats_get(src, e_RTE_METER_GREEN); + + dst->n_packets[e_RTE_METER_YELLOW] = + mtr_trtcm_data_stats_get(src, e_RTE_METER_YELLOW); + + dst->n_packets[e_RTE_METER_RED] = + mtr_trtcm_data_stats_get(src, e_RTE_METER_RED); + + dst->n_packets_valid = 1; + dst->n_bytes_valid = 0; + } + + stats->tc_mask = tc_mask; + } + + /* Clear */ + if (clear) + for (i = 0; i < RTE_TABLE_ACTION_TC_MAX; i++) { + struct mtr_trtcm_data *src = &mtr_data[i]; + + if ((tc_mask & (1 << i)) == 0) + continue; + + mtr_trtcm_data_stats_reset(src, e_RTE_METER_GREEN); + mtr_trtcm_data_stats_reset(src, e_RTE_METER_YELLOW); + mtr_trtcm_data_stats_reset(src, e_RTE_METER_RED); + } + + + return 0; +} + +int +rte_table_action_ttl_read(struct rte_table_action *action, + void *data, + struct rte_table_action_ttl_counters *stats, + int clear) +{ + struct ttl_data *ttl_data; + + /* Check input arguments */ + if ((action == NULL) || + ((action->cfg.action_mask & + (1LLU << RTE_TABLE_ACTION_TTL)) == 0) || + (data == NULL)) + return -EINVAL; + + ttl_data = action_data_get(data, action, RTE_TABLE_ACTION_TTL); + + /* Read */ + if (stats) + stats->n_packets = TTL_STATS_READ(ttl_data); + + /* Clear */ + if (clear) + TTL_STATS_RESET(ttl_data); + + return 0; +} + +int +rte_table_action_stats_read(struct rte_table_action *action, + void *data, + struct rte_table_action_stats_counters *stats, + int clear) +{ + struct stats_data *stats_data; + + /* Check input arguments */ + if ((action == NULL) || + ((action->cfg.action_mask & + (1LLU << RTE_TABLE_ACTION_STATS)) == 0) || + (data == NULL)) + return -EINVAL; + + stats_data = action_data_get(data, action, + RTE_TABLE_ACTION_STATS); + + /* Read */ + if (stats) { + stats->n_packets = stats_data->n_packets; + stats->n_bytes = stats_data->n_bytes; + stats->n_packets_valid = 1; + stats->n_bytes_valid = 1; + } + + /* Clear */ + if (clear) { + stats_data->n_packets = 0; + stats_data->n_bytes = 0; + } + + return 0; +} + +int +rte_table_action_time_read(struct rte_table_action *action, + void *data, + uint64_t *timestamp) +{ + struct time_data *time_data; + + /* Check input arguments */ + if ((action == NULL) || + ((action->cfg.action_mask & + (1LLU << RTE_TABLE_ACTION_TIME)) == 0) || + (data == NULL) || + (timestamp == NULL)) + return -EINVAL; + + time_data = action_data_get(data, action, RTE_TABLE_ACTION_TIME); + + /* Read */ + *timestamp = time_data->time; + + return 0; +} + +static __rte_always_inline uint64_t +pkt_work(struct rte_mbuf *mbuf, + struct rte_pipeline_table_entry *table_entry, + uint64_t time, + struct rte_table_action *action, + struct ap_config *cfg) +{ + uint64_t drop_mask = 0; + + uint32_t ip_offset = action->cfg.common.ip_offset; + void *ip = RTE_MBUF_METADATA_UINT32_PTR(mbuf, ip_offset); + + uint32_t dscp; + uint16_t total_length; + + if (cfg->common.ip_version) { + struct ipv4_hdr *hdr = ip; + + dscp = hdr->type_of_service >> 2; + total_length = rte_ntohs(hdr->total_length); + } else { + struct ipv6_hdr *hdr = ip; + + dscp = (rte_ntohl(hdr->vtc_flow) & 0x0F600000) >> 18; + total_length = + rte_ntohs(hdr->payload_len) + sizeof(struct ipv6_hdr); + } + + if (cfg->action_mask & (1LLU << RTE_TABLE_ACTION_LB)) { + void *data = + action_data_get(table_entry, action, RTE_TABLE_ACTION_LB); + + pkt_work_lb(mbuf, + data, + &cfg->lb); + } + if (cfg->action_mask & (1LLU << RTE_TABLE_ACTION_MTR)) { + void *data = + action_data_get(table_entry, action, RTE_TABLE_ACTION_MTR); + + drop_mask |= pkt_work_mtr(mbuf, + data, + &action->dscp_table, + action->mp, + time, + dscp, + total_length); + } + + if (cfg->action_mask & (1LLU << RTE_TABLE_ACTION_TM)) { + void *data = + action_data_get(table_entry, action, RTE_TABLE_ACTION_TM); + + pkt_work_tm(mbuf, + data, + &action->dscp_table, + dscp); + } + + if (cfg->action_mask & (1LLU << RTE_TABLE_ACTION_ENCAP)) { + void *data = + action_data_get(table_entry, action, RTE_TABLE_ACTION_ENCAP); + + pkt_work_encap(mbuf, + data, + &cfg->encap, + ip, + total_length, + ip_offset); + } + + if (cfg->action_mask & (1LLU << RTE_TABLE_ACTION_NAT)) { + void *data = + action_data_get(table_entry, action, RTE_TABLE_ACTION_NAT); + + if (cfg->common.ip_version) + pkt_ipv4_work_nat(ip, data, &cfg->nat); + else + pkt_ipv6_work_nat(ip, data, &cfg->nat); + } + + if (cfg->action_mask & (1LLU << RTE_TABLE_ACTION_TTL)) { + void *data = + action_data_get(table_entry, action, RTE_TABLE_ACTION_TTL); + + if (cfg->common.ip_version) + drop_mask |= pkt_ipv4_work_ttl(ip, data); + else + drop_mask |= pkt_ipv6_work_ttl(ip, data); + } + + if (cfg->action_mask & (1LLU << RTE_TABLE_ACTION_STATS)) { + void *data = + action_data_get(table_entry, action, RTE_TABLE_ACTION_STATS); + + pkt_work_stats(data, total_length); + } + + if (cfg->action_mask & (1LLU << RTE_TABLE_ACTION_TIME)) { + void *data = + action_data_get(table_entry, action, RTE_TABLE_ACTION_TIME); + + pkt_work_time(data, time); + } + + return drop_mask; +} + +static __rte_always_inline uint64_t +pkt4_work(struct rte_mbuf **mbufs, + struct rte_pipeline_table_entry **table_entries, + uint64_t time, + struct rte_table_action *action, + struct ap_config *cfg) +{ + uint64_t drop_mask0 = 0; + uint64_t drop_mask1 = 0; + uint64_t drop_mask2 = 0; + uint64_t drop_mask3 = 0; + + struct rte_mbuf *mbuf0 = mbufs[0]; + struct rte_mbuf *mbuf1 = mbufs[1]; + struct rte_mbuf *mbuf2 = mbufs[2]; + struct rte_mbuf *mbuf3 = mbufs[3]; + + struct rte_pipeline_table_entry *table_entry0 = table_entries[0]; + struct rte_pipeline_table_entry *table_entry1 = table_entries[1]; + struct rte_pipeline_table_entry *table_entry2 = table_entries[2]; + struct rte_pipeline_table_entry *table_entry3 = table_entries[3]; + + uint32_t ip_offset = action->cfg.common.ip_offset; + void *ip0 = RTE_MBUF_METADATA_UINT32_PTR(mbuf0, ip_offset); + void *ip1 = RTE_MBUF_METADATA_UINT32_PTR(mbuf1, ip_offset); + void *ip2 = RTE_MBUF_METADATA_UINT32_PTR(mbuf2, ip_offset); + void *ip3 = RTE_MBUF_METADATA_UINT32_PTR(mbuf3, ip_offset); + + uint32_t dscp0, dscp1, dscp2, dscp3; + uint16_t total_length0, total_length1, total_length2, total_length3; + + if (cfg->common.ip_version) { + struct ipv4_hdr *hdr0 = ip0; + struct ipv4_hdr *hdr1 = ip1; + struct ipv4_hdr *hdr2 = ip2; + struct ipv4_hdr *hdr3 = ip3; + + dscp0 = hdr0->type_of_service >> 2; + dscp1 = hdr1->type_of_service >> 2; + dscp2 = hdr2->type_of_service >> 2; + dscp3 = hdr3->type_of_service >> 2; + + total_length0 = rte_ntohs(hdr0->total_length); + total_length1 = rte_ntohs(hdr1->total_length); + total_length2 = rte_ntohs(hdr2->total_length); + total_length3 = rte_ntohs(hdr3->total_length); + } else { + struct ipv6_hdr *hdr0 = ip0; + struct ipv6_hdr *hdr1 = ip1; + struct ipv6_hdr *hdr2 = ip2; + struct ipv6_hdr *hdr3 = ip3; + + dscp0 = (rte_ntohl(hdr0->vtc_flow) & 0x0F600000) >> 18; + dscp1 = (rte_ntohl(hdr1->vtc_flow) & 0x0F600000) >> 18; + dscp2 = (rte_ntohl(hdr2->vtc_flow) & 0x0F600000) >> 18; + dscp3 = (rte_ntohl(hdr3->vtc_flow) & 0x0F600000) >> 18; + + total_length0 = + rte_ntohs(hdr0->payload_len) + sizeof(struct ipv6_hdr); + total_length1 = + rte_ntohs(hdr1->payload_len) + sizeof(struct ipv6_hdr); + total_length2 = + rte_ntohs(hdr2->payload_len) + sizeof(struct ipv6_hdr); + total_length3 = + rte_ntohs(hdr3->payload_len) + sizeof(struct ipv6_hdr); + } + + if (cfg->action_mask & (1LLU << RTE_TABLE_ACTION_LB)) { + void *data0 = + action_data_get(table_entry0, action, RTE_TABLE_ACTION_LB); + void *data1 = + action_data_get(table_entry1, action, RTE_TABLE_ACTION_LB); + void *data2 = + action_data_get(table_entry2, action, RTE_TABLE_ACTION_LB); + void *data3 = + action_data_get(table_entry3, action, RTE_TABLE_ACTION_LB); + + pkt_work_lb(mbuf0, + data0, + &cfg->lb); + + pkt_work_lb(mbuf1, + data1, + &cfg->lb); + + pkt_work_lb(mbuf2, + data2, + &cfg->lb); + + pkt_work_lb(mbuf3, + data3, + &cfg->lb); + } + + if (cfg->action_mask & (1LLU << RTE_TABLE_ACTION_MTR)) { + void *data0 = + action_data_get(table_entry0, action, RTE_TABLE_ACTION_MTR); + void *data1 = + action_data_get(table_entry1, action, RTE_TABLE_ACTION_MTR); + void *data2 = + action_data_get(table_entry2, action, RTE_TABLE_ACTION_MTR); + void *data3 = + action_data_get(table_entry3, action, RTE_TABLE_ACTION_MTR); + + drop_mask0 |= pkt_work_mtr(mbuf0, + data0, + &action->dscp_table, + action->mp, + time, + dscp0, + total_length0); + + drop_mask1 |= pkt_work_mtr(mbuf1, + data1, + &action->dscp_table, + action->mp, + time, + dscp1, + total_length1); + + drop_mask2 |= pkt_work_mtr(mbuf2, + data2, + &action->dscp_table, + action->mp, + time, + dscp2, + total_length2); + + drop_mask3 |= pkt_work_mtr(mbuf3, + data3, + &action->dscp_table, + action->mp, + time, + dscp3, + total_length3); + } + + if (cfg->action_mask & (1LLU << RTE_TABLE_ACTION_TM)) { + void *data0 = + action_data_get(table_entry0, action, RTE_TABLE_ACTION_TM); + void *data1 = + action_data_get(table_entry1, action, RTE_TABLE_ACTION_TM); + void *data2 = + action_data_get(table_entry2, action, RTE_TABLE_ACTION_TM); + void *data3 = + action_data_get(table_entry3, action, RTE_TABLE_ACTION_TM); + + pkt_work_tm(mbuf0, + data0, + &action->dscp_table, + dscp0); + + pkt_work_tm(mbuf1, + data1, + &action->dscp_table, + dscp1); + + pkt_work_tm(mbuf2, + data2, + &action->dscp_table, + dscp2); + + pkt_work_tm(mbuf3, + data3, + &action->dscp_table, + dscp3); + } + + if (cfg->action_mask & (1LLU << RTE_TABLE_ACTION_ENCAP)) { + void *data0 = + action_data_get(table_entry0, action, RTE_TABLE_ACTION_ENCAP); + void *data1 = + action_data_get(table_entry1, action, RTE_TABLE_ACTION_ENCAP); + void *data2 = + action_data_get(table_entry2, action, RTE_TABLE_ACTION_ENCAP); + void *data3 = + action_data_get(table_entry3, action, RTE_TABLE_ACTION_ENCAP); + + pkt_work_encap(mbuf0, + data0, + &cfg->encap, + ip0, + total_length0, + ip_offset); + + pkt_work_encap(mbuf1, + data1, + &cfg->encap, + ip1, + total_length1, + ip_offset); + + pkt_work_encap(mbuf2, + data2, + &cfg->encap, + ip2, + total_length2, + ip_offset); + + pkt_work_encap(mbuf3, + data3, + &cfg->encap, + ip3, + total_length3, + ip_offset); + } + + if (cfg->action_mask & (1LLU << RTE_TABLE_ACTION_NAT)) { + void *data0 = + action_data_get(table_entry0, action, RTE_TABLE_ACTION_NAT); + void *data1 = + action_data_get(table_entry1, action, RTE_TABLE_ACTION_NAT); + void *data2 = + action_data_get(table_entry2, action, RTE_TABLE_ACTION_NAT); + void *data3 = + action_data_get(table_entry3, action, RTE_TABLE_ACTION_NAT); + + if (cfg->common.ip_version) { + pkt_ipv4_work_nat(ip0, data0, &cfg->nat); + pkt_ipv4_work_nat(ip1, data1, &cfg->nat); + pkt_ipv4_work_nat(ip2, data2, &cfg->nat); + pkt_ipv4_work_nat(ip3, data3, &cfg->nat); + } else { + pkt_ipv6_work_nat(ip0, data0, &cfg->nat); + pkt_ipv6_work_nat(ip1, data1, &cfg->nat); + pkt_ipv6_work_nat(ip2, data2, &cfg->nat); + pkt_ipv6_work_nat(ip3, data3, &cfg->nat); + } + } + + if (cfg->action_mask & (1LLU << RTE_TABLE_ACTION_TTL)) { + void *data0 = + action_data_get(table_entry0, action, RTE_TABLE_ACTION_TTL); + void *data1 = + action_data_get(table_entry1, action, RTE_TABLE_ACTION_TTL); + void *data2 = + action_data_get(table_entry2, action, RTE_TABLE_ACTION_TTL); + void *data3 = + action_data_get(table_entry3, action, RTE_TABLE_ACTION_TTL); + + if (cfg->common.ip_version) { + drop_mask0 |= pkt_ipv4_work_ttl(ip0, data0); + drop_mask1 |= pkt_ipv4_work_ttl(ip1, data1); + drop_mask2 |= pkt_ipv4_work_ttl(ip2, data2); + drop_mask3 |= pkt_ipv4_work_ttl(ip3, data3); + } else { + drop_mask0 |= pkt_ipv6_work_ttl(ip0, data0); + drop_mask1 |= pkt_ipv6_work_ttl(ip1, data1); + drop_mask2 |= pkt_ipv6_work_ttl(ip2, data2); + drop_mask3 |= pkt_ipv6_work_ttl(ip3, data3); + } + } + + if (cfg->action_mask & (1LLU << RTE_TABLE_ACTION_STATS)) { + void *data0 = + action_data_get(table_entry0, action, RTE_TABLE_ACTION_STATS); + void *data1 = + action_data_get(table_entry1, action, RTE_TABLE_ACTION_STATS); + void *data2 = + action_data_get(table_entry2, action, RTE_TABLE_ACTION_STATS); + void *data3 = + action_data_get(table_entry3, action, RTE_TABLE_ACTION_STATS); + + pkt_work_stats(data0, total_length0); + pkt_work_stats(data1, total_length1); + pkt_work_stats(data2, total_length2); + pkt_work_stats(data3, total_length3); + } + + if (cfg->action_mask & (1LLU << RTE_TABLE_ACTION_TIME)) { + void *data0 = + action_data_get(table_entry0, action, RTE_TABLE_ACTION_TIME); + void *data1 = + action_data_get(table_entry1, action, RTE_TABLE_ACTION_TIME); + void *data2 = + action_data_get(table_entry2, action, RTE_TABLE_ACTION_TIME); + void *data3 = + action_data_get(table_entry3, action, RTE_TABLE_ACTION_TIME); + + pkt_work_time(data0, time); + pkt_work_time(data1, time); + pkt_work_time(data2, time); + pkt_work_time(data3, time); + } + + return drop_mask0 | + (drop_mask1 << 1) | + (drop_mask2 << 2) | + (drop_mask3 << 3); +} + +static __rte_always_inline int +ah(struct rte_pipeline *p, + struct rte_mbuf **pkts, + uint64_t pkts_mask, + struct rte_pipeline_table_entry **entries, + struct rte_table_action *action, + struct ap_config *cfg) +{ + uint64_t pkts_drop_mask = 0; + uint64_t time = 0; + + if (cfg->action_mask & ((1LLU << RTE_TABLE_ACTION_MTR) | + (1LLU << RTE_TABLE_ACTION_TIME))) + time = rte_rdtsc(); + + if ((pkts_mask & (pkts_mask + 1)) == 0) { + uint64_t n_pkts = __builtin_popcountll(pkts_mask); + uint32_t i; + + for (i = 0; i < (n_pkts & (~0x3LLU)); i += 4) { + uint64_t drop_mask; + + drop_mask = pkt4_work(&pkts[i], + &entries[i], + time, + action, + cfg); + + pkts_drop_mask |= drop_mask << i; + } + + for ( ; i < n_pkts; i++) { + uint64_t drop_mask; + + drop_mask = pkt_work(pkts[i], + entries[i], + time, + action, + cfg); + + pkts_drop_mask |= drop_mask << i; + } + } else + for ( ; pkts_mask; ) { + uint32_t pos = __builtin_ctzll(pkts_mask); + uint64_t pkt_mask = 1LLU << pos; + uint64_t drop_mask; + + drop_mask = pkt_work(pkts[pos], + entries[pos], + time, + action, + cfg); + + pkts_mask &= ~pkt_mask; + pkts_drop_mask |= drop_mask << pos; + } + + rte_pipeline_ah_packet_drop(p, pkts_drop_mask); + + return 0; +} + +static int +ah_default(struct rte_pipeline *p, + struct rte_mbuf **pkts, + uint64_t pkts_mask, + struct rte_pipeline_table_entry **entries, + void *arg) +{ + struct rte_table_action *action = arg; + + return ah(p, + pkts, + pkts_mask, + entries, + action, + &action->cfg); +} + +static rte_pipeline_table_action_handler_hit +ah_selector(struct rte_table_action *action) +{ + if (action->cfg.action_mask == (1LLU << RTE_TABLE_ACTION_FWD)) + return NULL; + + return ah_default; +} + +int +rte_table_action_table_params_get(struct rte_table_action *action, + struct rte_pipeline_table_params *params) +{ + rte_pipeline_table_action_handler_hit f_action_hit; + uint32_t total_size; + + /* Check input arguments */ + if ((action == NULL) || + (params == NULL)) + return -EINVAL; + + f_action_hit = ah_selector(action); + total_size = rte_align32pow2(action->data.total_size); + + /* Fill in params */ + params->f_action_hit = f_action_hit; + params->f_action_miss = NULL; + params->arg_ah = (f_action_hit) ? action : NULL; + params->action_data_size = total_size - + sizeof(struct rte_pipeline_table_entry); + + return 0; +} + +int +rte_table_action_free(struct rte_table_action *action) +{ + if (action == NULL) + return 0; + + rte_free(action); + + return 0; +} diff --git a/src/spdk/dpdk/lib/librte_pipeline/rte_table_action.h b/src/spdk/dpdk/lib/librte_pipeline/rte_table_action.h new file mode 100644 index 00000000..c7f751aa --- /dev/null +++ b/src/spdk/dpdk/lib/librte_pipeline/rte_table_action.h @@ -0,0 +1,905 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2018 Intel Corporation + */ + +#ifndef __INCLUDE_RTE_TABLE_ACTION_H__ +#define __INCLUDE_RTE_TABLE_ACTION_H__ + +/** + * @file + * RTE Pipeline Table Actions + * + * This API provides a common set of actions for pipeline tables to speed up + * application development. + * + * Each match-action rule added to a pipeline table has associated data that + * stores the action context. This data is input to the table action handler + * called for every input packet that hits the rule as part of the table lookup + * during the pipeline execution. The pipeline library allows the user to define + * his own table actions by providing customized table action handlers (table + * lookup) and complete freedom of setting the rules and their data (table rule + * add/delete). While the user can still follow this process, this API is + * intended to provide a quicker development alternative for a set of predefined + * actions. + * + * The typical steps to use this API are: + * - Define a table action profile. This is a configuration template that can + * potentially be shared by multiple tables from the same or different + * pipelines, with different tables from the same pipeline likely to use + * different action profiles. For every table using a given action profile, + * the profile defines the set of actions and the action configuration to be + * implemented for all the table rules. API functions: + * rte_table_action_profile_create(), + * rte_table_action_profile_action_register(), + * rte_table_action_profile_freeze(). + * + * - Instantiate the table action profile to create table action objects. Each + * pipeline table has its own table action object. API functions: + * rte_table_action_create(). + * + * - Use the table action object to generate the pipeline table action handlers + * (invoked by the pipeline table lookup operation). API functions: + * rte_table_action_table_params_get(). + * + * - Use the table action object to generate the rule data (for the pipeline + * table rule add operation) based on given action parameters. API functions: + * rte_table_action_apply(). + * + * - Use the table action object to read action data (e.g. stats counters) for + * any given rule. API functions: rte_table_action_XYZ_read(). + * + * @warning + * @b EXPERIMENTAL: this API may change without prior notice + */ + +#ifdef __cplusplus +extern "C" { +#endif + +#include <stdint.h> + +#include <rte_compat.h> +#include <rte_ether.h> +#include <rte_meter.h> +#include <rte_table_hash.h> + +#include "rte_pipeline.h" + +/** Table actions. */ +enum rte_table_action_type { + /** Forward to next pipeline table, output port or drop. */ + RTE_TABLE_ACTION_FWD = 0, + + /** Load balance. */ + RTE_TABLE_ACTION_LB, + + /** Traffic Metering and Policing. */ + RTE_TABLE_ACTION_MTR, + + /** Traffic Management. */ + RTE_TABLE_ACTION_TM, + + /** Packet encapsulations. */ + RTE_TABLE_ACTION_ENCAP, + + /** Network Address Translation (NAT). */ + RTE_TABLE_ACTION_NAT, + + /** Time to Live (TTL) update. */ + RTE_TABLE_ACTION_TTL, + + /** Statistics. */ + RTE_TABLE_ACTION_STATS, + + /** Timestamp. */ + RTE_TABLE_ACTION_TIME, +}; + +/** Common action configuration (per table action profile). */ +struct rte_table_action_common_config { + /** Input packet Internet Protocol (IP) version. Non-zero for IPv4, zero + * for IPv6. + */ + int ip_version; + + /** IP header offset within the input packet buffer. Offset 0 points to + * the first byte of the MBUF structure. + */ + uint32_t ip_offset; +}; + +/** + * RTE_TABLE_ACTION_FWD + */ +/** Forward action parameters (per table rule). */ +struct rte_table_action_fwd_params { + /** Forward action. */ + enum rte_pipeline_action action; + + /** Pipeline table ID or output port ID. */ + uint32_t id; +}; + +/** + * RTE_TABLE_ACTION_LB + */ +/** Load balance key size min (number of bytes). */ +#define RTE_TABLE_ACTION_LB_KEY_SIZE_MIN 8 + +/** Load balance key size max (number of bytes). */ +#define RTE_TABLE_ACTION_LB_KEY_SIZE_MAX 64 + +/** Load balance table size. */ +#define RTE_TABLE_ACTION_LB_TABLE_SIZE 8 + +/** Load balance action configuration (per table action profile). */ +struct rte_table_action_lb_config { + /** Key size (number of bytes). */ + uint32_t key_size; + + /** Key offset within the input packet buffer. Offset 0 points to the + * first byte of the MBUF structure. + */ + uint32_t key_offset; + + /** Key mask (*key_size* bytes are valid). */ + uint8_t key_mask[RTE_TABLE_ACTION_LB_KEY_SIZE_MAX]; + + /** Hash function. */ + rte_table_hash_op_hash f_hash; + + /** Seed value for *f_hash*. */ + uint64_t seed; + + /** Output value offset within the input packet buffer. Offset 0 points + * to the first byte of the MBUF structure. + */ + uint32_t out_offset; +}; + +/** Load balance action parameters (per table rule). */ +struct rte_table_action_lb_params { + /** Table defining the output values and their weights. The weights are + * set in 1/RTE_TABLE_ACTION_LB_TABLE_SIZE increments. To assign a + * weight of N/RTE_TABLE_ACTION_LB_TABLE_SIZE to a given output value + * (0 <= N <= RTE_TABLE_ACTION_LB_TABLE_SIZE), the same output value + * needs to show up exactly N times in this table. + */ + uint32_t out[RTE_TABLE_ACTION_LB_TABLE_SIZE]; +}; + +/** + * RTE_TABLE_ACTION_MTR + */ +/** Max number of traffic classes (TCs). */ +#define RTE_TABLE_ACTION_TC_MAX 4 + +/** Max number of queues per traffic class. */ +#define RTE_TABLE_ACTION_TC_QUEUE_MAX 4 + +/** Differentiated Services Code Point (DSCP) translation table entry. */ +struct rte_table_action_dscp_table_entry { + /** Traffic class. Used by the meter or the traffic management actions. + * Has to be strictly smaller than *RTE_TABLE_ACTION_TC_MAX*. Traffic + * class 0 is the highest priority. + */ + uint32_t tc_id; + + /** Traffic class queue. Used by the traffic management action. Has to + * be strictly smaller than *RTE_TABLE_ACTION_TC_QUEUE_MAX*. + */ + uint32_t tc_queue_id; + + /** Packet color. Used by the meter action as the packet input color + * for the color aware mode of the traffic metering algorithm. + */ + enum rte_meter_color color; +}; + +/** DSCP translation table. */ +struct rte_table_action_dscp_table { + /** Array of DSCP table entries */ + struct rte_table_action_dscp_table_entry entry[64]; +}; + +/** Supported traffic metering algorithms. */ +enum rte_table_action_meter_algorithm { + /** Single Rate Three Color Marker (srTCM) - IETF RFC 2697. */ + RTE_TABLE_ACTION_METER_SRTCM, + + /** Two Rate Three Color Marker (trTCM) - IETF RFC 2698. */ + RTE_TABLE_ACTION_METER_TRTCM, +}; + +/** Traffic metering profile (configuration template). */ +struct rte_table_action_meter_profile { + /** Traffic metering algorithm. */ + enum rte_table_action_meter_algorithm alg; + + RTE_STD_C11 + union { + /** Only valid when *alg* is set to srTCM - IETF RFC 2697. */ + struct rte_meter_srtcm_params srtcm; + + /** Only valid when *alg* is set to trTCM - IETF RFC 2698. */ + struct rte_meter_trtcm_params trtcm; + }; +}; + +/** Policer actions. */ +enum rte_table_action_policer { + /** Recolor the packet as green. */ + RTE_TABLE_ACTION_POLICER_COLOR_GREEN = 0, + + /** Recolor the packet as yellow. */ + RTE_TABLE_ACTION_POLICER_COLOR_YELLOW, + + /** Recolor the packet as red. */ + RTE_TABLE_ACTION_POLICER_COLOR_RED, + + /** Drop the packet. */ + RTE_TABLE_ACTION_POLICER_DROP, + + /** Number of policer actions. */ + RTE_TABLE_ACTION_POLICER_MAX +}; + +/** Meter action configuration per traffic class. */ +struct rte_table_action_mtr_tc_params { + /** Meter profile ID. */ + uint32_t meter_profile_id; + + /** Policer actions. */ + enum rte_table_action_policer policer[e_RTE_METER_COLORS]; +}; + +/** Meter action statistics counters per traffic class. */ +struct rte_table_action_mtr_counters_tc { + /** Number of packets per color at the output of the traffic metering + * and before the policer actions are executed. Only valid when + * *n_packets_valid* is non-zero. + */ + uint64_t n_packets[e_RTE_METER_COLORS]; + + /** Number of packet bytes per color at the output of the traffic + * metering and before the policer actions are executed. Only valid when + * *n_bytes_valid* is non-zero. + */ + uint64_t n_bytes[e_RTE_METER_COLORS]; + + /** When non-zero, the *n_packets* field is valid. */ + int n_packets_valid; + + /** When non-zero, the *n_bytes* field is valid. */ + int n_bytes_valid; +}; + +/** Meter action configuration (per table action profile). */ +struct rte_table_action_mtr_config { + /** Meter algorithm. */ + enum rte_table_action_meter_algorithm alg; + + /** Number of traffic classes. Each traffic class has its own traffic + * meter and policer instances. Needs to be equal to either 1 or to + * *RTE_TABLE_ACTION_TC_MAX*. + */ + uint32_t n_tc; + + /** When non-zero, the *n_packets* meter stats counter is enabled, + * otherwise it is disabled. + * + * @see struct rte_table_action_mtr_counters_tc + */ + int n_packets_enabled; + + /** When non-zero, the *n_bytes* meter stats counter is enabled, + * otherwise it is disabled. + * + * @see struct rte_table_action_mtr_counters_tc + */ + int n_bytes_enabled; +}; + +/** Meter action parameters (per table rule). */ +struct rte_table_action_mtr_params { + /** Traffic meter and policer parameters for each of the *tc_mask* + * traffic classes. + */ + struct rte_table_action_mtr_tc_params mtr[RTE_TABLE_ACTION_TC_MAX]; + + /** Bit mask defining which traffic class parameters are valid in *mtr*. + * If bit N is set in *tc_mask*, then parameters for traffic class N are + * valid in *mtr*. + */ + uint32_t tc_mask; +}; + +/** Meter action statistics counters (per table rule). */ +struct rte_table_action_mtr_counters { + /** Stats counters for each of the *tc_mask* traffic classes. */ + struct rte_table_action_mtr_counters_tc stats[RTE_TABLE_ACTION_TC_MAX]; + + /** Bit mask defining which traffic class parameters are valid in *mtr*. + * If bit N is set in *tc_mask*, then parameters for traffic class N are + * valid in *mtr*. + */ + uint32_t tc_mask; +}; + +/** + * RTE_TABLE_ACTION_TM + */ +/** Traffic management action configuration (per table action profile). */ +struct rte_table_action_tm_config { + /** Number of subports per port. */ + uint32_t n_subports_per_port; + + /** Number of pipes per subport. */ + uint32_t n_pipes_per_subport; +}; + +/** Traffic management action parameters (per table rule). */ +struct rte_table_action_tm_params { + /** Subport ID. */ + uint32_t subport_id; + + /** Pipe ID. */ + uint32_t pipe_id; +}; + +/** + * RTE_TABLE_ACTION_ENCAP + */ +/** Supported packet encapsulation types. */ +enum rte_table_action_encap_type { + /** IP -> { Ether | IP } */ + RTE_TABLE_ACTION_ENCAP_ETHER = 0, + + /** IP -> { Ether | VLAN | IP } */ + RTE_TABLE_ACTION_ENCAP_VLAN, + + /** IP -> { Ether | S-VLAN | C-VLAN | IP } */ + RTE_TABLE_ACTION_ENCAP_QINQ, + + /** IP -> { Ether | MPLS | IP } */ + RTE_TABLE_ACTION_ENCAP_MPLS, + + /** IP -> { Ether | PPPoE | PPP | IP } */ + RTE_TABLE_ACTION_ENCAP_PPPOE, +}; + +/** Pre-computed Ethernet header fields for encapsulation action. */ +struct rte_table_action_ether_hdr { + struct ether_addr da; /**< Destination address. */ + struct ether_addr sa; /**< Source address. */ +}; + +/** Pre-computed VLAN header fields for encapsulation action. */ +struct rte_table_action_vlan_hdr { + uint8_t pcp; /**< Priority Code Point (PCP). */ + uint8_t dei; /**< Drop Eligibility Indicator (DEI). */ + uint16_t vid; /**< VLAN Identifier (VID). */ +}; + +/** Pre-computed MPLS header fields for encapsulation action. */ +struct rte_table_action_mpls_hdr { + uint32_t label; /**< Label. */ + uint8_t tc; /**< Traffic Class (TC). */ + uint8_t ttl; /**< Time to Live (TTL). */ +}; + +/** Pre-computed PPPoE header fields for encapsulation action. */ +struct rte_table_action_pppoe_hdr { + uint16_t session_id; /**< Session ID. */ +}; + +/** Ether encap parameters. */ +struct rte_table_action_encap_ether_params { + struct rte_table_action_ether_hdr ether; /**< Ethernet header. */ +}; + +/** VLAN encap parameters. */ +struct rte_table_action_encap_vlan_params { + struct rte_table_action_ether_hdr ether; /**< Ethernet header. */ + struct rte_table_action_vlan_hdr vlan; /**< VLAN header. */ +}; + +/** QinQ encap parameters. */ +struct rte_table_action_encap_qinq_params { + struct rte_table_action_ether_hdr ether; /**< Ethernet header. */ + struct rte_table_action_vlan_hdr svlan; /**< Service VLAN header. */ + struct rte_table_action_vlan_hdr cvlan; /**< Customer VLAN header. */ +}; + +/** Max number of MPLS labels per output packet for MPLS encapsulation. */ +#ifndef RTE_TABLE_ACTION_MPLS_LABELS_MAX +#define RTE_TABLE_ACTION_MPLS_LABELS_MAX 4 +#endif + +/** MPLS encap parameters. */ +struct rte_table_action_encap_mpls_params { + /** Ethernet header. */ + struct rte_table_action_ether_hdr ether; + + /** MPLS header. */ + struct rte_table_action_mpls_hdr mpls[RTE_TABLE_ACTION_MPLS_LABELS_MAX]; + + /** Number of MPLS labels in MPLS header. */ + uint32_t mpls_count; + + /** Non-zero for MPLS unicast, zero for MPLS multicast. */ + int unicast; +}; + +/** PPPoE encap parameters. */ +struct rte_table_action_encap_pppoe_params { + struct rte_table_action_ether_hdr ether; /**< Ethernet header. */ + struct rte_table_action_pppoe_hdr pppoe; /**< PPPoE/PPP headers. */ +}; + +/** Encap action configuration (per table action profile). */ +struct rte_table_action_encap_config { + /** Bit mask defining the set of packet encapsulations enabled for the + * current table action profile. If bit (1 << N) is set in *encap_mask*, + * then packet encapsulation N is enabled, otherwise it is disabled. + * + * @see enum rte_table_action_encap_type + */ + uint64_t encap_mask; +}; + +/** Encap action parameters (per table rule). */ +struct rte_table_action_encap_params { + /** Encapsulation type. */ + enum rte_table_action_encap_type type; + + RTE_STD_C11 + union { + /** Only valid when *type* is set to Ether. */ + struct rte_table_action_encap_ether_params ether; + + /** Only valid when *type* is set to VLAN. */ + struct rte_table_action_encap_vlan_params vlan; + + /** Only valid when *type* is set to QinQ. */ + struct rte_table_action_encap_qinq_params qinq; + + /** Only valid when *type* is set to MPLS. */ + struct rte_table_action_encap_mpls_params mpls; + + /** Only valid when *type* is set to PPPoE. */ + struct rte_table_action_encap_pppoe_params pppoe; + }; +}; + +/** + * RTE_TABLE_ACTION_NAT + */ +/** NAT action configuration (per table action profile). */ +struct rte_table_action_nat_config { + /** When non-zero, the IP source address and L4 protocol source port are + * translated. When zero, the IP destination address and L4 protocol + * destination port are translated. + */ + int source_nat; + + /** Layer 4 protocol, for example TCP (0x06) or UDP (0x11). The checksum + * field is computed differently and placed at different header offset + * by each layer 4 protocol. + */ + uint8_t proto; +}; + +/** NAT action parameters (per table rule). */ +struct rte_table_action_nat_params { + /** IP version for *addr*: non-zero for IPv4, zero for IPv6. */ + int ip_version; + + /** IP address. */ + union { + /** IPv4 address; only valid when *ip_version* is non-zero. */ + uint32_t ipv4; + + /** IPv6 address; only valid when *ip_version* is set to 0. */ + uint8_t ipv6[16]; + } addr; + + /** Port. */ + uint16_t port; +}; + +/** + * RTE_TABLE_ACTION_TTL + */ +/** TTL action configuration (per table action profile). */ +struct rte_table_action_ttl_config { + /** When non-zero, the input packets whose updated IPv4 Time to Live + * (TTL) field or IPv6 Hop Limit (HL) field is zero are dropped. + * When zero, the input packets whose updated IPv4 TTL field or IPv6 HL + * field is zero are forwarded as usual (typically for debugging + * purpose). + */ + int drop; + + /** When non-zero, the *n_packets* stats counter for TTL action is + * enabled, otherwise disabled. + * + * @see struct rte_table_action_ttl_counters + */ + int n_packets_enabled; +}; + +/** TTL action parameters (per table rule). */ +struct rte_table_action_ttl_params { + /** When non-zero, decrement the IPv4 TTL field and update the checksum + * field, or decrement the IPv6 HL field. When zero, the IPv4 TTL field + * or the IPv6 HL field is not changed. + */ + int decrement; +}; + +/** TTL action statistics packets (per table rule). */ +struct rte_table_action_ttl_counters { + /** Number of IPv4 packets whose updated TTL field is zero or IPv6 + * packets whose updated HL field is zero. + */ + uint64_t n_packets; +}; + +/** + * RTE_TABLE_ACTION_STATS + */ +/** Stats action configuration (per table action profile). */ +struct rte_table_action_stats_config { + /** When non-zero, the *n_packets* stats counter is enabled, otherwise + * disabled. + * + * @see struct rte_table_action_stats_counters + */ + int n_packets_enabled; + + /** When non-zero, the *n_bytes* stats counter is enabled, otherwise + * disabled. + * + * @see struct rte_table_action_stats_counters + */ + int n_bytes_enabled; +}; + +/** Stats action parameters (per table rule). */ +struct rte_table_action_stats_params { + /** Initial value for the *n_packets* stats counter. Typically set to 0. + * + * @see struct rte_table_action_stats_counters + */ + uint64_t n_packets; + + /** Initial value for the *n_bytes* stats counter. Typically set to 0. + * + * @see struct rte_table_action_stats_counters + */ + uint64_t n_bytes; +}; + +/** Stats action counters (per table rule). */ +struct rte_table_action_stats_counters { + /** Number of packets. Valid only when *n_packets_valid* is non-zero. */ + uint64_t n_packets; + + /** Number of bytes. Valid only when *n_bytes_valid* is non-zero. */ + uint64_t n_bytes; + + /** When non-zero, the *n_packets* field is valid, otherwise invalid. */ + int n_packets_valid; + + /** When non-zero, the *n_bytes* field is valid, otherwise invalid. */ + int n_bytes_valid; +}; + +/** + * RTE_TABLE_ACTION_TIME + */ +/** Timestamp action parameters (per table rule). */ +struct rte_table_action_time_params { + /** Initial timestamp value. Typically set to current time. */ + uint64_t time; +}; + +/** + * Table action profile. + */ +struct rte_table_action_profile; + +/** + * Table action profile create. + * + * @param[in] common + * Common action configuration. + * @return + * Table action profile handle on success, NULL otherwise. + */ +struct rte_table_action_profile * __rte_experimental +rte_table_action_profile_create(struct rte_table_action_common_config *common); + +/** + * Table action profile free. + * + * @param[in] profile + * Table profile action handle (needs to be valid). + * @return + * Zero on success, non-zero error code otherwise. + */ +int __rte_experimental +rte_table_action_profile_free(struct rte_table_action_profile *profile); + +/** + * Table action profile action register. + * + * @param[in] profile + * Table profile action handle (needs to be valid and not in frozen state). + * @param[in] type + * Specific table action to be registered for *profile*. + * @param[in] action_config + * Configuration for the *type* action. + * If struct rte_table_action_*type*_config is defined by the Table Action + * API, it needs to point to a valid instance of this structure, otherwise it + * needs to be set to NULL. + * @return + * Zero on success, non-zero error code otherwise. + */ +int __rte_experimental +rte_table_action_profile_action_register(struct rte_table_action_profile *profile, + enum rte_table_action_type type, + void *action_config); + +/** + * Table action profile freeze. + * + * Once this function is called successfully, the given profile enters the + * frozen state with the following immediate effects: no more actions can be + * registered for this profile, so the profile can be instantiated to create + * table action objects. + * + * @param[in] profile + * Table profile action handle (needs to be valid and not in frozen state). + * @return + * Zero on success, non-zero error code otherwise. + * + * @see rte_table_action_create() + */ +int __rte_experimental +rte_table_action_profile_freeze(struct rte_table_action_profile *profile); + +/** + * Table action. + */ +struct rte_table_action; + +/** + * Table action create. + * + * Instantiates the given table action profile to create a table action object. + * + * @param[in] profile + * Table profile action handle (needs to be valid and in frozen state). + * @param[in] socket_id + * CPU socket ID where the internal data structures required by the new table + * action object should be allocated. + * @return + * Handle to table action object on success, NULL on error. + * + * @see rte_table_action_create() + */ +struct rte_table_action * __rte_experimental +rte_table_action_create(struct rte_table_action_profile *profile, + uint32_t socket_id); + +/** + * Table action free. + * + * @param[in] action + * Handle to table action object (needs to be valid). + * @return + * Zero on success, non-zero error code otherwise. + */ +int __rte_experimental +rte_table_action_free(struct rte_table_action *action); + +/** + * Table action table params get. + * + * @param[in] action + * Handle to table action object (needs to be valid). + * @param[inout] params + * Pipeline table parameters (needs to be pre-allocated). + * @return + * Zero on success, non-zero error code otherwise. + */ +int __rte_experimental +rte_table_action_table_params_get(struct rte_table_action *action, + struct rte_pipeline_table_params *params); + +/** + * Table action apply. + * + * @param[in] action + * Handle to table action object (needs to be valid). + * @param[in] data + * Data byte array (typically table rule data) to apply action *type* on. + * @param[in] type + * Specific table action previously registered for the table action profile of + * the *action* object. + * @param[in] action_params + * Parameters for the *type* action. + * If struct rte_table_action_*type*_params is defined by the Table Action + * API, it needs to point to a valid instance of this structure, otherwise it + * needs to be set to NULL. + * @return + * Zero on success, non-zero error code otherwise. + */ +int __rte_experimental +rte_table_action_apply(struct rte_table_action *action, + void *data, + enum rte_table_action_type type, + void *action_params); + +/** + * Table action DSCP table update. + * + * @param[in] action + * Handle to table action object (needs to be valid). + * @param[in] dscp_mask + * 64-bit mask defining the DSCP table entries to be updated. If bit N is set + * in this bit mask, then DSCP table entry N is to be updated, otherwise not. + * @param[in] table + * DSCP table. + * @return + * Zero on success, non-zero error code otherwise. + */ +int __rte_experimental +rte_table_action_dscp_table_update(struct rte_table_action *action, + uint64_t dscp_mask, + struct rte_table_action_dscp_table *table); + +/** + * Table action meter profile add. + * + * @param[in] action + * Handle to table action object (needs to be valid). + * @param[in] meter_profile_id + * Meter profile ID to be used for the *profile* once it is successfully added + * to the *action* object (needs to be unused by the set of meter profiles + * currently registered for the *action* object). + * @param[in] profile + * Meter profile to be added. + * @return + * Zero on success, non-zero error code otherwise. + */ +int __rte_experimental +rte_table_action_meter_profile_add(struct rte_table_action *action, + uint32_t meter_profile_id, + struct rte_table_action_meter_profile *profile); + +/** + * Table action meter profile delete. + * + * @param[in] action + * Handle to table action object (needs to be valid). + * @param[in] meter_profile_id + * Meter profile ID of the meter profile to be deleted from the *action* + * object (needs to be valid for the *action* object). + * @return + * Zero on success, non-zero error code otherwise. + */ +int __rte_experimental +rte_table_action_meter_profile_delete(struct rte_table_action *action, + uint32_t meter_profile_id); + +/** + * Table action meter read. + * + * @param[in] action + * Handle to table action object (needs to be valid). + * @param[in] data + * Data byte array (typically table rule data) with meter action previously + * applied on it. + * @param[in] tc_mask + * Bit mask defining which traffic classes should have the meter stats + * counters read from *data* and stored into *stats*. If bit N is set in this + * bit mask, then traffic class N is part of this operation, otherwise it is + * not. If bit N is set in this bit mask, then traffic class N must be one of + * the traffic classes that are enabled for the meter action in the table + * action profile used by the *action* object. + * @param[inout] stats + * When non-NULL, it points to the area where the meter stats counters read + * from *data* are saved. Only the meter stats counters for the *tc_mask* + * traffic classes are read and stored to *stats*. + * @param[in] clear + * When non-zero, the meter stats counters are cleared (i.e. set to zero), + * otherwise the counters are not modified. When the read operation is enabled + * (*stats* is non-NULL), the clear operation is performed after the read + * operation is completed. + * @return + * Zero on success, non-zero error code otherwise. + */ +int __rte_experimental +rte_table_action_meter_read(struct rte_table_action *action, + void *data, + uint32_t tc_mask, + struct rte_table_action_mtr_counters *stats, + int clear); + +/** + * Table action TTL read. + * + * @param[in] action + * Handle to table action object (needs to be valid). + * @param[in] data + * Data byte array (typically table rule data) with TTL action previously + * applied on it. + * @param[inout] stats + * When non-NULL, it points to the area where the TTL stats counters read from + * *data* are saved. + * @param[in] clear + * When non-zero, the TTL stats counters are cleared (i.e. set to zero), + * otherwise the counters are not modified. When the read operation is enabled + * (*stats* is non-NULL), the clear operation is performed after the read + * operation is completed. + * @return + * Zero on success, non-zero error code otherwise. + */ +int __rte_experimental +rte_table_action_ttl_read(struct rte_table_action *action, + void *data, + struct rte_table_action_ttl_counters *stats, + int clear); + +/** + * Table action stats read. + * + * @param[in] action + * Handle to table action object (needs to be valid). + * @param[in] data + * Data byte array (typically table rule data) with stats action previously + * applied on it. + * @param[inout] stats + * When non-NULL, it points to the area where the stats counters read from + * *data* are saved. + * @param[in] clear + * When non-zero, the stats counters are cleared (i.e. set to zero), otherwise + * the counters are not modified. When the read operation is enabled (*stats* + * is non-NULL), the clear operation is performed after the read operation is + * completed. + * @return + * Zero on success, non-zero error code otherwise. + */ +int __rte_experimental +rte_table_action_stats_read(struct rte_table_action *action, + void *data, + struct rte_table_action_stats_counters *stats, + int clear); + +/** + * Table action timestamp read. + * + * @param[in] action + * Handle to table action object (needs to be valid). + * @param[in] data + * Data byte array (typically table rule data) with timestamp action + * previously applied on it. + * @param[inout] timestamp + * Pre-allocated memory where the timestamp read from *data* is saved (has to + * be non-NULL). + * @return + * Zero on success, non-zero error code otherwise. + */ +int __rte_experimental +rte_table_action_time_read(struct rte_table_action *action, + void *data, + uint64_t *timestamp); + +#ifdef __cplusplus +} +#endif + +#endif /* __INCLUDE_RTE_TABLE_ACTION_H__ */ |