diff options
Diffstat (limited to 'drivers/net/ethernet/sfc/tc.c')
| -rw-r--r-- | drivers/net/ethernet/sfc/tc.c | 2769 |
1 files changed, 2769 insertions, 0 deletions
diff --git a/drivers/net/ethernet/sfc/tc.c b/drivers/net/ethernet/sfc/tc.c new file mode 100644 index 0000000000..30ebef8824 --- /dev/null +++ b/drivers/net/ethernet/sfc/tc.c @@ -0,0 +1,2769 @@ +// SPDX-License-Identifier: GPL-2.0-only +/**************************************************************************** + * Driver for Solarflare network controllers and boards + * Copyright 2019 Solarflare Communications Inc. + * Copyright 2020-2022 Xilinx Inc. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 as published + * by the Free Software Foundation, incorporated herein by reference. + */ + +#include <net/pkt_cls.h> +#include <net/vxlan.h> +#include <net/geneve.h> +#include <net/tc_act/tc_ct.h> +#include "tc.h" +#include "tc_bindings.h" +#include "tc_encap_actions.h" +#include "tc_conntrack.h" +#include "mae.h" +#include "ef100_rep.h" +#include "efx.h" + +enum efx_encap_type efx_tc_indr_netdev_type(struct net_device *net_dev) +{ + if (netif_is_vxlan(net_dev)) + return EFX_ENCAP_TYPE_VXLAN; + if (netif_is_geneve(net_dev)) + return EFX_ENCAP_TYPE_GENEVE; + + return EFX_ENCAP_TYPE_NONE; +} + +#define EFX_TC_HDR_TYPE_TTL_MASK ((u32)0xff) +/* Hoplimit is stored in the most significant byte in the pedit ipv6 header action */ +#define EFX_TC_HDR_TYPE_HLIMIT_MASK ~((u32)0xff000000) +#define EFX_EFV_PF NULL +/* Look up the representor information (efv) for a device. + * May return NULL for the PF (us), or an error pointer for a device that + * isn't supported as a TC offload endpoint + */ +struct efx_rep *efx_tc_flower_lookup_efv(struct efx_nic *efx, + struct net_device *dev) +{ + struct efx_rep *efv; + + if (!dev) + return ERR_PTR(-EOPNOTSUPP); + /* Is it us (the PF)? */ + if (dev == efx->net_dev) + return EFX_EFV_PF; + /* Is it an efx vfrep at all? */ + if (dev->netdev_ops != &efx_ef100_rep_netdev_ops) + return ERR_PTR(-EOPNOTSUPP); + /* Is it ours? We don't support TC rules that include another + * EF100's netdevices (not even on another port of the same NIC). + */ + efv = netdev_priv(dev); + if (efv->parent != efx) + return ERR_PTR(-EOPNOTSUPP); + return efv; +} + +/* Convert a driver-internal vport ID into an internal device (PF or VF) */ +static s64 efx_tc_flower_internal_mport(struct efx_nic *efx, struct efx_rep *efv) +{ + u32 mport; + + if (IS_ERR(efv)) + return PTR_ERR(efv); + if (!efv) /* device is PF (us) */ + efx_mae_mport_uplink(efx, &mport); + else /* device is repr */ + efx_mae_mport_mport(efx, efv->mport, &mport); + return mport; +} + +/* Convert a driver-internal vport ID into an external device (wire or VF) */ +s64 efx_tc_flower_external_mport(struct efx_nic *efx, struct efx_rep *efv) +{ + u32 mport; + + if (IS_ERR(efv)) + return PTR_ERR(efv); + if (!efv) /* device is PF (us) */ + efx_mae_mport_wire(efx, &mport); + else /* device is repr */ + efx_mae_mport_mport(efx, efv->mport, &mport); + return mport; +} + +static const struct rhashtable_params efx_tc_mac_ht_params = { + .key_len = offsetofend(struct efx_tc_mac_pedit_action, h_addr), + .key_offset = 0, + .head_offset = offsetof(struct efx_tc_mac_pedit_action, linkage), +}; + +static const struct rhashtable_params efx_tc_encap_match_ht_params = { + .key_len = offsetof(struct efx_tc_encap_match, linkage), + .key_offset = 0, + .head_offset = offsetof(struct efx_tc_encap_match, linkage), +}; + +static const struct rhashtable_params efx_tc_match_action_ht_params = { + .key_len = sizeof(unsigned long), + .key_offset = offsetof(struct efx_tc_flow_rule, cookie), + .head_offset = offsetof(struct efx_tc_flow_rule, linkage), +}; + +static const struct rhashtable_params efx_tc_lhs_rule_ht_params = { + .key_len = sizeof(unsigned long), + .key_offset = offsetof(struct efx_tc_lhs_rule, cookie), + .head_offset = offsetof(struct efx_tc_lhs_rule, linkage), +}; + +static const struct rhashtable_params efx_tc_recirc_ht_params = { + .key_len = offsetof(struct efx_tc_recirc_id, linkage), + .key_offset = 0, + .head_offset = offsetof(struct efx_tc_recirc_id, linkage), +}; + +static struct efx_tc_mac_pedit_action *efx_tc_flower_get_mac(struct efx_nic *efx, + unsigned char h_addr[ETH_ALEN], + struct netlink_ext_ack *extack) +{ + struct efx_tc_mac_pedit_action *ped, *old; + int rc; + + ped = kzalloc(sizeof(*ped), GFP_USER); + if (!ped) + return ERR_PTR(-ENOMEM); + memcpy(ped->h_addr, h_addr, ETH_ALEN); + old = rhashtable_lookup_get_insert_fast(&efx->tc->mac_ht, + &ped->linkage, + efx_tc_mac_ht_params); + if (old) { + /* don't need our new entry */ + kfree(ped); + if (IS_ERR(old)) /* oh dear, it's actually an error */ + return ERR_CAST(old); + if (!refcount_inc_not_zero(&old->ref)) + return ERR_PTR(-EAGAIN); + /* existing entry found, ref taken */ + return old; + } + + rc = efx_mae_allocate_pedit_mac(efx, ped); + if (rc < 0) { + NL_SET_ERR_MSG_MOD(extack, "Failed to store pedit MAC address in hw"); + goto out_remove; + } + + /* ref and return */ + refcount_set(&ped->ref, 1); + return ped; +out_remove: + rhashtable_remove_fast(&efx->tc->mac_ht, &ped->linkage, + efx_tc_mac_ht_params); + kfree(ped); + return ERR_PTR(rc); +} + +static void efx_tc_flower_put_mac(struct efx_nic *efx, + struct efx_tc_mac_pedit_action *ped) +{ + if (!refcount_dec_and_test(&ped->ref)) + return; /* still in use */ + rhashtable_remove_fast(&efx->tc->mac_ht, &ped->linkage, + efx_tc_mac_ht_params); + efx_mae_free_pedit_mac(efx, ped); + kfree(ped); +} + +static void efx_tc_free_action_set(struct efx_nic *efx, + struct efx_tc_action_set *act, bool in_hw) +{ + /* Failure paths calling this on the 'cursor' action set in_hw=false, + * because if the alloc had succeeded we'd've put it in acts.list and + * not still have it in act. + */ + if (in_hw) { + efx_mae_free_action_set(efx, act->fw_id); + /* in_hw is true iff we are on an acts.list; make sure to + * remove ourselves from that list before we are freed. + */ + list_del(&act->list); + } + if (act->count) { + spin_lock_bh(&act->count->cnt->lock); + if (!list_empty(&act->count_user)) + list_del(&act->count_user); + spin_unlock_bh(&act->count->cnt->lock); + efx_tc_flower_put_counter_index(efx, act->count); + } + if (act->encap_md) { + list_del(&act->encap_user); + efx_tc_flower_release_encap_md(efx, act->encap_md); + } + if (act->src_mac) + efx_tc_flower_put_mac(efx, act->src_mac); + if (act->dst_mac) + efx_tc_flower_put_mac(efx, act->dst_mac); + kfree(act); +} + +static void efx_tc_free_action_set_list(struct efx_nic *efx, + struct efx_tc_action_set_list *acts, + bool in_hw) +{ + struct efx_tc_action_set *act, *next; + + /* Failure paths set in_hw=false, because usually the acts didn't get + * to efx_mae_alloc_action_set_list(); if they did, the failure tree + * has a separate efx_mae_free_action_set_list() before calling us. + */ + if (in_hw) + efx_mae_free_action_set_list(efx, acts); + /* Any act that's on the list will be in_hw even if the list isn't */ + list_for_each_entry_safe(act, next, &acts->list, list) + efx_tc_free_action_set(efx, act, true); + /* Don't kfree, as acts is embedded inside a struct efx_tc_flow_rule */ +} + +/* Boilerplate for the simple 'copy a field' cases */ +#define _MAP_KEY_AND_MASK(_name, _type, _tcget, _tcfield, _field) \ +if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_##_name)) { \ + struct flow_match_##_type fm; \ + \ + flow_rule_match_##_tcget(rule, &fm); \ + match->value._field = fm.key->_tcfield; \ + match->mask._field = fm.mask->_tcfield; \ +} +#define MAP_KEY_AND_MASK(_name, _type, _tcfield, _field) \ + _MAP_KEY_AND_MASK(_name, _type, _type, _tcfield, _field) +#define MAP_ENC_KEY_AND_MASK(_name, _type, _tcget, _tcfield, _field) \ + _MAP_KEY_AND_MASK(ENC_##_name, _type, _tcget, _tcfield, _field) + +static int efx_tc_flower_parse_match(struct efx_nic *efx, + struct flow_rule *rule, + struct efx_tc_match *match, + struct netlink_ext_ack *extack) +{ + struct flow_dissector *dissector = rule->match.dissector; + unsigned char ipv = 0; + + /* Owing to internal TC infelicities, the IPV6_ADDRS key might be set + * even on IPv4 filters; so rather than relying on dissector->used_keys + * we check the addr_type in the CONTROL key. If we don't find it (or + * it's masked, which should never happen), we treat both IPV4_ADDRS + * and IPV6_ADDRS as absent. + */ + if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_CONTROL)) { + struct flow_match_control fm; + + flow_rule_match_control(rule, &fm); + if (IS_ALL_ONES(fm.mask->addr_type)) + switch (fm.key->addr_type) { + case FLOW_DISSECTOR_KEY_IPV4_ADDRS: + ipv = 4; + break; + case FLOW_DISSECTOR_KEY_IPV6_ADDRS: + ipv = 6; + break; + default: + break; + } + + if (fm.mask->flags & FLOW_DIS_IS_FRAGMENT) { + match->value.ip_frag = fm.key->flags & FLOW_DIS_IS_FRAGMENT; + match->mask.ip_frag = true; + } + if (fm.mask->flags & FLOW_DIS_FIRST_FRAG) { + match->value.ip_firstfrag = fm.key->flags & FLOW_DIS_FIRST_FRAG; + match->mask.ip_firstfrag = true; + } + if (fm.mask->flags & ~(FLOW_DIS_IS_FRAGMENT | FLOW_DIS_FIRST_FRAG)) { + NL_SET_ERR_MSG_FMT_MOD(extack, "Unsupported match on control.flags %#x", + fm.mask->flags); + return -EOPNOTSUPP; + } + } + if (dissector->used_keys & + ~(BIT_ULL(FLOW_DISSECTOR_KEY_CONTROL) | + BIT_ULL(FLOW_DISSECTOR_KEY_BASIC) | + BIT_ULL(FLOW_DISSECTOR_KEY_ETH_ADDRS) | + BIT_ULL(FLOW_DISSECTOR_KEY_VLAN) | + BIT_ULL(FLOW_DISSECTOR_KEY_CVLAN) | + BIT_ULL(FLOW_DISSECTOR_KEY_IPV4_ADDRS) | + BIT_ULL(FLOW_DISSECTOR_KEY_IPV6_ADDRS) | + BIT_ULL(FLOW_DISSECTOR_KEY_PORTS) | + BIT_ULL(FLOW_DISSECTOR_KEY_ENC_KEYID) | + BIT_ULL(FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS) | + BIT_ULL(FLOW_DISSECTOR_KEY_ENC_IPV6_ADDRS) | + BIT_ULL(FLOW_DISSECTOR_KEY_ENC_IP) | + BIT_ULL(FLOW_DISSECTOR_KEY_ENC_PORTS) | + BIT_ULL(FLOW_DISSECTOR_KEY_ENC_CONTROL) | + BIT_ULL(FLOW_DISSECTOR_KEY_CT) | + BIT_ULL(FLOW_DISSECTOR_KEY_TCP) | + BIT_ULL(FLOW_DISSECTOR_KEY_IP))) { + NL_SET_ERR_MSG_FMT_MOD(extack, "Unsupported flower keys %#llx", + dissector->used_keys); + return -EOPNOTSUPP; + } + + MAP_KEY_AND_MASK(BASIC, basic, n_proto, eth_proto); + /* Make sure we're IP if any L3/L4 keys used. */ + if (!IS_ALL_ONES(match->mask.eth_proto) || + !(match->value.eth_proto == htons(ETH_P_IP) || + match->value.eth_proto == htons(ETH_P_IPV6))) + if (dissector->used_keys & + (BIT_ULL(FLOW_DISSECTOR_KEY_IPV4_ADDRS) | + BIT_ULL(FLOW_DISSECTOR_KEY_IPV6_ADDRS) | + BIT_ULL(FLOW_DISSECTOR_KEY_PORTS) | + BIT_ULL(FLOW_DISSECTOR_KEY_IP) | + BIT_ULL(FLOW_DISSECTOR_KEY_TCP))) { + NL_SET_ERR_MSG_FMT_MOD(extack, + "L3/L4 flower keys %#llx require protocol ipv[46]", + dissector->used_keys); + return -EINVAL; + } + + if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_VLAN)) { + struct flow_match_vlan fm; + + flow_rule_match_vlan(rule, &fm); + if (fm.mask->vlan_id || fm.mask->vlan_priority || fm.mask->vlan_tpid) { + match->value.vlan_proto[0] = fm.key->vlan_tpid; + match->mask.vlan_proto[0] = fm.mask->vlan_tpid; + match->value.vlan_tci[0] = cpu_to_be16(fm.key->vlan_priority << 13 | + fm.key->vlan_id); + match->mask.vlan_tci[0] = cpu_to_be16(fm.mask->vlan_priority << 13 | + fm.mask->vlan_id); + } + } + + if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_CVLAN)) { + struct flow_match_vlan fm; + + flow_rule_match_cvlan(rule, &fm); + if (fm.mask->vlan_id || fm.mask->vlan_priority || fm.mask->vlan_tpid) { + match->value.vlan_proto[1] = fm.key->vlan_tpid; + match->mask.vlan_proto[1] = fm.mask->vlan_tpid; + match->value.vlan_tci[1] = cpu_to_be16(fm.key->vlan_priority << 13 | + fm.key->vlan_id); + match->mask.vlan_tci[1] = cpu_to_be16(fm.mask->vlan_priority << 13 | + fm.mask->vlan_id); + } + } + + if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ETH_ADDRS)) { + struct flow_match_eth_addrs fm; + + flow_rule_match_eth_addrs(rule, &fm); + ether_addr_copy(match->value.eth_saddr, fm.key->src); + ether_addr_copy(match->value.eth_daddr, fm.key->dst); + ether_addr_copy(match->mask.eth_saddr, fm.mask->src); + ether_addr_copy(match->mask.eth_daddr, fm.mask->dst); + } + + MAP_KEY_AND_MASK(BASIC, basic, ip_proto, ip_proto); + /* Make sure we're TCP/UDP if any L4 keys used. */ + if ((match->value.ip_proto != IPPROTO_UDP && + match->value.ip_proto != IPPROTO_TCP) || !IS_ALL_ONES(match->mask.ip_proto)) + if (dissector->used_keys & + (BIT_ULL(FLOW_DISSECTOR_KEY_PORTS) | + BIT_ULL(FLOW_DISSECTOR_KEY_TCP))) { + NL_SET_ERR_MSG_FMT_MOD(extack, + "L4 flower keys %#llx require ipproto udp or tcp", + dissector->used_keys); + return -EINVAL; + } + MAP_KEY_AND_MASK(IP, ip, tos, ip_tos); + MAP_KEY_AND_MASK(IP, ip, ttl, ip_ttl); + if (ipv == 4) { + MAP_KEY_AND_MASK(IPV4_ADDRS, ipv4_addrs, src, src_ip); + MAP_KEY_AND_MASK(IPV4_ADDRS, ipv4_addrs, dst, dst_ip); + } +#ifdef CONFIG_IPV6 + else if (ipv == 6) { + MAP_KEY_AND_MASK(IPV6_ADDRS, ipv6_addrs, src, src_ip6); + MAP_KEY_AND_MASK(IPV6_ADDRS, ipv6_addrs, dst, dst_ip6); + } +#endif + MAP_KEY_AND_MASK(PORTS, ports, src, l4_sport); + MAP_KEY_AND_MASK(PORTS, ports, dst, l4_dport); + MAP_KEY_AND_MASK(TCP, tcp, flags, tcp_flags); + if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ENC_CONTROL)) { + struct flow_match_control fm; + + flow_rule_match_enc_control(rule, &fm); + if (fm.mask->flags) { + NL_SET_ERR_MSG_FMT_MOD(extack, "Unsupported match on enc_control.flags %#x", + fm.mask->flags); + return -EOPNOTSUPP; + } + if (!IS_ALL_ONES(fm.mask->addr_type)) { + NL_SET_ERR_MSG_FMT_MOD(extack, "Unsupported enc addr_type mask %u (key %u)", + fm.mask->addr_type, + fm.key->addr_type); + return -EOPNOTSUPP; + } + switch (fm.key->addr_type) { + case FLOW_DISSECTOR_KEY_IPV4_ADDRS: + MAP_ENC_KEY_AND_MASK(IPV4_ADDRS, ipv4_addrs, enc_ipv4_addrs, + src, enc_src_ip); + MAP_ENC_KEY_AND_MASK(IPV4_ADDRS, ipv4_addrs, enc_ipv4_addrs, + dst, enc_dst_ip); + break; +#ifdef CONFIG_IPV6 + case FLOW_DISSECTOR_KEY_IPV6_ADDRS: + MAP_ENC_KEY_AND_MASK(IPV6_ADDRS, ipv6_addrs, enc_ipv6_addrs, + src, enc_src_ip6); + MAP_ENC_KEY_AND_MASK(IPV6_ADDRS, ipv6_addrs, enc_ipv6_addrs, + dst, enc_dst_ip6); + break; +#endif + default: + NL_SET_ERR_MSG_FMT_MOD(extack, + "Unsupported enc addr_type %u (supported are IPv4, IPv6)", + fm.key->addr_type); + return -EOPNOTSUPP; + } + MAP_ENC_KEY_AND_MASK(IP, ip, enc_ip, tos, enc_ip_tos); + MAP_ENC_KEY_AND_MASK(IP, ip, enc_ip, ttl, enc_ip_ttl); + MAP_ENC_KEY_AND_MASK(PORTS, ports, enc_ports, src, enc_sport); + MAP_ENC_KEY_AND_MASK(PORTS, ports, enc_ports, dst, enc_dport); + MAP_ENC_KEY_AND_MASK(KEYID, enc_keyid, enc_keyid, keyid, enc_keyid); + } else if (dissector->used_keys & + (BIT_ULL(FLOW_DISSECTOR_KEY_ENC_KEYID) | + BIT_ULL(FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS) | + BIT_ULL(FLOW_DISSECTOR_KEY_ENC_IPV6_ADDRS) | + BIT_ULL(FLOW_DISSECTOR_KEY_ENC_IP) | + BIT_ULL(FLOW_DISSECTOR_KEY_ENC_PORTS))) { + NL_SET_ERR_MSG_FMT_MOD(extack, + "Flower enc keys require enc_control (keys: %#llx)", + dissector->used_keys); + return -EOPNOTSUPP; + } + if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_CT)) { + struct flow_match_ct fm; + + flow_rule_match_ct(rule, &fm); + match->value.ct_state_trk = !!(fm.key->ct_state & TCA_FLOWER_KEY_CT_FLAGS_TRACKED); + match->mask.ct_state_trk = !!(fm.mask->ct_state & TCA_FLOWER_KEY_CT_FLAGS_TRACKED); + match->value.ct_state_est = !!(fm.key->ct_state & TCA_FLOWER_KEY_CT_FLAGS_ESTABLISHED); + match->mask.ct_state_est = !!(fm.mask->ct_state & TCA_FLOWER_KEY_CT_FLAGS_ESTABLISHED); + if (fm.mask->ct_state & ~(TCA_FLOWER_KEY_CT_FLAGS_TRACKED | + TCA_FLOWER_KEY_CT_FLAGS_ESTABLISHED)) { + NL_SET_ERR_MSG_FMT_MOD(extack, + "Unsupported ct_state match %#x", + fm.mask->ct_state); + return -EOPNOTSUPP; + } + match->value.ct_mark = fm.key->ct_mark; + match->mask.ct_mark = fm.mask->ct_mark; + match->value.ct_zone = fm.key->ct_zone; + match->mask.ct_zone = fm.mask->ct_zone; + + if (memchr_inv(fm.mask->ct_labels, 0, sizeof(fm.mask->ct_labels))) { + NL_SET_ERR_MSG_MOD(extack, "Matching on ct_label not supported"); + return -EOPNOTSUPP; + } + } + + return 0; +} + +static void efx_tc_flower_release_encap_match(struct efx_nic *efx, + struct efx_tc_encap_match *encap) +{ + int rc; + + if (!refcount_dec_and_test(&encap->ref)) + return; /* still in use */ + + if (encap->type == EFX_TC_EM_DIRECT) { + rc = efx_mae_unregister_encap_match(efx, encap); + if (rc) + /* Display message but carry on and remove entry from our + * SW tables, because there's not much we can do about it. + */ + netif_err(efx, drv, efx->net_dev, + "Failed to release encap match %#x, rc %d\n", + encap->fw_id, rc); + } + rhashtable_remove_fast(&efx->tc->encap_match_ht, &encap->linkage, + efx_tc_encap_match_ht_params); + if (encap->pseudo) + efx_tc_flower_release_encap_match(efx, encap->pseudo); + kfree(encap); +} + +static int efx_tc_flower_record_encap_match(struct efx_nic *efx, + struct efx_tc_match *match, + enum efx_encap_type type, + enum efx_tc_em_pseudo_type em_type, + u8 child_ip_tos_mask, + __be16 child_udp_sport_mask, + struct netlink_ext_ack *extack) +{ + struct efx_tc_encap_match *encap, *old, *pseudo = NULL; + bool ipv6 = false; + int rc; + + /* We require that the socket-defining fields (IP addrs and UDP dest + * port) are present and exact-match. Other fields may only be used + * if the field-set (and any masks) are the same for all encap + * matches on the same <sip,dip,dport> tuple; this is enforced by + * pseudo encap matches. + */ + if (match->mask.enc_dst_ip | match->mask.enc_src_ip) { + if (!IS_ALL_ONES(match->mask.enc_dst_ip)) { + NL_SET_ERR_MSG_MOD(extack, + "Egress encap match is not exact on dst IP address"); + return -EOPNOTSUPP; + } + if (!IS_ALL_ONES(match->mask.enc_src_ip)) { + NL_SET_ERR_MSG_MOD(extack, + "Egress encap match is not exact on src IP address"); + return -EOPNOTSUPP; + } +#ifdef CONFIG_IPV6 + if (!ipv6_addr_any(&match->mask.enc_dst_ip6) || + !ipv6_addr_any(&match->mask.enc_src_ip6)) { + NL_SET_ERR_MSG_MOD(extack, + "Egress encap match on both IPv4 and IPv6, don't understand"); + return -EOPNOTSUPP; + } + } else { + ipv6 = true; + if (!efx_ipv6_addr_all_ones(&match->mask.enc_dst_ip6)) { + NL_SET_ERR_MSG_MOD(extack, + "Egress encap match is not exact on dst IP address"); + return -EOPNOTSUPP; + } + if (!efx_ipv6_addr_all_ones(&match->mask.enc_src_ip6)) { + NL_SET_ERR_MSG_MOD(extack, + "Egress encap match is not exact on src IP address"); + return -EOPNOTSUPP; + } +#endif + } + if (!IS_ALL_ONES(match->mask.enc_dport)) { + NL_SET_ERR_MSG_MOD(extack, "Egress encap match is not exact on dst UDP port"); + return -EOPNOTSUPP; + } + if (match->mask.enc_sport || match->mask.enc_ip_tos) { + struct efx_tc_match pmatch = *match; + + if (em_type == EFX_TC_EM_PSEUDO_MASK) { /* can't happen */ + NL_SET_ERR_MSG_MOD(extack, "Bad recursion in egress encap match handler"); + return -EOPNOTSUPP; + } + pmatch.value.enc_ip_tos = 0; + pmatch.mask.enc_ip_tos = 0; + pmatch.value.enc_sport = 0; + pmatch.mask.enc_sport = 0; + rc = efx_tc_flower_record_encap_match(efx, &pmatch, type, + EFX_TC_EM_PSEUDO_MASK, + match->mask.enc_ip_tos, + match->mask.enc_sport, + extack); + if (rc) + return rc; + pseudo = pmatch.encap; + } + if (match->mask.enc_ip_ttl) { + NL_SET_ERR_MSG_MOD(extack, "Egress encap match on IP TTL not supported"); + rc = -EOPNOTSUPP; + goto fail_pseudo; + } + + rc = efx_mae_check_encap_match_caps(efx, ipv6, match->mask.enc_ip_tos, + match->mask.enc_sport, extack); + if (rc) + goto fail_pseudo; + + encap = kzalloc(sizeof(*encap), GFP_USER); + if (!encap) { + rc = -ENOMEM; + goto fail_pseudo; + } + encap->src_ip = match->value.enc_src_ip; + encap->dst_ip = match->value.enc_dst_ip; +#ifdef CONFIG_IPV6 + encap->src_ip6 = match->value.enc_src_ip6; + encap->dst_ip6 = match->value.enc_dst_ip6; +#endif + encap->udp_dport = match->value.enc_dport; + encap->tun_type = type; + encap->ip_tos = match->value.enc_ip_tos; + encap->ip_tos_mask = match->mask.enc_ip_tos; + encap->child_ip_tos_mask = child_ip_tos_mask; + encap->udp_sport = match->value.enc_sport; + encap->udp_sport_mask = match->mask.enc_sport; + encap->child_udp_sport_mask = child_udp_sport_mask; + encap->type = em_type; + encap->pseudo = pseudo; + old = rhashtable_lookup_get_insert_fast(&efx->tc->encap_match_ht, + &encap->linkage, + efx_tc_encap_match_ht_params); + if (old) { + /* don't need our new entry */ + kfree(encap); + if (pseudo) /* don't need our new pseudo either */ + efx_tc_flower_release_encap_match(efx, pseudo); + if (IS_ERR(old)) /* oh dear, it's actually an error */ + return PTR_ERR(old); + /* check old and new em_types are compatible */ + switch (old->type) { + case EFX_TC_EM_DIRECT: + /* old EM is in hardware, so mustn't overlap with a + * pseudo, but may be shared with another direct EM + */ + if (em_type == EFX_TC_EM_DIRECT) + break; + NL_SET_ERR_MSG_MOD(extack, "Pseudo encap match conflicts with existing direct entry"); + return -EEXIST; + case EFX_TC_EM_PSEUDO_MASK: + /* old EM is protecting a ToS- or src port-qualified + * filter, so may only be shared with another pseudo + * for the same ToS and src port masks. + */ + if (em_type != EFX_TC_EM_PSEUDO_MASK) { + NL_SET_ERR_MSG_FMT_MOD(extack, + "%s encap match conflicts with existing pseudo(MASK) entry", + em_type ? "Pseudo" : "Direct"); + return -EEXIST; + } + if (child_ip_tos_mask != old->child_ip_tos_mask) { + NL_SET_ERR_MSG_FMT_MOD(extack, + "Pseudo encap match for TOS mask %#04x conflicts with existing mask %#04x", + child_ip_tos_mask, + old->child_ip_tos_mask); + return -EEXIST; + } + if (child_udp_sport_mask != old->child_udp_sport_mask) { + NL_SET_ERR_MSG_FMT_MOD(extack, + "Pseudo encap match for UDP src port mask %#x conflicts with existing mask %#x", + child_udp_sport_mask, + old->child_udp_sport_mask); + return -EEXIST; + } + break; + default: /* Unrecognised pseudo-type. Just say no */ + NL_SET_ERR_MSG_FMT_MOD(extack, + "%s encap match conflicts with existing pseudo(%d) entry", + em_type ? "Pseudo" : "Direct", + old->type); + return -EEXIST; + } + /* check old and new tun_types are compatible */ + if (old->tun_type != type) { + NL_SET_ERR_MSG_FMT_MOD(extack, + "Egress encap match with conflicting tun_type %u != %u", + old->tun_type, type); + return -EEXIST; + } + if (!refcount_inc_not_zero(&old->ref)) + return -EAGAIN; + /* existing entry found */ + encap = old; + } else { + if (em_type == EFX_TC_EM_DIRECT) { + rc = efx_mae_register_encap_match(efx, encap); + if (rc) { + NL_SET_ERR_MSG_MOD(extack, "Failed to record egress encap match in HW"); + goto fail; + } + } + refcount_set(&encap->ref, 1); + } + match->encap = encap; + return 0; +fail: + rhashtable_remove_fast(&efx->tc->encap_match_ht, &encap->linkage, + efx_tc_encap_match_ht_params); + kfree(encap); +fail_pseudo: + if (pseudo) + efx_tc_flower_release_encap_match(efx, pseudo); + return rc; +} + +static struct efx_tc_recirc_id *efx_tc_get_recirc_id(struct efx_nic *efx, + u32 chain_index, + struct net_device *net_dev) +{ + struct efx_tc_recirc_id *rid, *old; + int rc; + + rid = kzalloc(sizeof(*rid), GFP_USER); + if (!rid) + return ERR_PTR(-ENOMEM); + rid->chain_index = chain_index; + /* We don't take a reference here, because it's implied - if there's + * a rule on the net_dev that's been offloaded to us, then the net_dev + * can't go away until the rule has been deoffloaded. + */ + rid->net_dev = net_dev; + old = rhashtable_lookup_get_insert_fast(&efx->tc->recirc_ht, + &rid->linkage, + efx_tc_recirc_ht_params); + if (old) { + /* don't need our new entry */ + kfree(rid); + if (IS_ERR(old)) /* oh dear, it's actually an error */ + return ERR_CAST(old); + if (!refcount_inc_not_zero(&old->ref)) + return ERR_PTR(-EAGAIN); + /* existing entry found */ + rid = old; + } else { + rc = ida_alloc_range(&efx->tc->recirc_ida, 1, U8_MAX, GFP_USER); + if (rc < 0) { + rhashtable_remove_fast(&efx->tc->recirc_ht, + &rid->linkage, + efx_tc_recirc_ht_params); + kfree(rid); + return ERR_PTR(rc); + } + rid->fw_id = rc; + refcount_set(&rid->ref, 1); + } + return rid; +} + +static void efx_tc_put_recirc_id(struct efx_nic *efx, struct efx_tc_recirc_id *rid) +{ + if (!refcount_dec_and_test(&rid->ref)) + return; /* still in use */ + rhashtable_remove_fast(&efx->tc->recirc_ht, &rid->linkage, + efx_tc_recirc_ht_params); + ida_free(&efx->tc->recirc_ida, rid->fw_id); + kfree(rid); +} + +static void efx_tc_delete_rule(struct efx_nic *efx, struct efx_tc_flow_rule *rule) +{ + efx_mae_delete_rule(efx, rule->fw_id); + + /* Release entries in subsidiary tables */ + efx_tc_free_action_set_list(efx, &rule->acts, true); + if (rule->match.rid) + efx_tc_put_recirc_id(efx, rule->match.rid); + if (rule->match.encap) + efx_tc_flower_release_encap_match(efx, rule->match.encap); + rule->fw_id = MC_CMD_MAE_ACTION_RULE_INSERT_OUT_ACTION_RULE_ID_NULL; +} + +static const char *efx_tc_encap_type_name(enum efx_encap_type typ) +{ + switch (typ) { + case EFX_ENCAP_TYPE_NONE: + return "none"; + case EFX_ENCAP_TYPE_VXLAN: + return "vxlan"; + case EFX_ENCAP_TYPE_GENEVE: + return "geneve"; + default: + pr_warn_once("Unknown efx_encap_type %d encountered\n", typ); + return "unknown"; + } +} + +/* For details of action order constraints refer to SF-123102-TC-1§12.6.1 */ +enum efx_tc_action_order { + EFX_TC_AO_DECAP, + EFX_TC_AO_DEC_TTL, + EFX_TC_AO_PEDIT_MAC_ADDRS, + EFX_TC_AO_VLAN_POP, + EFX_TC_AO_VLAN_PUSH, + EFX_TC_AO_COUNT, + EFX_TC_AO_ENCAP, + EFX_TC_AO_DELIVER +}; +/* Determine whether we can add @new action without violating order */ +static bool efx_tc_flower_action_order_ok(const struct efx_tc_action_set *act, + enum efx_tc_action_order new) +{ + switch (new) { + case EFX_TC_AO_DECAP: + if (act->decap) + return false; + /* PEDIT_MAC_ADDRS must not happen before DECAP, though it + * can wait until much later + */ + if (act->dst_mac || act->src_mac) + return false; + + /* Decrementing ttl must not happen before DECAP */ + if (act->do_ttl_dec) + return false; + fallthrough; + case EFX_TC_AO_VLAN_POP: + if (act->vlan_pop >= 2) + return false; + /* If we've already pushed a VLAN, we can't then pop it; + * the hardware would instead try to pop an existing VLAN + * before pushing the new one. + */ + if (act->vlan_push) + return false; + fallthrough; + case EFX_TC_AO_VLAN_PUSH: + if (act->vlan_push >= 2) + return false; + fallthrough; + case EFX_TC_AO_COUNT: + if (act->count) + return false; + fallthrough; + case EFX_TC_AO_PEDIT_MAC_ADDRS: + case EFX_TC_AO_ENCAP: + if (act->encap_md) + return false; + fallthrough; + case EFX_TC_AO_DELIVER: + return !act->deliver; + case EFX_TC_AO_DEC_TTL: + if (act->encap_md) + return false; + return !act->do_ttl_dec; + default: + /* Bad caller. Whatever they wanted to do, say they can't. */ + WARN_ON_ONCE(1); + return false; + } +} + +/** + * DOC: TC conntrack sequences + * + * The MAE hardware can handle at most two rounds of action rule matching, + * consequently we support conntrack through the notion of a "left-hand side + * rule". This is a rule which typically contains only the actions "ct" and + * "goto chain N", and corresponds to one or more "right-hand side rules" in + * chain N, which typically match on +trk+est, and may perform ct(nat) actions. + * RHS rules go in the Action Rule table as normal but with a nonzero recirc_id + * (the hardware equivalent of chain_index), while LHS rules may go in either + * the Action Rule or the Outer Rule table, the latter being preferred for + * performance reasons, and set both DO_CT and a recirc_id in their response. + * + * Besides the RHS rules, there are often also similar rules matching on + * +trk+new which perform the ct(commit) action. These are not offloaded. + */ + +static bool efx_tc_rule_is_lhs_rule(struct flow_rule *fr, + struct efx_tc_match *match) +{ + const struct flow_action_entry *fa; + int i; + + flow_action_for_each(i, fa, &fr->action) { + switch (fa->id) { + case FLOW_ACTION_GOTO: + return true; + case FLOW_ACTION_CT: + /* If rule is -trk, or doesn't mention trk at all, then + * a CT action implies a conntrack lookup (hence it's an + * LHS rule). If rule is +trk, then a CT action could + * just be ct(nat) or even ct(commit) (though the latter + * can't be offloaded). + */ + if (!match->mask.ct_state_trk || !match->value.ct_state_trk) + return true; + break; + default: + break; + } + } + return false; +} + +static int efx_tc_flower_handle_lhs_actions(struct efx_nic *efx, + struct flow_cls_offload *tc, + struct flow_rule *fr, + struct net_device *net_dev, + struct efx_tc_lhs_rule *rule) + +{ + struct netlink_ext_ack *extack = tc->common.extack; + struct efx_tc_lhs_action *act = &rule->lhs_act; + const struct flow_action_entry *fa; + bool pipe = true; + int i; + + flow_action_for_each(i, fa, &fr->action) { + struct efx_tc_ct_zone *ct_zone; + struct efx_tc_recirc_id *rid; + + if (!pipe) { + /* more actions after a non-pipe action */ + NL_SET_ERR_MSG_MOD(extack, "Action follows non-pipe action"); + return -EINVAL; + } + switch (fa->id) { + case FLOW_ACTION_GOTO: + if (!fa->chain_index) { + NL_SET_ERR_MSG_MOD(extack, "Can't goto chain 0, no looping in hw"); + return -EOPNOTSUPP; + } + rid = efx_tc_get_recirc_id(efx, fa->chain_index, + net_dev); + if (IS_ERR(rid)) { + NL_SET_ERR_MSG_MOD(extack, "Failed to allocate a hardware recirculation ID for this chain_index"); + return PTR_ERR(rid); + } + act->rid = rid; + if (fa->hw_stats) { + struct efx_tc_counter_index *cnt; + + if (!(fa->hw_stats & FLOW_ACTION_HW_STATS_DELAYED)) { + NL_SET_ERR_MSG_FMT_MOD(extack, + "hw_stats_type %u not supported (only 'delayed')", + fa->hw_stats); + return -EOPNOTSUPP; + } + cnt = efx_tc_flower_get_counter_index(efx, tc->cookie, + EFX_TC_COUNTER_TYPE_OR); + if (IS_ERR(cnt)) { + NL_SET_ERR_MSG_MOD(extack, "Failed to obtain a counter"); + return PTR_ERR(cnt); + } + WARN_ON(act->count); /* can't happen */ + act->count = cnt; + } + pipe = false; + break; + case FLOW_ACTION_CT: + if (act->zone) { + NL_SET_ERR_MSG_MOD(extack, "Can't offload multiple ct actions"); + return -EOPNOTSUPP; + } + if (fa->ct.action & (TCA_CT_ACT_COMMIT | + TCA_CT_ACT_FORCE)) { + NL_SET_ERR_MSG_MOD(extack, "Can't offload ct commit/force"); + return -EOPNOTSUPP; + } + if (fa->ct.action & TCA_CT_ACT_CLEAR) { + NL_SET_ERR_MSG_MOD(extack, "Can't clear ct in LHS rule"); + return -EOPNOTSUPP; + } + if (fa->ct.action & (TCA_CT_ACT_NAT | + TCA_CT_ACT_NAT_SRC | + TCA_CT_ACT_NAT_DST)) { + NL_SET_ERR_MSG_MOD(extack, "Can't perform NAT in LHS rule - packet isn't conntracked yet"); + return -EOPNOTSUPP; + } + if (fa->ct.action) { + NL_SET_ERR_MSG_FMT_MOD(extack, "Unhandled ct.action %u for LHS rule\n", + fa->ct.action); + return -EOPNOTSUPP; + } + ct_zone = efx_tc_ct_register_zone(efx, fa->ct.zone, + fa->ct.flow_table); + if (IS_ERR(ct_zone)) { + NL_SET_ERR_MSG_MOD(extack, "Failed to register for CT updates"); + return PTR_ERR(ct_zone); + } + act->zone = ct_zone; + break; + default: + NL_SET_ERR_MSG_FMT_MOD(extack, "Unhandled action %u for LHS rule\n", + fa->id); + return -EOPNOTSUPP; + } + } + + if (pipe) { + NL_SET_ERR_MSG_MOD(extack, "Missing goto chain in LHS rule"); + return -EOPNOTSUPP; + } + return 0; +} + +static void efx_tc_flower_release_lhs_actions(struct efx_nic *efx, + struct efx_tc_lhs_action *act) +{ + if (act->rid) + efx_tc_put_recirc_id(efx, act->rid); + if (act->zone) + efx_tc_ct_unregister_zone(efx, act->zone); + if (act->count) + efx_tc_flower_put_counter_index(efx, act->count); +} + +/** + * struct efx_tc_mangler_state - accumulates 32-bit pedits into fields + * + * @dst_mac_32: dst_mac[0:3] has been populated + * @dst_mac_16: dst_mac[4:5] has been populated + * @src_mac_16: src_mac[0:1] has been populated + * @src_mac_32: src_mac[2:5] has been populated + * @dst_mac: h_dest field of ethhdr + * @src_mac: h_source field of ethhdr + * + * Since FLOW_ACTION_MANGLE comes in 32-bit chunks that do not + * necessarily equate to whole fields of the packet header, this + * structure is used to hold the cumulative effect of the partial + * field pedits that have been processed so far. + */ +struct efx_tc_mangler_state { + u8 dst_mac_32:1; /* eth->h_dest[0:3] */ + u8 dst_mac_16:1; /* eth->h_dest[4:5] */ + u8 src_mac_16:1; /* eth->h_source[0:1] */ + u8 src_mac_32:1; /* eth->h_source[2:5] */ + unsigned char dst_mac[ETH_ALEN]; + unsigned char src_mac[ETH_ALEN]; +}; + +/** efx_tc_complete_mac_mangle() - pull complete field pedits out of @mung + * @efx: NIC we're installing a flow rule on + * @act: action set (cursor) to update + * @mung: accumulated partial mangles + * @extack: netlink extended ack for reporting errors + * + * Check @mung to find any combinations of partial mangles that can be + * combined into a complete packet field edit, add that edit to @act, + * and consume the partial mangles from @mung. + */ + +static int efx_tc_complete_mac_mangle(struct efx_nic *efx, + struct efx_tc_action_set *act, + struct efx_tc_mangler_state *mung, + struct netlink_ext_ack *extack) +{ + struct efx_tc_mac_pedit_action *ped; + + if (mung->dst_mac_32 && mung->dst_mac_16) { + ped = efx_tc_flower_get_mac(efx, mung->dst_mac, extack); + if (IS_ERR(ped)) + return PTR_ERR(ped); + + /* Check that we have not already populated dst_mac */ + if (act->dst_mac) + efx_tc_flower_put_mac(efx, act->dst_mac); + + act->dst_mac = ped; + + /* consume the incomplete state */ + mung->dst_mac_32 = 0; + mung->dst_mac_16 = 0; + } + if (mung->src_mac_16 && mung->src_mac_32) { + ped = efx_tc_flower_get_mac(efx, mung->src_mac, extack); + if (IS_ERR(ped)) + return PTR_ERR(ped); + + /* Check that we have not already populated src_mac */ + if (act->src_mac) + efx_tc_flower_put_mac(efx, act->src_mac); + + act->src_mac = ped; + + /* consume the incomplete state */ + mung->src_mac_32 = 0; + mung->src_mac_16 = 0; + } + return 0; +} + +static int efx_tc_pedit_add(struct efx_nic *efx, struct efx_tc_action_set *act, + const struct flow_action_entry *fa, + struct netlink_ext_ack *extack) +{ + switch (fa->mangle.htype) { + case FLOW_ACT_MANGLE_HDR_TYPE_IP4: + switch (fa->mangle.offset) { + case offsetof(struct iphdr, ttl): + /* check that pedit applies to ttl only */ + if (fa->mangle.mask != ~EFX_TC_HDR_TYPE_TTL_MASK) + break; + + /* Adding 0xff is equivalent to decrementing the ttl. + * Other added values are not supported. + */ + if ((fa->mangle.val & EFX_TC_HDR_TYPE_TTL_MASK) != U8_MAX) + break; + + /* check that we do not decrement ttl twice */ + if (!efx_tc_flower_action_order_ok(act, + EFX_TC_AO_DEC_TTL)) { + NL_SET_ERR_MSG_MOD(extack, "multiple dec ttl are not supported"); + return -EOPNOTSUPP; + } + act->do_ttl_dec = 1; + return 0; + default: + break; + } + break; + case FLOW_ACT_MANGLE_HDR_TYPE_IP6: + switch (fa->mangle.offset) { + case round_down(offsetof(struct ipv6hdr, hop_limit), 4): + /* check that pedit applies to hoplimit only */ + if (fa->mangle.mask != EFX_TC_HDR_TYPE_HLIMIT_MASK) + break; + + /* Adding 0xff is equivalent to decrementing the hoplimit. + * Other added values are not supported. + */ + if ((fa->mangle.val >> 24) != U8_MAX) + break; + + /* check that we do not decrement hoplimit twice */ + if (!efx_tc_flower_action_order_ok(act, + EFX_TC_AO_DEC_TTL)) { + NL_SET_ERR_MSG_MOD(extack, "multiple dec ttl are not supported"); + return -EOPNOTSUPP; + } + act->do_ttl_dec = 1; + return 0; + default: + break; + } + break; + default: + break; + } + + NL_SET_ERR_MSG_FMT_MOD(extack, + "ttl add action type %x %x %x/%x is not supported", + fa->mangle.htype, fa->mangle.offset, + fa->mangle.val, fa->mangle.mask); + return -EOPNOTSUPP; +} + +/** + * efx_tc_mangle() - handle a single 32-bit (or less) pedit + * @efx: NIC we're installing a flow rule on + * @act: action set (cursor) to update + * @fa: FLOW_ACTION_MANGLE action metadata + * @mung: accumulator for partial mangles + * @extack: netlink extended ack for reporting errors + * @match: original match used along with the mangle action + * + * Identify the fields written by a FLOW_ACTION_MANGLE, and record + * the partial mangle state in @mung. If this mangle completes an + * earlier partial mangle, consume and apply to @act by calling + * efx_tc_complete_mac_mangle(). + */ + +static int efx_tc_mangle(struct efx_nic *efx, struct efx_tc_action_set *act, + const struct flow_action_entry *fa, + struct efx_tc_mangler_state *mung, + struct netlink_ext_ack *extack, + struct efx_tc_match *match) +{ + __le32 mac32; + __le16 mac16; + u8 tr_ttl; + + switch (fa->mangle.htype) { + case FLOW_ACT_MANGLE_HDR_TYPE_ETH: + BUILD_BUG_ON(offsetof(struct ethhdr, h_dest) != 0); + BUILD_BUG_ON(offsetof(struct ethhdr, h_source) != 6); + if (!efx_tc_flower_action_order_ok(act, EFX_TC_AO_PEDIT_MAC_ADDRS)) { + NL_SET_ERR_MSG_MOD(extack, + "Pedit mangle mac action violates action order"); + return -EOPNOTSUPP; + } + switch (fa->mangle.offset) { + case 0: + if (fa->mangle.mask) { + NL_SET_ERR_MSG_FMT_MOD(extack, + "mask (%#x) of eth.dst32 mangle is not supported", + fa->mangle.mask); + return -EOPNOTSUPP; + } + /* Ethernet address is little-endian */ + mac32 = cpu_to_le32(fa->mangle.val); + memcpy(mung->dst_mac, &mac32, sizeof(mac32)); + mung->dst_mac_32 = 1; + return efx_tc_complete_mac_mangle(efx, act, mung, extack); + case 4: + if (fa->mangle.mask == 0xffff) { + mac16 = cpu_to_le16(fa->mangle.val >> 16); + memcpy(mung->src_mac, &mac16, sizeof(mac16)); + mung->src_mac_16 = 1; + } else if (fa->mangle.mask == 0xffff0000) { + mac16 = cpu_to_le16((u16)fa->mangle.val); + memcpy(mung->dst_mac + 4, &mac16, sizeof(mac16)); + mung->dst_mac_16 = 1; + } else { + NL_SET_ERR_MSG_FMT_MOD(extack, + "mask (%#x) of eth+4 mangle is not high or low 16b", + fa->mangle.mask); + return -EOPNOTSUPP; + } + return efx_tc_complete_mac_mangle(efx, act, mung, extack); + case 8: + if (fa->mangle.mask) { + NL_SET_ERR_MSG_FMT_MOD(extack, + "mask (%#x) of eth.src32 mangle is not supported", + fa->mangle.mask); + return -EOPNOTSUPP; + } + mac32 = cpu_to_le32(fa->mangle.val); + memcpy(mung->src_mac + 2, &mac32, sizeof(mac32)); + mung->src_mac_32 = 1; + return efx_tc_complete_mac_mangle(efx, act, mung, extack); + default: + NL_SET_ERR_MSG_FMT_MOD(extack, "mangle eth+%u %x/%x is not supported", + fa->mangle.offset, fa->mangle.val, fa->mangle.mask); + return -EOPNOTSUPP; + } + break; + case FLOW_ACT_MANGLE_HDR_TYPE_IP4: + switch (fa->mangle.offset) { + case offsetof(struct iphdr, ttl): + /* we currently only support pedit IP4 when it applies + * to TTL and then only when it can be achieved with a + * decrement ttl action + */ + + /* check that pedit applies to ttl only */ + if (fa->mangle.mask != ~EFX_TC_HDR_TYPE_TTL_MASK) { + NL_SET_ERR_MSG_FMT_MOD(extack, + "mask (%#x) out of range, only support mangle action on ipv4.ttl", + fa->mangle.mask); + return -EOPNOTSUPP; + } + + /* we can only convert to a dec ttl when we have an + * exact match on the ttl field + */ + if (match->mask.ip_ttl != U8_MAX) { + NL_SET_ERR_MSG_FMT_MOD(extack, + "only support mangle ttl when we have an exact match, current mask (%#x)", + match->mask.ip_ttl); + return -EOPNOTSUPP; + } + + /* check that we don't try to decrement 0, which equates + * to setting the ttl to 0xff + */ + if (match->value.ip_ttl == 0) { + NL_SET_ERR_MSG_MOD(extack, + "decrement ttl past 0 is not supported"); + return -EOPNOTSUPP; + } + + /* check that we do not decrement ttl twice */ + if (!efx_tc_flower_action_order_ok(act, + EFX_TC_AO_DEC_TTL)) { + NL_SET_ERR_MSG_MOD(extack, + "multiple dec ttl is not supported"); + return -EOPNOTSUPP; + } + + /* check pedit can be achieved with decrement action */ + tr_ttl = match->value.ip_ttl - 1; + if ((fa->mangle.val & EFX_TC_HDR_TYPE_TTL_MASK) == tr_ttl) { + act->do_ttl_dec = 1; + return 0; + } + + fallthrough; + default: + NL_SET_ERR_MSG_FMT_MOD(extack, + "only support mangle on the ttl field (offset is %u)", + fa->mangle.offset); + return -EOPNOTSUPP; + } + break; + case FLOW_ACT_MANGLE_HDR_TYPE_IP6: + switch (fa->mangle.offset) { + case round_down(offsetof(struct ipv6hdr, hop_limit), 4): + /* we currently only support pedit IP6 when it applies + * to the hoplimit and then only when it can be achieved + * with a decrement hoplimit action + */ + + /* check that pedit applies to ttl only */ + if (fa->mangle.mask != EFX_TC_HDR_TYPE_HLIMIT_MASK) { + NL_SET_ERR_MSG_FMT_MOD(extack, + "mask (%#x) out of range, only support mangle action on ipv6.hop_limit", + fa->mangle.mask); + + return -EOPNOTSUPP; + } + + /* we can only convert to a dec ttl when we have an + * exact match on the ttl field + */ + if (match->mask.ip_ttl != U8_MAX) { + NL_SET_ERR_MSG_FMT_MOD(extack, + "only support hop_limit when we have an exact match, current mask (%#x)", + match->mask.ip_ttl); + return -EOPNOTSUPP; + } + + /* check that we don't try to decrement 0, which equates + * to setting the ttl to 0xff + */ + if (match->value.ip_ttl == 0) { + NL_SET_ERR_MSG_MOD(extack, + "decrementing hop_limit past 0 is not supported"); + return -EOPNOTSUPP; + } + + /* check that we do not decrement hoplimit twice */ + if (!efx_tc_flower_action_order_ok(act, + EFX_TC_AO_DEC_TTL)) { + NL_SET_ERR_MSG_MOD(extack, + "multiple dec ttl is not supported"); + return -EOPNOTSUPP; + } + + /* check pedit can be achieved with decrement action */ + tr_ttl = match->value.ip_ttl - 1; + if ((fa->mangle.val >> 24) == tr_ttl) { + act->do_ttl_dec = 1; + return 0; + } + + fallthrough; + default: + NL_SET_ERR_MSG_FMT_MOD(extack, + "only support mangle on the hop_limit field"); + return -EOPNOTSUPP; + } + default: + NL_SET_ERR_MSG_FMT_MOD(extack, "Unhandled mangle htype %u for action rule", + fa->mangle.htype); + return -EOPNOTSUPP; + } + return 0; +} + +/** + * efx_tc_incomplete_mangle() - check for leftover partial pedits + * @mung: accumulator for partial mangles + * @extack: netlink extended ack for reporting errors + * + * Since the MAE can only overwrite whole fields, any partial + * field mangle left over on reaching packet delivery (mirred or + * end of TC actions) cannot be offloaded. Check for any such + * and reject them with -%EOPNOTSUPP. + */ + +static int efx_tc_incomplete_mangle(struct efx_tc_mangler_state *mung, + struct netlink_ext_ack *extack) +{ + if (mung->dst_mac_32 || mung->dst_mac_16) { + NL_SET_ERR_MSG_MOD(extack, "Incomplete pedit of destination MAC address"); + return -EOPNOTSUPP; + } + if (mung->src_mac_16 || mung->src_mac_32) { + NL_SET_ERR_MSG_MOD(extack, "Incomplete pedit of source MAC address"); + return -EOPNOTSUPP; + } + return 0; +} + +static int efx_tc_flower_replace_foreign(struct efx_nic *efx, + struct net_device *net_dev, + struct flow_cls_offload *tc) +{ + struct flow_rule *fr = flow_cls_offload_flow_rule(tc); + struct netlink_ext_ack *extack = tc->common.extack; + struct efx_tc_flow_rule *rule = NULL, *old = NULL; + struct efx_tc_action_set *act = NULL; + bool found = false, uplinked = false; + const struct flow_action_entry *fa; + struct efx_tc_match match; + struct efx_rep *to_efv; + s64 rc; + int i; + + /* Parse match */ + memset(&match, 0, sizeof(match)); + rc = efx_tc_flower_parse_match(efx, fr, &match, NULL); + if (rc) + return rc; + /* The rule as given to us doesn't specify a source netdevice. + * But, determining whether packets from a VF should match it is + * complicated, so leave those to the software slowpath: qualify + * the filter with source m-port == wire. + */ + rc = efx_tc_flower_external_mport(efx, EFX_EFV_PF); + if (rc < 0) { + NL_SET_ERR_MSG_MOD(extack, "Failed to identify ingress m-port for foreign filter"); + return rc; + } + match.value.ingress_port = rc; + match.mask.ingress_port = ~0; + + if (tc->common.chain_index) { + struct efx_tc_recirc_id *rid; + + rid = efx_tc_get_recirc_id(efx, tc->common.chain_index, net_dev); + if (IS_ERR(rid)) { + NL_SET_ERR_MSG_FMT_MOD(extack, + "Failed to allocate a hardware recirculation ID for chain_index %u", + tc->common.chain_index); + return PTR_ERR(rid); + } + match.rid = rid; + match.value.recirc_id = rid->fw_id; + } + match.mask.recirc_id = 0xff; + + /* AR table can't match on DO_CT (+trk). But a commonly used pattern is + * +trk+est, which is strictly implied by +est, so rewrite it to that. + */ + if (match.mask.ct_state_trk && match.value.ct_state_trk && + match.mask.ct_state_est && match.value.ct_state_est) + match.mask.ct_state_trk = 0; + /* Thanks to CT_TCP_FLAGS_INHIBIT, packets with interesting flags could + * match +trk-est (CT_HIT=0) despite being on an established connection. + * So make -est imply -tcp_syn_fin_rst match to ensure these packets + * still hit the software path. + */ + if (match.mask.ct_state_est && !match.value.ct_state_est) { + if (match.value.tcp_syn_fin_rst) { + /* Can't offload this combination */ + rc = -EOPNOTSUPP; + goto release; + } + match.mask.tcp_syn_fin_rst = true; + } + + flow_action_for_each(i, fa, &fr->action) { + switch (fa->id) { + case FLOW_ACTION_REDIRECT: + case FLOW_ACTION_MIRRED: /* mirred means mirror here */ + to_efv = efx_tc_flower_lookup_efv(efx, fa->dev); + if (IS_ERR(to_efv)) + continue; + found = true; + break; + default: + break; + } + } + if (!found) { /* We don't care. */ + netif_dbg(efx, drv, efx->net_dev, + "Ignoring foreign filter that doesn't egdev us\n"); + rc = -EOPNOTSUPP; + goto release; + } + + rc = efx_mae_match_check_caps(efx, &match.mask, NULL); + if (rc) + goto release; + + if (efx_tc_match_is_encap(&match.mask)) { + enum efx_encap_type type; + + type = efx_tc_indr_netdev_type(net_dev); + if (type == EFX_ENCAP_TYPE_NONE) { + NL_SET_ERR_MSG_MOD(extack, + "Egress encap match on unsupported tunnel device"); + rc = -EOPNOTSUPP; + goto release; + } + + rc = efx_mae_check_encap_type_supported(efx, type); + if (rc) { + NL_SET_ERR_MSG_FMT_MOD(extack, + "Firmware reports no support for %s encap match", + efx_tc_encap_type_name(type)); + goto release; + } + + rc = efx_tc_flower_record_encap_match(efx, &match, type, + EFX_TC_EM_DIRECT, 0, 0, + extack); + if (rc) + goto release; + } else { + /* This is not a tunnel decap rule, ignore it */ + netif_dbg(efx, drv, efx->net_dev, + "Ignoring foreign filter without encap match\n"); + rc = -EOPNOTSUPP; + goto release; + } + + rule = kzalloc(sizeof(*rule), GFP_USER); + if (!rule) { + rc = -ENOMEM; + goto release; + } + INIT_LIST_HEAD(&rule->acts.list); + rule->cookie = tc->cookie; + old = rhashtable_lookup_get_insert_fast(&efx->tc->match_action_ht, + &rule->linkage, + efx_tc_match_action_ht_params); + if (IS_ERR(old)) { + rc = PTR_ERR(old); + goto release; + } else if (old) { + netif_dbg(efx, drv, efx->net_dev, + "Ignoring already-offloaded rule (cookie %lx)\n", + tc->cookie); + rc = -EEXIST; + goto release; + } + + act = kzalloc(sizeof(*act), GFP_USER); + if (!act) { + rc = -ENOMEM; + goto release; + } + + /* Parse actions. For foreign rules we only support decap & redirect. + * See corresponding code in efx_tc_flower_replace() for theory of + * operation & how 'act' cursor is used. + */ + flow_action_for_each(i, fa, &fr->action) { + struct efx_tc_action_set save; + + switch (fa->id) { + case FLOW_ACTION_REDIRECT: + case FLOW_ACTION_MIRRED: + /* See corresponding code in efx_tc_flower_replace() for + * long explanations of what's going on here. + */ + save = *act; + if (fa->hw_stats) { + struct efx_tc_counter_index *ctr; + + if (!(fa->hw_stats & FLOW_ACTION_HW_STATS_DELAYED)) { + NL_SET_ERR_MSG_FMT_MOD(extack, + "hw_stats_type %u not supported (only 'delayed')", + fa->hw_stats); + rc = -EOPNOTSUPP; + goto release; + } + if (!efx_tc_flower_action_order_ok(act, EFX_TC_AO_COUNT)) { + rc = -EOPNOTSUPP; + goto release; + } + + ctr = efx_tc_flower_get_counter_index(efx, + tc->cookie, + EFX_TC_COUNTER_TYPE_AR); + if (IS_ERR(ctr)) { + rc = PTR_ERR(ctr); + NL_SET_ERR_MSG_MOD(extack, "Failed to obtain a counter"); + goto release; + } + act->count = ctr; + INIT_LIST_HEAD(&act->count_user); + } + + if (!efx_tc_flower_action_order_ok(act, EFX_TC_AO_DELIVER)) { + /* can't happen */ + rc = -EOPNOTSUPP; + NL_SET_ERR_MSG_MOD(extack, + "Deliver action violates action order (can't happen)"); + goto release; + } + to_efv = efx_tc_flower_lookup_efv(efx, fa->dev); + /* PF implies egdev is us, in which case we really + * want to deliver to the uplink (because this is an + * ingress filter). If we don't recognise the egdev + * at all, then we'd better trap so SW can handle it. + */ + if (IS_ERR(to_efv)) + to_efv = EFX_EFV_PF; + if (to_efv == EFX_EFV_PF) { + if (uplinked) + break; + uplinked = true; + } + rc = efx_tc_flower_internal_mport(efx, to_efv); + if (rc < 0) { + NL_SET_ERR_MSG_MOD(extack, "Failed to identify egress m-port"); + goto release; + } + act->dest_mport = rc; + act->deliver = 1; + rc = efx_mae_alloc_action_set(efx, act); + if (rc) { + NL_SET_ERR_MSG_MOD(extack, + "Failed to write action set to hw (mirred)"); + goto release; + } + list_add_tail(&act->list, &rule->acts.list); + act = NULL; + if (fa->id == FLOW_ACTION_REDIRECT) + break; /* end of the line */ + /* Mirror, so continue on with saved act */ + act = kzalloc(sizeof(*act), GFP_USER); + if (!act) { + rc = -ENOMEM; + goto release; + } + *act = save; + break; + case FLOW_ACTION_TUNNEL_DECAP: + if (!efx_tc_flower_action_order_ok(act, EFX_TC_AO_DECAP)) { + rc = -EINVAL; + NL_SET_ERR_MSG_MOD(extack, "Decap action violates action order"); + goto release; + } + act->decap = 1; + /* If we previously delivered/trapped to uplink, now + * that we've decapped we'll want another copy if we + * try to deliver/trap to uplink again. + */ + uplinked = false; + break; + default: + NL_SET_ERR_MSG_FMT_MOD(extack, "Unhandled action %u", + fa->id); + rc = -EOPNOTSUPP; + goto release; + } + } + + if (act) { + if (!uplinked) { + /* Not shot/redirected, so deliver to default dest (which is + * the uplink, as this is an ingress filter) + */ + efx_mae_mport_uplink(efx, &act->dest_mport); + act->deliver = 1; + } + rc = efx_mae_alloc_action_set(efx, act); + if (rc) { + NL_SET_ERR_MSG_MOD(extack, "Failed to write action set to hw (deliver)"); + goto release; + } + list_add_tail(&act->list, &rule->acts.list); + act = NULL; /* Prevent double-free in error path */ + } + + rule->match = match; + + netif_dbg(efx, drv, efx->net_dev, + "Successfully parsed foreign filter (cookie %lx)\n", + tc->cookie); + + rc = efx_mae_alloc_action_set_list(efx, &rule->acts); + if (rc) { + NL_SET_ERR_MSG_MOD(extack, "Failed to write action set list to hw"); + goto release; + } + rc = efx_mae_insert_rule(efx, &rule->match, EFX_TC_PRIO_TC, + rule->acts.fw_id, &rule->fw_id); + if (rc) { + NL_SET_ERR_MSG_MOD(extack, "Failed to insert rule in hw"); + goto release_acts; + } + return 0; + +release_acts: + efx_mae_free_action_set_list(efx, &rule->acts); +release: + /* We failed to insert the rule, so free up any entries we created in + * subsidiary tables. + */ + if (match.rid) + efx_tc_put_recirc_id(efx, match.rid); + if (act) + efx_tc_free_action_set(efx, act, false); + if (rule) { + if (!old) + rhashtable_remove_fast(&efx->tc->match_action_ht, + &rule->linkage, + efx_tc_match_action_ht_params); + efx_tc_free_action_set_list(efx, &rule->acts, false); + } + kfree(rule); + if (match.encap) + efx_tc_flower_release_encap_match(efx, match.encap); + return rc; +} + +static int efx_tc_flower_replace_lhs(struct efx_nic *efx, + struct flow_cls_offload *tc, + struct flow_rule *fr, + struct efx_tc_match *match, + struct efx_rep *efv, + struct net_device *net_dev) +{ + struct netlink_ext_ack *extack = tc->common.extack; + struct efx_tc_lhs_rule *rule, *old; + int rc; + + if (tc->common.chain_index) { + NL_SET_ERR_MSG_MOD(extack, "LHS rule only allowed in chain 0"); + return -EOPNOTSUPP; + } + + if (match->mask.ct_state_trk && match->value.ct_state_trk) { + NL_SET_ERR_MSG_MOD(extack, "LHS rule can never match +trk"); + return -EOPNOTSUPP; + } + /* LHS rules are always -trk, so we don't need to match on that */ + match->mask.ct_state_trk = 0; + match->value.ct_state_trk = 0; + + rc = efx_mae_match_check_caps_lhs(efx, &match->mask, extack); + if (rc) + return rc; + + rule = kzalloc(sizeof(*rule), GFP_USER); + if (!rule) + return -ENOMEM; + rule->cookie = tc->cookie; + old = rhashtable_lookup_get_insert_fast(&efx->tc->lhs_rule_ht, + &rule->linkage, + efx_tc_lhs_rule_ht_params); + if (IS_ERR(old)) { + rc = PTR_ERR(old); + goto release; + } else if (old) { + netif_dbg(efx, drv, efx->net_dev, + "Already offloaded rule (cookie %lx)\n", tc->cookie); + rc = -EEXIST; + NL_SET_ERR_MSG_MOD(extack, "Rule already offloaded"); + goto release; + } + + /* Parse actions */ + /* See note in efx_tc_flower_replace() regarding passed net_dev + * (used for efx_tc_get_recirc_id()). + */ + rc = efx_tc_flower_handle_lhs_actions(efx, tc, fr, efx->net_dev, rule); + if (rc) + goto release; + + rule->match = *match; + + rc = efx_mae_insert_lhs_rule(efx, rule, EFX_TC_PRIO_TC); + if (rc) { + NL_SET_ERR_MSG_MOD(extack, "Failed to insert rule in hw"); + goto release; + } + netif_dbg(efx, drv, efx->net_dev, + "Successfully parsed lhs rule (cookie %lx)\n", + tc->cookie); + return 0; + +release: + efx_tc_flower_release_lhs_actions(efx, &rule->lhs_act); + if (!old) + rhashtable_remove_fast(&efx->tc->lhs_rule_ht, &rule->linkage, + efx_tc_lhs_rule_ht_params); + kfree(rule); + return rc; +} + +static int efx_tc_flower_replace(struct efx_nic *efx, + struct net_device *net_dev, + struct flow_cls_offload *tc, + struct efx_rep *efv) +{ + struct flow_rule *fr = flow_cls_offload_flow_rule(tc); + struct netlink_ext_ack *extack = tc->common.extack; + const struct ip_tunnel_info *encap_info = NULL; + struct efx_tc_flow_rule *rule = NULL, *old; + struct efx_tc_mangler_state mung = {}; + struct efx_tc_action_set *act = NULL; + const struct flow_action_entry *fa; + struct efx_rep *from_efv, *to_efv; + struct efx_tc_match match; + u32 acts_id; + s64 rc; + int i; + + if (!tc_can_offload_extack(efx->net_dev, extack)) + return -EOPNOTSUPP; + if (WARN_ON(!efx->tc)) + return -ENETDOWN; + if (WARN_ON(!efx->tc->up)) + return -ENETDOWN; + + from_efv = efx_tc_flower_lookup_efv(efx, net_dev); + if (IS_ERR(from_efv)) { + /* Not from our PF or representors, so probably a tunnel dev */ + return efx_tc_flower_replace_foreign(efx, net_dev, tc); + } + + if (efv != from_efv) { + /* can't happen */ + NL_SET_ERR_MSG_FMT_MOD(extack, "for %s efv is %snull but from_efv is %snull (can't happen)", + netdev_name(net_dev), efv ? "non-" : "", + from_efv ? "non-" : ""); + return -EINVAL; + } + + /* Parse match */ + memset(&match, 0, sizeof(match)); + rc = efx_tc_flower_external_mport(efx, from_efv); + if (rc < 0) { + NL_SET_ERR_MSG_MOD(extack, "Failed to identify ingress m-port"); + return rc; + } + match.value.ingress_port = rc; + match.mask.ingress_port = ~0; + rc = efx_tc_flower_parse_match(efx, fr, &match, extack); + if (rc) + return rc; + if (efx_tc_match_is_encap(&match.mask)) { + NL_SET_ERR_MSG_MOD(extack, "Ingress enc_key matches not supported"); + return -EOPNOTSUPP; + } + + if (efx_tc_rule_is_lhs_rule(fr, &match)) + return efx_tc_flower_replace_lhs(efx, tc, fr, &match, efv, + net_dev); + + /* chain_index 0 is always recirc_id 0 (and does not appear in recirc_ht). + * Conveniently, match.rid == NULL and match.value.recirc_id == 0 owing + * to the initial memset(), so we don't need to do anything in that case. + */ + if (tc->common.chain_index) { + struct efx_tc_recirc_id *rid; + + /* Note regarding passed net_dev: + * VFreps and PF can share chain namespace, as they have + * distinct ingress_mports. So we don't need to burn an + * extra recirc_id if both use the same chain_index. + * (Strictly speaking, we could give each VFrep its own + * recirc_id namespace that doesn't take IDs away from the + * PF, but that would require a bunch of additional IDAs - + * one for each representor - and that's not likely to be + * the main cause of recirc_id exhaustion anyway.) + */ + rid = efx_tc_get_recirc_id(efx, tc->common.chain_index, + efx->net_dev); + if (IS_ERR(rid)) { + NL_SET_ERR_MSG_FMT_MOD(extack, + "Failed to allocate a hardware recirculation ID for chain_index %u", + tc->common.chain_index); + return PTR_ERR(rid); + } + match.rid = rid; + match.value.recirc_id = rid->fw_id; + } + match.mask.recirc_id = 0xff; + + /* AR table can't match on DO_CT (+trk). But a commonly used pattern is + * +trk+est, which is strictly implied by +est, so rewrite it to that. + */ + if (match.mask.ct_state_trk && match.value.ct_state_trk && + match.mask.ct_state_est && match.value.ct_state_est) + match.mask.ct_state_trk = 0; + /* Thanks to CT_TCP_FLAGS_INHIBIT, packets with interesting flags could + * match +trk-est (CT_HIT=0) despite being on an established connection. + * So make -est imply -tcp_syn_fin_rst match to ensure these packets + * still hit the software path. + */ + if (match.mask.ct_state_est && !match.value.ct_state_est) { + if (match.value.tcp_syn_fin_rst) { + /* Can't offload this combination */ + rc = -EOPNOTSUPP; + goto release; + } + match.mask.tcp_syn_fin_rst = true; + } + + rc = efx_mae_match_check_caps(efx, &match.mask, extack); + if (rc) + goto release; + + rule = kzalloc(sizeof(*rule), GFP_USER); + if (!rule) { + rc = -ENOMEM; + goto release; + } + INIT_LIST_HEAD(&rule->acts.list); + rule->cookie = tc->cookie; + old = rhashtable_lookup_get_insert_fast(&efx->tc->match_action_ht, + &rule->linkage, + efx_tc_match_action_ht_params); + if (IS_ERR(old)) { + rc = PTR_ERR(old); + goto release; + } else if (old) { + netif_dbg(efx, drv, efx->net_dev, + "Already offloaded rule (cookie %lx)\n", tc->cookie); + NL_SET_ERR_MSG_MOD(extack, "Rule already offloaded"); + rc = -EEXIST; + goto release; + } + + /* Parse actions */ + act = kzalloc(sizeof(*act), GFP_USER); + if (!act) { + rc = -ENOMEM; + goto release; + } + + /** + * DOC: TC action translation + * + * Actions in TC are sequential and cumulative, with delivery actions + * potentially anywhere in the order. The EF100 MAE, however, takes + * an 'action set list' consisting of 'action sets', each of which is + * applied to the _original_ packet, and consists of a set of optional + * actions in a fixed order with delivery at the end. + * To translate between these two models, we maintain a 'cursor', @act, + * which describes the cumulative effect of all the packet-mutating + * actions encountered so far; on handling a delivery (mirred or drop) + * action, once the action-set has been inserted into hardware, we + * append @act to the action-set list (@rule->acts); if this is a pipe + * action (mirred mirror) we then allocate a new @act with a copy of + * the cursor state _before_ the delivery action, otherwise we set @act + * to %NULL. + * This ensures that every allocated action-set is either attached to + * @rule->acts or pointed to by @act (and never both), and that only + * those action-sets in @rule->acts exist in hardware. Consequently, + * in the failure path, @act only needs to be freed in memory, whereas + * for @rule->acts we remove each action-set from hardware before + * freeing it (efx_tc_free_action_set_list()), even if the action-set + * list itself is not in hardware. + */ + flow_action_for_each(i, fa, &fr->action) { + struct efx_tc_action_set save; + u16 tci; + + if (!act) { + /* more actions after a non-pipe action */ + NL_SET_ERR_MSG_MOD(extack, "Action follows non-pipe action"); + rc = -EINVAL; + goto release; + } + + if ((fa->id == FLOW_ACTION_REDIRECT || + fa->id == FLOW_ACTION_MIRRED || + fa->id == FLOW_ACTION_DROP) && fa->hw_stats) { + struct efx_tc_counter_index *ctr; + + /* Currently the only actions that want stats are + * mirred and gact (ok, shot, trap, goto-chain), which + * means we want stats just before delivery. Also, + * note that tunnel_key set shouldn't change the length + * — it's only the subsequent mirred that does that, + * and the stats are taken _before_ the mirred action + * happens. + */ + if (!efx_tc_flower_action_order_ok(act, EFX_TC_AO_COUNT)) { + /* All supported actions that count either steal + * (gact shot, mirred redirect) or clone act + * (mirred mirror), so we should never get two + * count actions on one action_set. + */ + NL_SET_ERR_MSG_MOD(extack, "Count-action conflict (can't happen)"); + rc = -EOPNOTSUPP; + goto release; + } + + if (!(fa->hw_stats & FLOW_ACTION_HW_STATS_DELAYED)) { + NL_SET_ERR_MSG_FMT_MOD(extack, "hw_stats_type %u not supported (only 'delayed')", + fa->hw_stats); + rc = -EOPNOTSUPP; + goto release; + } + + ctr = efx_tc_flower_get_counter_index(efx, tc->cookie, + EFX_TC_COUNTER_TYPE_AR); + if (IS_ERR(ctr)) { + rc = PTR_ERR(ctr); + NL_SET_ERR_MSG_MOD(extack, "Failed to obtain a counter"); + goto release; + } + act->count = ctr; + INIT_LIST_HEAD(&act->count_user); + } + + switch (fa->id) { + case FLOW_ACTION_DROP: + rc = efx_mae_alloc_action_set(efx, act); + if (rc) { + NL_SET_ERR_MSG_MOD(extack, "Failed to write action set to hw (drop)"); + goto release; + } + list_add_tail(&act->list, &rule->acts.list); + act = NULL; /* end of the line */ + break; + case FLOW_ACTION_REDIRECT: + case FLOW_ACTION_MIRRED: + save = *act; + + if (encap_info) { + struct efx_tc_encap_action *encap; + + if (!efx_tc_flower_action_order_ok(act, + EFX_TC_AO_ENCAP)) { + rc = -EOPNOTSUPP; + NL_SET_ERR_MSG_MOD(extack, "Encap action violates action order"); + goto release; + } + encap = efx_tc_flower_create_encap_md( + efx, encap_info, fa->dev, extack); + if (IS_ERR_OR_NULL(encap)) { + rc = PTR_ERR(encap); + if (!rc) + rc = -EIO; /* arbitrary */ + goto release; + } + act->encap_md = encap; + list_add_tail(&act->encap_user, &encap->users); + act->dest_mport = encap->dest_mport; + act->deliver = 1; + if (act->count && !WARN_ON(!act->count->cnt)) { + /* This counter is used by an encap + * action, which needs a reference back + * so it can prod neighbouring whenever + * traffic is seen. + */ + spin_lock_bh(&act->count->cnt->lock); + list_add_tail(&act->count_user, + &act->count->cnt->users); + spin_unlock_bh(&act->count->cnt->lock); + } + rc = efx_mae_alloc_action_set(efx, act); + if (rc) { + NL_SET_ERR_MSG_MOD(extack, "Failed to write action set to hw (encap)"); + goto release; + } + list_add_tail(&act->list, &rule->acts.list); + act->user = &rule->acts; + act = NULL; + if (fa->id == FLOW_ACTION_REDIRECT) + break; /* end of the line */ + /* Mirror, so continue on with saved act */ + save.count = NULL; + act = kzalloc(sizeof(*act), GFP_USER); + if (!act) { + rc = -ENOMEM; + goto release; + } + *act = save; + break; + } + + if (!efx_tc_flower_action_order_ok(act, EFX_TC_AO_DELIVER)) { + /* can't happen */ + rc = -EOPNOTSUPP; + NL_SET_ERR_MSG_MOD(extack, "Deliver action violates action order (can't happen)"); + goto release; + } + + to_efv = efx_tc_flower_lookup_efv(efx, fa->dev); + if (IS_ERR(to_efv)) { + NL_SET_ERR_MSG_MOD(extack, "Mirred egress device not on switch"); + rc = PTR_ERR(to_efv); + goto release; + } + rc = efx_tc_flower_external_mport(efx, to_efv); + if (rc < 0) { + NL_SET_ERR_MSG_MOD(extack, "Failed to identify egress m-port"); + goto release; + } + act->dest_mport = rc; + act->deliver = 1; + rc = efx_mae_alloc_action_set(efx, act); + if (rc) { + NL_SET_ERR_MSG_MOD(extack, "Failed to write action set to hw (mirred)"); + goto release; + } + list_add_tail(&act->list, &rule->acts.list); + act = NULL; + if (fa->id == FLOW_ACTION_REDIRECT) + break; /* end of the line */ + /* Mirror, so continue on with saved act */ + save.count = NULL; + act = kzalloc(sizeof(*act), GFP_USER); + if (!act) { + rc = -ENOMEM; + goto release; + } + *act = save; + break; + case FLOW_ACTION_VLAN_POP: + if (act->vlan_push) { + act->vlan_push--; + } else if (efx_tc_flower_action_order_ok(act, EFX_TC_AO_VLAN_POP)) { + act->vlan_pop++; + } else { + NL_SET_ERR_MSG_MOD(extack, + "More than two VLAN pops, or action order violated"); + rc = -EINVAL; + goto release; + } + break; + case FLOW_ACTION_VLAN_PUSH: + if (!efx_tc_flower_action_order_ok(act, EFX_TC_AO_VLAN_PUSH)) { + rc = -EINVAL; + NL_SET_ERR_MSG_MOD(extack, + "More than two VLAN pushes, or action order violated"); + goto release; + } + tci = fa->vlan.vid & VLAN_VID_MASK; + tci |= fa->vlan.prio << VLAN_PRIO_SHIFT; + act->vlan_tci[act->vlan_push] = cpu_to_be16(tci); + act->vlan_proto[act->vlan_push] = fa->vlan.proto; + act->vlan_push++; + break; + case FLOW_ACTION_ADD: + rc = efx_tc_pedit_add(efx, act, fa, extack); + if (rc < 0) + goto release; + break; + case FLOW_ACTION_MANGLE: + rc = efx_tc_mangle(efx, act, fa, &mung, extack, &match); + if (rc < 0) + goto release; + break; + case FLOW_ACTION_TUNNEL_ENCAP: + if (encap_info) { + /* Can't specify encap multiple times. + * If you want to overwrite an existing + * encap_info, use an intervening + * FLOW_ACTION_TUNNEL_DECAP to clear it. + */ + NL_SET_ERR_MSG_MOD(extack, "Tunnel key set when already set"); + rc = -EINVAL; + goto release; + } + if (!fa->tunnel) { + NL_SET_ERR_MSG_MOD(extack, "Tunnel key set is missing key"); + rc = -EOPNOTSUPP; + goto release; + } + encap_info = fa->tunnel; + break; + case FLOW_ACTION_TUNNEL_DECAP: + if (encap_info) { + encap_info = NULL; + break; + } + /* Since we don't support enc_key matches on ingress + * (and if we did there'd be no tunnel-device to give + * us a type), we can't offload a decap that's not + * just undoing a previous encap action. + */ + NL_SET_ERR_MSG_MOD(extack, "Cannot offload tunnel decap action without tunnel device"); + rc = -EOPNOTSUPP; + goto release; + default: + NL_SET_ERR_MSG_FMT_MOD(extack, "Unhandled action %u", + fa->id); + rc = -EOPNOTSUPP; + goto release; + } + } + + rc = efx_tc_incomplete_mangle(&mung, extack); + if (rc < 0) + goto release; + if (act) { + /* Not shot/redirected, so deliver to default dest */ + if (from_efv == EFX_EFV_PF) + /* Rule applies to traffic from the wire, + * and default dest is thus the PF + */ + efx_mae_mport_uplink(efx, &act->dest_mport); + else + /* Representor, so rule applies to traffic from + * representee, and default dest is thus the rep. + * All reps use the same mport for delivery + */ + efx_mae_mport_mport(efx, efx->tc->reps_mport_id, + &act->dest_mport); + act->deliver = 1; + rc = efx_mae_alloc_action_set(efx, act); + if (rc) { + NL_SET_ERR_MSG_MOD(extack, "Failed to write action set to hw (deliver)"); + goto release; + } + list_add_tail(&act->list, &rule->acts.list); + act = NULL; /* Prevent double-free in error path */ + } + + netif_dbg(efx, drv, efx->net_dev, + "Successfully parsed filter (cookie %lx)\n", + tc->cookie); + + rule->match = match; + + rc = efx_mae_alloc_action_set_list(efx, &rule->acts); + if (rc) { + NL_SET_ERR_MSG_MOD(extack, "Failed to write action set list to hw"); + goto release; + } + if (from_efv == EFX_EFV_PF) + /* PF netdev, so rule applies to traffic from wire */ + rule->fallback = &efx->tc->facts.pf; + else + /* repdev, so rule applies to traffic from representee */ + rule->fallback = &efx->tc->facts.reps; + if (!efx_tc_check_ready(efx, rule)) { + netif_dbg(efx, drv, efx->net_dev, "action not ready for hw\n"); + acts_id = rule->fallback->fw_id; + } else { + netif_dbg(efx, drv, efx->net_dev, "ready for hw\n"); + acts_id = rule->acts.fw_id; + } + rc = efx_mae_insert_rule(efx, &rule->match, EFX_TC_PRIO_TC, + acts_id, &rule->fw_id); + if (rc) { + NL_SET_ERR_MSG_MOD(extack, "Failed to insert rule in hw"); + goto release_acts; + } + return 0; + +release_acts: + efx_mae_free_action_set_list(efx, &rule->acts); +release: + /* We failed to insert the rule, so free up any entries we created in + * subsidiary tables. + */ + if (match.rid) + efx_tc_put_recirc_id(efx, match.rid); + if (act) + efx_tc_free_action_set(efx, act, false); + if (rule) { + if (!old) + rhashtable_remove_fast(&efx->tc->match_action_ht, + &rule->linkage, + efx_tc_match_action_ht_params); + efx_tc_free_action_set_list(efx, &rule->acts, false); + } + kfree(rule); + return rc; +} + +static int efx_tc_flower_destroy(struct efx_nic *efx, + struct net_device *net_dev, + struct flow_cls_offload *tc) +{ + struct netlink_ext_ack *extack = tc->common.extack; + struct efx_tc_lhs_rule *lhs_rule; + struct efx_tc_flow_rule *rule; + + lhs_rule = rhashtable_lookup_fast(&efx->tc->lhs_rule_ht, &tc->cookie, + efx_tc_lhs_rule_ht_params); + if (lhs_rule) { + /* Remove it from HW */ + efx_mae_remove_lhs_rule(efx, lhs_rule); + /* Delete it from SW */ + efx_tc_flower_release_lhs_actions(efx, &lhs_rule->lhs_act); + rhashtable_remove_fast(&efx->tc->lhs_rule_ht, &lhs_rule->linkage, + efx_tc_lhs_rule_ht_params); + if (lhs_rule->match.encap) + efx_tc_flower_release_encap_match(efx, lhs_rule->match.encap); + netif_dbg(efx, drv, efx->net_dev, "Removed (lhs) filter %lx\n", + lhs_rule->cookie); + kfree(lhs_rule); + return 0; + } + + rule = rhashtable_lookup_fast(&efx->tc->match_action_ht, &tc->cookie, + efx_tc_match_action_ht_params); + if (!rule) { + /* Only log a message if we're the ingress device. Otherwise + * it's a foreign filter and we might just not have been + * interested (e.g. we might not have been the egress device + * either). + */ + if (!IS_ERR(efx_tc_flower_lookup_efv(efx, net_dev))) + netif_warn(efx, drv, efx->net_dev, + "Filter %lx not found to remove\n", tc->cookie); + NL_SET_ERR_MSG_MOD(extack, "Flow cookie not found in offloaded rules"); + return -ENOENT; + } + + /* Remove it from HW */ + efx_tc_delete_rule(efx, rule); + /* Delete it from SW */ + rhashtable_remove_fast(&efx->tc->match_action_ht, &rule->linkage, + efx_tc_match_action_ht_params); + netif_dbg(efx, drv, efx->net_dev, "Removed filter %lx\n", rule->cookie); + kfree(rule); + return 0; +} + +static int efx_tc_flower_stats(struct efx_nic *efx, struct net_device *net_dev, + struct flow_cls_offload *tc) +{ + struct netlink_ext_ack *extack = tc->common.extack; + struct efx_tc_counter_index *ctr; + struct efx_tc_counter *cnt; + u64 packets, bytes; + + ctr = efx_tc_flower_find_counter_index(efx, tc->cookie); + if (!ctr) { + /* See comment in efx_tc_flower_destroy() */ + if (!IS_ERR(efx_tc_flower_lookup_efv(efx, net_dev))) + if (net_ratelimit()) + netif_warn(efx, drv, efx->net_dev, + "Filter %lx not found for stats\n", + tc->cookie); + NL_SET_ERR_MSG_MOD(extack, "Flow cookie not found in offloaded rules"); + return -ENOENT; + } + if (WARN_ON(!ctr->cnt)) /* can't happen */ + return -EIO; + cnt = ctr->cnt; + + spin_lock_bh(&cnt->lock); + /* Report only new pkts/bytes since last time TC asked */ + packets = cnt->packets; + bytes = cnt->bytes; + flow_stats_update(&tc->stats, bytes - cnt->old_bytes, + packets - cnt->old_packets, 0, cnt->touched, + FLOW_ACTION_HW_STATS_DELAYED); + cnt->old_packets = packets; + cnt->old_bytes = bytes; + spin_unlock_bh(&cnt->lock); + return 0; +} + +int efx_tc_flower(struct efx_nic *efx, struct net_device *net_dev, + struct flow_cls_offload *tc, struct efx_rep *efv) +{ + int rc; + + if (!efx->tc) + return -EOPNOTSUPP; + + mutex_lock(&efx->tc->mutex); + switch (tc->command) { + case FLOW_CLS_REPLACE: + rc = efx_tc_flower_replace(efx, net_dev, tc, efv); + break; + case FLOW_CLS_DESTROY: + rc = efx_tc_flower_destroy(efx, net_dev, tc); + break; + case FLOW_CLS_STATS: + rc = efx_tc_flower_stats(efx, net_dev, tc); + break; + default: + rc = -EOPNOTSUPP; + break; + } + mutex_unlock(&efx->tc->mutex); + return rc; +} + +static int efx_tc_configure_default_rule(struct efx_nic *efx, u32 ing_port, + u32 eg_port, struct efx_tc_flow_rule *rule) +{ + struct efx_tc_action_set_list *acts = &rule->acts; + struct efx_tc_match *match = &rule->match; + struct efx_tc_action_set *act; + int rc; + + match->value.ingress_port = ing_port; + match->mask.ingress_port = ~0; + act = kzalloc(sizeof(*act), GFP_KERNEL); + if (!act) + return -ENOMEM; + act->deliver = 1; + act->dest_mport = eg_port; + rc = efx_mae_alloc_action_set(efx, act); + if (rc) + goto fail1; + EFX_WARN_ON_PARANOID(!list_empty(&acts->list)); + list_add_tail(&act->list, &acts->list); + rc = efx_mae_alloc_action_set_list(efx, acts); + if (rc) + goto fail2; + rc = efx_mae_insert_rule(efx, match, EFX_TC_PRIO_DFLT, + acts->fw_id, &rule->fw_id); + if (rc) + goto fail3; + return 0; +fail3: + efx_mae_free_action_set_list(efx, acts); +fail2: + list_del(&act->list); + efx_mae_free_action_set(efx, act->fw_id); +fail1: + kfree(act); + return rc; +} + +static int efx_tc_configure_default_rule_pf(struct efx_nic *efx) +{ + struct efx_tc_flow_rule *rule = &efx->tc->dflt.pf; + u32 ing_port, eg_port; + + efx_mae_mport_uplink(efx, &ing_port); + efx_mae_mport_wire(efx, &eg_port); + return efx_tc_configure_default_rule(efx, ing_port, eg_port, rule); +} + +static int efx_tc_configure_default_rule_wire(struct efx_nic *efx) +{ + struct efx_tc_flow_rule *rule = &efx->tc->dflt.wire; + u32 ing_port, eg_port; + + efx_mae_mport_wire(efx, &ing_port); + efx_mae_mport_uplink(efx, &eg_port); + return efx_tc_configure_default_rule(efx, ing_port, eg_port, rule); +} + +int efx_tc_configure_default_rule_rep(struct efx_rep *efv) +{ + struct efx_tc_flow_rule *rule = &efv->dflt; + struct efx_nic *efx = efv->parent; + u32 ing_port, eg_port; + + efx_mae_mport_mport(efx, efv->mport, &ing_port); + efx_mae_mport_mport(efx, efx->tc->reps_mport_id, &eg_port); + return efx_tc_configure_default_rule(efx, ing_port, eg_port, rule); +} + +void efx_tc_deconfigure_default_rule(struct efx_nic *efx, + struct efx_tc_flow_rule *rule) +{ + if (rule->fw_id != MC_CMD_MAE_ACTION_RULE_INSERT_OUT_ACTION_RULE_ID_NULL) + efx_tc_delete_rule(efx, rule); + rule->fw_id = MC_CMD_MAE_ACTION_RULE_INSERT_OUT_ACTION_RULE_ID_NULL; +} + +static int efx_tc_configure_fallback_acts(struct efx_nic *efx, u32 eg_port, + struct efx_tc_action_set_list *acts) +{ + struct efx_tc_action_set *act; + int rc; + + act = kzalloc(sizeof(*act), GFP_KERNEL); + if (!act) + return -ENOMEM; + act->deliver = 1; + act->dest_mport = eg_port; + rc = efx_mae_alloc_action_set(efx, act); + if (rc) + goto fail1; + EFX_WARN_ON_PARANOID(!list_empty(&acts->list)); + list_add_tail(&act->list, &acts->list); + rc = efx_mae_alloc_action_set_list(efx, acts); + if (rc) + goto fail2; + return 0; +fail2: + list_del(&act->list); + efx_mae_free_action_set(efx, act->fw_id); +fail1: + kfree(act); + return rc; +} + +static int efx_tc_configure_fallback_acts_pf(struct efx_nic *efx) +{ + struct efx_tc_action_set_list *acts = &efx->tc->facts.pf; + u32 eg_port; + + efx_mae_mport_uplink(efx, &eg_port); + return efx_tc_configure_fallback_acts(efx, eg_port, acts); +} + +static int efx_tc_configure_fallback_acts_reps(struct efx_nic *efx) +{ + struct efx_tc_action_set_list *acts = &efx->tc->facts.reps; + u32 eg_port; + + efx_mae_mport_mport(efx, efx->tc->reps_mport_id, &eg_port); + return efx_tc_configure_fallback_acts(efx, eg_port, acts); +} + +static void efx_tc_deconfigure_fallback_acts(struct efx_nic *efx, + struct efx_tc_action_set_list *acts) +{ + efx_tc_free_action_set_list(efx, acts, true); +} + +static int efx_tc_configure_rep_mport(struct efx_nic *efx) +{ + u32 rep_mport_label; + int rc; + + rc = efx_mae_allocate_mport(efx, &efx->tc->reps_mport_id, &rep_mport_label); + if (rc) + return rc; + pci_dbg(efx->pci_dev, "created rep mport 0x%08x (0x%04x)\n", + efx->tc->reps_mport_id, rep_mport_label); + /* Use mport *selector* as vport ID */ + efx_mae_mport_mport(efx, efx->tc->reps_mport_id, + &efx->tc->reps_mport_vport_id); + return 0; +} + +static void efx_tc_deconfigure_rep_mport(struct efx_nic *efx) +{ + efx_mae_free_mport(efx, efx->tc->reps_mport_id); + efx->tc->reps_mport_id = MAE_MPORT_SELECTOR_NULL; +} + +int efx_tc_insert_rep_filters(struct efx_nic *efx) +{ + struct efx_filter_spec promisc, allmulti; + int rc; + + if (efx->type->is_vf) + return 0; + if (!efx->tc) + return 0; + efx_filter_init_rx(&promisc, EFX_FILTER_PRI_REQUIRED, 0, 0); + efx_filter_set_uc_def(&promisc); + efx_filter_set_vport_id(&promisc, efx->tc->reps_mport_vport_id); + rc = efx_filter_insert_filter(efx, &promisc, false); + if (rc < 0) + return rc; + efx->tc->reps_filter_uc = rc; + efx_filter_init_rx(&allmulti, EFX_FILTER_PRI_REQUIRED, 0, 0); + efx_filter_set_mc_def(&allmulti); + efx_filter_set_vport_id(&allmulti, efx->tc->reps_mport_vport_id); + rc = efx_filter_insert_filter(efx, &allmulti, false); + if (rc < 0) + return rc; + efx->tc->reps_filter_mc = rc; + return 0; +} + +void efx_tc_remove_rep_filters(struct efx_nic *efx) +{ + if (efx->type->is_vf) + return; + if (!efx->tc) + return; + if (efx->tc->reps_filter_mc >= 0) + efx_filter_remove_id_safe(efx, EFX_FILTER_PRI_REQUIRED, efx->tc->reps_filter_mc); + efx->tc->reps_filter_mc = -1; + if (efx->tc->reps_filter_uc >= 0) + efx_filter_remove_id_safe(efx, EFX_FILTER_PRI_REQUIRED, efx->tc->reps_filter_uc); + efx->tc->reps_filter_uc = -1; +} + +int efx_init_tc(struct efx_nic *efx) +{ + int rc; + + rc = efx_mae_get_caps(efx, efx->tc->caps); + if (rc) + return rc; + if (efx->tc->caps->match_field_count > MAE_NUM_FIELDS) + /* Firmware supports some match fields the driver doesn't know + * about. Not fatal, unless any of those fields are required + * (MAE_FIELD_SUPPORTED_MATCH_ALWAYS) but if so we don't know. + */ + netif_warn(efx, probe, efx->net_dev, + "FW reports additional match fields %u\n", + efx->tc->caps->match_field_count); + if (efx->tc->caps->action_prios < EFX_TC_PRIO__NUM) { + netif_err(efx, probe, efx->net_dev, + "Too few action prios supported (have %u, need %u)\n", + efx->tc->caps->action_prios, EFX_TC_PRIO__NUM); + return -EIO; + } + rc = efx_tc_configure_default_rule_pf(efx); + if (rc) + return rc; + rc = efx_tc_configure_default_rule_wire(efx); + if (rc) + return rc; + rc = efx_tc_configure_rep_mport(efx); + if (rc) + return rc; + rc = efx_tc_configure_fallback_acts_pf(efx); + if (rc) + return rc; + rc = efx_tc_configure_fallback_acts_reps(efx); + if (rc) + return rc; + rc = efx_mae_get_tables(efx); + if (rc) + return rc; + rc = flow_indr_dev_register(efx_tc_indr_setup_cb, efx); + if (rc) + goto out_free; + efx->tc->up = true; + return 0; +out_free: + efx_mae_free_tables(efx); + return rc; +} + +void efx_fini_tc(struct efx_nic *efx) +{ + /* We can get called even if efx_init_struct_tc() failed */ + if (!efx->tc) + return; + if (efx->tc->up) + flow_indr_dev_unregister(efx_tc_indr_setup_cb, efx, efx_tc_block_unbind); + efx_tc_deconfigure_rep_mport(efx); + efx_tc_deconfigure_default_rule(efx, &efx->tc->dflt.pf); + efx_tc_deconfigure_default_rule(efx, &efx->tc->dflt.wire); + efx_tc_deconfigure_fallback_acts(efx, &efx->tc->facts.pf); + efx_tc_deconfigure_fallback_acts(efx, &efx->tc->facts.reps); + efx->tc->up = false; + efx_mae_free_tables(efx); +} + +/* At teardown time, all TC filter rules (and thus all resources they created) + * should already have been removed. If we find any in our hashtables, make a + * cursory attempt to clean up the software side. + */ +static void efx_tc_encap_match_free(void *ptr, void *__unused) +{ + struct efx_tc_encap_match *encap = ptr; + + WARN_ON(refcount_read(&encap->ref)); + kfree(encap); +} + +static void efx_tc_recirc_free(void *ptr, void *arg) +{ + struct efx_tc_recirc_id *rid = ptr; + struct efx_nic *efx = arg; + + WARN_ON(refcount_read(&rid->ref)); + ida_free(&efx->tc->recirc_ida, rid->fw_id); + kfree(rid); +} + +static void efx_tc_lhs_free(void *ptr, void *arg) +{ + struct efx_tc_lhs_rule *rule = ptr; + struct efx_nic *efx = arg; + + netif_err(efx, drv, efx->net_dev, + "tc lhs_rule %lx still present at teardown, removing\n", + rule->cookie); + + if (rule->lhs_act.zone) + efx_tc_ct_unregister_zone(efx, rule->lhs_act.zone); + if (rule->lhs_act.count) + efx_tc_flower_put_counter_index(efx, rule->lhs_act.count); + efx_mae_remove_lhs_rule(efx, rule); + + kfree(rule); +} + +static void efx_tc_mac_free(void *ptr, void *__unused) +{ + struct efx_tc_mac_pedit_action *ped = ptr; + + WARN_ON(refcount_read(&ped->ref)); + kfree(ped); +} + +static void efx_tc_flow_free(void *ptr, void *arg) +{ + struct efx_tc_flow_rule *rule = ptr; + struct efx_nic *efx = arg; + + netif_err(efx, drv, efx->net_dev, + "tc rule %lx still present at teardown, removing\n", + rule->cookie); + + /* Also releases entries in subsidiary tables */ + efx_tc_delete_rule(efx, rule); + + kfree(rule); +} + +int efx_init_struct_tc(struct efx_nic *efx) +{ + int rc; + + if (efx->type->is_vf) + return 0; + + efx->tc = kzalloc(sizeof(*efx->tc), GFP_KERNEL); + if (!efx->tc) + return -ENOMEM; + efx->tc->caps = kzalloc(sizeof(struct mae_caps), GFP_KERNEL); + if (!efx->tc->caps) { + rc = -ENOMEM; + goto fail_alloc_caps; + } + INIT_LIST_HEAD(&efx->tc->block_list); + + mutex_init(&efx->tc->mutex); + init_waitqueue_head(&efx->tc->flush_wq); + rc = efx_tc_init_encap_actions(efx); + if (rc < 0) + goto fail_encap_actions; + rc = efx_tc_init_counters(efx); + if (rc < 0) + goto fail_counters; + rc = rhashtable_init(&efx->tc->mac_ht, &efx_tc_mac_ht_params); + if (rc < 0) + goto fail_mac_ht; + rc = rhashtable_init(&efx->tc->encap_match_ht, &efx_tc_encap_match_ht_params); + if (rc < 0) + goto fail_encap_match_ht; + rc = rhashtable_init(&efx->tc->match_action_ht, &efx_tc_match_action_ht_params); + if (rc < 0) + goto fail_match_action_ht; + rc = rhashtable_init(&efx->tc->lhs_rule_ht, &efx_tc_lhs_rule_ht_params); + if (rc < 0) + goto fail_lhs_rule_ht; + rc = efx_tc_init_conntrack(efx); + if (rc < 0) + goto fail_conntrack; + rc = rhashtable_init(&efx->tc->recirc_ht, &efx_tc_recirc_ht_params); + if (rc < 0) + goto fail_recirc_ht; + ida_init(&efx->tc->recirc_ida); + efx->tc->reps_filter_uc = -1; + efx->tc->reps_filter_mc = -1; + INIT_LIST_HEAD(&efx->tc->dflt.pf.acts.list); + efx->tc->dflt.pf.fw_id = MC_CMD_MAE_ACTION_RULE_INSERT_OUT_ACTION_RULE_ID_NULL; + INIT_LIST_HEAD(&efx->tc->dflt.wire.acts.list); + efx->tc->dflt.wire.fw_id = MC_CMD_MAE_ACTION_RULE_INSERT_OUT_ACTION_RULE_ID_NULL; + INIT_LIST_HEAD(&efx->tc->facts.pf.list); + efx->tc->facts.pf.fw_id = MC_CMD_MAE_ACTION_SET_ALLOC_OUT_ACTION_SET_ID_NULL; + INIT_LIST_HEAD(&efx->tc->facts.reps.list); + efx->tc->facts.reps.fw_id = MC_CMD_MAE_ACTION_SET_ALLOC_OUT_ACTION_SET_ID_NULL; + efx->extra_channel_type[EFX_EXTRA_CHANNEL_TC] = &efx_tc_channel_type; + return 0; +fail_recirc_ht: + efx_tc_destroy_conntrack(efx); +fail_conntrack: + rhashtable_destroy(&efx->tc->lhs_rule_ht); +fail_lhs_rule_ht: + rhashtable_destroy(&efx->tc->match_action_ht); +fail_match_action_ht: + rhashtable_destroy(&efx->tc->encap_match_ht); +fail_encap_match_ht: + rhashtable_destroy(&efx->tc->mac_ht); +fail_mac_ht: + efx_tc_destroy_counters(efx); +fail_counters: + efx_tc_destroy_encap_actions(efx); +fail_encap_actions: + mutex_destroy(&efx->tc->mutex); + kfree(efx->tc->caps); +fail_alloc_caps: + kfree(efx->tc); + efx->tc = NULL; + return rc; +} + +void efx_fini_struct_tc(struct efx_nic *efx) +{ + if (!efx->tc) + return; + + mutex_lock(&efx->tc->mutex); + EFX_WARN_ON_PARANOID(efx->tc->dflt.pf.fw_id != + MC_CMD_MAE_ACTION_RULE_INSERT_OUT_ACTION_RULE_ID_NULL); + EFX_WARN_ON_PARANOID(efx->tc->dflt.wire.fw_id != + MC_CMD_MAE_ACTION_RULE_INSERT_OUT_ACTION_RULE_ID_NULL); + EFX_WARN_ON_PARANOID(efx->tc->facts.pf.fw_id != + MC_CMD_MAE_ACTION_SET_LIST_ALLOC_OUT_ACTION_SET_LIST_ID_NULL); + EFX_WARN_ON_PARANOID(efx->tc->facts.reps.fw_id != + MC_CMD_MAE_ACTION_SET_LIST_ALLOC_OUT_ACTION_SET_LIST_ID_NULL); + rhashtable_free_and_destroy(&efx->tc->lhs_rule_ht, efx_tc_lhs_free, efx); + rhashtable_free_and_destroy(&efx->tc->match_action_ht, efx_tc_flow_free, + efx); + rhashtable_free_and_destroy(&efx->tc->encap_match_ht, + efx_tc_encap_match_free, NULL); + efx_tc_fini_conntrack(efx); + rhashtable_free_and_destroy(&efx->tc->recirc_ht, efx_tc_recirc_free, efx); + WARN_ON(!ida_is_empty(&efx->tc->recirc_ida)); + ida_destroy(&efx->tc->recirc_ida); + rhashtable_free_and_destroy(&efx->tc->mac_ht, efx_tc_mac_free, NULL); + efx_tc_fini_counters(efx); + efx_tc_fini_encap_actions(efx); + mutex_unlock(&efx->tc->mutex); + mutex_destroy(&efx->tc->mutex); + kfree(efx->tc->caps); + kfree(efx->tc); + efx->tc = NULL; +} |