summaryrefslogtreecommitdiffstats
path: root/net/xfrm
diff options
context:
space:
mode:
Diffstat (limited to 'net/xfrm')
-rw-r--r--net/xfrm/Kconfig140
-rw-r--r--net/xfrm/Makefile17
-rw-r--r--net/xfrm/espintcp.c579
-rw-r--r--net/xfrm/xfrm_algo.c866
-rw-r--r--net/xfrm/xfrm_compat.c675
-rw-r--r--net/xfrm/xfrm_device.c444
-rw-r--r--net/xfrm/xfrm_hash.c40
-rw-r--r--net/xfrm/xfrm_hash.h199
-rw-r--r--net/xfrm/xfrm_inout.h70
-rw-r--r--net/xfrm/xfrm_input.c834
-rw-r--r--net/xfrm/xfrm_interface_core.c1242
-rw-r--r--net/xfrm/xfrm_ipcomp.c378
-rw-r--r--net/xfrm/xfrm_output.c909
-rw-r--r--net/xfrm/xfrm_policy.c4490
-rw-r--r--net/xfrm/xfrm_proc.c75
-rw-r--r--net/xfrm/xfrm_replay.c795
-rw-r--r--net/xfrm/xfrm_state.c2933
-rw-r--r--net/xfrm/xfrm_sysctl.c87
-rw-r--r--net/xfrm/xfrm_user.c3830
19 files changed, 18603 insertions, 0 deletions
diff --git a/net/xfrm/Kconfig b/net/xfrm/Kconfig
new file mode 100644
index 000000000..3adf31a83
--- /dev/null
+++ b/net/xfrm/Kconfig
@@ -0,0 +1,140 @@
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# XFRM configuration
+#
+config XFRM
+ bool
+ depends on INET
+ select GRO_CELLS
+ select SKB_EXTENSIONS
+
+config XFRM_OFFLOAD
+ bool
+
+config XFRM_ALGO
+ tristate
+ select XFRM
+ select CRYPTO
+ select CRYPTO_HASH
+ select CRYPTO_SKCIPHER
+
+if INET
+config XFRM_USER
+ tristate "Transformation user configuration interface"
+ select XFRM_ALGO
+ help
+ Support for Transformation(XFRM) user configuration interface
+ like IPsec used by native Linux tools.
+
+ If unsure, say Y.
+
+config XFRM_USER_COMPAT
+ tristate "Compatible ABI support"
+ depends on XFRM_USER && COMPAT_FOR_U64_ALIGNMENT && \
+ HAVE_EFFICIENT_UNALIGNED_ACCESS
+ select WANT_COMPAT_NETLINK_MESSAGES
+ help
+ Transformation(XFRM) user configuration interface like IPsec
+ used by compatible Linux applications.
+
+ If unsure, say N.
+
+config XFRM_INTERFACE
+ tristate "Transformation virtual interface"
+ depends on XFRM && IPV6
+ help
+ This provides a virtual interface to route IPsec traffic.
+
+ If unsure, say N.
+
+config XFRM_SUB_POLICY
+ bool "Transformation sub policy support"
+ depends on XFRM
+ help
+ Support sub policy for developers. By using sub policy with main
+ one, two policies can be applied to the same packet at once.
+ Policy which lives shorter time in kernel should be a sub.
+
+ If unsure, say N.
+
+config XFRM_MIGRATE
+ bool "Transformation migrate database"
+ depends on XFRM
+ help
+ A feature to update locator(s) of a given IPsec security
+ association dynamically. This feature is required, for
+ instance, in a Mobile IPv6 environment with IPsec configuration
+ where mobile nodes change their attachment point to the Internet.
+
+ If unsure, say N.
+
+config XFRM_STATISTICS
+ bool "Transformation statistics"
+ depends on XFRM && PROC_FS
+ help
+ This statistics is not a SNMP/MIB specification but shows
+ statistics about transformation error (or almost error) factor
+ at packet processing for developer.
+
+ If unsure, say N.
+
+# This option selects XFRM_ALGO along with the AH authentication algorithms that
+# RFC 8221 lists as MUST be implemented.
+config XFRM_AH
+ tristate
+ select XFRM_ALGO
+ select CRYPTO
+ select CRYPTO_HMAC
+ select CRYPTO_SHA256
+
+# This option selects XFRM_ALGO along with the ESP encryption and authentication
+# algorithms that RFC 8221 lists as MUST be implemented.
+config XFRM_ESP
+ tristate
+ select XFRM_ALGO
+ select CRYPTO
+ select CRYPTO_AES
+ select CRYPTO_AUTHENC
+ select CRYPTO_CBC
+ select CRYPTO_ECHAINIV
+ select CRYPTO_GCM
+ select CRYPTO_HMAC
+ select CRYPTO_SEQIV
+ select CRYPTO_SHA256
+
+config XFRM_IPCOMP
+ tristate
+ select XFRM_ALGO
+ select CRYPTO
+ select CRYPTO_DEFLATE
+
+config NET_KEY
+ tristate "PF_KEY sockets"
+ select XFRM_ALGO
+ help
+ PF_KEYv2 socket family, compatible to KAME ones.
+ They are required if you are going to use IPsec tools ported
+ from KAME.
+
+ Say Y unless you know what you are doing.
+
+config NET_KEY_MIGRATE
+ bool "PF_KEY MIGRATE"
+ depends on NET_KEY
+ select XFRM_MIGRATE
+ help
+ Add a PF_KEY MIGRATE message to PF_KEYv2 socket family.
+ The PF_KEY MIGRATE message is used to dynamically update
+ locator(s) of a given IPsec security association.
+ This feature is required, for instance, in a Mobile IPv6
+ environment with IPsec configuration where mobile nodes
+ change their attachment point to the Internet. Detail
+ information can be found in the internet-draft
+ <draft-sugimoto-mip6-pfkey-migrate>.
+
+ If unsure, say N.
+
+config XFRM_ESPINTCP
+ bool
+
+endif # INET
diff --git a/net/xfrm/Makefile b/net/xfrm/Makefile
new file mode 100644
index 000000000..08a2870fd
--- /dev/null
+++ b/net/xfrm/Makefile
@@ -0,0 +1,17 @@
+# SPDX-License-Identifier: GPL-2.0
+#
+# Makefile for the XFRM subsystem.
+#
+
+xfrm_interface-$(CONFIG_XFRM_INTERFACE) += xfrm_interface_core.o
+
+obj-$(CONFIG_XFRM) := xfrm_policy.o xfrm_state.o xfrm_hash.o \
+ xfrm_input.o xfrm_output.o \
+ xfrm_sysctl.o xfrm_replay.o xfrm_device.o
+obj-$(CONFIG_XFRM_STATISTICS) += xfrm_proc.o
+obj-$(CONFIG_XFRM_ALGO) += xfrm_algo.o
+obj-$(CONFIG_XFRM_USER) += xfrm_user.o
+obj-$(CONFIG_XFRM_USER_COMPAT) += xfrm_compat.o
+obj-$(CONFIG_XFRM_IPCOMP) += xfrm_ipcomp.o
+obj-$(CONFIG_XFRM_INTERFACE) += xfrm_interface.o
+obj-$(CONFIG_XFRM_ESPINTCP) += espintcp.o
diff --git a/net/xfrm/espintcp.c b/net/xfrm/espintcp.c
new file mode 100644
index 000000000..d6fece1ed
--- /dev/null
+++ b/net/xfrm/espintcp.c
@@ -0,0 +1,579 @@
+// SPDX-License-Identifier: GPL-2.0
+#include <net/tcp.h>
+#include <net/strparser.h>
+#include <net/xfrm.h>
+#include <net/esp.h>
+#include <net/espintcp.h>
+#include <linux/skmsg.h>
+#include <net/inet_common.h>
+#if IS_ENABLED(CONFIG_IPV6)
+#include <net/ipv6_stubs.h>
+#endif
+
+static void handle_nonesp(struct espintcp_ctx *ctx, struct sk_buff *skb,
+ struct sock *sk)
+{
+ if (atomic_read(&sk->sk_rmem_alloc) >= sk->sk_rcvbuf ||
+ !sk_rmem_schedule(sk, skb, skb->truesize)) {
+ XFRM_INC_STATS(sock_net(sk), LINUX_MIB_XFRMINERROR);
+ kfree_skb(skb);
+ return;
+ }
+
+ skb_set_owner_r(skb, sk);
+
+ memset(skb->cb, 0, sizeof(skb->cb));
+ skb_queue_tail(&ctx->ike_queue, skb);
+ ctx->saved_data_ready(sk);
+}
+
+static void handle_esp(struct sk_buff *skb, struct sock *sk)
+{
+ struct tcp_skb_cb *tcp_cb = (struct tcp_skb_cb *)skb->cb;
+
+ skb_reset_transport_header(skb);
+
+ /* restore IP CB, we need at least IP6CB->nhoff */
+ memmove(skb->cb, &tcp_cb->header, sizeof(tcp_cb->header));
+
+ rcu_read_lock();
+ skb->dev = dev_get_by_index_rcu(sock_net(sk), skb->skb_iif);
+ local_bh_disable();
+#if IS_ENABLED(CONFIG_IPV6)
+ if (sk->sk_family == AF_INET6)
+ ipv6_stub->xfrm6_rcv_encap(skb, IPPROTO_ESP, 0, TCP_ENCAP_ESPINTCP);
+ else
+#endif
+ xfrm4_rcv_encap(skb, IPPROTO_ESP, 0, TCP_ENCAP_ESPINTCP);
+ local_bh_enable();
+ rcu_read_unlock();
+}
+
+static void espintcp_rcv(struct strparser *strp, struct sk_buff *skb)
+{
+ struct espintcp_ctx *ctx = container_of(strp, struct espintcp_ctx,
+ strp);
+ struct strp_msg *rxm = strp_msg(skb);
+ int len = rxm->full_len - 2;
+ u32 nonesp_marker;
+ int err;
+
+ /* keepalive packet? */
+ if (unlikely(len == 1)) {
+ u8 data;
+
+ err = skb_copy_bits(skb, rxm->offset + 2, &data, 1);
+ if (err < 0) {
+ XFRM_INC_STATS(sock_net(strp->sk), LINUX_MIB_XFRMINHDRERROR);
+ kfree_skb(skb);
+ return;
+ }
+
+ if (data == 0xff) {
+ kfree_skb(skb);
+ return;
+ }
+ }
+
+ /* drop other short messages */
+ if (unlikely(len <= sizeof(nonesp_marker))) {
+ XFRM_INC_STATS(sock_net(strp->sk), LINUX_MIB_XFRMINHDRERROR);
+ kfree_skb(skb);
+ return;
+ }
+
+ err = skb_copy_bits(skb, rxm->offset + 2, &nonesp_marker,
+ sizeof(nonesp_marker));
+ if (err < 0) {
+ XFRM_INC_STATS(sock_net(strp->sk), LINUX_MIB_XFRMINHDRERROR);
+ kfree_skb(skb);
+ return;
+ }
+
+ /* remove header, leave non-ESP marker/SPI */
+ if (!pskb_pull(skb, rxm->offset + 2)) {
+ XFRM_INC_STATS(sock_net(strp->sk), LINUX_MIB_XFRMINERROR);
+ kfree_skb(skb);
+ return;
+ }
+
+ if (pskb_trim(skb, rxm->full_len - 2) != 0) {
+ XFRM_INC_STATS(sock_net(strp->sk), LINUX_MIB_XFRMINERROR);
+ kfree_skb(skb);
+ return;
+ }
+
+ if (nonesp_marker == 0)
+ handle_nonesp(ctx, skb, strp->sk);
+ else
+ handle_esp(skb, strp->sk);
+}
+
+static int espintcp_parse(struct strparser *strp, struct sk_buff *skb)
+{
+ struct strp_msg *rxm = strp_msg(skb);
+ __be16 blen;
+ u16 len;
+ int err;
+
+ if (skb->len < rxm->offset + 2)
+ return 0;
+
+ err = skb_copy_bits(skb, rxm->offset, &blen, sizeof(blen));
+ if (err < 0)
+ return err;
+
+ len = be16_to_cpu(blen);
+ if (len < 2)
+ return -EINVAL;
+
+ return len;
+}
+
+static int espintcp_recvmsg(struct sock *sk, struct msghdr *msg, size_t len,
+ int flags, int *addr_len)
+{
+ struct espintcp_ctx *ctx = espintcp_getctx(sk);
+ struct sk_buff *skb;
+ int err = 0;
+ int copied;
+ int off = 0;
+
+ skb = __skb_recv_datagram(sk, &ctx->ike_queue, flags, &off, &err);
+ if (!skb) {
+ if (err == -EAGAIN && sk->sk_shutdown & RCV_SHUTDOWN)
+ return 0;
+ return err;
+ }
+
+ copied = len;
+ if (copied > skb->len)
+ copied = skb->len;
+ else if (copied < skb->len)
+ msg->msg_flags |= MSG_TRUNC;
+
+ err = skb_copy_datagram_msg(skb, 0, msg, copied);
+ if (unlikely(err)) {
+ kfree_skb(skb);
+ return err;
+ }
+
+ if (flags & MSG_TRUNC)
+ copied = skb->len;
+ kfree_skb(skb);
+ return copied;
+}
+
+int espintcp_queue_out(struct sock *sk, struct sk_buff *skb)
+{
+ struct espintcp_ctx *ctx = espintcp_getctx(sk);
+
+ if (skb_queue_len(&ctx->out_queue) >= READ_ONCE(netdev_max_backlog))
+ return -ENOBUFS;
+
+ __skb_queue_tail(&ctx->out_queue, skb);
+
+ return 0;
+}
+EXPORT_SYMBOL_GPL(espintcp_queue_out);
+
+/* espintcp length field is 2B and length includes the length field's size */
+#define MAX_ESPINTCP_MSG (((1 << 16) - 1) - 2)
+
+static int espintcp_sendskb_locked(struct sock *sk, struct espintcp_msg *emsg,
+ int flags)
+{
+ do {
+ int ret;
+
+ ret = skb_send_sock_locked(sk, emsg->skb,
+ emsg->offset, emsg->len);
+ if (ret < 0)
+ return ret;
+
+ emsg->len -= ret;
+ emsg->offset += ret;
+ } while (emsg->len > 0);
+
+ kfree_skb(emsg->skb);
+ memset(emsg, 0, sizeof(*emsg));
+
+ return 0;
+}
+
+static int espintcp_sendskmsg_locked(struct sock *sk,
+ struct espintcp_msg *emsg, int flags)
+{
+ struct sk_msg *skmsg = &emsg->skmsg;
+ struct scatterlist *sg;
+ int done = 0;
+ int ret;
+
+ flags |= MSG_SENDPAGE_NOTLAST;
+ sg = &skmsg->sg.data[skmsg->sg.start];
+ do {
+ size_t size = sg->length - emsg->offset;
+ int offset = sg->offset + emsg->offset;
+ struct page *p;
+
+ emsg->offset = 0;
+
+ if (sg_is_last(sg))
+ flags &= ~MSG_SENDPAGE_NOTLAST;
+
+ p = sg_page(sg);
+retry:
+ ret = do_tcp_sendpages(sk, p, offset, size, flags);
+ if (ret < 0) {
+ emsg->offset = offset - sg->offset;
+ skmsg->sg.start += done;
+ return ret;
+ }
+
+ if (ret != size) {
+ offset += ret;
+ size -= ret;
+ goto retry;
+ }
+
+ done++;
+ put_page(p);
+ sk_mem_uncharge(sk, sg->length);
+ sg = sg_next(sg);
+ } while (sg);
+
+ memset(emsg, 0, sizeof(*emsg));
+
+ return 0;
+}
+
+static int espintcp_push_msgs(struct sock *sk, int flags)
+{
+ struct espintcp_ctx *ctx = espintcp_getctx(sk);
+ struct espintcp_msg *emsg = &ctx->partial;
+ int err;
+
+ if (!emsg->len)
+ return 0;
+
+ if (ctx->tx_running)
+ return -EAGAIN;
+ ctx->tx_running = 1;
+
+ if (emsg->skb)
+ err = espintcp_sendskb_locked(sk, emsg, flags);
+ else
+ err = espintcp_sendskmsg_locked(sk, emsg, flags);
+ if (err == -EAGAIN) {
+ ctx->tx_running = 0;
+ return flags & MSG_DONTWAIT ? -EAGAIN : 0;
+ }
+ if (!err)
+ memset(emsg, 0, sizeof(*emsg));
+
+ ctx->tx_running = 0;
+
+ return err;
+}
+
+int espintcp_push_skb(struct sock *sk, struct sk_buff *skb)
+{
+ struct espintcp_ctx *ctx = espintcp_getctx(sk);
+ struct espintcp_msg *emsg = &ctx->partial;
+ unsigned int len;
+ int offset;
+
+ if (sk->sk_state != TCP_ESTABLISHED) {
+ kfree_skb(skb);
+ return -ECONNRESET;
+ }
+
+ offset = skb_transport_offset(skb);
+ len = skb->len - offset;
+
+ espintcp_push_msgs(sk, 0);
+
+ if (emsg->len) {
+ kfree_skb(skb);
+ return -ENOBUFS;
+ }
+
+ skb_set_owner_w(skb, sk);
+
+ emsg->offset = offset;
+ emsg->len = len;
+ emsg->skb = skb;
+
+ espintcp_push_msgs(sk, 0);
+
+ return 0;
+}
+EXPORT_SYMBOL_GPL(espintcp_push_skb);
+
+static int espintcp_sendmsg(struct sock *sk, struct msghdr *msg, size_t size)
+{
+ long timeo = sock_sndtimeo(sk, msg->msg_flags & MSG_DONTWAIT);
+ struct espintcp_ctx *ctx = espintcp_getctx(sk);
+ struct espintcp_msg *emsg = &ctx->partial;
+ struct iov_iter pfx_iter;
+ struct kvec pfx_iov = {};
+ size_t msglen = size + 2;
+ char buf[2] = {0};
+ int err, end;
+
+ if (msg->msg_flags & ~MSG_DONTWAIT)
+ return -EOPNOTSUPP;
+
+ if (size > MAX_ESPINTCP_MSG)
+ return -EMSGSIZE;
+
+ if (msg->msg_controllen)
+ return -EOPNOTSUPP;
+
+ lock_sock(sk);
+
+ err = espintcp_push_msgs(sk, msg->msg_flags & MSG_DONTWAIT);
+ if (err < 0) {
+ if (err != -EAGAIN || !(msg->msg_flags & MSG_DONTWAIT))
+ err = -ENOBUFS;
+ goto unlock;
+ }
+
+ sk_msg_init(&emsg->skmsg);
+ while (1) {
+ /* only -ENOMEM is possible since we don't coalesce */
+ err = sk_msg_alloc(sk, &emsg->skmsg, msglen, 0);
+ if (!err)
+ break;
+
+ err = sk_stream_wait_memory(sk, &timeo);
+ if (err)
+ goto fail;
+ }
+
+ *((__be16 *)buf) = cpu_to_be16(msglen);
+ pfx_iov.iov_base = buf;
+ pfx_iov.iov_len = sizeof(buf);
+ iov_iter_kvec(&pfx_iter, ITER_SOURCE, &pfx_iov, 1, pfx_iov.iov_len);
+
+ err = sk_msg_memcopy_from_iter(sk, &pfx_iter, &emsg->skmsg,
+ pfx_iov.iov_len);
+ if (err < 0)
+ goto fail;
+
+ err = sk_msg_memcopy_from_iter(sk, &msg->msg_iter, &emsg->skmsg, size);
+ if (err < 0)
+ goto fail;
+
+ end = emsg->skmsg.sg.end;
+ emsg->len = size;
+ sk_msg_iter_var_prev(end);
+ sg_mark_end(sk_msg_elem(&emsg->skmsg, end));
+
+ tcp_rate_check_app_limited(sk);
+
+ err = espintcp_push_msgs(sk, msg->msg_flags & MSG_DONTWAIT);
+ /* this message could be partially sent, keep it */
+
+ release_sock(sk);
+
+ return size;
+
+fail:
+ sk_msg_free(sk, &emsg->skmsg);
+ memset(emsg, 0, sizeof(*emsg));
+unlock:
+ release_sock(sk);
+ return err;
+}
+
+static struct proto espintcp_prot __ro_after_init;
+static struct proto_ops espintcp_ops __ro_after_init;
+static struct proto espintcp6_prot;
+static struct proto_ops espintcp6_ops;
+static DEFINE_MUTEX(tcpv6_prot_mutex);
+
+static void espintcp_data_ready(struct sock *sk)
+{
+ struct espintcp_ctx *ctx = espintcp_getctx(sk);
+
+ strp_data_ready(&ctx->strp);
+}
+
+static void espintcp_tx_work(struct work_struct *work)
+{
+ struct espintcp_ctx *ctx = container_of(work,
+ struct espintcp_ctx, work);
+ struct sock *sk = ctx->strp.sk;
+
+ lock_sock(sk);
+ if (!ctx->tx_running)
+ espintcp_push_msgs(sk, 0);
+ release_sock(sk);
+}
+
+static void espintcp_write_space(struct sock *sk)
+{
+ struct espintcp_ctx *ctx = espintcp_getctx(sk);
+
+ schedule_work(&ctx->work);
+ ctx->saved_write_space(sk);
+}
+
+static void espintcp_destruct(struct sock *sk)
+{
+ struct espintcp_ctx *ctx = espintcp_getctx(sk);
+
+ ctx->saved_destruct(sk);
+ kfree(ctx);
+}
+
+bool tcp_is_ulp_esp(struct sock *sk)
+{
+ return sk->sk_prot == &espintcp_prot || sk->sk_prot == &espintcp6_prot;
+}
+EXPORT_SYMBOL_GPL(tcp_is_ulp_esp);
+
+static void build_protos(struct proto *espintcp_prot,
+ struct proto_ops *espintcp_ops,
+ const struct proto *orig_prot,
+ const struct proto_ops *orig_ops);
+static int espintcp_init_sk(struct sock *sk)
+{
+ struct inet_connection_sock *icsk = inet_csk(sk);
+ struct strp_callbacks cb = {
+ .rcv_msg = espintcp_rcv,
+ .parse_msg = espintcp_parse,
+ };
+ struct espintcp_ctx *ctx;
+ int err;
+
+ /* sockmap is not compatible with espintcp */
+ if (sk->sk_user_data)
+ return -EBUSY;
+
+ ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
+ if (!ctx)
+ return -ENOMEM;
+
+ err = strp_init(&ctx->strp, sk, &cb);
+ if (err)
+ goto free;
+
+ __sk_dst_reset(sk);
+
+ strp_check_rcv(&ctx->strp);
+ skb_queue_head_init(&ctx->ike_queue);
+ skb_queue_head_init(&ctx->out_queue);
+
+ if (sk->sk_family == AF_INET) {
+ sk->sk_prot = &espintcp_prot;
+ sk->sk_socket->ops = &espintcp_ops;
+ } else {
+ mutex_lock(&tcpv6_prot_mutex);
+ if (!espintcp6_prot.recvmsg)
+ build_protos(&espintcp6_prot, &espintcp6_ops, sk->sk_prot, sk->sk_socket->ops);
+ mutex_unlock(&tcpv6_prot_mutex);
+
+ sk->sk_prot = &espintcp6_prot;
+ sk->sk_socket->ops = &espintcp6_ops;
+ }
+ ctx->saved_data_ready = sk->sk_data_ready;
+ ctx->saved_write_space = sk->sk_write_space;
+ ctx->saved_destruct = sk->sk_destruct;
+ sk->sk_data_ready = espintcp_data_ready;
+ sk->sk_write_space = espintcp_write_space;
+ sk->sk_destruct = espintcp_destruct;
+ rcu_assign_pointer(icsk->icsk_ulp_data, ctx);
+ INIT_WORK(&ctx->work, espintcp_tx_work);
+
+ /* avoid using task_frag */
+ sk->sk_allocation = GFP_ATOMIC;
+
+ return 0;
+
+free:
+ kfree(ctx);
+ return err;
+}
+
+static void espintcp_release(struct sock *sk)
+{
+ struct espintcp_ctx *ctx = espintcp_getctx(sk);
+ struct sk_buff_head queue;
+ struct sk_buff *skb;
+
+ __skb_queue_head_init(&queue);
+ skb_queue_splice_init(&ctx->out_queue, &queue);
+
+ while ((skb = __skb_dequeue(&queue)))
+ espintcp_push_skb(sk, skb);
+
+ tcp_release_cb(sk);
+}
+
+static void espintcp_close(struct sock *sk, long timeout)
+{
+ struct espintcp_ctx *ctx = espintcp_getctx(sk);
+ struct espintcp_msg *emsg = &ctx->partial;
+
+ strp_stop(&ctx->strp);
+
+ sk->sk_prot = &tcp_prot;
+ barrier();
+
+ cancel_work_sync(&ctx->work);
+ strp_done(&ctx->strp);
+
+ skb_queue_purge(&ctx->out_queue);
+ skb_queue_purge(&ctx->ike_queue);
+
+ if (emsg->len) {
+ if (emsg->skb)
+ kfree_skb(emsg->skb);
+ else
+ sk_msg_free(sk, &emsg->skmsg);
+ }
+
+ tcp_close(sk, timeout);
+}
+
+static __poll_t espintcp_poll(struct file *file, struct socket *sock,
+ poll_table *wait)
+{
+ __poll_t mask = datagram_poll(file, sock, wait);
+ struct sock *sk = sock->sk;
+ struct espintcp_ctx *ctx = espintcp_getctx(sk);
+
+ if (!skb_queue_empty(&ctx->ike_queue))
+ mask |= EPOLLIN | EPOLLRDNORM;
+
+ return mask;
+}
+
+static void build_protos(struct proto *espintcp_prot,
+ struct proto_ops *espintcp_ops,
+ const struct proto *orig_prot,
+ const struct proto_ops *orig_ops)
+{
+ memcpy(espintcp_prot, orig_prot, sizeof(struct proto));
+ memcpy(espintcp_ops, orig_ops, sizeof(struct proto_ops));
+ espintcp_prot->sendmsg = espintcp_sendmsg;
+ espintcp_prot->recvmsg = espintcp_recvmsg;
+ espintcp_prot->close = espintcp_close;
+ espintcp_prot->release_cb = espintcp_release;
+ espintcp_ops->poll = espintcp_poll;
+}
+
+static struct tcp_ulp_ops espintcp_ulp __read_mostly = {
+ .name = "espintcp",
+ .owner = THIS_MODULE,
+ .init = espintcp_init_sk,
+};
+
+void __init espintcp_init(void)
+{
+ build_protos(&espintcp_prot, &espintcp_ops, &tcp_prot, &inet_stream_ops);
+
+ tcp_register_ulp(&espintcp_ulp);
+}
diff --git a/net/xfrm/xfrm_algo.c b/net/xfrm/xfrm_algo.c
new file mode 100644
index 000000000..094734fbe
--- /dev/null
+++ b/net/xfrm/xfrm_algo.c
@@ -0,0 +1,866 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * xfrm algorithm interface
+ *
+ * Copyright (c) 2002 James Morris <jmorris@intercode.com.au>
+ */
+
+#include <crypto/hash.h>
+#include <crypto/skcipher.h>
+#include <linux/module.h>
+#include <linux/kernel.h>
+#include <linux/pfkeyv2.h>
+#include <linux/crypto.h>
+#include <linux/scatterlist.h>
+#include <net/xfrm.h>
+#if IS_ENABLED(CONFIG_INET_ESP) || IS_ENABLED(CONFIG_INET6_ESP)
+#include <net/esp.h>
+#endif
+
+/*
+ * Algorithms supported by IPsec. These entries contain properties which
+ * are used in key negotiation and xfrm processing, and are used to verify
+ * that instantiated crypto transforms have correct parameters for IPsec
+ * purposes.
+ */
+static struct xfrm_algo_desc aead_list[] = {
+{
+ .name = "rfc4106(gcm(aes))",
+
+ .uinfo = {
+ .aead = {
+ .geniv = "seqiv",
+ .icv_truncbits = 64,
+ }
+ },
+
+ .pfkey_supported = 1,
+
+ .desc = {
+ .sadb_alg_id = SADB_X_EALG_AES_GCM_ICV8,
+ .sadb_alg_ivlen = 8,
+ .sadb_alg_minbits = 128,
+ .sadb_alg_maxbits = 256
+ }
+},
+{
+ .name = "rfc4106(gcm(aes))",
+
+ .uinfo = {
+ .aead = {
+ .geniv = "seqiv",
+ .icv_truncbits = 96,
+ }
+ },
+
+ .pfkey_supported = 1,
+
+ .desc = {
+ .sadb_alg_id = SADB_X_EALG_AES_GCM_ICV12,
+ .sadb_alg_ivlen = 8,
+ .sadb_alg_minbits = 128,
+ .sadb_alg_maxbits = 256
+ }
+},
+{
+ .name = "rfc4106(gcm(aes))",
+
+ .uinfo = {
+ .aead = {
+ .geniv = "seqiv",
+ .icv_truncbits = 128,
+ }
+ },
+
+ .pfkey_supported = 1,
+
+ .desc = {
+ .sadb_alg_id = SADB_X_EALG_AES_GCM_ICV16,
+ .sadb_alg_ivlen = 8,
+ .sadb_alg_minbits = 128,
+ .sadb_alg_maxbits = 256
+ }
+},
+{
+ .name = "rfc4309(ccm(aes))",
+
+ .uinfo = {
+ .aead = {
+ .geniv = "seqiv",
+ .icv_truncbits = 64,
+ }
+ },
+
+ .pfkey_supported = 1,
+
+ .desc = {
+ .sadb_alg_id = SADB_X_EALG_AES_CCM_ICV8,
+ .sadb_alg_ivlen = 8,
+ .sadb_alg_minbits = 128,
+ .sadb_alg_maxbits = 256
+ }
+},
+{
+ .name = "rfc4309(ccm(aes))",
+
+ .uinfo = {
+ .aead = {
+ .geniv = "seqiv",
+ .icv_truncbits = 96,
+ }
+ },
+
+ .pfkey_supported = 1,
+
+ .desc = {
+ .sadb_alg_id = SADB_X_EALG_AES_CCM_ICV12,
+ .sadb_alg_ivlen = 8,
+ .sadb_alg_minbits = 128,
+ .sadb_alg_maxbits = 256
+ }
+},
+{
+ .name = "rfc4309(ccm(aes))",
+
+ .uinfo = {
+ .aead = {
+ .geniv = "seqiv",
+ .icv_truncbits = 128,
+ }
+ },
+
+ .pfkey_supported = 1,
+
+ .desc = {
+ .sadb_alg_id = SADB_X_EALG_AES_CCM_ICV16,
+ .sadb_alg_ivlen = 8,
+ .sadb_alg_minbits = 128,
+ .sadb_alg_maxbits = 256
+ }
+},
+{
+ .name = "rfc4543(gcm(aes))",
+
+ .uinfo = {
+ .aead = {
+ .geniv = "seqiv",
+ .icv_truncbits = 128,
+ }
+ },
+
+ .pfkey_supported = 1,
+
+ .desc = {
+ .sadb_alg_id = SADB_X_EALG_NULL_AES_GMAC,
+ .sadb_alg_ivlen = 8,
+ .sadb_alg_minbits = 128,
+ .sadb_alg_maxbits = 256
+ }
+},
+{
+ .name = "rfc7539esp(chacha20,poly1305)",
+
+ .uinfo = {
+ .aead = {
+ .geniv = "seqiv",
+ .icv_truncbits = 128,
+ }
+ },
+
+ .pfkey_supported = 0,
+},
+};
+
+static struct xfrm_algo_desc aalg_list[] = {
+{
+ .name = "digest_null",
+
+ .uinfo = {
+ .auth = {
+ .icv_truncbits = 0,
+ .icv_fullbits = 0,
+ }
+ },
+
+ .pfkey_supported = 1,
+
+ .desc = {
+ .sadb_alg_id = SADB_X_AALG_NULL,
+ .sadb_alg_ivlen = 0,
+ .sadb_alg_minbits = 0,
+ .sadb_alg_maxbits = 0
+ }
+},
+{
+ .name = "hmac(md5)",
+ .compat = "md5",
+
+ .uinfo = {
+ .auth = {
+ .icv_truncbits = 96,
+ .icv_fullbits = 128,
+ }
+ },
+
+ .pfkey_supported = 1,
+
+ .desc = {
+ .sadb_alg_id = SADB_AALG_MD5HMAC,
+ .sadb_alg_ivlen = 0,
+ .sadb_alg_minbits = 128,
+ .sadb_alg_maxbits = 128
+ }
+},
+{
+ .name = "hmac(sha1)",
+ .compat = "sha1",
+
+ .uinfo = {
+ .auth = {
+ .icv_truncbits = 96,
+ .icv_fullbits = 160,
+ }
+ },
+
+ .pfkey_supported = 1,
+
+ .desc = {
+ .sadb_alg_id = SADB_AALG_SHA1HMAC,
+ .sadb_alg_ivlen = 0,
+ .sadb_alg_minbits = 160,
+ .sadb_alg_maxbits = 160
+ }
+},
+{
+ .name = "hmac(sha256)",
+ .compat = "sha256",
+
+ .uinfo = {
+ .auth = {
+ .icv_truncbits = 96,
+ .icv_fullbits = 256,
+ }
+ },
+
+ .pfkey_supported = 1,
+
+ .desc = {
+ .sadb_alg_id = SADB_X_AALG_SHA2_256HMAC,
+ .sadb_alg_ivlen = 0,
+ .sadb_alg_minbits = 256,
+ .sadb_alg_maxbits = 256
+ }
+},
+{
+ .name = "hmac(sha384)",
+
+ .uinfo = {
+ .auth = {
+ .icv_truncbits = 192,
+ .icv_fullbits = 384,
+ }
+ },
+
+ .pfkey_supported = 1,
+
+ .desc = {
+ .sadb_alg_id = SADB_X_AALG_SHA2_384HMAC,
+ .sadb_alg_ivlen = 0,
+ .sadb_alg_minbits = 384,
+ .sadb_alg_maxbits = 384
+ }
+},
+{
+ .name = "hmac(sha512)",
+
+ .uinfo = {
+ .auth = {
+ .icv_truncbits = 256,
+ .icv_fullbits = 512,
+ }
+ },
+
+ .pfkey_supported = 1,
+
+ .desc = {
+ .sadb_alg_id = SADB_X_AALG_SHA2_512HMAC,
+ .sadb_alg_ivlen = 0,
+ .sadb_alg_minbits = 512,
+ .sadb_alg_maxbits = 512
+ }
+},
+{
+ .name = "hmac(rmd160)",
+ .compat = "rmd160",
+
+ .uinfo = {
+ .auth = {
+ .icv_truncbits = 96,
+ .icv_fullbits = 160,
+ }
+ },
+
+ .pfkey_supported = 1,
+
+ .desc = {
+ .sadb_alg_id = SADB_X_AALG_RIPEMD160HMAC,
+ .sadb_alg_ivlen = 0,
+ .sadb_alg_minbits = 160,
+ .sadb_alg_maxbits = 160
+ }
+},
+{
+ .name = "xcbc(aes)",
+
+ .uinfo = {
+ .auth = {
+ .icv_truncbits = 96,
+ .icv_fullbits = 128,
+ }
+ },
+
+ .pfkey_supported = 1,
+
+ .desc = {
+ .sadb_alg_id = SADB_X_AALG_AES_XCBC_MAC,
+ .sadb_alg_ivlen = 0,
+ .sadb_alg_minbits = 128,
+ .sadb_alg_maxbits = 128
+ }
+},
+{
+ /* rfc4494 */
+ .name = "cmac(aes)",
+
+ .uinfo = {
+ .auth = {
+ .icv_truncbits = 96,
+ .icv_fullbits = 128,
+ }
+ },
+
+ .pfkey_supported = 0,
+},
+{
+ .name = "hmac(sm3)",
+ .compat = "sm3",
+
+ .uinfo = {
+ .auth = {
+ .icv_truncbits = 256,
+ .icv_fullbits = 256,
+ }
+ },
+
+ .pfkey_supported = 1,
+
+ .desc = {
+ .sadb_alg_id = SADB_X_AALG_SM3_256HMAC,
+ .sadb_alg_ivlen = 0,
+ .sadb_alg_minbits = 256,
+ .sadb_alg_maxbits = 256
+ }
+},
+};
+
+static struct xfrm_algo_desc ealg_list[] = {
+{
+ .name = "ecb(cipher_null)",
+ .compat = "cipher_null",
+
+ .uinfo = {
+ .encr = {
+ .blockbits = 8,
+ .defkeybits = 0,
+ }
+ },
+
+ .pfkey_supported = 1,
+
+ .desc = {
+ .sadb_alg_id = SADB_EALG_NULL,
+ .sadb_alg_ivlen = 0,
+ .sadb_alg_minbits = 0,
+ .sadb_alg_maxbits = 0
+ }
+},
+{
+ .name = "cbc(des)",
+ .compat = "des",
+
+ .uinfo = {
+ .encr = {
+ .geniv = "echainiv",
+ .blockbits = 64,
+ .defkeybits = 64,
+ }
+ },
+
+ .pfkey_supported = 1,
+
+ .desc = {
+ .sadb_alg_id = SADB_EALG_DESCBC,
+ .sadb_alg_ivlen = 8,
+ .sadb_alg_minbits = 64,
+ .sadb_alg_maxbits = 64
+ }
+},
+{
+ .name = "cbc(des3_ede)",
+ .compat = "des3_ede",
+
+ .uinfo = {
+ .encr = {
+ .geniv = "echainiv",
+ .blockbits = 64,
+ .defkeybits = 192,
+ }
+ },
+
+ .pfkey_supported = 1,
+
+ .desc = {
+ .sadb_alg_id = SADB_EALG_3DESCBC,
+ .sadb_alg_ivlen = 8,
+ .sadb_alg_minbits = 192,
+ .sadb_alg_maxbits = 192
+ }
+},
+{
+ .name = "cbc(cast5)",
+ .compat = "cast5",
+
+ .uinfo = {
+ .encr = {
+ .geniv = "echainiv",
+ .blockbits = 64,
+ .defkeybits = 128,
+ }
+ },
+
+ .pfkey_supported = 1,
+
+ .desc = {
+ .sadb_alg_id = SADB_X_EALG_CASTCBC,
+ .sadb_alg_ivlen = 8,
+ .sadb_alg_minbits = 40,
+ .sadb_alg_maxbits = 128
+ }
+},
+{
+ .name = "cbc(blowfish)",
+ .compat = "blowfish",
+
+ .uinfo = {
+ .encr = {
+ .geniv = "echainiv",
+ .blockbits = 64,
+ .defkeybits = 128,
+ }
+ },
+
+ .pfkey_supported = 1,
+
+ .desc = {
+ .sadb_alg_id = SADB_X_EALG_BLOWFISHCBC,
+ .sadb_alg_ivlen = 8,
+ .sadb_alg_minbits = 40,
+ .sadb_alg_maxbits = 448
+ }
+},
+{
+ .name = "cbc(aes)",
+ .compat = "aes",
+
+ .uinfo = {
+ .encr = {
+ .geniv = "echainiv",
+ .blockbits = 128,
+ .defkeybits = 128,
+ }
+ },
+
+ .pfkey_supported = 1,
+
+ .desc = {
+ .sadb_alg_id = SADB_X_EALG_AESCBC,
+ .sadb_alg_ivlen = 8,
+ .sadb_alg_minbits = 128,
+ .sadb_alg_maxbits = 256
+ }
+},
+{
+ .name = "cbc(serpent)",
+ .compat = "serpent",
+
+ .uinfo = {
+ .encr = {
+ .geniv = "echainiv",
+ .blockbits = 128,
+ .defkeybits = 128,
+ }
+ },
+
+ .pfkey_supported = 1,
+
+ .desc = {
+ .sadb_alg_id = SADB_X_EALG_SERPENTCBC,
+ .sadb_alg_ivlen = 8,
+ .sadb_alg_minbits = 128,
+ .sadb_alg_maxbits = 256,
+ }
+},
+{
+ .name = "cbc(camellia)",
+ .compat = "camellia",
+
+ .uinfo = {
+ .encr = {
+ .geniv = "echainiv",
+ .blockbits = 128,
+ .defkeybits = 128,
+ }
+ },
+
+ .pfkey_supported = 1,
+
+ .desc = {
+ .sadb_alg_id = SADB_X_EALG_CAMELLIACBC,
+ .sadb_alg_ivlen = 8,
+ .sadb_alg_minbits = 128,
+ .sadb_alg_maxbits = 256
+ }
+},
+{
+ .name = "cbc(twofish)",
+ .compat = "twofish",
+
+ .uinfo = {
+ .encr = {
+ .geniv = "echainiv",
+ .blockbits = 128,
+ .defkeybits = 128,
+ }
+ },
+
+ .pfkey_supported = 1,
+
+ .desc = {
+ .sadb_alg_id = SADB_X_EALG_TWOFISHCBC,
+ .sadb_alg_ivlen = 8,
+ .sadb_alg_minbits = 128,
+ .sadb_alg_maxbits = 256
+ }
+},
+{
+ .name = "rfc3686(ctr(aes))",
+
+ .uinfo = {
+ .encr = {
+ .geniv = "seqiv",
+ .blockbits = 128,
+ .defkeybits = 160, /* 128-bit key + 32-bit nonce */
+ }
+ },
+
+ .pfkey_supported = 1,
+
+ .desc = {
+ .sadb_alg_id = SADB_X_EALG_AESCTR,
+ .sadb_alg_ivlen = 8,
+ .sadb_alg_minbits = 160,
+ .sadb_alg_maxbits = 288
+ }
+},
+{
+ .name = "cbc(sm4)",
+ .compat = "sm4",
+
+ .uinfo = {
+ .encr = {
+ .geniv = "echainiv",
+ .blockbits = 128,
+ .defkeybits = 128,
+ }
+ },
+
+ .pfkey_supported = 1,
+
+ .desc = {
+ .sadb_alg_id = SADB_X_EALG_SM4CBC,
+ .sadb_alg_ivlen = 16,
+ .sadb_alg_minbits = 128,
+ .sadb_alg_maxbits = 256
+ }
+},
+};
+
+static struct xfrm_algo_desc calg_list[] = {
+{
+ .name = "deflate",
+ .uinfo = {
+ .comp = {
+ .threshold = 90,
+ }
+ },
+ .pfkey_supported = 1,
+ .desc = { .sadb_alg_id = SADB_X_CALG_DEFLATE }
+},
+{
+ .name = "lzs",
+ .uinfo = {
+ .comp = {
+ .threshold = 90,
+ }
+ },
+ .pfkey_supported = 1,
+ .desc = { .sadb_alg_id = SADB_X_CALG_LZS }
+},
+{
+ .name = "lzjh",
+ .uinfo = {
+ .comp = {
+ .threshold = 50,
+ }
+ },
+ .pfkey_supported = 1,
+ .desc = { .sadb_alg_id = SADB_X_CALG_LZJH }
+},
+};
+
+static inline int aalg_entries(void)
+{
+ return ARRAY_SIZE(aalg_list);
+}
+
+static inline int ealg_entries(void)
+{
+ return ARRAY_SIZE(ealg_list);
+}
+
+static inline int calg_entries(void)
+{
+ return ARRAY_SIZE(calg_list);
+}
+
+struct xfrm_algo_list {
+ struct xfrm_algo_desc *algs;
+ int entries;
+ u32 type;
+ u32 mask;
+};
+
+static const struct xfrm_algo_list xfrm_aead_list = {
+ .algs = aead_list,
+ .entries = ARRAY_SIZE(aead_list),
+ .type = CRYPTO_ALG_TYPE_AEAD,
+ .mask = CRYPTO_ALG_TYPE_MASK,
+};
+
+static const struct xfrm_algo_list xfrm_aalg_list = {
+ .algs = aalg_list,
+ .entries = ARRAY_SIZE(aalg_list),
+ .type = CRYPTO_ALG_TYPE_HASH,
+ .mask = CRYPTO_ALG_TYPE_HASH_MASK,
+};
+
+static const struct xfrm_algo_list xfrm_ealg_list = {
+ .algs = ealg_list,
+ .entries = ARRAY_SIZE(ealg_list),
+ .type = CRYPTO_ALG_TYPE_SKCIPHER,
+ .mask = CRYPTO_ALG_TYPE_MASK,
+};
+
+static const struct xfrm_algo_list xfrm_calg_list = {
+ .algs = calg_list,
+ .entries = ARRAY_SIZE(calg_list),
+ .type = CRYPTO_ALG_TYPE_COMPRESS,
+ .mask = CRYPTO_ALG_TYPE_MASK,
+};
+
+static struct xfrm_algo_desc *xfrm_find_algo(
+ const struct xfrm_algo_list *algo_list,
+ int match(const struct xfrm_algo_desc *entry, const void *data),
+ const void *data, int probe)
+{
+ struct xfrm_algo_desc *list = algo_list->algs;
+ int i, status;
+
+ for (i = 0; i < algo_list->entries; i++) {
+ if (!match(list + i, data))
+ continue;
+
+ if (list[i].available)
+ return &list[i];
+
+ if (!probe)
+ break;
+
+ status = crypto_has_alg(list[i].name, algo_list->type,
+ algo_list->mask);
+ if (!status)
+ break;
+
+ list[i].available = status;
+ return &list[i];
+ }
+ return NULL;
+}
+
+static int xfrm_alg_id_match(const struct xfrm_algo_desc *entry,
+ const void *data)
+{
+ return entry->desc.sadb_alg_id == (unsigned long)data;
+}
+
+struct xfrm_algo_desc *xfrm_aalg_get_byid(int alg_id)
+{
+ return xfrm_find_algo(&xfrm_aalg_list, xfrm_alg_id_match,
+ (void *)(unsigned long)alg_id, 1);
+}
+EXPORT_SYMBOL_GPL(xfrm_aalg_get_byid);
+
+struct xfrm_algo_desc *xfrm_ealg_get_byid(int alg_id)
+{
+ return xfrm_find_algo(&xfrm_ealg_list, xfrm_alg_id_match,
+ (void *)(unsigned long)alg_id, 1);
+}
+EXPORT_SYMBOL_GPL(xfrm_ealg_get_byid);
+
+struct xfrm_algo_desc *xfrm_calg_get_byid(int alg_id)
+{
+ return xfrm_find_algo(&xfrm_calg_list, xfrm_alg_id_match,
+ (void *)(unsigned long)alg_id, 1);
+}
+EXPORT_SYMBOL_GPL(xfrm_calg_get_byid);
+
+static int xfrm_alg_name_match(const struct xfrm_algo_desc *entry,
+ const void *data)
+{
+ const char *name = data;
+
+ return name && (!strcmp(name, entry->name) ||
+ (entry->compat && !strcmp(name, entry->compat)));
+}
+
+struct xfrm_algo_desc *xfrm_aalg_get_byname(const char *name, int probe)
+{
+ return xfrm_find_algo(&xfrm_aalg_list, xfrm_alg_name_match, name,
+ probe);
+}
+EXPORT_SYMBOL_GPL(xfrm_aalg_get_byname);
+
+struct xfrm_algo_desc *xfrm_ealg_get_byname(const char *name, int probe)
+{
+ return xfrm_find_algo(&xfrm_ealg_list, xfrm_alg_name_match, name,
+ probe);
+}
+EXPORT_SYMBOL_GPL(xfrm_ealg_get_byname);
+
+struct xfrm_algo_desc *xfrm_calg_get_byname(const char *name, int probe)
+{
+ return xfrm_find_algo(&xfrm_calg_list, xfrm_alg_name_match, name,
+ probe);
+}
+EXPORT_SYMBOL_GPL(xfrm_calg_get_byname);
+
+struct xfrm_aead_name {
+ const char *name;
+ int icvbits;
+};
+
+static int xfrm_aead_name_match(const struct xfrm_algo_desc *entry,
+ const void *data)
+{
+ const struct xfrm_aead_name *aead = data;
+ const char *name = aead->name;
+
+ return aead->icvbits == entry->uinfo.aead.icv_truncbits && name &&
+ !strcmp(name, entry->name);
+}
+
+struct xfrm_algo_desc *xfrm_aead_get_byname(const char *name, int icv_len, int probe)
+{
+ struct xfrm_aead_name data = {
+ .name = name,
+ .icvbits = icv_len,
+ };
+
+ return xfrm_find_algo(&xfrm_aead_list, xfrm_aead_name_match, &data,
+ probe);
+}
+EXPORT_SYMBOL_GPL(xfrm_aead_get_byname);
+
+struct xfrm_algo_desc *xfrm_aalg_get_byidx(unsigned int idx)
+{
+ if (idx >= aalg_entries())
+ return NULL;
+
+ return &aalg_list[idx];
+}
+EXPORT_SYMBOL_GPL(xfrm_aalg_get_byidx);
+
+struct xfrm_algo_desc *xfrm_ealg_get_byidx(unsigned int idx)
+{
+ if (idx >= ealg_entries())
+ return NULL;
+
+ return &ealg_list[idx];
+}
+EXPORT_SYMBOL_GPL(xfrm_ealg_get_byidx);
+
+/*
+ * Probe for the availability of crypto algorithms, and set the available
+ * flag for any algorithms found on the system. This is typically called by
+ * pfkey during userspace SA add, update or register.
+ */
+void xfrm_probe_algs(void)
+{
+ int i, status;
+
+ BUG_ON(in_softirq());
+
+ for (i = 0; i < aalg_entries(); i++) {
+ status = crypto_has_ahash(aalg_list[i].name, 0, 0);
+ if (aalg_list[i].available != status)
+ aalg_list[i].available = status;
+ }
+
+ for (i = 0; i < ealg_entries(); i++) {
+ status = crypto_has_skcipher(ealg_list[i].name, 0, 0);
+ if (ealg_list[i].available != status)
+ ealg_list[i].available = status;
+ }
+
+ for (i = 0; i < calg_entries(); i++) {
+ status = crypto_has_comp(calg_list[i].name, 0,
+ CRYPTO_ALG_ASYNC);
+ if (calg_list[i].available != status)
+ calg_list[i].available = status;
+ }
+}
+EXPORT_SYMBOL_GPL(xfrm_probe_algs);
+
+int xfrm_count_pfkey_auth_supported(void)
+{
+ int i, n;
+
+ for (i = 0, n = 0; i < aalg_entries(); i++)
+ if (aalg_list[i].available && aalg_list[i].pfkey_supported)
+ n++;
+ return n;
+}
+EXPORT_SYMBOL_GPL(xfrm_count_pfkey_auth_supported);
+
+int xfrm_count_pfkey_enc_supported(void)
+{
+ int i, n;
+
+ for (i = 0, n = 0; i < ealg_entries(); i++)
+ if (ealg_list[i].available && ealg_list[i].pfkey_supported)
+ n++;
+ return n;
+}
+EXPORT_SYMBOL_GPL(xfrm_count_pfkey_enc_supported);
+
+MODULE_LICENSE("GPL");
diff --git a/net/xfrm/xfrm_compat.c b/net/xfrm/xfrm_compat.c
new file mode 100644
index 000000000..655fe4ff8
--- /dev/null
+++ b/net/xfrm/xfrm_compat.c
@@ -0,0 +1,675 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * XFRM compat layer
+ * Author: Dmitry Safonov <dima@arista.com>
+ * Based on code and translator idea by: Florian Westphal <fw@strlen.de>
+ */
+#include <linux/compat.h>
+#include <linux/nospec.h>
+#include <linux/xfrm.h>
+#include <net/xfrm.h>
+
+struct compat_xfrm_lifetime_cfg {
+ compat_u64 soft_byte_limit, hard_byte_limit;
+ compat_u64 soft_packet_limit, hard_packet_limit;
+ compat_u64 soft_add_expires_seconds, hard_add_expires_seconds;
+ compat_u64 soft_use_expires_seconds, hard_use_expires_seconds;
+}; /* same size on 32bit, but only 4 byte alignment required */
+
+struct compat_xfrm_lifetime_cur {
+ compat_u64 bytes, packets, add_time, use_time;
+}; /* same size on 32bit, but only 4 byte alignment required */
+
+struct compat_xfrm_userpolicy_info {
+ struct xfrm_selector sel;
+ struct compat_xfrm_lifetime_cfg lft;
+ struct compat_xfrm_lifetime_cur curlft;
+ __u32 priority, index;
+ u8 dir, action, flags, share;
+ /* 4 bytes additional padding on 64bit */
+};
+
+struct compat_xfrm_usersa_info {
+ struct xfrm_selector sel;
+ struct xfrm_id id;
+ xfrm_address_t saddr;
+ struct compat_xfrm_lifetime_cfg lft;
+ struct compat_xfrm_lifetime_cur curlft;
+ struct xfrm_stats stats;
+ __u32 seq, reqid;
+ u16 family;
+ u8 mode, replay_window, flags;
+ /* 4 bytes additional padding on 64bit */
+};
+
+struct compat_xfrm_user_acquire {
+ struct xfrm_id id;
+ xfrm_address_t saddr;
+ struct xfrm_selector sel;
+ struct compat_xfrm_userpolicy_info policy;
+ /* 4 bytes additional padding on 64bit */
+ __u32 aalgos, ealgos, calgos, seq;
+};
+
+struct compat_xfrm_userspi_info {
+ struct compat_xfrm_usersa_info info;
+ /* 4 bytes additional padding on 64bit */
+ __u32 min, max;
+};
+
+struct compat_xfrm_user_expire {
+ struct compat_xfrm_usersa_info state;
+ /* 8 bytes additional padding on 64bit */
+ u8 hard;
+};
+
+struct compat_xfrm_user_polexpire {
+ struct compat_xfrm_userpolicy_info pol;
+ /* 8 bytes additional padding on 64bit */
+ u8 hard;
+};
+
+#define XMSGSIZE(type) sizeof(struct type)
+
+static const int compat_msg_min[XFRM_NR_MSGTYPES] = {
+ [XFRM_MSG_NEWSA - XFRM_MSG_BASE] = XMSGSIZE(compat_xfrm_usersa_info),
+ [XFRM_MSG_DELSA - XFRM_MSG_BASE] = XMSGSIZE(xfrm_usersa_id),
+ [XFRM_MSG_GETSA - XFRM_MSG_BASE] = XMSGSIZE(xfrm_usersa_id),
+ [XFRM_MSG_NEWPOLICY - XFRM_MSG_BASE] = XMSGSIZE(compat_xfrm_userpolicy_info),
+ [XFRM_MSG_DELPOLICY - XFRM_MSG_BASE] = XMSGSIZE(xfrm_userpolicy_id),
+ [XFRM_MSG_GETPOLICY - XFRM_MSG_BASE] = XMSGSIZE(xfrm_userpolicy_id),
+ [XFRM_MSG_ALLOCSPI - XFRM_MSG_BASE] = XMSGSIZE(compat_xfrm_userspi_info),
+ [XFRM_MSG_ACQUIRE - XFRM_MSG_BASE] = XMSGSIZE(compat_xfrm_user_acquire),
+ [XFRM_MSG_EXPIRE - XFRM_MSG_BASE] = XMSGSIZE(compat_xfrm_user_expire),
+ [XFRM_MSG_UPDPOLICY - XFRM_MSG_BASE] = XMSGSIZE(compat_xfrm_userpolicy_info),
+ [XFRM_MSG_UPDSA - XFRM_MSG_BASE] = XMSGSIZE(compat_xfrm_usersa_info),
+ [XFRM_MSG_POLEXPIRE - XFRM_MSG_BASE] = XMSGSIZE(compat_xfrm_user_polexpire),
+ [XFRM_MSG_FLUSHSA - XFRM_MSG_BASE] = XMSGSIZE(xfrm_usersa_flush),
+ [XFRM_MSG_FLUSHPOLICY - XFRM_MSG_BASE] = 0,
+ [XFRM_MSG_NEWAE - XFRM_MSG_BASE] = XMSGSIZE(xfrm_aevent_id),
+ [XFRM_MSG_GETAE - XFRM_MSG_BASE] = XMSGSIZE(xfrm_aevent_id),
+ [XFRM_MSG_REPORT - XFRM_MSG_BASE] = XMSGSIZE(xfrm_user_report),
+ [XFRM_MSG_MIGRATE - XFRM_MSG_BASE] = XMSGSIZE(xfrm_userpolicy_id),
+ [XFRM_MSG_NEWSADINFO - XFRM_MSG_BASE] = sizeof(u32),
+ [XFRM_MSG_GETSADINFO - XFRM_MSG_BASE] = sizeof(u32),
+ [XFRM_MSG_NEWSPDINFO - XFRM_MSG_BASE] = sizeof(u32),
+ [XFRM_MSG_GETSPDINFO - XFRM_MSG_BASE] = sizeof(u32),
+ [XFRM_MSG_MAPPING - XFRM_MSG_BASE] = XMSGSIZE(xfrm_user_mapping)
+};
+
+static const struct nla_policy compat_policy[XFRMA_MAX+1] = {
+ [XFRMA_SA] = { .len = XMSGSIZE(compat_xfrm_usersa_info)},
+ [XFRMA_POLICY] = { .len = XMSGSIZE(compat_xfrm_userpolicy_info)},
+ [XFRMA_LASTUSED] = { .type = NLA_U64},
+ [XFRMA_ALG_AUTH_TRUNC] = { .len = sizeof(struct xfrm_algo_auth)},
+ [XFRMA_ALG_AEAD] = { .len = sizeof(struct xfrm_algo_aead) },
+ [XFRMA_ALG_AUTH] = { .len = sizeof(struct xfrm_algo) },
+ [XFRMA_ALG_CRYPT] = { .len = sizeof(struct xfrm_algo) },
+ [XFRMA_ALG_COMP] = { .len = sizeof(struct xfrm_algo) },
+ [XFRMA_ENCAP] = { .len = sizeof(struct xfrm_encap_tmpl) },
+ [XFRMA_TMPL] = { .len = sizeof(struct xfrm_user_tmpl) },
+ [XFRMA_SEC_CTX] = { .len = sizeof(struct xfrm_user_sec_ctx) },
+ [XFRMA_LTIME_VAL] = { .len = sizeof(struct xfrm_lifetime_cur) },
+ [XFRMA_REPLAY_VAL] = { .len = sizeof(struct xfrm_replay_state) },
+ [XFRMA_REPLAY_THRESH] = { .type = NLA_U32 },
+ [XFRMA_ETIMER_THRESH] = { .type = NLA_U32 },
+ [XFRMA_SRCADDR] = { .len = sizeof(xfrm_address_t) },
+ [XFRMA_COADDR] = { .len = sizeof(xfrm_address_t) },
+ [XFRMA_POLICY_TYPE] = { .len = sizeof(struct xfrm_userpolicy_type)},
+ [XFRMA_MIGRATE] = { .len = sizeof(struct xfrm_user_migrate) },
+ [XFRMA_KMADDRESS] = { .len = sizeof(struct xfrm_user_kmaddress) },
+ [XFRMA_MARK] = { .len = sizeof(struct xfrm_mark) },
+ [XFRMA_TFCPAD] = { .type = NLA_U32 },
+ [XFRMA_REPLAY_ESN_VAL] = { .len = sizeof(struct xfrm_replay_state_esn) },
+ [XFRMA_SA_EXTRA_FLAGS] = { .type = NLA_U32 },
+ [XFRMA_PROTO] = { .type = NLA_U8 },
+ [XFRMA_ADDRESS_FILTER] = { .len = sizeof(struct xfrm_address_filter) },
+ [XFRMA_OFFLOAD_DEV] = { .len = sizeof(struct xfrm_user_offload) },
+ [XFRMA_SET_MARK] = { .type = NLA_U32 },
+ [XFRMA_SET_MARK_MASK] = { .type = NLA_U32 },
+ [XFRMA_IF_ID] = { .type = NLA_U32 },
+ [XFRMA_MTIMER_THRESH] = { .type = NLA_U32 },
+};
+
+static struct nlmsghdr *xfrm_nlmsg_put_compat(struct sk_buff *skb,
+ const struct nlmsghdr *nlh_src, u16 type)
+{
+ int payload = compat_msg_min[type];
+ int src_len = xfrm_msg_min[type];
+ struct nlmsghdr *nlh_dst;
+
+ /* Compat messages are shorter or equal to native (+padding) */
+ if (WARN_ON_ONCE(src_len < payload))
+ return ERR_PTR(-EMSGSIZE);
+
+ nlh_dst = nlmsg_put(skb, nlh_src->nlmsg_pid, nlh_src->nlmsg_seq,
+ nlh_src->nlmsg_type, payload, nlh_src->nlmsg_flags);
+ if (!nlh_dst)
+ return ERR_PTR(-EMSGSIZE);
+
+ memset(nlmsg_data(nlh_dst), 0, payload);
+
+ switch (nlh_src->nlmsg_type) {
+ /* Compat message has the same layout as native */
+ case XFRM_MSG_DELSA:
+ case XFRM_MSG_DELPOLICY:
+ case XFRM_MSG_FLUSHSA:
+ case XFRM_MSG_FLUSHPOLICY:
+ case XFRM_MSG_NEWAE:
+ case XFRM_MSG_REPORT:
+ case XFRM_MSG_MIGRATE:
+ case XFRM_MSG_NEWSADINFO:
+ case XFRM_MSG_NEWSPDINFO:
+ case XFRM_MSG_MAPPING:
+ WARN_ON_ONCE(src_len != payload);
+ memcpy(nlmsg_data(nlh_dst), nlmsg_data(nlh_src), src_len);
+ break;
+ /* 4 byte alignment for trailing u64 on native, but not on compat */
+ case XFRM_MSG_NEWSA:
+ case XFRM_MSG_NEWPOLICY:
+ case XFRM_MSG_UPDSA:
+ case XFRM_MSG_UPDPOLICY:
+ WARN_ON_ONCE(src_len != payload + 4);
+ memcpy(nlmsg_data(nlh_dst), nlmsg_data(nlh_src), payload);
+ break;
+ case XFRM_MSG_EXPIRE: {
+ const struct xfrm_user_expire *src_ue = nlmsg_data(nlh_src);
+ struct compat_xfrm_user_expire *dst_ue = nlmsg_data(nlh_dst);
+
+ /* compat_xfrm_user_expire has 4-byte smaller state */
+ memcpy(dst_ue, src_ue, sizeof(dst_ue->state));
+ dst_ue->hard = src_ue->hard;
+ break;
+ }
+ case XFRM_MSG_ACQUIRE: {
+ const struct xfrm_user_acquire *src_ua = nlmsg_data(nlh_src);
+ struct compat_xfrm_user_acquire *dst_ua = nlmsg_data(nlh_dst);
+
+ memcpy(dst_ua, src_ua, offsetof(struct compat_xfrm_user_acquire, aalgos));
+ dst_ua->aalgos = src_ua->aalgos;
+ dst_ua->ealgos = src_ua->ealgos;
+ dst_ua->calgos = src_ua->calgos;
+ dst_ua->seq = src_ua->seq;
+ break;
+ }
+ case XFRM_MSG_POLEXPIRE: {
+ const struct xfrm_user_polexpire *src_upe = nlmsg_data(nlh_src);
+ struct compat_xfrm_user_polexpire *dst_upe = nlmsg_data(nlh_dst);
+
+ /* compat_xfrm_user_polexpire has 4-byte smaller state */
+ memcpy(dst_upe, src_upe, sizeof(dst_upe->pol));
+ dst_upe->hard = src_upe->hard;
+ break;
+ }
+ case XFRM_MSG_ALLOCSPI: {
+ const struct xfrm_userspi_info *src_usi = nlmsg_data(nlh_src);
+ struct compat_xfrm_userspi_info *dst_usi = nlmsg_data(nlh_dst);
+
+ /* compat_xfrm_user_polexpire has 4-byte smaller state */
+ memcpy(dst_usi, src_usi, sizeof(src_usi->info));
+ dst_usi->min = src_usi->min;
+ dst_usi->max = src_usi->max;
+ break;
+ }
+ /* Not being sent by kernel */
+ case XFRM_MSG_GETSA:
+ case XFRM_MSG_GETPOLICY:
+ case XFRM_MSG_GETAE:
+ case XFRM_MSG_GETSADINFO:
+ case XFRM_MSG_GETSPDINFO:
+ default:
+ pr_warn_once("unsupported nlmsg_type %d\n", nlh_src->nlmsg_type);
+ return ERR_PTR(-EOPNOTSUPP);
+ }
+
+ return nlh_dst;
+}
+
+static int xfrm_nla_cpy(struct sk_buff *dst, const struct nlattr *src, int len)
+{
+ return nla_put(dst, src->nla_type, len, nla_data(src));
+}
+
+static int xfrm_xlate64_attr(struct sk_buff *dst, const struct nlattr *src)
+{
+ switch (src->nla_type) {
+ case XFRMA_PAD:
+ /* Ignore */
+ return 0;
+ case XFRMA_UNSPEC:
+ case XFRMA_ALG_AUTH:
+ case XFRMA_ALG_CRYPT:
+ case XFRMA_ALG_COMP:
+ case XFRMA_ENCAP:
+ case XFRMA_TMPL:
+ return xfrm_nla_cpy(dst, src, nla_len(src));
+ case XFRMA_SA:
+ return xfrm_nla_cpy(dst, src, XMSGSIZE(compat_xfrm_usersa_info));
+ case XFRMA_POLICY:
+ return xfrm_nla_cpy(dst, src, XMSGSIZE(compat_xfrm_userpolicy_info));
+ case XFRMA_SEC_CTX:
+ return xfrm_nla_cpy(dst, src, nla_len(src));
+ case XFRMA_LTIME_VAL:
+ return nla_put_64bit(dst, src->nla_type, nla_len(src),
+ nla_data(src), XFRMA_PAD);
+ case XFRMA_REPLAY_VAL:
+ case XFRMA_REPLAY_THRESH:
+ case XFRMA_ETIMER_THRESH:
+ case XFRMA_SRCADDR:
+ case XFRMA_COADDR:
+ return xfrm_nla_cpy(dst, src, nla_len(src));
+ case XFRMA_LASTUSED:
+ return nla_put_64bit(dst, src->nla_type, nla_len(src),
+ nla_data(src), XFRMA_PAD);
+ case XFRMA_POLICY_TYPE:
+ case XFRMA_MIGRATE:
+ case XFRMA_ALG_AEAD:
+ case XFRMA_KMADDRESS:
+ case XFRMA_ALG_AUTH_TRUNC:
+ case XFRMA_MARK:
+ case XFRMA_TFCPAD:
+ case XFRMA_REPLAY_ESN_VAL:
+ case XFRMA_SA_EXTRA_FLAGS:
+ case XFRMA_PROTO:
+ case XFRMA_ADDRESS_FILTER:
+ case XFRMA_OFFLOAD_DEV:
+ case XFRMA_SET_MARK:
+ case XFRMA_SET_MARK_MASK:
+ case XFRMA_IF_ID:
+ case XFRMA_MTIMER_THRESH:
+ return xfrm_nla_cpy(dst, src, nla_len(src));
+ default:
+ BUILD_BUG_ON(XFRMA_MAX != XFRMA_MTIMER_THRESH);
+ pr_warn_once("unsupported nla_type %d\n", src->nla_type);
+ return -EOPNOTSUPP;
+ }
+}
+
+/* Take kernel-built (64bit layout) and create 32bit layout for userspace */
+static int xfrm_xlate64(struct sk_buff *dst, const struct nlmsghdr *nlh_src)
+{
+ u16 type = nlh_src->nlmsg_type - XFRM_MSG_BASE;
+ const struct nlattr *nla, *attrs;
+ struct nlmsghdr *nlh_dst;
+ int len, remaining;
+
+ nlh_dst = xfrm_nlmsg_put_compat(dst, nlh_src, type);
+ if (IS_ERR(nlh_dst))
+ return PTR_ERR(nlh_dst);
+
+ attrs = nlmsg_attrdata(nlh_src, xfrm_msg_min[type]);
+ len = nlmsg_attrlen(nlh_src, xfrm_msg_min[type]);
+
+ nla_for_each_attr(nla, attrs, len, remaining) {
+ int err;
+
+ switch (nlh_src->nlmsg_type) {
+ case XFRM_MSG_NEWSPDINFO:
+ err = xfrm_nla_cpy(dst, nla, nla_len(nla));
+ break;
+ default:
+ err = xfrm_xlate64_attr(dst, nla);
+ break;
+ }
+ if (err)
+ return err;
+ }
+
+ nlmsg_end(dst, nlh_dst);
+
+ return 0;
+}
+
+static int xfrm_alloc_compat(struct sk_buff *skb, const struct nlmsghdr *nlh_src)
+{
+ u16 type = nlh_src->nlmsg_type - XFRM_MSG_BASE;
+ struct sk_buff *new = NULL;
+ int err;
+
+ if (type >= ARRAY_SIZE(xfrm_msg_min)) {
+ pr_warn_once("unsupported nlmsg_type %d\n", nlh_src->nlmsg_type);
+ return -EOPNOTSUPP;
+ }
+
+ if (skb_shinfo(skb)->frag_list == NULL) {
+ new = alloc_skb(skb->len + skb_tailroom(skb), GFP_ATOMIC);
+ if (!new)
+ return -ENOMEM;
+ skb_shinfo(skb)->frag_list = new;
+ }
+
+ err = xfrm_xlate64(skb_shinfo(skb)->frag_list, nlh_src);
+ if (err) {
+ if (new) {
+ kfree_skb(new);
+ skb_shinfo(skb)->frag_list = NULL;
+ }
+ return err;
+ }
+
+ return 0;
+}
+
+/* Calculates len of translated 64-bit message. */
+static size_t xfrm_user_rcv_calculate_len64(const struct nlmsghdr *src,
+ struct nlattr *attrs[XFRMA_MAX + 1],
+ int maxtype)
+{
+ size_t len = nlmsg_len(src);
+
+ switch (src->nlmsg_type) {
+ case XFRM_MSG_NEWSA:
+ case XFRM_MSG_NEWPOLICY:
+ case XFRM_MSG_ALLOCSPI:
+ case XFRM_MSG_ACQUIRE:
+ case XFRM_MSG_UPDPOLICY:
+ case XFRM_MSG_UPDSA:
+ len += 4;
+ break;
+ case XFRM_MSG_EXPIRE:
+ case XFRM_MSG_POLEXPIRE:
+ len += 8;
+ break;
+ case XFRM_MSG_NEWSPDINFO:
+ /* attirbutes are xfrm_spdattr_type_t, not xfrm_attr_type_t */
+ return len;
+ default:
+ break;
+ }
+
+ /* Unexpected for anything, but XFRM_MSG_NEWSPDINFO, please
+ * correct both 64=>32-bit and 32=>64-bit translators to copy
+ * new attributes.
+ */
+ if (WARN_ON_ONCE(maxtype))
+ return len;
+
+ if (attrs[XFRMA_SA])
+ len += 4;
+ if (attrs[XFRMA_POLICY])
+ len += 4;
+
+ /* XXX: some attrs may need to be realigned
+ * if !CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS
+ */
+
+ return len;
+}
+
+static int xfrm_attr_cpy32(void *dst, size_t *pos, const struct nlattr *src,
+ size_t size, int copy_len, int payload)
+{
+ struct nlmsghdr *nlmsg = dst;
+ struct nlattr *nla;
+
+ /* xfrm_user_rcv_msg_compat() relies on fact that 32-bit messages
+ * have the same len or shorted than 64-bit ones.
+ * 32-bit translation that is bigger than 64-bit original is unexpected.
+ */
+ if (WARN_ON_ONCE(copy_len > payload))
+ copy_len = payload;
+
+ if (size - *pos < nla_attr_size(payload))
+ return -ENOBUFS;
+
+ nla = dst + *pos;
+
+ memcpy(nla, src, nla_attr_size(copy_len));
+ nla->nla_len = nla_attr_size(payload);
+ *pos += nla_attr_size(copy_len);
+ nlmsg->nlmsg_len += nla->nla_len;
+
+ memset(dst + *pos, 0, payload - copy_len);
+ *pos += payload - copy_len;
+
+ return 0;
+}
+
+static int xfrm_xlate32_attr(void *dst, const struct nlattr *nla,
+ size_t *pos, size_t size,
+ struct netlink_ext_ack *extack)
+{
+ int type = nla_type(nla);
+ u16 pol_len32, pol_len64;
+ int err;
+
+ if (type > XFRMA_MAX) {
+ BUILD_BUG_ON(XFRMA_MAX != XFRMA_MTIMER_THRESH);
+ NL_SET_ERR_MSG(extack, "Bad attribute");
+ return -EOPNOTSUPP;
+ }
+ type = array_index_nospec(type, XFRMA_MAX + 1);
+ if (nla_len(nla) < compat_policy[type].len) {
+ NL_SET_ERR_MSG(extack, "Attribute bad length");
+ return -EOPNOTSUPP;
+ }
+
+ pol_len32 = compat_policy[type].len;
+ pol_len64 = xfrma_policy[type].len;
+
+ /* XFRMA_SA and XFRMA_POLICY - need to know how-to translate */
+ if (pol_len32 != pol_len64) {
+ if (nla_len(nla) != compat_policy[type].len) {
+ NL_SET_ERR_MSG(extack, "Attribute bad length");
+ return -EOPNOTSUPP;
+ }
+ err = xfrm_attr_cpy32(dst, pos, nla, size, pol_len32, pol_len64);
+ if (err)
+ return err;
+ }
+
+ return xfrm_attr_cpy32(dst, pos, nla, size, nla_len(nla), nla_len(nla));
+}
+
+static int xfrm_xlate32(struct nlmsghdr *dst, const struct nlmsghdr *src,
+ struct nlattr *attrs[XFRMA_MAX+1],
+ size_t size, u8 type, int maxtype,
+ struct netlink_ext_ack *extack)
+{
+ size_t pos;
+ int i;
+
+ memcpy(dst, src, NLMSG_HDRLEN);
+ dst->nlmsg_len = NLMSG_HDRLEN + xfrm_msg_min[type];
+ memset(nlmsg_data(dst), 0, xfrm_msg_min[type]);
+
+ switch (src->nlmsg_type) {
+ /* Compat message has the same layout as native */
+ case XFRM_MSG_DELSA:
+ case XFRM_MSG_GETSA:
+ case XFRM_MSG_DELPOLICY:
+ case XFRM_MSG_GETPOLICY:
+ case XFRM_MSG_FLUSHSA:
+ case XFRM_MSG_FLUSHPOLICY:
+ case XFRM_MSG_NEWAE:
+ case XFRM_MSG_GETAE:
+ case XFRM_MSG_REPORT:
+ case XFRM_MSG_MIGRATE:
+ case XFRM_MSG_NEWSADINFO:
+ case XFRM_MSG_GETSADINFO:
+ case XFRM_MSG_NEWSPDINFO:
+ case XFRM_MSG_GETSPDINFO:
+ case XFRM_MSG_MAPPING:
+ memcpy(nlmsg_data(dst), nlmsg_data(src), compat_msg_min[type]);
+ break;
+ /* 4 byte alignment for trailing u64 on native, but not on compat */
+ case XFRM_MSG_NEWSA:
+ case XFRM_MSG_NEWPOLICY:
+ case XFRM_MSG_UPDSA:
+ case XFRM_MSG_UPDPOLICY:
+ memcpy(nlmsg_data(dst), nlmsg_data(src), compat_msg_min[type]);
+ break;
+ case XFRM_MSG_EXPIRE: {
+ const struct compat_xfrm_user_expire *src_ue = nlmsg_data(src);
+ struct xfrm_user_expire *dst_ue = nlmsg_data(dst);
+
+ /* compat_xfrm_user_expire has 4-byte smaller state */
+ memcpy(dst_ue, src_ue, sizeof(src_ue->state));
+ dst_ue->hard = src_ue->hard;
+ break;
+ }
+ case XFRM_MSG_ACQUIRE: {
+ const struct compat_xfrm_user_acquire *src_ua = nlmsg_data(src);
+ struct xfrm_user_acquire *dst_ua = nlmsg_data(dst);
+
+ memcpy(dst_ua, src_ua, offsetof(struct compat_xfrm_user_acquire, aalgos));
+ dst_ua->aalgos = src_ua->aalgos;
+ dst_ua->ealgos = src_ua->ealgos;
+ dst_ua->calgos = src_ua->calgos;
+ dst_ua->seq = src_ua->seq;
+ break;
+ }
+ case XFRM_MSG_POLEXPIRE: {
+ const struct compat_xfrm_user_polexpire *src_upe = nlmsg_data(src);
+ struct xfrm_user_polexpire *dst_upe = nlmsg_data(dst);
+
+ /* compat_xfrm_user_polexpire has 4-byte smaller state */
+ memcpy(dst_upe, src_upe, sizeof(src_upe->pol));
+ dst_upe->hard = src_upe->hard;
+ break;
+ }
+ case XFRM_MSG_ALLOCSPI: {
+ const struct compat_xfrm_userspi_info *src_usi = nlmsg_data(src);
+ struct xfrm_userspi_info *dst_usi = nlmsg_data(dst);
+
+ /* compat_xfrm_user_polexpire has 4-byte smaller state */
+ memcpy(dst_usi, src_usi, sizeof(src_usi->info));
+ dst_usi->min = src_usi->min;
+ dst_usi->max = src_usi->max;
+ break;
+ }
+ default:
+ NL_SET_ERR_MSG(extack, "Unsupported message type");
+ return -EOPNOTSUPP;
+ }
+ pos = dst->nlmsg_len;
+
+ if (maxtype) {
+ /* attirbutes are xfrm_spdattr_type_t, not xfrm_attr_type_t */
+ WARN_ON_ONCE(src->nlmsg_type != XFRM_MSG_NEWSPDINFO);
+
+ for (i = 1; i <= maxtype; i++) {
+ int err;
+
+ if (!attrs[i])
+ continue;
+
+ /* just copy - no need for translation */
+ err = xfrm_attr_cpy32(dst, &pos, attrs[i], size,
+ nla_len(attrs[i]), nla_len(attrs[i]));
+ if (err)
+ return err;
+ }
+ return 0;
+ }
+
+ for (i = 1; i < XFRMA_MAX + 1; i++) {
+ int err;
+
+ if (i == XFRMA_PAD)
+ continue;
+
+ if (!attrs[i])
+ continue;
+
+ err = xfrm_xlate32_attr(dst, attrs[i], &pos, size, extack);
+ if (err)
+ return err;
+ }
+
+ return 0;
+}
+
+static struct nlmsghdr *xfrm_user_rcv_msg_compat(const struct nlmsghdr *h32,
+ int maxtype, const struct nla_policy *policy,
+ struct netlink_ext_ack *extack)
+{
+ /* netlink_rcv_skb() checks if a message has full (struct nlmsghdr) */
+ u16 type = h32->nlmsg_type - XFRM_MSG_BASE;
+ struct nlattr *attrs[XFRMA_MAX+1];
+ struct nlmsghdr *h64;
+ size_t len;
+ int err;
+
+ BUILD_BUG_ON(ARRAY_SIZE(xfrm_msg_min) != ARRAY_SIZE(compat_msg_min));
+
+ if (type >= ARRAY_SIZE(xfrm_msg_min))
+ return ERR_PTR(-EINVAL);
+
+ /* Don't call parse: the message might have only nlmsg header */
+ if ((h32->nlmsg_type == XFRM_MSG_GETSA ||
+ h32->nlmsg_type == XFRM_MSG_GETPOLICY) &&
+ (h32->nlmsg_flags & NLM_F_DUMP))
+ return NULL;
+
+ err = nlmsg_parse_deprecated(h32, compat_msg_min[type], attrs,
+ maxtype ? : XFRMA_MAX, policy ? : compat_policy, extack);
+ if (err < 0)
+ return ERR_PTR(err);
+
+ len = xfrm_user_rcv_calculate_len64(h32, attrs, maxtype);
+ /* The message doesn't need translation */
+ if (len == nlmsg_len(h32))
+ return NULL;
+
+ len += NLMSG_HDRLEN;
+ h64 = kvmalloc(len, GFP_KERNEL);
+ if (!h64)
+ return ERR_PTR(-ENOMEM);
+
+ err = xfrm_xlate32(h64, h32, attrs, len, type, maxtype, extack);
+ if (err < 0) {
+ kvfree(h64);
+ return ERR_PTR(err);
+ }
+
+ return h64;
+}
+
+static int xfrm_user_policy_compat(u8 **pdata32, int optlen)
+{
+ struct compat_xfrm_userpolicy_info *p = (void *)*pdata32;
+ u8 *src_templates, *dst_templates;
+ u8 *data64;
+
+ if (optlen < sizeof(*p))
+ return -EINVAL;
+
+ data64 = kmalloc_track_caller(optlen + 4, GFP_USER | __GFP_NOWARN);
+ if (!data64)
+ return -ENOMEM;
+
+ memcpy(data64, *pdata32, sizeof(*p));
+ memset(data64 + sizeof(*p), 0, 4);
+
+ src_templates = *pdata32 + sizeof(*p);
+ dst_templates = data64 + sizeof(*p) + 4;
+ memcpy(dst_templates, src_templates, optlen - sizeof(*p));
+
+ kfree(*pdata32);
+ *pdata32 = data64;
+ return 0;
+}
+
+static struct xfrm_translator xfrm_translator = {
+ .owner = THIS_MODULE,
+ .alloc_compat = xfrm_alloc_compat,
+ .rcv_msg_compat = xfrm_user_rcv_msg_compat,
+ .xlate_user_policy_sockptr = xfrm_user_policy_compat,
+};
+
+static int __init xfrm_compat_init(void)
+{
+ return xfrm_register_translator(&xfrm_translator);
+}
+
+static void __exit xfrm_compat_exit(void)
+{
+ xfrm_unregister_translator(&xfrm_translator);
+}
+
+module_init(xfrm_compat_init);
+module_exit(xfrm_compat_exit);
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR("Dmitry Safonov");
+MODULE_DESCRIPTION("XFRM 32-bit compatibility layer");
diff --git a/net/xfrm/xfrm_device.c b/net/xfrm/xfrm_device.c
new file mode 100644
index 000000000..21269e8f2
--- /dev/null
+++ b/net/xfrm/xfrm_device.c
@@ -0,0 +1,444 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * xfrm_device.c - IPsec device offloading code.
+ *
+ * Copyright (c) 2015 secunet Security Networks AG
+ *
+ * Author:
+ * Steffen Klassert <steffen.klassert@secunet.com>
+ */
+
+#include <linux/errno.h>
+#include <linux/module.h>
+#include <linux/netdevice.h>
+#include <linux/skbuff.h>
+#include <linux/slab.h>
+#include <linux/spinlock.h>
+#include <net/dst.h>
+#include <net/xfrm.h>
+#include <linux/notifier.h>
+
+#ifdef CONFIG_XFRM_OFFLOAD
+static void __xfrm_transport_prep(struct xfrm_state *x, struct sk_buff *skb,
+ unsigned int hsize)
+{
+ struct xfrm_offload *xo = xfrm_offload(skb);
+
+ skb_reset_mac_len(skb);
+ if (xo->flags & XFRM_GSO_SEGMENT)
+ skb->transport_header -= x->props.header_len;
+
+ pskb_pull(skb, skb_transport_offset(skb) + x->props.header_len);
+}
+
+static void __xfrm_mode_tunnel_prep(struct xfrm_state *x, struct sk_buff *skb,
+ unsigned int hsize)
+
+{
+ struct xfrm_offload *xo = xfrm_offload(skb);
+
+ if (xo->flags & XFRM_GSO_SEGMENT)
+ skb->transport_header = skb->network_header + hsize;
+
+ skb_reset_mac_len(skb);
+ pskb_pull(skb, skb->mac_len + x->props.header_len);
+}
+
+static void __xfrm_mode_beet_prep(struct xfrm_state *x, struct sk_buff *skb,
+ unsigned int hsize)
+{
+ struct xfrm_offload *xo = xfrm_offload(skb);
+ int phlen = 0;
+
+ if (xo->flags & XFRM_GSO_SEGMENT)
+ skb->transport_header = skb->network_header + hsize;
+
+ skb_reset_mac_len(skb);
+ if (x->sel.family != AF_INET6) {
+ phlen = IPV4_BEET_PHMAXLEN;
+ if (x->outer_mode.family == AF_INET6)
+ phlen += sizeof(struct ipv6hdr) - sizeof(struct iphdr);
+ }
+
+ pskb_pull(skb, skb->mac_len + hsize + (x->props.header_len - phlen));
+}
+
+/* Adjust pointers into the packet when IPsec is done at layer2 */
+static void xfrm_outer_mode_prep(struct xfrm_state *x, struct sk_buff *skb)
+{
+ switch (x->outer_mode.encap) {
+ case XFRM_MODE_TUNNEL:
+ if (x->outer_mode.family == AF_INET)
+ return __xfrm_mode_tunnel_prep(x, skb,
+ sizeof(struct iphdr));
+ if (x->outer_mode.family == AF_INET6)
+ return __xfrm_mode_tunnel_prep(x, skb,
+ sizeof(struct ipv6hdr));
+ break;
+ case XFRM_MODE_TRANSPORT:
+ if (x->outer_mode.family == AF_INET)
+ return __xfrm_transport_prep(x, skb,
+ sizeof(struct iphdr));
+ if (x->outer_mode.family == AF_INET6)
+ return __xfrm_transport_prep(x, skb,
+ sizeof(struct ipv6hdr));
+ break;
+ case XFRM_MODE_BEET:
+ if (x->outer_mode.family == AF_INET)
+ return __xfrm_mode_beet_prep(x, skb,
+ sizeof(struct iphdr));
+ if (x->outer_mode.family == AF_INET6)
+ return __xfrm_mode_beet_prep(x, skb,
+ sizeof(struct ipv6hdr));
+ break;
+ case XFRM_MODE_ROUTEOPTIMIZATION:
+ case XFRM_MODE_IN_TRIGGER:
+ break;
+ }
+}
+
+static inline bool xmit_xfrm_check_overflow(struct sk_buff *skb)
+{
+ struct xfrm_offload *xo = xfrm_offload(skb);
+ __u32 seq = xo->seq.low;
+
+ seq += skb_shinfo(skb)->gso_segs;
+ if (unlikely(seq < xo->seq.low))
+ return true;
+
+ return false;
+}
+
+struct sk_buff *validate_xmit_xfrm(struct sk_buff *skb, netdev_features_t features, bool *again)
+{
+ int err;
+ unsigned long flags;
+ struct xfrm_state *x;
+ struct softnet_data *sd;
+ struct sk_buff *skb2, *nskb, *pskb = NULL;
+ netdev_features_t esp_features = features;
+ struct xfrm_offload *xo = xfrm_offload(skb);
+ struct net_device *dev = skb->dev;
+ struct sec_path *sp;
+
+ if (!xo || (xo->flags & XFRM_XMIT))
+ return skb;
+
+ if (!(features & NETIF_F_HW_ESP))
+ esp_features = features & ~(NETIF_F_SG | NETIF_F_CSUM_MASK);
+
+ sp = skb_sec_path(skb);
+ x = sp->xvec[sp->len - 1];
+ if (xo->flags & XFRM_GRO || x->xso.dir == XFRM_DEV_OFFLOAD_IN)
+ return skb;
+
+ /* This skb was already validated on the upper/virtual dev */
+ if ((x->xso.dev != dev) && (x->xso.real_dev == dev))
+ return skb;
+
+ local_irq_save(flags);
+ sd = this_cpu_ptr(&softnet_data);
+ err = !skb_queue_empty(&sd->xfrm_backlog);
+ local_irq_restore(flags);
+
+ if (err) {
+ *again = true;
+ return skb;
+ }
+
+ if (skb_is_gso(skb) && (unlikely(x->xso.dev != dev) ||
+ unlikely(xmit_xfrm_check_overflow(skb)))) {
+ struct sk_buff *segs;
+
+ /* Packet got rerouted, fixup features and segment it. */
+ esp_features = esp_features & ~(NETIF_F_HW_ESP | NETIF_F_GSO_ESP);
+
+ segs = skb_gso_segment(skb, esp_features);
+ if (IS_ERR(segs)) {
+ kfree_skb(skb);
+ dev_core_stats_tx_dropped_inc(dev);
+ return NULL;
+ } else {
+ consume_skb(skb);
+ skb = segs;
+ }
+ }
+
+ if (!skb->next) {
+ esp_features |= skb->dev->gso_partial_features;
+ xfrm_outer_mode_prep(x, skb);
+
+ xo->flags |= XFRM_DEV_RESUME;
+
+ err = x->type_offload->xmit(x, skb, esp_features);
+ if (err) {
+ if (err == -EINPROGRESS)
+ return NULL;
+
+ XFRM_INC_STATS(xs_net(x), LINUX_MIB_XFRMOUTSTATEPROTOERROR);
+ kfree_skb(skb);
+ return NULL;
+ }
+
+ skb_push(skb, skb->data - skb_mac_header(skb));
+
+ return skb;
+ }
+
+ skb_list_walk_safe(skb, skb2, nskb) {
+ esp_features |= skb->dev->gso_partial_features;
+ skb_mark_not_on_list(skb2);
+
+ xo = xfrm_offload(skb2);
+ xo->flags |= XFRM_DEV_RESUME;
+
+ xfrm_outer_mode_prep(x, skb2);
+
+ err = x->type_offload->xmit(x, skb2, esp_features);
+ if (!err) {
+ skb2->next = nskb;
+ } else if (err != -EINPROGRESS) {
+ XFRM_INC_STATS(xs_net(x), LINUX_MIB_XFRMOUTSTATEPROTOERROR);
+ skb2->next = nskb;
+ kfree_skb_list(skb2);
+ return NULL;
+ } else {
+ if (skb == skb2)
+ skb = nskb;
+ else
+ pskb->next = nskb;
+
+ continue;
+ }
+
+ skb_push(skb2, skb2->data - skb_mac_header(skb2));
+ pskb = skb2;
+ }
+
+ return skb;
+}
+EXPORT_SYMBOL_GPL(validate_xmit_xfrm);
+
+int xfrm_dev_state_add(struct net *net, struct xfrm_state *x,
+ struct xfrm_user_offload *xuo,
+ struct netlink_ext_ack *extack)
+{
+ int err;
+ struct dst_entry *dst;
+ struct net_device *dev;
+ struct xfrm_dev_offload *xso = &x->xso;
+ xfrm_address_t *saddr;
+ xfrm_address_t *daddr;
+
+ if (!x->type_offload) {
+ NL_SET_ERR_MSG(extack, "Type doesn't support offload");
+ return -EINVAL;
+ }
+
+ /* We don't yet support UDP encapsulation and TFC padding. */
+ if (x->encap || x->tfcpad) {
+ NL_SET_ERR_MSG(extack, "Encapsulation and TFC padding can't be offloaded");
+ return -EINVAL;
+ }
+
+ if (xuo->flags & ~(XFRM_OFFLOAD_IPV6 | XFRM_OFFLOAD_INBOUND)) {
+ NL_SET_ERR_MSG(extack, "Unrecognized flags in offload request");
+ return -EINVAL;
+ }
+
+ dev = dev_get_by_index(net, xuo->ifindex);
+ if (!dev) {
+ if (!(xuo->flags & XFRM_OFFLOAD_INBOUND)) {
+ saddr = &x->props.saddr;
+ daddr = &x->id.daddr;
+ } else {
+ saddr = &x->id.daddr;
+ daddr = &x->props.saddr;
+ }
+
+ dst = __xfrm_dst_lookup(net, 0, 0, saddr, daddr,
+ x->props.family,
+ xfrm_smark_get(0, x));
+ if (IS_ERR(dst))
+ return 0;
+
+ dev = dst->dev;
+
+ dev_hold(dev);
+ dst_release(dst);
+ }
+
+ if (!dev->xfrmdev_ops || !dev->xfrmdev_ops->xdo_dev_state_add) {
+ xso->dev = NULL;
+ dev_put(dev);
+ return 0;
+ }
+
+ if (x->props.flags & XFRM_STATE_ESN &&
+ !dev->xfrmdev_ops->xdo_dev_state_advance_esn) {
+ NL_SET_ERR_MSG(extack, "Device doesn't support offload with ESN");
+ xso->dev = NULL;
+ dev_put(dev);
+ return -EINVAL;
+ }
+
+ xso->dev = dev;
+ netdev_tracker_alloc(dev, &xso->dev_tracker, GFP_ATOMIC);
+ xso->real_dev = dev;
+
+ if (xuo->flags & XFRM_OFFLOAD_INBOUND)
+ xso->dir = XFRM_DEV_OFFLOAD_IN;
+ else
+ xso->dir = XFRM_DEV_OFFLOAD_OUT;
+
+ err = dev->xfrmdev_ops->xdo_dev_state_add(x);
+ if (err) {
+ xso->dev = NULL;
+ xso->dir = 0;
+ xso->real_dev = NULL;
+ netdev_put(dev, &xso->dev_tracker);
+
+ if (err != -EOPNOTSUPP) {
+ NL_SET_ERR_MSG(extack, "Device failed to offload this state");
+ return err;
+ }
+ }
+
+ return 0;
+}
+EXPORT_SYMBOL_GPL(xfrm_dev_state_add);
+
+bool xfrm_dev_offload_ok(struct sk_buff *skb, struct xfrm_state *x)
+{
+ int mtu;
+ struct dst_entry *dst = skb_dst(skb);
+ struct xfrm_dst *xdst = (struct xfrm_dst *)dst;
+ struct net_device *dev = x->xso.dev;
+
+ if (!x->type_offload || x->encap)
+ return false;
+
+ if ((!dev || (dev == xfrm_dst_path(dst)->dev)) &&
+ (!xdst->child->xfrm)) {
+ mtu = xfrm_state_mtu(x, xdst->child_mtu_cached);
+ if (skb->len <= mtu)
+ goto ok;
+
+ if (skb_is_gso(skb) && skb_gso_validate_network_len(skb, mtu))
+ goto ok;
+ }
+
+ return false;
+
+ok:
+ if (dev && dev->xfrmdev_ops && dev->xfrmdev_ops->xdo_dev_offload_ok)
+ return x->xso.dev->xfrmdev_ops->xdo_dev_offload_ok(skb, x);
+
+ return true;
+}
+EXPORT_SYMBOL_GPL(xfrm_dev_offload_ok);
+
+void xfrm_dev_resume(struct sk_buff *skb)
+{
+ struct net_device *dev = skb->dev;
+ int ret = NETDEV_TX_BUSY;
+ struct netdev_queue *txq;
+ struct softnet_data *sd;
+ unsigned long flags;
+
+ rcu_read_lock();
+ txq = netdev_core_pick_tx(dev, skb, NULL);
+
+ HARD_TX_LOCK(dev, txq, smp_processor_id());
+ if (!netif_xmit_frozen_or_stopped(txq))
+ skb = dev_hard_start_xmit(skb, dev, txq, &ret);
+ HARD_TX_UNLOCK(dev, txq);
+
+ if (!dev_xmit_complete(ret)) {
+ local_irq_save(flags);
+ sd = this_cpu_ptr(&softnet_data);
+ skb_queue_tail(&sd->xfrm_backlog, skb);
+ raise_softirq_irqoff(NET_TX_SOFTIRQ);
+ local_irq_restore(flags);
+ }
+ rcu_read_unlock();
+}
+EXPORT_SYMBOL_GPL(xfrm_dev_resume);
+
+void xfrm_dev_backlog(struct softnet_data *sd)
+{
+ struct sk_buff_head *xfrm_backlog = &sd->xfrm_backlog;
+ struct sk_buff_head list;
+ struct sk_buff *skb;
+
+ if (skb_queue_empty(xfrm_backlog))
+ return;
+
+ __skb_queue_head_init(&list);
+
+ spin_lock(&xfrm_backlog->lock);
+ skb_queue_splice_init(xfrm_backlog, &list);
+ spin_unlock(&xfrm_backlog->lock);
+
+ while (!skb_queue_empty(&list)) {
+ skb = __skb_dequeue(&list);
+ xfrm_dev_resume(skb);
+ }
+
+}
+#endif
+
+static int xfrm_api_check(struct net_device *dev)
+{
+#ifdef CONFIG_XFRM_OFFLOAD
+ if ((dev->features & NETIF_F_HW_ESP_TX_CSUM) &&
+ !(dev->features & NETIF_F_HW_ESP))
+ return NOTIFY_BAD;
+
+ if ((dev->features & NETIF_F_HW_ESP) &&
+ (!(dev->xfrmdev_ops &&
+ dev->xfrmdev_ops->xdo_dev_state_add &&
+ dev->xfrmdev_ops->xdo_dev_state_delete)))
+ return NOTIFY_BAD;
+#else
+ if (dev->features & (NETIF_F_HW_ESP | NETIF_F_HW_ESP_TX_CSUM))
+ return NOTIFY_BAD;
+#endif
+
+ return NOTIFY_DONE;
+}
+
+static int xfrm_dev_down(struct net_device *dev)
+{
+ if (dev->features & NETIF_F_HW_ESP)
+ xfrm_dev_state_flush(dev_net(dev), dev, true);
+
+ return NOTIFY_DONE;
+}
+
+static int xfrm_dev_event(struct notifier_block *this, unsigned long event, void *ptr)
+{
+ struct net_device *dev = netdev_notifier_info_to_dev(ptr);
+
+ switch (event) {
+ case NETDEV_REGISTER:
+ return xfrm_api_check(dev);
+
+ case NETDEV_FEAT_CHANGE:
+ return xfrm_api_check(dev);
+
+ case NETDEV_DOWN:
+ case NETDEV_UNREGISTER:
+ return xfrm_dev_down(dev);
+ }
+ return NOTIFY_DONE;
+}
+
+static struct notifier_block xfrm_dev_notifier = {
+ .notifier_call = xfrm_dev_event,
+};
+
+void __init xfrm_dev_init(void)
+{
+ register_netdevice_notifier(&xfrm_dev_notifier);
+}
diff --git a/net/xfrm/xfrm_hash.c b/net/xfrm/xfrm_hash.c
new file mode 100644
index 000000000..eca8d84d9
--- /dev/null
+++ b/net/xfrm/xfrm_hash.c
@@ -0,0 +1,40 @@
+// SPDX-License-Identifier: GPL-2.0
+/* xfrm_hash.c: Common hash table code.
+ *
+ * Copyright (C) 2006 David S. Miller (davem@davemloft.net)
+ */
+
+#include <linux/kernel.h>
+#include <linux/mm.h>
+#include <linux/memblock.h>
+#include <linux/vmalloc.h>
+#include <linux/slab.h>
+#include <linux/xfrm.h>
+
+#include "xfrm_hash.h"
+
+struct hlist_head *xfrm_hash_alloc(unsigned int sz)
+{
+ struct hlist_head *n;
+
+ if (sz <= PAGE_SIZE)
+ n = kzalloc(sz, GFP_KERNEL);
+ else if (hashdist)
+ n = vzalloc(sz);
+ else
+ n = (struct hlist_head *)
+ __get_free_pages(GFP_KERNEL | __GFP_NOWARN | __GFP_ZERO,
+ get_order(sz));
+
+ return n;
+}
+
+void xfrm_hash_free(struct hlist_head *n, unsigned int sz)
+{
+ if (sz <= PAGE_SIZE)
+ kfree(n);
+ else if (hashdist)
+ vfree(n);
+ else
+ free_pages((unsigned long)n, get_order(sz));
+}
diff --git a/net/xfrm/xfrm_hash.h b/net/xfrm/xfrm_hash.h
new file mode 100644
index 000000000..d12bb906c
--- /dev/null
+++ b/net/xfrm/xfrm_hash.h
@@ -0,0 +1,199 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef _XFRM_HASH_H
+#define _XFRM_HASH_H
+
+#include <linux/xfrm.h>
+#include <linux/socket.h>
+#include <linux/jhash.h>
+
+static inline unsigned int __xfrm4_addr_hash(const xfrm_address_t *addr)
+{
+ return ntohl(addr->a4);
+}
+
+static inline unsigned int __xfrm6_addr_hash(const xfrm_address_t *addr)
+{
+ return jhash2((__force u32 *)addr->a6, 4, 0);
+}
+
+static inline unsigned int __xfrm4_daddr_saddr_hash(const xfrm_address_t *daddr,
+ const xfrm_address_t *saddr)
+{
+ u32 sum = (__force u32)daddr->a4 + (__force u32)saddr->a4;
+ return ntohl((__force __be32)sum);
+}
+
+static inline unsigned int __xfrm6_daddr_saddr_hash(const xfrm_address_t *daddr,
+ const xfrm_address_t *saddr)
+{
+ return __xfrm6_addr_hash(daddr) ^ __xfrm6_addr_hash(saddr);
+}
+
+static inline u32 __bits2mask32(__u8 bits)
+{
+ u32 mask32 = 0xffffffff;
+
+ if (bits == 0)
+ mask32 = 0;
+ else if (bits < 32)
+ mask32 <<= (32 - bits);
+
+ return mask32;
+}
+
+static inline unsigned int __xfrm4_dpref_spref_hash(const xfrm_address_t *daddr,
+ const xfrm_address_t *saddr,
+ __u8 dbits,
+ __u8 sbits)
+{
+ return jhash_2words(ntohl(daddr->a4) & __bits2mask32(dbits),
+ ntohl(saddr->a4) & __bits2mask32(sbits),
+ 0);
+}
+
+static inline unsigned int __xfrm6_pref_hash(const xfrm_address_t *addr,
+ __u8 prefixlen)
+{
+ unsigned int pdw;
+ unsigned int pbi;
+ u32 initval = 0;
+
+ pdw = prefixlen >> 5; /* num of whole u32 in prefix */
+ pbi = prefixlen & 0x1f; /* num of bits in incomplete u32 in prefix */
+
+ if (pbi) {
+ __be32 mask;
+
+ mask = htonl((0xffffffff) << (32 - pbi));
+
+ initval = (__force u32)(addr->a6[pdw] & mask);
+ }
+
+ return jhash2((__force u32 *)addr->a6, pdw, initval);
+}
+
+static inline unsigned int __xfrm6_dpref_spref_hash(const xfrm_address_t *daddr,
+ const xfrm_address_t *saddr,
+ __u8 dbits,
+ __u8 sbits)
+{
+ return __xfrm6_pref_hash(daddr, dbits) ^
+ __xfrm6_pref_hash(saddr, sbits);
+}
+
+static inline unsigned int __xfrm_dst_hash(const xfrm_address_t *daddr,
+ const xfrm_address_t *saddr,
+ u32 reqid, unsigned short family,
+ unsigned int hmask)
+{
+ unsigned int h = family ^ reqid;
+ switch (family) {
+ case AF_INET:
+ h ^= __xfrm4_daddr_saddr_hash(daddr, saddr);
+ break;
+ case AF_INET6:
+ h ^= __xfrm6_daddr_saddr_hash(daddr, saddr);
+ break;
+ }
+ return (h ^ (h >> 16)) & hmask;
+}
+
+static inline unsigned int __xfrm_src_hash(const xfrm_address_t *daddr,
+ const xfrm_address_t *saddr,
+ unsigned short family,
+ unsigned int hmask)
+{
+ unsigned int h = family;
+ switch (family) {
+ case AF_INET:
+ h ^= __xfrm4_daddr_saddr_hash(daddr, saddr);
+ break;
+ case AF_INET6:
+ h ^= __xfrm6_daddr_saddr_hash(daddr, saddr);
+ break;
+ }
+ return (h ^ (h >> 16)) & hmask;
+}
+
+static inline unsigned int
+__xfrm_spi_hash(const xfrm_address_t *daddr, __be32 spi, u8 proto,
+ unsigned short family, unsigned int hmask)
+{
+ unsigned int h = (__force u32)spi ^ proto;
+ switch (family) {
+ case AF_INET:
+ h ^= __xfrm4_addr_hash(daddr);
+ break;
+ case AF_INET6:
+ h ^= __xfrm6_addr_hash(daddr);
+ break;
+ }
+ return (h ^ (h >> 10) ^ (h >> 20)) & hmask;
+}
+
+static inline unsigned int
+__xfrm_seq_hash(u32 seq, unsigned int hmask)
+{
+ unsigned int h = seq;
+ return (h ^ (h >> 10) ^ (h >> 20)) & hmask;
+}
+
+static inline unsigned int __idx_hash(u32 index, unsigned int hmask)
+{
+ return (index ^ (index >> 8)) & hmask;
+}
+
+static inline unsigned int __sel_hash(const struct xfrm_selector *sel,
+ unsigned short family, unsigned int hmask,
+ u8 dbits, u8 sbits)
+{
+ const xfrm_address_t *daddr = &sel->daddr;
+ const xfrm_address_t *saddr = &sel->saddr;
+ unsigned int h = 0;
+
+ switch (family) {
+ case AF_INET:
+ if (sel->prefixlen_d < dbits ||
+ sel->prefixlen_s < sbits)
+ return hmask + 1;
+
+ h = __xfrm4_dpref_spref_hash(daddr, saddr, dbits, sbits);
+ break;
+
+ case AF_INET6:
+ if (sel->prefixlen_d < dbits ||
+ sel->prefixlen_s < sbits)
+ return hmask + 1;
+
+ h = __xfrm6_dpref_spref_hash(daddr, saddr, dbits, sbits);
+ break;
+ }
+ h ^= (h >> 16);
+ return h & hmask;
+}
+
+static inline unsigned int __addr_hash(const xfrm_address_t *daddr,
+ const xfrm_address_t *saddr,
+ unsigned short family,
+ unsigned int hmask,
+ u8 dbits, u8 sbits)
+{
+ unsigned int h = 0;
+
+ switch (family) {
+ case AF_INET:
+ h = __xfrm4_dpref_spref_hash(daddr, saddr, dbits, sbits);
+ break;
+
+ case AF_INET6:
+ h = __xfrm6_dpref_spref_hash(daddr, saddr, dbits, sbits);
+ break;
+ }
+ h ^= (h >> 16);
+ return h & hmask;
+}
+
+struct hlist_head *xfrm_hash_alloc(unsigned int sz);
+void xfrm_hash_free(struct hlist_head *n, unsigned int sz);
+
+#endif /* _XFRM_HASH_H */
diff --git a/net/xfrm/xfrm_inout.h b/net/xfrm/xfrm_inout.h
new file mode 100644
index 000000000..efc5e6b2e
--- /dev/null
+++ b/net/xfrm/xfrm_inout.h
@@ -0,0 +1,70 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#include <linux/ipv6.h>
+#include <net/dsfield.h>
+#include <net/xfrm.h>
+
+#ifndef XFRM_INOUT_H
+#define XFRM_INOUT_H 1
+
+static inline void xfrm4_extract_header(struct sk_buff *skb)
+{
+ const struct iphdr *iph = ip_hdr(skb);
+
+ XFRM_MODE_SKB_CB(skb)->ihl = sizeof(*iph);
+ XFRM_MODE_SKB_CB(skb)->id = iph->id;
+ XFRM_MODE_SKB_CB(skb)->frag_off = iph->frag_off;
+ XFRM_MODE_SKB_CB(skb)->tos = iph->tos;
+ XFRM_MODE_SKB_CB(skb)->ttl = iph->ttl;
+ XFRM_MODE_SKB_CB(skb)->optlen = iph->ihl * 4 - sizeof(*iph);
+ memset(XFRM_MODE_SKB_CB(skb)->flow_lbl, 0,
+ sizeof(XFRM_MODE_SKB_CB(skb)->flow_lbl));
+}
+
+static inline void xfrm6_extract_header(struct sk_buff *skb)
+{
+#if IS_ENABLED(CONFIG_IPV6)
+ struct ipv6hdr *iph = ipv6_hdr(skb);
+
+ XFRM_MODE_SKB_CB(skb)->ihl = sizeof(*iph);
+ XFRM_MODE_SKB_CB(skb)->id = 0;
+ XFRM_MODE_SKB_CB(skb)->frag_off = htons(IP_DF);
+ XFRM_MODE_SKB_CB(skb)->tos = ipv6_get_dsfield(iph);
+ XFRM_MODE_SKB_CB(skb)->ttl = iph->hop_limit;
+ XFRM_MODE_SKB_CB(skb)->optlen = 0;
+ memcpy(XFRM_MODE_SKB_CB(skb)->flow_lbl, iph->flow_lbl,
+ sizeof(XFRM_MODE_SKB_CB(skb)->flow_lbl));
+#else
+ WARN_ON_ONCE(1);
+#endif
+}
+
+static inline void xfrm6_beet_make_header(struct sk_buff *skb)
+{
+ struct ipv6hdr *iph = ipv6_hdr(skb);
+
+ iph->version = 6;
+
+ memcpy(iph->flow_lbl, XFRM_MODE_SKB_CB(skb)->flow_lbl,
+ sizeof(iph->flow_lbl));
+ iph->nexthdr = XFRM_MODE_SKB_CB(skb)->protocol;
+
+ ipv6_change_dsfield(iph, 0, XFRM_MODE_SKB_CB(skb)->tos);
+ iph->hop_limit = XFRM_MODE_SKB_CB(skb)->ttl;
+}
+
+static inline void xfrm4_beet_make_header(struct sk_buff *skb)
+{
+ struct iphdr *iph = ip_hdr(skb);
+
+ iph->ihl = 5;
+ iph->version = 4;
+
+ iph->protocol = XFRM_MODE_SKB_CB(skb)->protocol;
+ iph->tos = XFRM_MODE_SKB_CB(skb)->tos;
+
+ iph->id = XFRM_MODE_SKB_CB(skb)->id;
+ iph->frag_off = XFRM_MODE_SKB_CB(skb)->frag_off;
+ iph->ttl = XFRM_MODE_SKB_CB(skb)->ttl;
+}
+
+#endif
diff --git a/net/xfrm/xfrm_input.c b/net/xfrm/xfrm_input.c
new file mode 100644
index 000000000..ac1a645af
--- /dev/null
+++ b/net/xfrm/xfrm_input.c
@@ -0,0 +1,834 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * xfrm_input.c
+ *
+ * Changes:
+ * YOSHIFUJI Hideaki @USAGI
+ * Split up af-specific portion
+ *
+ */
+
+#include <linux/bottom_half.h>
+#include <linux/cache.h>
+#include <linux/interrupt.h>
+#include <linux/slab.h>
+#include <linux/module.h>
+#include <linux/netdevice.h>
+#include <linux/percpu.h>
+#include <net/dst.h>
+#include <net/ip.h>
+#include <net/xfrm.h>
+#include <net/ip_tunnels.h>
+#include <net/ip6_tunnel.h>
+#include <net/dst_metadata.h>
+
+#include "xfrm_inout.h"
+
+struct xfrm_trans_tasklet {
+ struct work_struct work;
+ spinlock_t queue_lock;
+ struct sk_buff_head queue;
+};
+
+struct xfrm_trans_cb {
+ union {
+ struct inet_skb_parm h4;
+#if IS_ENABLED(CONFIG_IPV6)
+ struct inet6_skb_parm h6;
+#endif
+ } header;
+ int (*finish)(struct net *net, struct sock *sk, struct sk_buff *skb);
+ struct net *net;
+};
+
+#define XFRM_TRANS_SKB_CB(__skb) ((struct xfrm_trans_cb *)&((__skb)->cb[0]))
+
+static DEFINE_SPINLOCK(xfrm_input_afinfo_lock);
+static struct xfrm_input_afinfo const __rcu *xfrm_input_afinfo[2][AF_INET6 + 1];
+
+static struct gro_cells gro_cells;
+static struct net_device xfrm_napi_dev;
+
+static DEFINE_PER_CPU(struct xfrm_trans_tasklet, xfrm_trans_tasklet);
+
+int xfrm_input_register_afinfo(const struct xfrm_input_afinfo *afinfo)
+{
+ int err = 0;
+
+ if (WARN_ON(afinfo->family > AF_INET6))
+ return -EAFNOSUPPORT;
+
+ spin_lock_bh(&xfrm_input_afinfo_lock);
+ if (unlikely(xfrm_input_afinfo[afinfo->is_ipip][afinfo->family]))
+ err = -EEXIST;
+ else
+ rcu_assign_pointer(xfrm_input_afinfo[afinfo->is_ipip][afinfo->family], afinfo);
+ spin_unlock_bh(&xfrm_input_afinfo_lock);
+ return err;
+}
+EXPORT_SYMBOL(xfrm_input_register_afinfo);
+
+int xfrm_input_unregister_afinfo(const struct xfrm_input_afinfo *afinfo)
+{
+ int err = 0;
+
+ spin_lock_bh(&xfrm_input_afinfo_lock);
+ if (likely(xfrm_input_afinfo[afinfo->is_ipip][afinfo->family])) {
+ if (unlikely(xfrm_input_afinfo[afinfo->is_ipip][afinfo->family] != afinfo))
+ err = -EINVAL;
+ else
+ RCU_INIT_POINTER(xfrm_input_afinfo[afinfo->is_ipip][afinfo->family], NULL);
+ }
+ spin_unlock_bh(&xfrm_input_afinfo_lock);
+ synchronize_rcu();
+ return err;
+}
+EXPORT_SYMBOL(xfrm_input_unregister_afinfo);
+
+static const struct xfrm_input_afinfo *xfrm_input_get_afinfo(u8 family, bool is_ipip)
+{
+ const struct xfrm_input_afinfo *afinfo;
+
+ if (WARN_ON_ONCE(family > AF_INET6))
+ return NULL;
+
+ rcu_read_lock();
+ afinfo = rcu_dereference(xfrm_input_afinfo[is_ipip][family]);
+ if (unlikely(!afinfo))
+ rcu_read_unlock();
+ return afinfo;
+}
+
+static int xfrm_rcv_cb(struct sk_buff *skb, unsigned int family, u8 protocol,
+ int err)
+{
+ bool is_ipip = (protocol == IPPROTO_IPIP || protocol == IPPROTO_IPV6);
+ const struct xfrm_input_afinfo *afinfo;
+ int ret;
+
+ afinfo = xfrm_input_get_afinfo(family, is_ipip);
+ if (!afinfo)
+ return -EAFNOSUPPORT;
+
+ ret = afinfo->callback(skb, protocol, err);
+ rcu_read_unlock();
+
+ return ret;
+}
+
+struct sec_path *secpath_set(struct sk_buff *skb)
+{
+ struct sec_path *sp, *tmp = skb_ext_find(skb, SKB_EXT_SEC_PATH);
+
+ sp = skb_ext_add(skb, SKB_EXT_SEC_PATH);
+ if (!sp)
+ return NULL;
+
+ if (tmp) /* reused existing one (was COW'd if needed) */
+ return sp;
+
+ /* allocated new secpath */
+ memset(sp->ovec, 0, sizeof(sp->ovec));
+ sp->olen = 0;
+ sp->len = 0;
+ sp->verified_cnt = 0;
+
+ return sp;
+}
+EXPORT_SYMBOL(secpath_set);
+
+/* Fetch spi and seq from ipsec header */
+
+int xfrm_parse_spi(struct sk_buff *skb, u8 nexthdr, __be32 *spi, __be32 *seq)
+{
+ int offset, offset_seq;
+ int hlen;
+
+ switch (nexthdr) {
+ case IPPROTO_AH:
+ hlen = sizeof(struct ip_auth_hdr);
+ offset = offsetof(struct ip_auth_hdr, spi);
+ offset_seq = offsetof(struct ip_auth_hdr, seq_no);
+ break;
+ case IPPROTO_ESP:
+ hlen = sizeof(struct ip_esp_hdr);
+ offset = offsetof(struct ip_esp_hdr, spi);
+ offset_seq = offsetof(struct ip_esp_hdr, seq_no);
+ break;
+ case IPPROTO_COMP:
+ if (!pskb_may_pull(skb, sizeof(struct ip_comp_hdr)))
+ return -EINVAL;
+ *spi = htonl(ntohs(*(__be16 *)(skb_transport_header(skb) + 2)));
+ *seq = 0;
+ return 0;
+ default:
+ return 1;
+ }
+
+ if (!pskb_may_pull(skb, hlen))
+ return -EINVAL;
+
+ *spi = *(__be32 *)(skb_transport_header(skb) + offset);
+ *seq = *(__be32 *)(skb_transport_header(skb) + offset_seq);
+ return 0;
+}
+EXPORT_SYMBOL(xfrm_parse_spi);
+
+static int xfrm4_remove_beet_encap(struct xfrm_state *x, struct sk_buff *skb)
+{
+ struct iphdr *iph;
+ int optlen = 0;
+ int err = -EINVAL;
+
+ if (unlikely(XFRM_MODE_SKB_CB(skb)->protocol == IPPROTO_BEETPH)) {
+ struct ip_beet_phdr *ph;
+ int phlen;
+
+ if (!pskb_may_pull(skb, sizeof(*ph)))
+ goto out;
+
+ ph = (struct ip_beet_phdr *)skb->data;
+
+ phlen = sizeof(*ph) + ph->padlen;
+ optlen = ph->hdrlen * 8 + (IPV4_BEET_PHMAXLEN - phlen);
+ if (optlen < 0 || optlen & 3 || optlen > 250)
+ goto out;
+
+ XFRM_MODE_SKB_CB(skb)->protocol = ph->nexthdr;
+
+ if (!pskb_may_pull(skb, phlen))
+ goto out;
+ __skb_pull(skb, phlen);
+ }
+
+ skb_push(skb, sizeof(*iph));
+ skb_reset_network_header(skb);
+ skb_mac_header_rebuild(skb);
+
+ xfrm4_beet_make_header(skb);
+
+ iph = ip_hdr(skb);
+
+ iph->ihl += optlen / 4;
+ iph->tot_len = htons(skb->len);
+ iph->daddr = x->sel.daddr.a4;
+ iph->saddr = x->sel.saddr.a4;
+ iph->check = 0;
+ iph->check = ip_fast_csum(skb_network_header(skb), iph->ihl);
+ err = 0;
+out:
+ return err;
+}
+
+static void ipip_ecn_decapsulate(struct sk_buff *skb)
+{
+ struct iphdr *inner_iph = ipip_hdr(skb);
+
+ if (INET_ECN_is_ce(XFRM_MODE_SKB_CB(skb)->tos))
+ IP_ECN_set_ce(inner_iph);
+}
+
+static int xfrm4_remove_tunnel_encap(struct xfrm_state *x, struct sk_buff *skb)
+{
+ int err = -EINVAL;
+
+ if (XFRM_MODE_SKB_CB(skb)->protocol != IPPROTO_IPIP)
+ goto out;
+
+ if (!pskb_may_pull(skb, sizeof(struct iphdr)))
+ goto out;
+
+ err = skb_unclone(skb, GFP_ATOMIC);
+ if (err)
+ goto out;
+
+ if (x->props.flags & XFRM_STATE_DECAP_DSCP)
+ ipv4_copy_dscp(XFRM_MODE_SKB_CB(skb)->tos, ipip_hdr(skb));
+ if (!(x->props.flags & XFRM_STATE_NOECN))
+ ipip_ecn_decapsulate(skb);
+
+ skb_reset_network_header(skb);
+ skb_mac_header_rebuild(skb);
+ if (skb->mac_len)
+ eth_hdr(skb)->h_proto = skb->protocol;
+
+ err = 0;
+
+out:
+ return err;
+}
+
+static void ipip6_ecn_decapsulate(struct sk_buff *skb)
+{
+ struct ipv6hdr *inner_iph = ipipv6_hdr(skb);
+
+ if (INET_ECN_is_ce(XFRM_MODE_SKB_CB(skb)->tos))
+ IP6_ECN_set_ce(skb, inner_iph);
+}
+
+static int xfrm6_remove_tunnel_encap(struct xfrm_state *x, struct sk_buff *skb)
+{
+ int err = -EINVAL;
+
+ if (XFRM_MODE_SKB_CB(skb)->protocol != IPPROTO_IPV6)
+ goto out;
+ if (!pskb_may_pull(skb, sizeof(struct ipv6hdr)))
+ goto out;
+
+ err = skb_unclone(skb, GFP_ATOMIC);
+ if (err)
+ goto out;
+
+ if (x->props.flags & XFRM_STATE_DECAP_DSCP)
+ ipv6_copy_dscp(XFRM_MODE_SKB_CB(skb)->tos, ipipv6_hdr(skb));
+ if (!(x->props.flags & XFRM_STATE_NOECN))
+ ipip6_ecn_decapsulate(skb);
+
+ skb_reset_network_header(skb);
+ skb_mac_header_rebuild(skb);
+ if (skb->mac_len)
+ eth_hdr(skb)->h_proto = skb->protocol;
+
+ err = 0;
+
+out:
+ return err;
+}
+
+static int xfrm6_remove_beet_encap(struct xfrm_state *x, struct sk_buff *skb)
+{
+ struct ipv6hdr *ip6h;
+ int size = sizeof(struct ipv6hdr);
+ int err;
+
+ err = skb_cow_head(skb, size + skb->mac_len);
+ if (err)
+ goto out;
+
+ __skb_push(skb, size);
+ skb_reset_network_header(skb);
+ skb_mac_header_rebuild(skb);
+
+ xfrm6_beet_make_header(skb);
+
+ ip6h = ipv6_hdr(skb);
+ ip6h->payload_len = htons(skb->len - size);
+ ip6h->daddr = x->sel.daddr.in6;
+ ip6h->saddr = x->sel.saddr.in6;
+ err = 0;
+out:
+ return err;
+}
+
+/* Remove encapsulation header.
+ *
+ * The IP header will be moved over the top of the encapsulation
+ * header.
+ *
+ * On entry, the transport header shall point to where the IP header
+ * should be and the network header shall be set to where the IP
+ * header currently is. skb->data shall point to the start of the
+ * payload.
+ */
+static int
+xfrm_inner_mode_encap_remove(struct xfrm_state *x,
+ const struct xfrm_mode *inner_mode,
+ struct sk_buff *skb)
+{
+ switch (inner_mode->encap) {
+ case XFRM_MODE_BEET:
+ if (inner_mode->family == AF_INET)
+ return xfrm4_remove_beet_encap(x, skb);
+ if (inner_mode->family == AF_INET6)
+ return xfrm6_remove_beet_encap(x, skb);
+ break;
+ case XFRM_MODE_TUNNEL:
+ if (inner_mode->family == AF_INET)
+ return xfrm4_remove_tunnel_encap(x, skb);
+ if (inner_mode->family == AF_INET6)
+ return xfrm6_remove_tunnel_encap(x, skb);
+ break;
+ }
+
+ WARN_ON_ONCE(1);
+ return -EOPNOTSUPP;
+}
+
+static int xfrm_prepare_input(struct xfrm_state *x, struct sk_buff *skb)
+{
+ const struct xfrm_mode *inner_mode = &x->inner_mode;
+
+ switch (x->outer_mode.family) {
+ case AF_INET:
+ xfrm4_extract_header(skb);
+ break;
+ case AF_INET6:
+ xfrm6_extract_header(skb);
+ break;
+ default:
+ WARN_ON_ONCE(1);
+ return -EAFNOSUPPORT;
+ }
+
+ if (x->sel.family == AF_UNSPEC) {
+ inner_mode = xfrm_ip2inner_mode(x, XFRM_MODE_SKB_CB(skb)->protocol);
+ if (!inner_mode)
+ return -EAFNOSUPPORT;
+ }
+
+ switch (inner_mode->family) {
+ case AF_INET:
+ skb->protocol = htons(ETH_P_IP);
+ break;
+ case AF_INET6:
+ skb->protocol = htons(ETH_P_IPV6);
+ break;
+ default:
+ WARN_ON_ONCE(1);
+ break;
+ }
+
+ return xfrm_inner_mode_encap_remove(x, inner_mode, skb);
+}
+
+/* Remove encapsulation header.
+ *
+ * The IP header will be moved over the top of the encapsulation header.
+ *
+ * On entry, skb_transport_header() shall point to where the IP header
+ * should be and skb_network_header() shall be set to where the IP header
+ * currently is. skb->data shall point to the start of the payload.
+ */
+static int xfrm4_transport_input(struct xfrm_state *x, struct sk_buff *skb)
+{
+ int ihl = skb->data - skb_transport_header(skb);
+
+ if (skb->transport_header != skb->network_header) {
+ memmove(skb_transport_header(skb),
+ skb_network_header(skb), ihl);
+ skb->network_header = skb->transport_header;
+ }
+ ip_hdr(skb)->tot_len = htons(skb->len + ihl);
+ skb_reset_transport_header(skb);
+ return 0;
+}
+
+static int xfrm6_transport_input(struct xfrm_state *x, struct sk_buff *skb)
+{
+#if IS_ENABLED(CONFIG_IPV6)
+ int ihl = skb->data - skb_transport_header(skb);
+
+ if (skb->transport_header != skb->network_header) {
+ memmove(skb_transport_header(skb),
+ skb_network_header(skb), ihl);
+ skb->network_header = skb->transport_header;
+ }
+ ipv6_hdr(skb)->payload_len = htons(skb->len + ihl -
+ sizeof(struct ipv6hdr));
+ skb_reset_transport_header(skb);
+ return 0;
+#else
+ WARN_ON_ONCE(1);
+ return -EAFNOSUPPORT;
+#endif
+}
+
+static int xfrm_inner_mode_input(struct xfrm_state *x,
+ const struct xfrm_mode *inner_mode,
+ struct sk_buff *skb)
+{
+ switch (inner_mode->encap) {
+ case XFRM_MODE_BEET:
+ case XFRM_MODE_TUNNEL:
+ return xfrm_prepare_input(x, skb);
+ case XFRM_MODE_TRANSPORT:
+ if (inner_mode->family == AF_INET)
+ return xfrm4_transport_input(x, skb);
+ if (inner_mode->family == AF_INET6)
+ return xfrm6_transport_input(x, skb);
+ break;
+ case XFRM_MODE_ROUTEOPTIMIZATION:
+ WARN_ON_ONCE(1);
+ break;
+ default:
+ WARN_ON_ONCE(1);
+ break;
+ }
+
+ return -EOPNOTSUPP;
+}
+
+int xfrm_input(struct sk_buff *skb, int nexthdr, __be32 spi, int encap_type)
+{
+ const struct xfrm_state_afinfo *afinfo;
+ struct net *net = dev_net(skb->dev);
+ const struct xfrm_mode *inner_mode;
+ int err;
+ __be32 seq;
+ __be32 seq_hi;
+ struct xfrm_state *x = NULL;
+ xfrm_address_t *daddr;
+ u32 mark = skb->mark;
+ unsigned int family = AF_UNSPEC;
+ int decaps = 0;
+ int async = 0;
+ bool xfrm_gro = false;
+ bool crypto_done = false;
+ struct xfrm_offload *xo = xfrm_offload(skb);
+ struct sec_path *sp;
+
+ if (encap_type < 0) {
+ x = xfrm_input_state(skb);
+
+ if (unlikely(x->km.state != XFRM_STATE_VALID)) {
+ if (x->km.state == XFRM_STATE_ACQ)
+ XFRM_INC_STATS(net, LINUX_MIB_XFRMACQUIREERROR);
+ else
+ XFRM_INC_STATS(net,
+ LINUX_MIB_XFRMINSTATEINVALID);
+
+ if (encap_type == -1)
+ dev_put(skb->dev);
+ goto drop;
+ }
+
+ family = x->outer_mode.family;
+
+ /* An encap_type of -1 indicates async resumption. */
+ if (encap_type == -1) {
+ async = 1;
+ seq = XFRM_SKB_CB(skb)->seq.input.low;
+ goto resume;
+ }
+
+ /* encap_type < -1 indicates a GRO call. */
+ encap_type = 0;
+ seq = XFRM_SPI_SKB_CB(skb)->seq;
+
+ if (xo && (xo->flags & CRYPTO_DONE)) {
+ crypto_done = true;
+ family = XFRM_SPI_SKB_CB(skb)->family;
+
+ if (!(xo->status & CRYPTO_SUCCESS)) {
+ if (xo->status &
+ (CRYPTO_TRANSPORT_AH_AUTH_FAILED |
+ CRYPTO_TRANSPORT_ESP_AUTH_FAILED |
+ CRYPTO_TUNNEL_AH_AUTH_FAILED |
+ CRYPTO_TUNNEL_ESP_AUTH_FAILED)) {
+
+ xfrm_audit_state_icvfail(x, skb,
+ x->type->proto);
+ x->stats.integrity_failed++;
+ XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATEPROTOERROR);
+ goto drop;
+ }
+
+ if (xo->status & CRYPTO_INVALID_PROTOCOL) {
+ XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATEPROTOERROR);
+ goto drop;
+ }
+
+ XFRM_INC_STATS(net, LINUX_MIB_XFRMINBUFFERERROR);
+ goto drop;
+ }
+
+ if (xfrm_parse_spi(skb, nexthdr, &spi, &seq)) {
+ XFRM_INC_STATS(net, LINUX_MIB_XFRMINHDRERROR);
+ goto drop;
+ }
+ }
+
+ goto lock;
+ }
+
+ family = XFRM_SPI_SKB_CB(skb)->family;
+
+ /* if tunnel is present override skb->mark value with tunnel i_key */
+ switch (family) {
+ case AF_INET:
+ if (XFRM_TUNNEL_SKB_CB(skb)->tunnel.ip4)
+ mark = be32_to_cpu(XFRM_TUNNEL_SKB_CB(skb)->tunnel.ip4->parms.i_key);
+ break;
+ case AF_INET6:
+ if (XFRM_TUNNEL_SKB_CB(skb)->tunnel.ip6)
+ mark = be32_to_cpu(XFRM_TUNNEL_SKB_CB(skb)->tunnel.ip6->parms.i_key);
+ break;
+ }
+
+ sp = secpath_set(skb);
+ if (!sp) {
+ XFRM_INC_STATS(net, LINUX_MIB_XFRMINERROR);
+ goto drop;
+ }
+
+ seq = 0;
+ if (!spi && xfrm_parse_spi(skb, nexthdr, &spi, &seq)) {
+ secpath_reset(skb);
+ XFRM_INC_STATS(net, LINUX_MIB_XFRMINHDRERROR);
+ goto drop;
+ }
+
+ daddr = (xfrm_address_t *)(skb_network_header(skb) +
+ XFRM_SPI_SKB_CB(skb)->daddroff);
+ do {
+ sp = skb_sec_path(skb);
+
+ if (sp->len == XFRM_MAX_DEPTH) {
+ secpath_reset(skb);
+ XFRM_INC_STATS(net, LINUX_MIB_XFRMINBUFFERERROR);
+ goto drop;
+ }
+
+ x = xfrm_state_lookup(net, mark, daddr, spi, nexthdr, family);
+ if (x == NULL) {
+ secpath_reset(skb);
+ XFRM_INC_STATS(net, LINUX_MIB_XFRMINNOSTATES);
+ xfrm_audit_state_notfound(skb, family, spi, seq);
+ goto drop;
+ }
+
+ skb->mark = xfrm_smark_get(skb->mark, x);
+
+ sp->xvec[sp->len++] = x;
+
+ skb_dst_force(skb);
+ if (!skb_dst(skb)) {
+ XFRM_INC_STATS(net, LINUX_MIB_XFRMINERROR);
+ goto drop;
+ }
+
+lock:
+ spin_lock(&x->lock);
+
+ if (unlikely(x->km.state != XFRM_STATE_VALID)) {
+ if (x->km.state == XFRM_STATE_ACQ)
+ XFRM_INC_STATS(net, LINUX_MIB_XFRMACQUIREERROR);
+ else
+ XFRM_INC_STATS(net,
+ LINUX_MIB_XFRMINSTATEINVALID);
+ goto drop_unlock;
+ }
+
+ if ((x->encap ? x->encap->encap_type : 0) != encap_type) {
+ XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATEMISMATCH);
+ goto drop_unlock;
+ }
+
+ if (xfrm_replay_check(x, skb, seq)) {
+ XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATESEQERROR);
+ goto drop_unlock;
+ }
+
+ if (xfrm_state_check_expire(x)) {
+ XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATEEXPIRED);
+ goto drop_unlock;
+ }
+
+ spin_unlock(&x->lock);
+
+ if (xfrm_tunnel_check(skb, x, family)) {
+ XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATEMODEERROR);
+ goto drop;
+ }
+
+ seq_hi = htonl(xfrm_replay_seqhi(x, seq));
+
+ XFRM_SKB_CB(skb)->seq.input.low = seq;
+ XFRM_SKB_CB(skb)->seq.input.hi = seq_hi;
+
+ dev_hold(skb->dev);
+
+ if (crypto_done)
+ nexthdr = x->type_offload->input_tail(x, skb);
+ else
+ nexthdr = x->type->input(x, skb);
+
+ if (nexthdr == -EINPROGRESS)
+ return 0;
+resume:
+ dev_put(skb->dev);
+
+ spin_lock(&x->lock);
+ if (nexthdr < 0) {
+ if (nexthdr == -EBADMSG) {
+ xfrm_audit_state_icvfail(x, skb,
+ x->type->proto);
+ x->stats.integrity_failed++;
+ }
+ XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATEPROTOERROR);
+ goto drop_unlock;
+ }
+
+ /* only the first xfrm gets the encap type */
+ encap_type = 0;
+
+ if (xfrm_replay_recheck(x, skb, seq)) {
+ XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATESEQERROR);
+ goto drop_unlock;
+ }
+
+ xfrm_replay_advance(x, seq);
+
+ x->curlft.bytes += skb->len;
+ x->curlft.packets++;
+
+ spin_unlock(&x->lock);
+
+ XFRM_MODE_SKB_CB(skb)->protocol = nexthdr;
+
+ inner_mode = &x->inner_mode;
+
+ if (x->sel.family == AF_UNSPEC) {
+ inner_mode = xfrm_ip2inner_mode(x, XFRM_MODE_SKB_CB(skb)->protocol);
+ if (inner_mode == NULL) {
+ XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATEMODEERROR);
+ goto drop;
+ }
+ }
+
+ if (xfrm_inner_mode_input(x, inner_mode, skb)) {
+ XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATEMODEERROR);
+ goto drop;
+ }
+
+ if (x->outer_mode.flags & XFRM_MODE_FLAG_TUNNEL) {
+ decaps = 1;
+ break;
+ }
+
+ /*
+ * We need the inner address. However, we only get here for
+ * transport mode so the outer address is identical.
+ */
+ daddr = &x->id.daddr;
+ family = x->outer_mode.family;
+
+ err = xfrm_parse_spi(skb, nexthdr, &spi, &seq);
+ if (err < 0) {
+ XFRM_INC_STATS(net, LINUX_MIB_XFRMINHDRERROR);
+ goto drop;
+ }
+ crypto_done = false;
+ } while (!err);
+
+ err = xfrm_rcv_cb(skb, family, x->type->proto, 0);
+ if (err)
+ goto drop;
+
+ nf_reset_ct(skb);
+
+ if (decaps) {
+ sp = skb_sec_path(skb);
+ if (sp)
+ sp->olen = 0;
+ if (skb_valid_dst(skb))
+ skb_dst_drop(skb);
+ gro_cells_receive(&gro_cells, skb);
+ return 0;
+ } else {
+ xo = xfrm_offload(skb);
+ if (xo)
+ xfrm_gro = xo->flags & XFRM_GRO;
+
+ err = -EAFNOSUPPORT;
+ rcu_read_lock();
+ afinfo = xfrm_state_afinfo_get_rcu(x->inner_mode.family);
+ if (likely(afinfo))
+ err = afinfo->transport_finish(skb, xfrm_gro || async);
+ rcu_read_unlock();
+ if (xfrm_gro) {
+ sp = skb_sec_path(skb);
+ if (sp)
+ sp->olen = 0;
+ if (skb_valid_dst(skb))
+ skb_dst_drop(skb);
+ gro_cells_receive(&gro_cells, skb);
+ return err;
+ }
+
+ return err;
+ }
+
+drop_unlock:
+ spin_unlock(&x->lock);
+drop:
+ xfrm_rcv_cb(skb, family, x && x->type ? x->type->proto : nexthdr, -1);
+ kfree_skb(skb);
+ return 0;
+}
+EXPORT_SYMBOL(xfrm_input);
+
+int xfrm_input_resume(struct sk_buff *skb, int nexthdr)
+{
+ return xfrm_input(skb, nexthdr, 0, -1);
+}
+EXPORT_SYMBOL(xfrm_input_resume);
+
+static void xfrm_trans_reinject(struct work_struct *work)
+{
+ struct xfrm_trans_tasklet *trans = container_of(work, struct xfrm_trans_tasklet, work);
+ struct sk_buff_head queue;
+ struct sk_buff *skb;
+
+ __skb_queue_head_init(&queue);
+ spin_lock_bh(&trans->queue_lock);
+ skb_queue_splice_init(&trans->queue, &queue);
+ spin_unlock_bh(&trans->queue_lock);
+
+ local_bh_disable();
+ while ((skb = __skb_dequeue(&queue)))
+ XFRM_TRANS_SKB_CB(skb)->finish(XFRM_TRANS_SKB_CB(skb)->net,
+ NULL, skb);
+ local_bh_enable();
+}
+
+int xfrm_trans_queue_net(struct net *net, struct sk_buff *skb,
+ int (*finish)(struct net *, struct sock *,
+ struct sk_buff *))
+{
+ struct xfrm_trans_tasklet *trans;
+
+ trans = this_cpu_ptr(&xfrm_trans_tasklet);
+
+ if (skb_queue_len(&trans->queue) >= READ_ONCE(netdev_max_backlog))
+ return -ENOBUFS;
+
+ BUILD_BUG_ON(sizeof(struct xfrm_trans_cb) > sizeof(skb->cb));
+
+ XFRM_TRANS_SKB_CB(skb)->finish = finish;
+ XFRM_TRANS_SKB_CB(skb)->net = net;
+ spin_lock_bh(&trans->queue_lock);
+ __skb_queue_tail(&trans->queue, skb);
+ spin_unlock_bh(&trans->queue_lock);
+ schedule_work(&trans->work);
+ return 0;
+}
+EXPORT_SYMBOL(xfrm_trans_queue_net);
+
+int xfrm_trans_queue(struct sk_buff *skb,
+ int (*finish)(struct net *, struct sock *,
+ struct sk_buff *))
+{
+ return xfrm_trans_queue_net(dev_net(skb->dev), skb, finish);
+}
+EXPORT_SYMBOL(xfrm_trans_queue);
+
+void __init xfrm_input_init(void)
+{
+ int err;
+ int i;
+
+ init_dummy_netdev(&xfrm_napi_dev);
+ err = gro_cells_init(&gro_cells, &xfrm_napi_dev);
+ if (err)
+ gro_cells.cells = NULL;
+
+ for_each_possible_cpu(i) {
+ struct xfrm_trans_tasklet *trans;
+
+ trans = &per_cpu(xfrm_trans_tasklet, i);
+ spin_lock_init(&trans->queue_lock);
+ __skb_queue_head_init(&trans->queue);
+ INIT_WORK(&trans->work, xfrm_trans_reinject);
+ }
+}
diff --git a/net/xfrm/xfrm_interface_core.c b/net/xfrm/xfrm_interface_core.c
new file mode 100644
index 000000000..85501b77f
--- /dev/null
+++ b/net/xfrm/xfrm_interface_core.c
@@ -0,0 +1,1242 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * XFRM virtual interface
+ *
+ * Copyright (C) 2018 secunet Security Networks AG
+ *
+ * Author:
+ * Steffen Klassert <steffen.klassert@secunet.com>
+ */
+
+#include <linux/module.h>
+#include <linux/capability.h>
+#include <linux/errno.h>
+#include <linux/types.h>
+#include <linux/sockios.h>
+#include <linux/icmp.h>
+#include <linux/if.h>
+#include <linux/in.h>
+#include <linux/ip.h>
+#include <linux/net.h>
+#include <linux/in6.h>
+#include <linux/netdevice.h>
+#include <linux/if_link.h>
+#include <linux/if_arp.h>
+#include <linux/icmpv6.h>
+#include <linux/init.h>
+#include <linux/route.h>
+#include <linux/rtnetlink.h>
+#include <linux/netfilter_ipv6.h>
+#include <linux/slab.h>
+#include <linux/hash.h>
+
+#include <linux/uaccess.h>
+#include <linux/atomic.h>
+
+#include <net/icmp.h>
+#include <net/ip.h>
+#include <net/ipv6.h>
+#include <net/ip6_route.h>
+#include <net/ip_tunnels.h>
+#include <net/addrconf.h>
+#include <net/xfrm.h>
+#include <net/net_namespace.h>
+#include <net/dst_metadata.h>
+#include <net/netns/generic.h>
+#include <linux/etherdevice.h>
+
+static int xfrmi_dev_init(struct net_device *dev);
+static void xfrmi_dev_setup(struct net_device *dev);
+static struct rtnl_link_ops xfrmi_link_ops __read_mostly;
+static unsigned int xfrmi_net_id __read_mostly;
+static const struct net_device_ops xfrmi_netdev_ops;
+
+#define XFRMI_HASH_BITS 8
+#define XFRMI_HASH_SIZE BIT(XFRMI_HASH_BITS)
+
+struct xfrmi_net {
+ /* lists for storing interfaces in use */
+ struct xfrm_if __rcu *xfrmi[XFRMI_HASH_SIZE];
+ struct xfrm_if __rcu *collect_md_xfrmi;
+};
+
+static const struct nla_policy xfrm_lwt_policy[LWT_XFRM_MAX + 1] = {
+ [LWT_XFRM_IF_ID] = NLA_POLICY_MIN(NLA_U32, 1),
+ [LWT_XFRM_LINK] = NLA_POLICY_MIN(NLA_U32, 1),
+};
+
+static void xfrmi_destroy_state(struct lwtunnel_state *lwt)
+{
+}
+
+static int xfrmi_build_state(struct net *net, struct nlattr *nla,
+ unsigned int family, const void *cfg,
+ struct lwtunnel_state **ts,
+ struct netlink_ext_ack *extack)
+{
+ struct nlattr *tb[LWT_XFRM_MAX + 1];
+ struct lwtunnel_state *new_state;
+ struct xfrm_md_info *info;
+ int ret;
+
+ ret = nla_parse_nested(tb, LWT_XFRM_MAX, nla, xfrm_lwt_policy, extack);
+ if (ret < 0)
+ return ret;
+
+ if (!tb[LWT_XFRM_IF_ID]) {
+ NL_SET_ERR_MSG(extack, "if_id must be set");
+ return -EINVAL;
+ }
+
+ new_state = lwtunnel_state_alloc(sizeof(*info));
+ if (!new_state) {
+ NL_SET_ERR_MSG(extack, "failed to create encap info");
+ return -ENOMEM;
+ }
+
+ new_state->type = LWTUNNEL_ENCAP_XFRM;
+
+ info = lwt_xfrm_info(new_state);
+
+ info->if_id = nla_get_u32(tb[LWT_XFRM_IF_ID]);
+
+ if (tb[LWT_XFRM_LINK])
+ info->link = nla_get_u32(tb[LWT_XFRM_LINK]);
+
+ *ts = new_state;
+ return 0;
+}
+
+static int xfrmi_fill_encap_info(struct sk_buff *skb,
+ struct lwtunnel_state *lwt)
+{
+ struct xfrm_md_info *info = lwt_xfrm_info(lwt);
+
+ if (nla_put_u32(skb, LWT_XFRM_IF_ID, info->if_id) ||
+ (info->link && nla_put_u32(skb, LWT_XFRM_LINK, info->link)))
+ return -EMSGSIZE;
+
+ return 0;
+}
+
+static int xfrmi_encap_nlsize(struct lwtunnel_state *lwtstate)
+{
+ return nla_total_size(sizeof(u32)) + /* LWT_XFRM_IF_ID */
+ nla_total_size(sizeof(u32)); /* LWT_XFRM_LINK */
+}
+
+static int xfrmi_encap_cmp(struct lwtunnel_state *a, struct lwtunnel_state *b)
+{
+ struct xfrm_md_info *a_info = lwt_xfrm_info(a);
+ struct xfrm_md_info *b_info = lwt_xfrm_info(b);
+
+ return memcmp(a_info, b_info, sizeof(*a_info));
+}
+
+static const struct lwtunnel_encap_ops xfrmi_encap_ops = {
+ .build_state = xfrmi_build_state,
+ .destroy_state = xfrmi_destroy_state,
+ .fill_encap = xfrmi_fill_encap_info,
+ .get_encap_size = xfrmi_encap_nlsize,
+ .cmp_encap = xfrmi_encap_cmp,
+ .owner = THIS_MODULE,
+};
+
+#define for_each_xfrmi_rcu(start, xi) \
+ for (xi = rcu_dereference(start); xi; xi = rcu_dereference(xi->next))
+
+static u32 xfrmi_hash(u32 if_id)
+{
+ return hash_32(if_id, XFRMI_HASH_BITS);
+}
+
+static struct xfrm_if *xfrmi_lookup(struct net *net, struct xfrm_state *x)
+{
+ struct xfrmi_net *xfrmn = net_generic(net, xfrmi_net_id);
+ struct xfrm_if *xi;
+
+ for_each_xfrmi_rcu(xfrmn->xfrmi[xfrmi_hash(x->if_id)], xi) {
+ if (x->if_id == xi->p.if_id &&
+ (xi->dev->flags & IFF_UP))
+ return xi;
+ }
+
+ xi = rcu_dereference(xfrmn->collect_md_xfrmi);
+ if (xi && (xi->dev->flags & IFF_UP))
+ return xi;
+
+ return NULL;
+}
+
+static bool xfrmi_decode_session(struct sk_buff *skb,
+ unsigned short family,
+ struct xfrm_if_decode_session_result *res)
+{
+ struct net_device *dev;
+ struct xfrm_if *xi;
+ int ifindex = 0;
+
+ if (!secpath_exists(skb) || !skb->dev)
+ return false;
+
+ switch (family) {
+ case AF_INET6:
+ ifindex = inet6_sdif(skb);
+ break;
+ case AF_INET:
+ ifindex = inet_sdif(skb);
+ break;
+ }
+
+ if (ifindex) {
+ struct net *net = xs_net(xfrm_input_state(skb));
+
+ dev = dev_get_by_index_rcu(net, ifindex);
+ } else {
+ dev = skb->dev;
+ }
+
+ if (!dev || !(dev->flags & IFF_UP))
+ return false;
+ if (dev->netdev_ops != &xfrmi_netdev_ops)
+ return false;
+
+ xi = netdev_priv(dev);
+ res->net = xi->net;
+
+ if (xi->p.collect_md)
+ res->if_id = xfrm_input_state(skb)->if_id;
+ else
+ res->if_id = xi->p.if_id;
+ return true;
+}
+
+static void xfrmi_link(struct xfrmi_net *xfrmn, struct xfrm_if *xi)
+{
+ struct xfrm_if __rcu **xip = &xfrmn->xfrmi[xfrmi_hash(xi->p.if_id)];
+
+ rcu_assign_pointer(xi->next , rtnl_dereference(*xip));
+ rcu_assign_pointer(*xip, xi);
+}
+
+static void xfrmi_unlink(struct xfrmi_net *xfrmn, struct xfrm_if *xi)
+{
+ struct xfrm_if __rcu **xip;
+ struct xfrm_if *iter;
+
+ for (xip = &xfrmn->xfrmi[xfrmi_hash(xi->p.if_id)];
+ (iter = rtnl_dereference(*xip)) != NULL;
+ xip = &iter->next) {
+ if (xi == iter) {
+ rcu_assign_pointer(*xip, xi->next);
+ break;
+ }
+ }
+}
+
+static void xfrmi_dev_free(struct net_device *dev)
+{
+ struct xfrm_if *xi = netdev_priv(dev);
+
+ gro_cells_destroy(&xi->gro_cells);
+ free_percpu(dev->tstats);
+}
+
+static int xfrmi_create(struct net_device *dev)
+{
+ struct xfrm_if *xi = netdev_priv(dev);
+ struct net *net = dev_net(dev);
+ struct xfrmi_net *xfrmn = net_generic(net, xfrmi_net_id);
+ int err;
+
+ dev->rtnl_link_ops = &xfrmi_link_ops;
+ err = register_netdevice(dev);
+ if (err < 0)
+ goto out;
+
+ if (xi->p.collect_md)
+ rcu_assign_pointer(xfrmn->collect_md_xfrmi, xi);
+ else
+ xfrmi_link(xfrmn, xi);
+
+ return 0;
+
+out:
+ return err;
+}
+
+static struct xfrm_if *xfrmi_locate(struct net *net, struct xfrm_if_parms *p)
+{
+ struct xfrm_if __rcu **xip;
+ struct xfrm_if *xi;
+ struct xfrmi_net *xfrmn = net_generic(net, xfrmi_net_id);
+
+ for (xip = &xfrmn->xfrmi[xfrmi_hash(p->if_id)];
+ (xi = rtnl_dereference(*xip)) != NULL;
+ xip = &xi->next)
+ if (xi->p.if_id == p->if_id)
+ return xi;
+
+ return NULL;
+}
+
+static void xfrmi_dev_uninit(struct net_device *dev)
+{
+ struct xfrm_if *xi = netdev_priv(dev);
+ struct xfrmi_net *xfrmn = net_generic(xi->net, xfrmi_net_id);
+
+ if (xi->p.collect_md)
+ RCU_INIT_POINTER(xfrmn->collect_md_xfrmi, NULL);
+ else
+ xfrmi_unlink(xfrmn, xi);
+}
+
+static void xfrmi_scrub_packet(struct sk_buff *skb, bool xnet)
+{
+ skb_clear_tstamp(skb);
+ skb->pkt_type = PACKET_HOST;
+ skb->skb_iif = 0;
+ skb->ignore_df = 0;
+ skb_dst_drop(skb);
+ nf_reset_ct(skb);
+ nf_reset_trace(skb);
+
+ if (!xnet)
+ return;
+
+ ipvs_reset(skb);
+ secpath_reset(skb);
+ skb_orphan(skb);
+ skb->mark = 0;
+}
+
+static int xfrmi_input(struct sk_buff *skb, int nexthdr, __be32 spi,
+ int encap_type, unsigned short family)
+{
+ struct sec_path *sp;
+
+ sp = skb_sec_path(skb);
+ if (sp && (sp->len || sp->olen) &&
+ !xfrm_policy_check(NULL, XFRM_POLICY_IN, skb, family))
+ goto discard;
+
+ XFRM_SPI_SKB_CB(skb)->family = family;
+ if (family == AF_INET) {
+ XFRM_SPI_SKB_CB(skb)->daddroff = offsetof(struct iphdr, daddr);
+ XFRM_TUNNEL_SKB_CB(skb)->tunnel.ip4 = NULL;
+ } else {
+ XFRM_SPI_SKB_CB(skb)->daddroff = offsetof(struct ipv6hdr, daddr);
+ XFRM_TUNNEL_SKB_CB(skb)->tunnel.ip6 = NULL;
+ }
+
+ return xfrm_input(skb, nexthdr, spi, encap_type);
+discard:
+ kfree_skb(skb);
+ return 0;
+}
+
+static int xfrmi4_rcv(struct sk_buff *skb)
+{
+ return xfrmi_input(skb, ip_hdr(skb)->protocol, 0, 0, AF_INET);
+}
+
+static int xfrmi6_rcv(struct sk_buff *skb)
+{
+ return xfrmi_input(skb, skb_network_header(skb)[IP6CB(skb)->nhoff],
+ 0, 0, AF_INET6);
+}
+
+static int xfrmi4_input(struct sk_buff *skb, int nexthdr, __be32 spi, int encap_type)
+{
+ return xfrmi_input(skb, nexthdr, spi, encap_type, AF_INET);
+}
+
+static int xfrmi6_input(struct sk_buff *skb, int nexthdr, __be32 spi, int encap_type)
+{
+ return xfrmi_input(skb, nexthdr, spi, encap_type, AF_INET6);
+}
+
+static int xfrmi_rcv_cb(struct sk_buff *skb, int err)
+{
+ const struct xfrm_mode *inner_mode;
+ struct net_device *dev;
+ struct xfrm_state *x;
+ struct xfrm_if *xi;
+ bool xnet;
+ int link;
+
+ if (err && !secpath_exists(skb))
+ return 0;
+
+ x = xfrm_input_state(skb);
+
+ xi = xfrmi_lookup(xs_net(x), x);
+ if (!xi)
+ return 1;
+
+ link = skb->dev->ifindex;
+ dev = xi->dev;
+ skb->dev = dev;
+
+ if (err) {
+ DEV_STATS_INC(dev, rx_errors);
+ DEV_STATS_INC(dev, rx_dropped);
+
+ return 0;
+ }
+
+ xnet = !net_eq(xi->net, dev_net(skb->dev));
+
+ if (xnet) {
+ inner_mode = &x->inner_mode;
+
+ if (x->sel.family == AF_UNSPEC) {
+ inner_mode = xfrm_ip2inner_mode(x, XFRM_MODE_SKB_CB(skb)->protocol);
+ if (inner_mode == NULL) {
+ XFRM_INC_STATS(dev_net(skb->dev),
+ LINUX_MIB_XFRMINSTATEMODEERROR);
+ return -EINVAL;
+ }
+ }
+
+ if (!xfrm_policy_check(NULL, XFRM_POLICY_IN, skb,
+ inner_mode->family))
+ return -EPERM;
+ }
+
+ xfrmi_scrub_packet(skb, xnet);
+ if (xi->p.collect_md) {
+ struct metadata_dst *md_dst;
+
+ md_dst = metadata_dst_alloc(0, METADATA_XFRM, GFP_ATOMIC);
+ if (!md_dst)
+ return -ENOMEM;
+
+ md_dst->u.xfrm_info.if_id = x->if_id;
+ md_dst->u.xfrm_info.link = link;
+ skb_dst_set(skb, (struct dst_entry *)md_dst);
+ }
+ dev_sw_netstats_rx_add(dev, skb->len);
+
+ return 0;
+}
+
+static int
+xfrmi_xmit2(struct sk_buff *skb, struct net_device *dev, struct flowi *fl)
+{
+ struct xfrm_if *xi = netdev_priv(dev);
+ struct dst_entry *dst = skb_dst(skb);
+ unsigned int length = skb->len;
+ struct net_device *tdev;
+ struct xfrm_state *x;
+ int err = -1;
+ u32 if_id;
+ int mtu;
+
+ if (xi->p.collect_md) {
+ struct xfrm_md_info *md_info = skb_xfrm_md_info(skb);
+
+ if (unlikely(!md_info))
+ return -EINVAL;
+
+ if_id = md_info->if_id;
+ fl->flowi_oif = md_info->link;
+ } else {
+ if_id = xi->p.if_id;
+ }
+
+ dst_hold(dst);
+ dst = xfrm_lookup_with_ifid(xi->net, dst, fl, NULL, 0, if_id);
+ if (IS_ERR(dst)) {
+ err = PTR_ERR(dst);
+ dst = NULL;
+ goto tx_err_link_failure;
+ }
+
+ x = dst->xfrm;
+ if (!x)
+ goto tx_err_link_failure;
+
+ if (x->if_id != if_id)
+ goto tx_err_link_failure;
+
+ tdev = dst->dev;
+
+ if (tdev == dev) {
+ DEV_STATS_INC(dev, collisions);
+ net_warn_ratelimited("%s: Local routing loop detected!\n",
+ dev->name);
+ goto tx_err_dst_release;
+ }
+
+ mtu = dst_mtu(dst);
+ if ((!skb_is_gso(skb) && skb->len > mtu) ||
+ (skb_is_gso(skb) && !skb_gso_validate_network_len(skb, mtu))) {
+ skb_dst_update_pmtu_no_confirm(skb, mtu);
+
+ if (skb->protocol == htons(ETH_P_IPV6)) {
+ if (mtu < IPV6_MIN_MTU)
+ mtu = IPV6_MIN_MTU;
+
+ if (skb->len > 1280)
+ icmpv6_ndo_send(skb, ICMPV6_PKT_TOOBIG, 0, mtu);
+ else
+ goto xmit;
+ } else {
+ if (!(ip_hdr(skb)->frag_off & htons(IP_DF)))
+ goto xmit;
+ icmp_ndo_send(skb, ICMP_DEST_UNREACH, ICMP_FRAG_NEEDED,
+ htonl(mtu));
+ }
+
+ dst_release(dst);
+ return -EMSGSIZE;
+ }
+
+xmit:
+ xfrmi_scrub_packet(skb, !net_eq(xi->net, dev_net(dev)));
+ skb_dst_set(skb, dst);
+ skb->dev = tdev;
+
+ err = dst_output(xi->net, skb->sk, skb);
+ if (net_xmit_eval(err) == 0) {
+ dev_sw_netstats_tx_add(dev, 1, length);
+ } else {
+ DEV_STATS_INC(dev, tx_errors);
+ DEV_STATS_INC(dev, tx_aborted_errors);
+ }
+
+ return 0;
+tx_err_link_failure:
+ DEV_STATS_INC(dev, tx_carrier_errors);
+ dst_link_failure(skb);
+tx_err_dst_release:
+ dst_release(dst);
+ return err;
+}
+
+static netdev_tx_t xfrmi_xmit(struct sk_buff *skb, struct net_device *dev)
+{
+ struct xfrm_if *xi = netdev_priv(dev);
+ struct dst_entry *dst = skb_dst(skb);
+ struct flowi fl;
+ int ret;
+
+ memset(&fl, 0, sizeof(fl));
+
+ switch (skb->protocol) {
+ case htons(ETH_P_IPV6):
+ memset(IP6CB(skb), 0, sizeof(*IP6CB(skb)));
+ xfrm_decode_session(skb, &fl, AF_INET6);
+ if (!dst) {
+ fl.u.ip6.flowi6_oif = dev->ifindex;
+ fl.u.ip6.flowi6_flags |= FLOWI_FLAG_ANYSRC;
+ dst = ip6_route_output(dev_net(dev), NULL, &fl.u.ip6);
+ if (dst->error) {
+ dst_release(dst);
+ DEV_STATS_INC(dev, tx_carrier_errors);
+ goto tx_err;
+ }
+ skb_dst_set(skb, dst);
+ }
+ break;
+ case htons(ETH_P_IP):
+ memset(IPCB(skb), 0, sizeof(*IPCB(skb)));
+ xfrm_decode_session(skb, &fl, AF_INET);
+ if (!dst) {
+ struct rtable *rt;
+
+ fl.u.ip4.flowi4_oif = dev->ifindex;
+ fl.u.ip4.flowi4_flags |= FLOWI_FLAG_ANYSRC;
+ rt = __ip_route_output_key(dev_net(dev), &fl.u.ip4);
+ if (IS_ERR(rt)) {
+ DEV_STATS_INC(dev, tx_carrier_errors);
+ goto tx_err;
+ }
+ skb_dst_set(skb, &rt->dst);
+ }
+ break;
+ default:
+ goto tx_err;
+ }
+
+ fl.flowi_oif = xi->p.link;
+
+ ret = xfrmi_xmit2(skb, dev, &fl);
+ if (ret < 0)
+ goto tx_err;
+
+ return NETDEV_TX_OK;
+
+tx_err:
+ DEV_STATS_INC(dev, tx_errors);
+ DEV_STATS_INC(dev, tx_dropped);
+ kfree_skb(skb);
+ return NETDEV_TX_OK;
+}
+
+static int xfrmi4_err(struct sk_buff *skb, u32 info)
+{
+ const struct iphdr *iph = (const struct iphdr *)skb->data;
+ struct net *net = dev_net(skb->dev);
+ int protocol = iph->protocol;
+ struct ip_comp_hdr *ipch;
+ struct ip_esp_hdr *esph;
+ struct ip_auth_hdr *ah ;
+ struct xfrm_state *x;
+ struct xfrm_if *xi;
+ __be32 spi;
+
+ switch (protocol) {
+ case IPPROTO_ESP:
+ esph = (struct ip_esp_hdr *)(skb->data+(iph->ihl<<2));
+ spi = esph->spi;
+ break;
+ case IPPROTO_AH:
+ ah = (struct ip_auth_hdr *)(skb->data+(iph->ihl<<2));
+ spi = ah->spi;
+ break;
+ case IPPROTO_COMP:
+ ipch = (struct ip_comp_hdr *)(skb->data+(iph->ihl<<2));
+ spi = htonl(ntohs(ipch->cpi));
+ break;
+ default:
+ return 0;
+ }
+
+ switch (icmp_hdr(skb)->type) {
+ case ICMP_DEST_UNREACH:
+ if (icmp_hdr(skb)->code != ICMP_FRAG_NEEDED)
+ return 0;
+ break;
+ case ICMP_REDIRECT:
+ break;
+ default:
+ return 0;
+ }
+
+ x = xfrm_state_lookup(net, skb->mark, (const xfrm_address_t *)&iph->daddr,
+ spi, protocol, AF_INET);
+ if (!x)
+ return 0;
+
+ xi = xfrmi_lookup(net, x);
+ if (!xi) {
+ xfrm_state_put(x);
+ return -1;
+ }
+
+ if (icmp_hdr(skb)->type == ICMP_DEST_UNREACH)
+ ipv4_update_pmtu(skb, net, info, 0, protocol);
+ else
+ ipv4_redirect(skb, net, 0, protocol);
+ xfrm_state_put(x);
+
+ return 0;
+}
+
+static int xfrmi6_err(struct sk_buff *skb, struct inet6_skb_parm *opt,
+ u8 type, u8 code, int offset, __be32 info)
+{
+ const struct ipv6hdr *iph = (const struct ipv6hdr *)skb->data;
+ struct net *net = dev_net(skb->dev);
+ int protocol = iph->nexthdr;
+ struct ip_comp_hdr *ipch;
+ struct ip_esp_hdr *esph;
+ struct ip_auth_hdr *ah;
+ struct xfrm_state *x;
+ struct xfrm_if *xi;
+ __be32 spi;
+
+ switch (protocol) {
+ case IPPROTO_ESP:
+ esph = (struct ip_esp_hdr *)(skb->data + offset);
+ spi = esph->spi;
+ break;
+ case IPPROTO_AH:
+ ah = (struct ip_auth_hdr *)(skb->data + offset);
+ spi = ah->spi;
+ break;
+ case IPPROTO_COMP:
+ ipch = (struct ip_comp_hdr *)(skb->data + offset);
+ spi = htonl(ntohs(ipch->cpi));
+ break;
+ default:
+ return 0;
+ }
+
+ if (type != ICMPV6_PKT_TOOBIG &&
+ type != NDISC_REDIRECT)
+ return 0;
+
+ x = xfrm_state_lookup(net, skb->mark, (const xfrm_address_t *)&iph->daddr,
+ spi, protocol, AF_INET6);
+ if (!x)
+ return 0;
+
+ xi = xfrmi_lookup(net, x);
+ if (!xi) {
+ xfrm_state_put(x);
+ return -1;
+ }
+
+ if (type == NDISC_REDIRECT)
+ ip6_redirect(skb, net, skb->dev->ifindex, 0,
+ sock_net_uid(net, NULL));
+ else
+ ip6_update_pmtu(skb, net, info, 0, 0, sock_net_uid(net, NULL));
+ xfrm_state_put(x);
+
+ return 0;
+}
+
+static int xfrmi_change(struct xfrm_if *xi, const struct xfrm_if_parms *p)
+{
+ if (xi->p.link != p->link)
+ return -EINVAL;
+
+ xi->p.if_id = p->if_id;
+
+ return 0;
+}
+
+static int xfrmi_update(struct xfrm_if *xi, struct xfrm_if_parms *p)
+{
+ struct net *net = xi->net;
+ struct xfrmi_net *xfrmn = net_generic(net, xfrmi_net_id);
+ int err;
+
+ xfrmi_unlink(xfrmn, xi);
+ synchronize_net();
+ err = xfrmi_change(xi, p);
+ xfrmi_link(xfrmn, xi);
+ netdev_state_change(xi->dev);
+ return err;
+}
+
+static int xfrmi_get_iflink(const struct net_device *dev)
+{
+ struct xfrm_if *xi = netdev_priv(dev);
+
+ return xi->p.link;
+}
+
+static const struct net_device_ops xfrmi_netdev_ops = {
+ .ndo_init = xfrmi_dev_init,
+ .ndo_uninit = xfrmi_dev_uninit,
+ .ndo_start_xmit = xfrmi_xmit,
+ .ndo_get_stats64 = dev_get_tstats64,
+ .ndo_get_iflink = xfrmi_get_iflink,
+};
+
+static void xfrmi_dev_setup(struct net_device *dev)
+{
+ dev->netdev_ops = &xfrmi_netdev_ops;
+ dev->header_ops = &ip_tunnel_header_ops;
+ dev->type = ARPHRD_NONE;
+ dev->mtu = ETH_DATA_LEN;
+ dev->min_mtu = ETH_MIN_MTU;
+ dev->max_mtu = IP_MAX_MTU;
+ dev->flags = IFF_NOARP;
+ dev->needs_free_netdev = true;
+ dev->priv_destructor = xfrmi_dev_free;
+ netif_keep_dst(dev);
+
+ eth_broadcast_addr(dev->broadcast);
+}
+
+#define XFRMI_FEATURES (NETIF_F_SG | \
+ NETIF_F_FRAGLIST | \
+ NETIF_F_GSO_SOFTWARE | \
+ NETIF_F_HW_CSUM)
+
+static int xfrmi_dev_init(struct net_device *dev)
+{
+ struct xfrm_if *xi = netdev_priv(dev);
+ struct net_device *phydev = __dev_get_by_index(xi->net, xi->p.link);
+ int err;
+
+ dev->tstats = netdev_alloc_pcpu_stats(struct pcpu_sw_netstats);
+ if (!dev->tstats)
+ return -ENOMEM;
+
+ err = gro_cells_init(&xi->gro_cells, dev);
+ if (err) {
+ free_percpu(dev->tstats);
+ return err;
+ }
+
+ dev->features |= NETIF_F_LLTX;
+ dev->features |= XFRMI_FEATURES;
+ dev->hw_features |= XFRMI_FEATURES;
+
+ if (phydev) {
+ dev->needed_headroom = phydev->needed_headroom;
+ dev->needed_tailroom = phydev->needed_tailroom;
+
+ if (is_zero_ether_addr(dev->dev_addr))
+ eth_hw_addr_inherit(dev, phydev);
+ if (is_zero_ether_addr(dev->broadcast))
+ memcpy(dev->broadcast, phydev->broadcast,
+ dev->addr_len);
+ } else {
+ eth_hw_addr_random(dev);
+ eth_broadcast_addr(dev->broadcast);
+ }
+
+ return 0;
+}
+
+static int xfrmi_validate(struct nlattr *tb[], struct nlattr *data[],
+ struct netlink_ext_ack *extack)
+{
+ return 0;
+}
+
+static void xfrmi_netlink_parms(struct nlattr *data[],
+ struct xfrm_if_parms *parms)
+{
+ memset(parms, 0, sizeof(*parms));
+
+ if (!data)
+ return;
+
+ if (data[IFLA_XFRM_LINK])
+ parms->link = nla_get_u32(data[IFLA_XFRM_LINK]);
+
+ if (data[IFLA_XFRM_IF_ID])
+ parms->if_id = nla_get_u32(data[IFLA_XFRM_IF_ID]);
+
+ if (data[IFLA_XFRM_COLLECT_METADATA])
+ parms->collect_md = true;
+}
+
+static int xfrmi_newlink(struct net *src_net, struct net_device *dev,
+ struct nlattr *tb[], struct nlattr *data[],
+ struct netlink_ext_ack *extack)
+{
+ struct net *net = dev_net(dev);
+ struct xfrm_if_parms p = {};
+ struct xfrm_if *xi;
+ int err;
+
+ xfrmi_netlink_parms(data, &p);
+ if (p.collect_md) {
+ struct xfrmi_net *xfrmn = net_generic(net, xfrmi_net_id);
+
+ if (p.link || p.if_id) {
+ NL_SET_ERR_MSG(extack, "link and if_id must be zero");
+ return -EINVAL;
+ }
+
+ if (rtnl_dereference(xfrmn->collect_md_xfrmi))
+ return -EEXIST;
+
+ } else {
+ if (!p.if_id) {
+ NL_SET_ERR_MSG(extack, "if_id must be non zero");
+ return -EINVAL;
+ }
+
+ xi = xfrmi_locate(net, &p);
+ if (xi)
+ return -EEXIST;
+ }
+
+ xi = netdev_priv(dev);
+ xi->p = p;
+ xi->net = net;
+ xi->dev = dev;
+
+ err = xfrmi_create(dev);
+ return err;
+}
+
+static void xfrmi_dellink(struct net_device *dev, struct list_head *head)
+{
+ unregister_netdevice_queue(dev, head);
+}
+
+static int xfrmi_changelink(struct net_device *dev, struct nlattr *tb[],
+ struct nlattr *data[],
+ struct netlink_ext_ack *extack)
+{
+ struct xfrm_if *xi = netdev_priv(dev);
+ struct net *net = xi->net;
+ struct xfrm_if_parms p = {};
+
+ xfrmi_netlink_parms(data, &p);
+ if (!p.if_id) {
+ NL_SET_ERR_MSG(extack, "if_id must be non zero");
+ return -EINVAL;
+ }
+
+ if (p.collect_md) {
+ NL_SET_ERR_MSG(extack, "collect_md can't be changed");
+ return -EINVAL;
+ }
+
+ xi = xfrmi_locate(net, &p);
+ if (!xi) {
+ xi = netdev_priv(dev);
+ } else {
+ if (xi->dev != dev)
+ return -EEXIST;
+ if (xi->p.collect_md) {
+ NL_SET_ERR_MSG(extack,
+ "device can't be changed to collect_md");
+ return -EINVAL;
+ }
+ }
+
+ return xfrmi_update(xi, &p);
+}
+
+static size_t xfrmi_get_size(const struct net_device *dev)
+{
+ return
+ /* IFLA_XFRM_LINK */
+ nla_total_size(4) +
+ /* IFLA_XFRM_IF_ID */
+ nla_total_size(4) +
+ /* IFLA_XFRM_COLLECT_METADATA */
+ nla_total_size(0) +
+ 0;
+}
+
+static int xfrmi_fill_info(struct sk_buff *skb, const struct net_device *dev)
+{
+ struct xfrm_if *xi = netdev_priv(dev);
+ struct xfrm_if_parms *parm = &xi->p;
+
+ if (nla_put_u32(skb, IFLA_XFRM_LINK, parm->link) ||
+ nla_put_u32(skb, IFLA_XFRM_IF_ID, parm->if_id) ||
+ (xi->p.collect_md && nla_put_flag(skb, IFLA_XFRM_COLLECT_METADATA)))
+ goto nla_put_failure;
+ return 0;
+
+nla_put_failure:
+ return -EMSGSIZE;
+}
+
+static struct net *xfrmi_get_link_net(const struct net_device *dev)
+{
+ struct xfrm_if *xi = netdev_priv(dev);
+
+ return xi->net;
+}
+
+static const struct nla_policy xfrmi_policy[IFLA_XFRM_MAX + 1] = {
+ [IFLA_XFRM_UNSPEC] = { .strict_start_type = IFLA_XFRM_COLLECT_METADATA },
+ [IFLA_XFRM_LINK] = { .type = NLA_U32 },
+ [IFLA_XFRM_IF_ID] = { .type = NLA_U32 },
+ [IFLA_XFRM_COLLECT_METADATA] = { .type = NLA_FLAG },
+};
+
+static struct rtnl_link_ops xfrmi_link_ops __read_mostly = {
+ .kind = "xfrm",
+ .maxtype = IFLA_XFRM_MAX,
+ .policy = xfrmi_policy,
+ .priv_size = sizeof(struct xfrm_if),
+ .setup = xfrmi_dev_setup,
+ .validate = xfrmi_validate,
+ .newlink = xfrmi_newlink,
+ .dellink = xfrmi_dellink,
+ .changelink = xfrmi_changelink,
+ .get_size = xfrmi_get_size,
+ .fill_info = xfrmi_fill_info,
+ .get_link_net = xfrmi_get_link_net,
+};
+
+static void __net_exit xfrmi_exit_batch_net(struct list_head *net_exit_list)
+{
+ struct net *net;
+ LIST_HEAD(list);
+
+ rtnl_lock();
+ list_for_each_entry(net, net_exit_list, exit_list) {
+ struct xfrmi_net *xfrmn = net_generic(net, xfrmi_net_id);
+ struct xfrm_if __rcu **xip;
+ struct xfrm_if *xi;
+ int i;
+
+ for (i = 0; i < XFRMI_HASH_SIZE; i++) {
+ for (xip = &xfrmn->xfrmi[i];
+ (xi = rtnl_dereference(*xip)) != NULL;
+ xip = &xi->next)
+ unregister_netdevice_queue(xi->dev, &list);
+ }
+ xi = rtnl_dereference(xfrmn->collect_md_xfrmi);
+ if (xi)
+ unregister_netdevice_queue(xi->dev, &list);
+ }
+ unregister_netdevice_many(&list);
+ rtnl_unlock();
+}
+
+static struct pernet_operations xfrmi_net_ops = {
+ .exit_batch = xfrmi_exit_batch_net,
+ .id = &xfrmi_net_id,
+ .size = sizeof(struct xfrmi_net),
+};
+
+static struct xfrm6_protocol xfrmi_esp6_protocol __read_mostly = {
+ .handler = xfrmi6_rcv,
+ .input_handler = xfrmi6_input,
+ .cb_handler = xfrmi_rcv_cb,
+ .err_handler = xfrmi6_err,
+ .priority = 10,
+};
+
+static struct xfrm6_protocol xfrmi_ah6_protocol __read_mostly = {
+ .handler = xfrm6_rcv,
+ .input_handler = xfrm_input,
+ .cb_handler = xfrmi_rcv_cb,
+ .err_handler = xfrmi6_err,
+ .priority = 10,
+};
+
+static struct xfrm6_protocol xfrmi_ipcomp6_protocol __read_mostly = {
+ .handler = xfrm6_rcv,
+ .input_handler = xfrm_input,
+ .cb_handler = xfrmi_rcv_cb,
+ .err_handler = xfrmi6_err,
+ .priority = 10,
+};
+
+#if IS_REACHABLE(CONFIG_INET6_XFRM_TUNNEL)
+static int xfrmi6_rcv_tunnel(struct sk_buff *skb)
+{
+ const xfrm_address_t *saddr;
+ __be32 spi;
+
+ saddr = (const xfrm_address_t *)&ipv6_hdr(skb)->saddr;
+ spi = xfrm6_tunnel_spi_lookup(dev_net(skb->dev), saddr);
+
+ return xfrm6_rcv_spi(skb, IPPROTO_IPV6, spi, NULL);
+}
+
+static struct xfrm6_tunnel xfrmi_ipv6_handler __read_mostly = {
+ .handler = xfrmi6_rcv_tunnel,
+ .cb_handler = xfrmi_rcv_cb,
+ .err_handler = xfrmi6_err,
+ .priority = 2,
+};
+
+static struct xfrm6_tunnel xfrmi_ip6ip_handler __read_mostly = {
+ .handler = xfrmi6_rcv_tunnel,
+ .cb_handler = xfrmi_rcv_cb,
+ .err_handler = xfrmi6_err,
+ .priority = 2,
+};
+#endif
+
+static struct xfrm4_protocol xfrmi_esp4_protocol __read_mostly = {
+ .handler = xfrmi4_rcv,
+ .input_handler = xfrmi4_input,
+ .cb_handler = xfrmi_rcv_cb,
+ .err_handler = xfrmi4_err,
+ .priority = 10,
+};
+
+static struct xfrm4_protocol xfrmi_ah4_protocol __read_mostly = {
+ .handler = xfrm4_rcv,
+ .input_handler = xfrm_input,
+ .cb_handler = xfrmi_rcv_cb,
+ .err_handler = xfrmi4_err,
+ .priority = 10,
+};
+
+static struct xfrm4_protocol xfrmi_ipcomp4_protocol __read_mostly = {
+ .handler = xfrm4_rcv,
+ .input_handler = xfrm_input,
+ .cb_handler = xfrmi_rcv_cb,
+ .err_handler = xfrmi4_err,
+ .priority = 10,
+};
+
+#if IS_REACHABLE(CONFIG_INET_XFRM_TUNNEL)
+static int xfrmi4_rcv_tunnel(struct sk_buff *skb)
+{
+ return xfrm4_rcv_spi(skb, IPPROTO_IPIP, ip_hdr(skb)->saddr);
+}
+
+static struct xfrm_tunnel xfrmi_ipip_handler __read_mostly = {
+ .handler = xfrmi4_rcv_tunnel,
+ .cb_handler = xfrmi_rcv_cb,
+ .err_handler = xfrmi4_err,
+ .priority = 3,
+};
+
+static struct xfrm_tunnel xfrmi_ipip6_handler __read_mostly = {
+ .handler = xfrmi4_rcv_tunnel,
+ .cb_handler = xfrmi_rcv_cb,
+ .err_handler = xfrmi4_err,
+ .priority = 2,
+};
+#endif
+
+static int __init xfrmi4_init(void)
+{
+ int err;
+
+ err = xfrm4_protocol_register(&xfrmi_esp4_protocol, IPPROTO_ESP);
+ if (err < 0)
+ goto xfrm_proto_esp_failed;
+ err = xfrm4_protocol_register(&xfrmi_ah4_protocol, IPPROTO_AH);
+ if (err < 0)
+ goto xfrm_proto_ah_failed;
+ err = xfrm4_protocol_register(&xfrmi_ipcomp4_protocol, IPPROTO_COMP);
+ if (err < 0)
+ goto xfrm_proto_comp_failed;
+#if IS_REACHABLE(CONFIG_INET_XFRM_TUNNEL)
+ err = xfrm4_tunnel_register(&xfrmi_ipip_handler, AF_INET);
+ if (err < 0)
+ goto xfrm_tunnel_ipip_failed;
+ err = xfrm4_tunnel_register(&xfrmi_ipip6_handler, AF_INET6);
+ if (err < 0)
+ goto xfrm_tunnel_ipip6_failed;
+#endif
+
+ return 0;
+
+#if IS_REACHABLE(CONFIG_INET_XFRM_TUNNEL)
+xfrm_tunnel_ipip6_failed:
+ xfrm4_tunnel_deregister(&xfrmi_ipip_handler, AF_INET);
+xfrm_tunnel_ipip_failed:
+ xfrm4_protocol_deregister(&xfrmi_ipcomp4_protocol, IPPROTO_COMP);
+#endif
+xfrm_proto_comp_failed:
+ xfrm4_protocol_deregister(&xfrmi_ah4_protocol, IPPROTO_AH);
+xfrm_proto_ah_failed:
+ xfrm4_protocol_deregister(&xfrmi_esp4_protocol, IPPROTO_ESP);
+xfrm_proto_esp_failed:
+ return err;
+}
+
+static void xfrmi4_fini(void)
+{
+#if IS_REACHABLE(CONFIG_INET_XFRM_TUNNEL)
+ xfrm4_tunnel_deregister(&xfrmi_ipip6_handler, AF_INET6);
+ xfrm4_tunnel_deregister(&xfrmi_ipip_handler, AF_INET);
+#endif
+ xfrm4_protocol_deregister(&xfrmi_ipcomp4_protocol, IPPROTO_COMP);
+ xfrm4_protocol_deregister(&xfrmi_ah4_protocol, IPPROTO_AH);
+ xfrm4_protocol_deregister(&xfrmi_esp4_protocol, IPPROTO_ESP);
+}
+
+static int __init xfrmi6_init(void)
+{
+ int err;
+
+ err = xfrm6_protocol_register(&xfrmi_esp6_protocol, IPPROTO_ESP);
+ if (err < 0)
+ goto xfrm_proto_esp_failed;
+ err = xfrm6_protocol_register(&xfrmi_ah6_protocol, IPPROTO_AH);
+ if (err < 0)
+ goto xfrm_proto_ah_failed;
+ err = xfrm6_protocol_register(&xfrmi_ipcomp6_protocol, IPPROTO_COMP);
+ if (err < 0)
+ goto xfrm_proto_comp_failed;
+#if IS_REACHABLE(CONFIG_INET6_XFRM_TUNNEL)
+ err = xfrm6_tunnel_register(&xfrmi_ipv6_handler, AF_INET6);
+ if (err < 0)
+ goto xfrm_tunnel_ipv6_failed;
+ err = xfrm6_tunnel_register(&xfrmi_ip6ip_handler, AF_INET);
+ if (err < 0)
+ goto xfrm_tunnel_ip6ip_failed;
+#endif
+
+ return 0;
+
+#if IS_REACHABLE(CONFIG_INET6_XFRM_TUNNEL)
+xfrm_tunnel_ip6ip_failed:
+ xfrm6_tunnel_deregister(&xfrmi_ipv6_handler, AF_INET6);
+xfrm_tunnel_ipv6_failed:
+ xfrm6_protocol_deregister(&xfrmi_ipcomp6_protocol, IPPROTO_COMP);
+#endif
+xfrm_proto_comp_failed:
+ xfrm6_protocol_deregister(&xfrmi_ah6_protocol, IPPROTO_AH);
+xfrm_proto_ah_failed:
+ xfrm6_protocol_deregister(&xfrmi_esp6_protocol, IPPROTO_ESP);
+xfrm_proto_esp_failed:
+ return err;
+}
+
+static void xfrmi6_fini(void)
+{
+#if IS_REACHABLE(CONFIG_INET6_XFRM_TUNNEL)
+ xfrm6_tunnel_deregister(&xfrmi_ip6ip_handler, AF_INET);
+ xfrm6_tunnel_deregister(&xfrmi_ipv6_handler, AF_INET6);
+#endif
+ xfrm6_protocol_deregister(&xfrmi_ipcomp6_protocol, IPPROTO_COMP);
+ xfrm6_protocol_deregister(&xfrmi_ah6_protocol, IPPROTO_AH);
+ xfrm6_protocol_deregister(&xfrmi_esp6_protocol, IPPROTO_ESP);
+}
+
+static const struct xfrm_if_cb xfrm_if_cb = {
+ .decode_session = xfrmi_decode_session,
+};
+
+static int __init xfrmi_init(void)
+{
+ const char *msg;
+ int err;
+
+ pr_info("IPsec XFRM device driver\n");
+
+ msg = "tunnel device";
+ err = register_pernet_device(&xfrmi_net_ops);
+ if (err < 0)
+ goto pernet_dev_failed;
+
+ msg = "xfrm4 protocols";
+ err = xfrmi4_init();
+ if (err < 0)
+ goto xfrmi4_failed;
+
+ msg = "xfrm6 protocols";
+ err = xfrmi6_init();
+ if (err < 0)
+ goto xfrmi6_failed;
+
+
+ msg = "netlink interface";
+ err = rtnl_link_register(&xfrmi_link_ops);
+ if (err < 0)
+ goto rtnl_link_failed;
+
+ lwtunnel_encap_add_ops(&xfrmi_encap_ops, LWTUNNEL_ENCAP_XFRM);
+
+ xfrm_if_register_cb(&xfrm_if_cb);
+
+ return err;
+
+rtnl_link_failed:
+ xfrmi6_fini();
+xfrmi6_failed:
+ xfrmi4_fini();
+xfrmi4_failed:
+ unregister_pernet_device(&xfrmi_net_ops);
+pernet_dev_failed:
+ pr_err("xfrmi init: failed to register %s\n", msg);
+ return err;
+}
+
+static void __exit xfrmi_fini(void)
+{
+ xfrm_if_unregister_cb();
+ lwtunnel_encap_del_ops(&xfrmi_encap_ops, LWTUNNEL_ENCAP_XFRM);
+ rtnl_link_unregister(&xfrmi_link_ops);
+ xfrmi4_fini();
+ xfrmi6_fini();
+ unregister_pernet_device(&xfrmi_net_ops);
+}
+
+module_init(xfrmi_init);
+module_exit(xfrmi_fini);
+MODULE_LICENSE("GPL");
+MODULE_ALIAS_RTNL_LINK("xfrm");
+MODULE_ALIAS_NETDEV("xfrm0");
+MODULE_AUTHOR("Steffen Klassert");
+MODULE_DESCRIPTION("XFRM virtual interface");
diff --git a/net/xfrm/xfrm_ipcomp.c b/net/xfrm/xfrm_ipcomp.c
new file mode 100644
index 000000000..80143360b
--- /dev/null
+++ b/net/xfrm/xfrm_ipcomp.c
@@ -0,0 +1,378 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * IP Payload Compression Protocol (IPComp) - RFC3173.
+ *
+ * Copyright (c) 2003 James Morris <jmorris@intercode.com.au>
+ * Copyright (c) 2003-2008 Herbert Xu <herbert@gondor.apana.org.au>
+ *
+ * Todo:
+ * - Tunable compression parameters.
+ * - Compression stats.
+ * - Adaptive compression.
+ */
+
+#include <linux/crypto.h>
+#include <linux/err.h>
+#include <linux/list.h>
+#include <linux/module.h>
+#include <linux/mutex.h>
+#include <linux/percpu.h>
+#include <linux/slab.h>
+#include <linux/smp.h>
+#include <linux/vmalloc.h>
+#include <net/ip.h>
+#include <net/ipcomp.h>
+#include <net/xfrm.h>
+
+struct ipcomp_tfms {
+ struct list_head list;
+ struct crypto_comp * __percpu *tfms;
+ int users;
+};
+
+static DEFINE_MUTEX(ipcomp_resource_mutex);
+static void * __percpu *ipcomp_scratches;
+static int ipcomp_scratch_users;
+static LIST_HEAD(ipcomp_tfms_list);
+
+static int ipcomp_decompress(struct xfrm_state *x, struct sk_buff *skb)
+{
+ struct ipcomp_data *ipcd = x->data;
+ const int plen = skb->len;
+ int dlen = IPCOMP_SCRATCH_SIZE;
+ const u8 *start = skb->data;
+ u8 *scratch = *this_cpu_ptr(ipcomp_scratches);
+ struct crypto_comp *tfm = *this_cpu_ptr(ipcd->tfms);
+ int err = crypto_comp_decompress(tfm, start, plen, scratch, &dlen);
+ int len;
+
+ if (err)
+ return err;
+
+ if (dlen < (plen + sizeof(struct ip_comp_hdr)))
+ return -EINVAL;
+
+ len = dlen - plen;
+ if (len > skb_tailroom(skb))
+ len = skb_tailroom(skb);
+
+ __skb_put(skb, len);
+
+ len += plen;
+ skb_copy_to_linear_data(skb, scratch, len);
+
+ while ((scratch += len, dlen -= len) > 0) {
+ skb_frag_t *frag;
+ struct page *page;
+
+ if (WARN_ON(skb_shinfo(skb)->nr_frags >= MAX_SKB_FRAGS))
+ return -EMSGSIZE;
+
+ frag = skb_shinfo(skb)->frags + skb_shinfo(skb)->nr_frags;
+ page = alloc_page(GFP_ATOMIC);
+
+ if (!page)
+ return -ENOMEM;
+
+ __skb_frag_set_page(frag, page);
+
+ len = PAGE_SIZE;
+ if (dlen < len)
+ len = dlen;
+
+ skb_frag_off_set(frag, 0);
+ skb_frag_size_set(frag, len);
+ memcpy(skb_frag_address(frag), scratch, len);
+
+ skb->truesize += len;
+ skb->data_len += len;
+ skb->len += len;
+
+ skb_shinfo(skb)->nr_frags++;
+ }
+
+ return 0;
+}
+
+int ipcomp_input(struct xfrm_state *x, struct sk_buff *skb)
+{
+ int nexthdr;
+ int err = -ENOMEM;
+ struct ip_comp_hdr *ipch;
+
+ if (skb_linearize_cow(skb))
+ goto out;
+
+ skb->ip_summed = CHECKSUM_NONE;
+
+ /* Remove ipcomp header and decompress original payload */
+ ipch = (void *)skb->data;
+ nexthdr = ipch->nexthdr;
+
+ skb->transport_header = skb->network_header + sizeof(*ipch);
+ __skb_pull(skb, sizeof(*ipch));
+ err = ipcomp_decompress(x, skb);
+ if (err)
+ goto out;
+
+ err = nexthdr;
+
+out:
+ return err;
+}
+EXPORT_SYMBOL_GPL(ipcomp_input);
+
+static int ipcomp_compress(struct xfrm_state *x, struct sk_buff *skb)
+{
+ struct ipcomp_data *ipcd = x->data;
+ const int plen = skb->len;
+ int dlen = IPCOMP_SCRATCH_SIZE;
+ u8 *start = skb->data;
+ struct crypto_comp *tfm;
+ u8 *scratch;
+ int err;
+
+ local_bh_disable();
+ scratch = *this_cpu_ptr(ipcomp_scratches);
+ tfm = *this_cpu_ptr(ipcd->tfms);
+ err = crypto_comp_compress(tfm, start, plen, scratch, &dlen);
+ if (err)
+ goto out;
+
+ if ((dlen + sizeof(struct ip_comp_hdr)) >= plen) {
+ err = -EMSGSIZE;
+ goto out;
+ }
+
+ memcpy(start + sizeof(struct ip_comp_hdr), scratch, dlen);
+ local_bh_enable();
+
+ pskb_trim(skb, dlen + sizeof(struct ip_comp_hdr));
+ return 0;
+
+out:
+ local_bh_enable();
+ return err;
+}
+
+int ipcomp_output(struct xfrm_state *x, struct sk_buff *skb)
+{
+ int err;
+ struct ip_comp_hdr *ipch;
+ struct ipcomp_data *ipcd = x->data;
+
+ if (skb->len < ipcd->threshold) {
+ /* Don't bother compressing */
+ goto out_ok;
+ }
+
+ if (skb_linearize_cow(skb))
+ goto out_ok;
+
+ err = ipcomp_compress(x, skb);
+
+ if (err) {
+ goto out_ok;
+ }
+
+ /* Install ipcomp header, convert into ipcomp datagram. */
+ ipch = ip_comp_hdr(skb);
+ ipch->nexthdr = *skb_mac_header(skb);
+ ipch->flags = 0;
+ ipch->cpi = htons((u16 )ntohl(x->id.spi));
+ *skb_mac_header(skb) = IPPROTO_COMP;
+out_ok:
+ skb_push(skb, -skb_network_offset(skb));
+ return 0;
+}
+EXPORT_SYMBOL_GPL(ipcomp_output);
+
+static void ipcomp_free_scratches(void)
+{
+ int i;
+ void * __percpu *scratches;
+
+ if (--ipcomp_scratch_users)
+ return;
+
+ scratches = ipcomp_scratches;
+ if (!scratches)
+ return;
+
+ for_each_possible_cpu(i)
+ vfree(*per_cpu_ptr(scratches, i));
+
+ free_percpu(scratches);
+ ipcomp_scratches = NULL;
+}
+
+static void * __percpu *ipcomp_alloc_scratches(void)
+{
+ void * __percpu *scratches;
+ int i;
+
+ if (ipcomp_scratch_users++)
+ return ipcomp_scratches;
+
+ scratches = alloc_percpu(void *);
+ if (!scratches)
+ return NULL;
+
+ ipcomp_scratches = scratches;
+
+ for_each_possible_cpu(i) {
+ void *scratch;
+
+ scratch = vmalloc_node(IPCOMP_SCRATCH_SIZE, cpu_to_node(i));
+ if (!scratch)
+ return NULL;
+ *per_cpu_ptr(scratches, i) = scratch;
+ }
+
+ return scratches;
+}
+
+static void ipcomp_free_tfms(struct crypto_comp * __percpu *tfms)
+{
+ struct ipcomp_tfms *pos;
+ int cpu;
+
+ list_for_each_entry(pos, &ipcomp_tfms_list, list) {
+ if (pos->tfms == tfms)
+ break;
+ }
+
+ WARN_ON(list_entry_is_head(pos, &ipcomp_tfms_list, list));
+
+ if (--pos->users)
+ return;
+
+ list_del(&pos->list);
+ kfree(pos);
+
+ if (!tfms)
+ return;
+
+ for_each_possible_cpu(cpu) {
+ struct crypto_comp *tfm = *per_cpu_ptr(tfms, cpu);
+ crypto_free_comp(tfm);
+ }
+ free_percpu(tfms);
+}
+
+static struct crypto_comp * __percpu *ipcomp_alloc_tfms(const char *alg_name)
+{
+ struct ipcomp_tfms *pos;
+ struct crypto_comp * __percpu *tfms;
+ int cpu;
+
+
+ list_for_each_entry(pos, &ipcomp_tfms_list, list) {
+ struct crypto_comp *tfm;
+
+ /* This can be any valid CPU ID so we don't need locking. */
+ tfm = this_cpu_read(*pos->tfms);
+
+ if (!strcmp(crypto_comp_name(tfm), alg_name)) {
+ pos->users++;
+ return pos->tfms;
+ }
+ }
+
+ pos = kmalloc(sizeof(*pos), GFP_KERNEL);
+ if (!pos)
+ return NULL;
+
+ pos->users = 1;
+ INIT_LIST_HEAD(&pos->list);
+ list_add(&pos->list, &ipcomp_tfms_list);
+
+ pos->tfms = tfms = alloc_percpu(struct crypto_comp *);
+ if (!tfms)
+ goto error;
+
+ for_each_possible_cpu(cpu) {
+ struct crypto_comp *tfm = crypto_alloc_comp(alg_name, 0,
+ CRYPTO_ALG_ASYNC);
+ if (IS_ERR(tfm))
+ goto error;
+ *per_cpu_ptr(tfms, cpu) = tfm;
+ }
+
+ return tfms;
+
+error:
+ ipcomp_free_tfms(tfms);
+ return NULL;
+}
+
+static void ipcomp_free_data(struct ipcomp_data *ipcd)
+{
+ if (ipcd->tfms)
+ ipcomp_free_tfms(ipcd->tfms);
+ ipcomp_free_scratches();
+}
+
+void ipcomp_destroy(struct xfrm_state *x)
+{
+ struct ipcomp_data *ipcd = x->data;
+ if (!ipcd)
+ return;
+ xfrm_state_delete_tunnel(x);
+ mutex_lock(&ipcomp_resource_mutex);
+ ipcomp_free_data(ipcd);
+ mutex_unlock(&ipcomp_resource_mutex);
+ kfree(ipcd);
+}
+EXPORT_SYMBOL_GPL(ipcomp_destroy);
+
+int ipcomp_init_state(struct xfrm_state *x, struct netlink_ext_ack *extack)
+{
+ int err;
+ struct ipcomp_data *ipcd;
+ struct xfrm_algo_desc *calg_desc;
+
+ err = -EINVAL;
+ if (!x->calg) {
+ NL_SET_ERR_MSG(extack, "Missing required compression algorithm");
+ goto out;
+ }
+
+ if (x->encap) {
+ NL_SET_ERR_MSG(extack, "IPComp is not compatible with encapsulation");
+ goto out;
+ }
+
+ err = -ENOMEM;
+ ipcd = kzalloc(sizeof(*ipcd), GFP_KERNEL);
+ if (!ipcd)
+ goto out;
+
+ mutex_lock(&ipcomp_resource_mutex);
+ if (!ipcomp_alloc_scratches())
+ goto error;
+
+ ipcd->tfms = ipcomp_alloc_tfms(x->calg->alg_name);
+ if (!ipcd->tfms)
+ goto error;
+ mutex_unlock(&ipcomp_resource_mutex);
+
+ calg_desc = xfrm_calg_get_byname(x->calg->alg_name, 0);
+ BUG_ON(!calg_desc);
+ ipcd->threshold = calg_desc->uinfo.comp.threshold;
+ x->data = ipcd;
+ err = 0;
+out:
+ return err;
+
+error:
+ ipcomp_free_data(ipcd);
+ mutex_unlock(&ipcomp_resource_mutex);
+ kfree(ipcd);
+ goto out;
+}
+EXPORT_SYMBOL_GPL(ipcomp_init_state);
+
+MODULE_LICENSE("GPL");
+MODULE_DESCRIPTION("IP Payload Compression Protocol (IPComp) - RFC3173");
+MODULE_AUTHOR("James Morris <jmorris@intercode.com.au>");
diff --git a/net/xfrm/xfrm_output.c b/net/xfrm/xfrm_output.c
new file mode 100644
index 000000000..9a5e79a38
--- /dev/null
+++ b/net/xfrm/xfrm_output.c
@@ -0,0 +1,909 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * xfrm_output.c - Common IPsec encapsulation code.
+ *
+ * Copyright (c) 2007 Herbert Xu <herbert@gondor.apana.org.au>
+ */
+
+#include <linux/errno.h>
+#include <linux/module.h>
+#include <linux/netdevice.h>
+#include <linux/netfilter.h>
+#include <linux/skbuff.h>
+#include <linux/slab.h>
+#include <linux/spinlock.h>
+#include <net/dst.h>
+#include <net/icmp.h>
+#include <net/inet_ecn.h>
+#include <net/xfrm.h>
+
+#if IS_ENABLED(CONFIG_IPV6)
+#include <net/ip6_route.h>
+#include <net/ipv6_stubs.h>
+#endif
+
+#include "xfrm_inout.h"
+
+static int xfrm_output2(struct net *net, struct sock *sk, struct sk_buff *skb);
+static int xfrm_inner_extract_output(struct xfrm_state *x, struct sk_buff *skb);
+
+static int xfrm_skb_check_space(struct sk_buff *skb)
+{
+ struct dst_entry *dst = skb_dst(skb);
+ int nhead = dst->header_len + LL_RESERVED_SPACE(dst->dev)
+ - skb_headroom(skb);
+ int ntail = dst->dev->needed_tailroom - skb_tailroom(skb);
+
+ if (nhead <= 0) {
+ if (ntail <= 0)
+ return 0;
+ nhead = 0;
+ } else if (ntail < 0)
+ ntail = 0;
+
+ return pskb_expand_head(skb, nhead, ntail, GFP_ATOMIC);
+}
+
+/* Children define the path of the packet through the
+ * Linux networking. Thus, destinations are stackable.
+ */
+
+static struct dst_entry *skb_dst_pop(struct sk_buff *skb)
+{
+ struct dst_entry *child = dst_clone(xfrm_dst_child(skb_dst(skb)));
+
+ skb_dst_drop(skb);
+ return child;
+}
+
+/* Add encapsulation header.
+ *
+ * The IP header will be moved forward to make space for the encapsulation
+ * header.
+ */
+static int xfrm4_transport_output(struct xfrm_state *x, struct sk_buff *skb)
+{
+ struct iphdr *iph = ip_hdr(skb);
+ int ihl = iph->ihl * 4;
+
+ skb_set_inner_transport_header(skb, skb_transport_offset(skb));
+
+ skb_set_network_header(skb, -x->props.header_len);
+ skb->mac_header = skb->network_header +
+ offsetof(struct iphdr, protocol);
+ skb->transport_header = skb->network_header + ihl;
+ __skb_pull(skb, ihl);
+ memmove(skb_network_header(skb), iph, ihl);
+ return 0;
+}
+
+#if IS_ENABLED(CONFIG_IPV6_MIP6)
+static int mip6_rthdr_offset(struct sk_buff *skb, u8 **nexthdr, int type)
+{
+ const unsigned char *nh = skb_network_header(skb);
+ unsigned int offset = sizeof(struct ipv6hdr);
+ unsigned int packet_len;
+ int found_rhdr = 0;
+
+ packet_len = skb_tail_pointer(skb) - nh;
+ *nexthdr = &ipv6_hdr(skb)->nexthdr;
+
+ while (offset <= packet_len) {
+ struct ipv6_opt_hdr *exthdr;
+
+ switch (**nexthdr) {
+ case NEXTHDR_HOP:
+ break;
+ case NEXTHDR_ROUTING:
+ if (type == IPPROTO_ROUTING && offset + 3 <= packet_len) {
+ struct ipv6_rt_hdr *rt;
+
+ rt = (struct ipv6_rt_hdr *)(nh + offset);
+ if (rt->type != 0)
+ return offset;
+ }
+ found_rhdr = 1;
+ break;
+ case NEXTHDR_DEST:
+ /* HAO MUST NOT appear more than once.
+ * XXX: It is better to try to find by the end of
+ * XXX: packet if HAO exists.
+ */
+ if (ipv6_find_tlv(skb, offset, IPV6_TLV_HAO) >= 0) {
+ net_dbg_ratelimited("mip6: hao exists already, override\n");
+ return offset;
+ }
+
+ if (found_rhdr)
+ return offset;
+
+ break;
+ default:
+ return offset;
+ }
+
+ if (offset + sizeof(struct ipv6_opt_hdr) > packet_len)
+ return -EINVAL;
+
+ exthdr = (struct ipv6_opt_hdr *)(skb_network_header(skb) +
+ offset);
+ offset += ipv6_optlen(exthdr);
+ if (offset > IPV6_MAXPLEN)
+ return -EINVAL;
+ *nexthdr = &exthdr->nexthdr;
+ }
+
+ return -EINVAL;
+}
+#endif
+
+#if IS_ENABLED(CONFIG_IPV6)
+static int xfrm6_hdr_offset(struct xfrm_state *x, struct sk_buff *skb, u8 **prevhdr)
+{
+ switch (x->type->proto) {
+#if IS_ENABLED(CONFIG_IPV6_MIP6)
+ case IPPROTO_DSTOPTS:
+ case IPPROTO_ROUTING:
+ return mip6_rthdr_offset(skb, prevhdr, x->type->proto);
+#endif
+ default:
+ break;
+ }
+
+ return ip6_find_1stfragopt(skb, prevhdr);
+}
+#endif
+
+/* Add encapsulation header.
+ *
+ * The IP header and mutable extension headers will be moved forward to make
+ * space for the encapsulation header.
+ */
+static int xfrm6_transport_output(struct xfrm_state *x, struct sk_buff *skb)
+{
+#if IS_ENABLED(CONFIG_IPV6)
+ struct ipv6hdr *iph;
+ u8 *prevhdr;
+ int hdr_len;
+
+ iph = ipv6_hdr(skb);
+ skb_set_inner_transport_header(skb, skb_transport_offset(skb));
+
+ hdr_len = xfrm6_hdr_offset(x, skb, &prevhdr);
+ if (hdr_len < 0)
+ return hdr_len;
+ skb_set_mac_header(skb,
+ (prevhdr - x->props.header_len) - skb->data);
+ skb_set_network_header(skb, -x->props.header_len);
+ skb->transport_header = skb->network_header + hdr_len;
+ __skb_pull(skb, hdr_len);
+ memmove(ipv6_hdr(skb), iph, hdr_len);
+ return 0;
+#else
+ WARN_ON_ONCE(1);
+ return -EAFNOSUPPORT;
+#endif
+}
+
+/* Add route optimization header space.
+ *
+ * The IP header and mutable extension headers will be moved forward to make
+ * space for the route optimization header.
+ */
+static int xfrm6_ro_output(struct xfrm_state *x, struct sk_buff *skb)
+{
+#if IS_ENABLED(CONFIG_IPV6)
+ struct ipv6hdr *iph;
+ u8 *prevhdr;
+ int hdr_len;
+
+ iph = ipv6_hdr(skb);
+
+ hdr_len = xfrm6_hdr_offset(x, skb, &prevhdr);
+ if (hdr_len < 0)
+ return hdr_len;
+ skb_set_mac_header(skb,
+ (prevhdr - x->props.header_len) - skb->data);
+ skb_set_network_header(skb, -x->props.header_len);
+ skb->transport_header = skb->network_header + hdr_len;
+ __skb_pull(skb, hdr_len);
+ memmove(ipv6_hdr(skb), iph, hdr_len);
+
+ x->lastused = ktime_get_real_seconds();
+
+ return 0;
+#else
+ WARN_ON_ONCE(1);
+ return -EAFNOSUPPORT;
+#endif
+}
+
+/* Add encapsulation header.
+ *
+ * The top IP header will be constructed per draft-nikander-esp-beet-mode-06.txt.
+ */
+static int xfrm4_beet_encap_add(struct xfrm_state *x, struct sk_buff *skb)
+{
+ struct ip_beet_phdr *ph;
+ struct iphdr *top_iph;
+ int hdrlen, optlen;
+
+ hdrlen = 0;
+ optlen = XFRM_MODE_SKB_CB(skb)->optlen;
+ if (unlikely(optlen))
+ hdrlen += IPV4_BEET_PHMAXLEN - (optlen & 4);
+
+ skb_set_network_header(skb, -x->props.header_len - hdrlen +
+ (XFRM_MODE_SKB_CB(skb)->ihl - sizeof(*top_iph)));
+ if (x->sel.family != AF_INET6)
+ skb->network_header += IPV4_BEET_PHMAXLEN;
+ skb->mac_header = skb->network_header +
+ offsetof(struct iphdr, protocol);
+ skb->transport_header = skb->network_header + sizeof(*top_iph);
+
+ xfrm4_beet_make_header(skb);
+
+ ph = __skb_pull(skb, XFRM_MODE_SKB_CB(skb)->ihl - hdrlen);
+
+ top_iph = ip_hdr(skb);
+
+ if (unlikely(optlen)) {
+ if (WARN_ON(optlen < 0))
+ return -EINVAL;
+
+ ph->padlen = 4 - (optlen & 4);
+ ph->hdrlen = optlen / 8;
+ ph->nexthdr = top_iph->protocol;
+ if (ph->padlen)
+ memset(ph + 1, IPOPT_NOP, ph->padlen);
+
+ top_iph->protocol = IPPROTO_BEETPH;
+ top_iph->ihl = sizeof(struct iphdr) / 4;
+ }
+
+ top_iph->saddr = x->props.saddr.a4;
+ top_iph->daddr = x->id.daddr.a4;
+
+ return 0;
+}
+
+/* Add encapsulation header.
+ *
+ * The top IP header will be constructed per RFC 2401.
+ */
+static int xfrm4_tunnel_encap_add(struct xfrm_state *x, struct sk_buff *skb)
+{
+ bool small_ipv6 = (skb->protocol == htons(ETH_P_IPV6)) && (skb->len <= IPV6_MIN_MTU);
+ struct dst_entry *dst = skb_dst(skb);
+ struct iphdr *top_iph;
+ int flags;
+
+ skb_set_inner_network_header(skb, skb_network_offset(skb));
+ skb_set_inner_transport_header(skb, skb_transport_offset(skb));
+
+ skb_set_network_header(skb, -x->props.header_len);
+ skb->mac_header = skb->network_header +
+ offsetof(struct iphdr, protocol);
+ skb->transport_header = skb->network_header + sizeof(*top_iph);
+ top_iph = ip_hdr(skb);
+
+ top_iph->ihl = 5;
+ top_iph->version = 4;
+
+ top_iph->protocol = xfrm_af2proto(skb_dst(skb)->ops->family);
+
+ /* DS disclosing depends on XFRM_SA_XFLAG_DONT_ENCAP_DSCP */
+ if (x->props.extra_flags & XFRM_SA_XFLAG_DONT_ENCAP_DSCP)
+ top_iph->tos = 0;
+ else
+ top_iph->tos = XFRM_MODE_SKB_CB(skb)->tos;
+ top_iph->tos = INET_ECN_encapsulate(top_iph->tos,
+ XFRM_MODE_SKB_CB(skb)->tos);
+
+ flags = x->props.flags;
+ if (flags & XFRM_STATE_NOECN)
+ IP_ECN_clear(top_iph);
+
+ top_iph->frag_off = (flags & XFRM_STATE_NOPMTUDISC) || small_ipv6 ?
+ 0 : (XFRM_MODE_SKB_CB(skb)->frag_off & htons(IP_DF));
+
+ top_iph->ttl = ip4_dst_hoplimit(xfrm_dst_child(dst));
+
+ top_iph->saddr = x->props.saddr.a4;
+ top_iph->daddr = x->id.daddr.a4;
+ ip_select_ident(dev_net(dst->dev), skb, NULL);
+
+ return 0;
+}
+
+#if IS_ENABLED(CONFIG_IPV6)
+static int xfrm6_tunnel_encap_add(struct xfrm_state *x, struct sk_buff *skb)
+{
+ struct dst_entry *dst = skb_dst(skb);
+ struct ipv6hdr *top_iph;
+ int dsfield;
+
+ skb_set_inner_network_header(skb, skb_network_offset(skb));
+ skb_set_inner_transport_header(skb, skb_transport_offset(skb));
+
+ skb_set_network_header(skb, -x->props.header_len);
+ skb->mac_header = skb->network_header +
+ offsetof(struct ipv6hdr, nexthdr);
+ skb->transport_header = skb->network_header + sizeof(*top_iph);
+ top_iph = ipv6_hdr(skb);
+
+ top_iph->version = 6;
+
+ memcpy(top_iph->flow_lbl, XFRM_MODE_SKB_CB(skb)->flow_lbl,
+ sizeof(top_iph->flow_lbl));
+ top_iph->nexthdr = xfrm_af2proto(skb_dst(skb)->ops->family);
+
+ if (x->props.extra_flags & XFRM_SA_XFLAG_DONT_ENCAP_DSCP)
+ dsfield = 0;
+ else
+ dsfield = XFRM_MODE_SKB_CB(skb)->tos;
+ dsfield = INET_ECN_encapsulate(dsfield, XFRM_MODE_SKB_CB(skb)->tos);
+ if (x->props.flags & XFRM_STATE_NOECN)
+ dsfield &= ~INET_ECN_MASK;
+ ipv6_change_dsfield(top_iph, 0, dsfield);
+ top_iph->hop_limit = ip6_dst_hoplimit(xfrm_dst_child(dst));
+ top_iph->saddr = *(struct in6_addr *)&x->props.saddr;
+ top_iph->daddr = *(struct in6_addr *)&x->id.daddr;
+ return 0;
+}
+
+static int xfrm6_beet_encap_add(struct xfrm_state *x, struct sk_buff *skb)
+{
+ struct ipv6hdr *top_iph;
+ struct ip_beet_phdr *ph;
+ int optlen, hdr_len;
+
+ hdr_len = 0;
+ optlen = XFRM_MODE_SKB_CB(skb)->optlen;
+ if (unlikely(optlen))
+ hdr_len += IPV4_BEET_PHMAXLEN - (optlen & 4);
+
+ skb_set_network_header(skb, -x->props.header_len - hdr_len);
+ if (x->sel.family != AF_INET6)
+ skb->network_header += IPV4_BEET_PHMAXLEN;
+ skb->mac_header = skb->network_header +
+ offsetof(struct ipv6hdr, nexthdr);
+ skb->transport_header = skb->network_header + sizeof(*top_iph);
+ ph = __skb_pull(skb, XFRM_MODE_SKB_CB(skb)->ihl - hdr_len);
+
+ xfrm6_beet_make_header(skb);
+
+ top_iph = ipv6_hdr(skb);
+ if (unlikely(optlen)) {
+ if (WARN_ON(optlen < 0))
+ return -EINVAL;
+
+ ph->padlen = 4 - (optlen & 4);
+ ph->hdrlen = optlen / 8;
+ ph->nexthdr = top_iph->nexthdr;
+ if (ph->padlen)
+ memset(ph + 1, IPOPT_NOP, ph->padlen);
+
+ top_iph->nexthdr = IPPROTO_BEETPH;
+ }
+
+ top_iph->saddr = *(struct in6_addr *)&x->props.saddr;
+ top_iph->daddr = *(struct in6_addr *)&x->id.daddr;
+ return 0;
+}
+#endif
+
+/* Add encapsulation header.
+ *
+ * On exit, the transport header will be set to the start of the
+ * encapsulation header to be filled in by x->type->output and the mac
+ * header will be set to the nextheader (protocol for IPv4) field of the
+ * extension header directly preceding the encapsulation header, or in
+ * its absence, that of the top IP header.
+ * The value of the network header will always point to the top IP header
+ * while skb->data will point to the payload.
+ */
+static int xfrm4_prepare_output(struct xfrm_state *x, struct sk_buff *skb)
+{
+ int err;
+
+ err = xfrm_inner_extract_output(x, skb);
+ if (err)
+ return err;
+
+ IPCB(skb)->flags |= IPSKB_XFRM_TUNNEL_SIZE;
+ skb->protocol = htons(ETH_P_IP);
+
+ switch (x->outer_mode.encap) {
+ case XFRM_MODE_BEET:
+ return xfrm4_beet_encap_add(x, skb);
+ case XFRM_MODE_TUNNEL:
+ return xfrm4_tunnel_encap_add(x, skb);
+ }
+
+ WARN_ON_ONCE(1);
+ return -EOPNOTSUPP;
+}
+
+static int xfrm6_prepare_output(struct xfrm_state *x, struct sk_buff *skb)
+{
+#if IS_ENABLED(CONFIG_IPV6)
+ int err;
+
+ err = xfrm_inner_extract_output(x, skb);
+ if (err)
+ return err;
+
+ skb->ignore_df = 1;
+ skb->protocol = htons(ETH_P_IPV6);
+
+ switch (x->outer_mode.encap) {
+ case XFRM_MODE_BEET:
+ return xfrm6_beet_encap_add(x, skb);
+ case XFRM_MODE_TUNNEL:
+ return xfrm6_tunnel_encap_add(x, skb);
+ default:
+ WARN_ON_ONCE(1);
+ return -EOPNOTSUPP;
+ }
+#endif
+ WARN_ON_ONCE(1);
+ return -EAFNOSUPPORT;
+}
+
+static int xfrm_outer_mode_output(struct xfrm_state *x, struct sk_buff *skb)
+{
+ switch (x->outer_mode.encap) {
+ case XFRM_MODE_BEET:
+ case XFRM_MODE_TUNNEL:
+ if (x->outer_mode.family == AF_INET)
+ return xfrm4_prepare_output(x, skb);
+ if (x->outer_mode.family == AF_INET6)
+ return xfrm6_prepare_output(x, skb);
+ break;
+ case XFRM_MODE_TRANSPORT:
+ if (x->outer_mode.family == AF_INET)
+ return xfrm4_transport_output(x, skb);
+ if (x->outer_mode.family == AF_INET6)
+ return xfrm6_transport_output(x, skb);
+ break;
+ case XFRM_MODE_ROUTEOPTIMIZATION:
+ if (x->outer_mode.family == AF_INET6)
+ return xfrm6_ro_output(x, skb);
+ WARN_ON_ONCE(1);
+ break;
+ default:
+ WARN_ON_ONCE(1);
+ break;
+ }
+
+ return -EOPNOTSUPP;
+}
+
+#if IS_ENABLED(CONFIG_NET_PKTGEN)
+int pktgen_xfrm_outer_mode_output(struct xfrm_state *x, struct sk_buff *skb)
+{
+ return xfrm_outer_mode_output(x, skb);
+}
+EXPORT_SYMBOL_GPL(pktgen_xfrm_outer_mode_output);
+#endif
+
+static int xfrm_output_one(struct sk_buff *skb, int err)
+{
+ struct dst_entry *dst = skb_dst(skb);
+ struct xfrm_state *x = dst->xfrm;
+ struct net *net = xs_net(x);
+
+ if (err <= 0)
+ goto resume;
+
+ do {
+ err = xfrm_skb_check_space(skb);
+ if (err) {
+ XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTERROR);
+ goto error_nolock;
+ }
+
+ skb->mark = xfrm_smark_get(skb->mark, x);
+
+ err = xfrm_outer_mode_output(x, skb);
+ if (err) {
+ XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTSTATEMODEERROR);
+ goto error_nolock;
+ }
+
+ spin_lock_bh(&x->lock);
+
+ if (unlikely(x->km.state != XFRM_STATE_VALID)) {
+ XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTSTATEINVALID);
+ err = -EINVAL;
+ goto error;
+ }
+
+ err = xfrm_state_check_expire(x);
+ if (err) {
+ XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTSTATEEXPIRED);
+ goto error;
+ }
+
+ err = xfrm_replay_overflow(x, skb);
+ if (err) {
+ XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTSTATESEQERROR);
+ goto error;
+ }
+
+ x->curlft.bytes += skb->len;
+ x->curlft.packets++;
+
+ spin_unlock_bh(&x->lock);
+
+ skb_dst_force(skb);
+ if (!skb_dst(skb)) {
+ XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTERROR);
+ err = -EHOSTUNREACH;
+ goto error_nolock;
+ }
+
+ if (xfrm_offload(skb)) {
+ x->type_offload->encap(x, skb);
+ } else {
+ /* Inner headers are invalid now. */
+ skb->encapsulation = 0;
+
+ err = x->type->output(x, skb);
+ if (err == -EINPROGRESS)
+ goto out;
+ }
+
+resume:
+ if (err) {
+ XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTSTATEPROTOERROR);
+ goto error_nolock;
+ }
+
+ dst = skb_dst_pop(skb);
+ if (!dst) {
+ XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTERROR);
+ err = -EHOSTUNREACH;
+ goto error_nolock;
+ }
+ skb_dst_set(skb, dst);
+ x = dst->xfrm;
+ } while (x && !(x->outer_mode.flags & XFRM_MODE_FLAG_TUNNEL));
+
+ return 0;
+
+error:
+ spin_unlock_bh(&x->lock);
+error_nolock:
+ kfree_skb(skb);
+out:
+ return err;
+}
+
+int xfrm_output_resume(struct sock *sk, struct sk_buff *skb, int err)
+{
+ struct net *net = xs_net(skb_dst(skb)->xfrm);
+
+ while (likely((err = xfrm_output_one(skb, err)) == 0)) {
+ nf_reset_ct(skb);
+
+ err = skb_dst(skb)->ops->local_out(net, sk, skb);
+ if (unlikely(err != 1))
+ goto out;
+
+ if (!skb_dst(skb)->xfrm)
+ return dst_output(net, sk, skb);
+
+ err = nf_hook(skb_dst(skb)->ops->family,
+ NF_INET_POST_ROUTING, net, sk, skb,
+ NULL, skb_dst(skb)->dev, xfrm_output2);
+ if (unlikely(err != 1))
+ goto out;
+ }
+
+ if (err == -EINPROGRESS)
+ err = 0;
+
+out:
+ return err;
+}
+EXPORT_SYMBOL_GPL(xfrm_output_resume);
+
+static int xfrm_output2(struct net *net, struct sock *sk, struct sk_buff *skb)
+{
+ return xfrm_output_resume(sk, skb, 1);
+}
+
+static int xfrm_output_gso(struct net *net, struct sock *sk, struct sk_buff *skb)
+{
+ struct sk_buff *segs, *nskb;
+
+ BUILD_BUG_ON(sizeof(*IPCB(skb)) > SKB_GSO_CB_OFFSET);
+ BUILD_BUG_ON(sizeof(*IP6CB(skb)) > SKB_GSO_CB_OFFSET);
+ segs = skb_gso_segment(skb, 0);
+ kfree_skb(skb);
+ if (IS_ERR(segs))
+ return PTR_ERR(segs);
+ if (segs == NULL)
+ return -EINVAL;
+
+ skb_list_walk_safe(segs, segs, nskb) {
+ int err;
+
+ skb_mark_not_on_list(segs);
+ err = xfrm_output2(net, sk, segs);
+
+ if (unlikely(err)) {
+ kfree_skb_list(nskb);
+ return err;
+ }
+ }
+
+ return 0;
+}
+
+/* For partial checksum offload, the outer header checksum is calculated
+ * by software and the inner header checksum is calculated by hardware.
+ * This requires hardware to know the inner packet type to calculate
+ * the inner header checksum. Save inner ip protocol here to avoid
+ * traversing the packet in the vendor's xmit code.
+ * For IPsec tunnel mode save the ip protocol from the IP header of the
+ * plain text packet. Otherwise If the encap type is IPIP, just save
+ * skb->inner_ipproto in any other case get the ip protocol from the IP
+ * header.
+ */
+static void xfrm_get_inner_ipproto(struct sk_buff *skb, struct xfrm_state *x)
+{
+ struct xfrm_offload *xo = xfrm_offload(skb);
+ const struct ethhdr *eth;
+
+ if (!xo)
+ return;
+
+ if (x->outer_mode.encap == XFRM_MODE_TUNNEL) {
+ switch (x->outer_mode.family) {
+ case AF_INET:
+ xo->inner_ipproto = ip_hdr(skb)->protocol;
+ break;
+ case AF_INET6:
+ xo->inner_ipproto = ipv6_hdr(skb)->nexthdr;
+ break;
+ default:
+ break;
+ }
+
+ return;
+ }
+
+ /* non-Tunnel Mode */
+ if (!skb->encapsulation)
+ return;
+
+ if (skb->inner_protocol_type == ENCAP_TYPE_IPPROTO) {
+ xo->inner_ipproto = skb->inner_ipproto;
+ return;
+ }
+
+ if (skb->inner_protocol_type != ENCAP_TYPE_ETHER)
+ return;
+
+ eth = (struct ethhdr *)skb_inner_mac_header(skb);
+
+ switch (ntohs(eth->h_proto)) {
+ case ETH_P_IPV6:
+ xo->inner_ipproto = inner_ipv6_hdr(skb)->nexthdr;
+ break;
+ case ETH_P_IP:
+ xo->inner_ipproto = inner_ip_hdr(skb)->protocol;
+ break;
+ }
+}
+
+int xfrm_output(struct sock *sk, struct sk_buff *skb)
+{
+ struct net *net = dev_net(skb_dst(skb)->dev);
+ struct xfrm_state *x = skb_dst(skb)->xfrm;
+ int err;
+
+ switch (x->outer_mode.family) {
+ case AF_INET:
+ memset(IPCB(skb), 0, sizeof(*IPCB(skb)));
+ IPCB(skb)->flags |= IPSKB_XFRM_TRANSFORMED;
+ break;
+ case AF_INET6:
+ memset(IP6CB(skb), 0, sizeof(*IP6CB(skb)));
+
+ IP6CB(skb)->flags |= IP6SKB_XFRM_TRANSFORMED;
+ break;
+ }
+
+ secpath_reset(skb);
+
+ if (xfrm_dev_offload_ok(skb, x)) {
+ struct sec_path *sp;
+
+ sp = secpath_set(skb);
+ if (!sp) {
+ XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTERROR);
+ kfree_skb(skb);
+ return -ENOMEM;
+ }
+
+ sp->olen++;
+ sp->xvec[sp->len++] = x;
+ xfrm_state_hold(x);
+
+ xfrm_get_inner_ipproto(skb, x);
+ skb->encapsulation = 1;
+
+ if (skb_is_gso(skb)) {
+ if (skb->inner_protocol)
+ return xfrm_output_gso(net, sk, skb);
+
+ skb_shinfo(skb)->gso_type |= SKB_GSO_ESP;
+ goto out;
+ }
+
+ if (x->xso.dev && x->xso.dev->features & NETIF_F_HW_ESP_TX_CSUM)
+ goto out;
+ } else {
+ if (skb_is_gso(skb))
+ return xfrm_output_gso(net, sk, skb);
+ }
+
+ if (skb->ip_summed == CHECKSUM_PARTIAL) {
+ err = skb_checksum_help(skb);
+ if (err) {
+ XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTERROR);
+ kfree_skb(skb);
+ return err;
+ }
+ }
+
+out:
+ return xfrm_output2(net, sk, skb);
+}
+EXPORT_SYMBOL_GPL(xfrm_output);
+
+static int xfrm4_tunnel_check_size(struct sk_buff *skb)
+{
+ int mtu, ret = 0;
+
+ if (IPCB(skb)->flags & IPSKB_XFRM_TUNNEL_SIZE)
+ goto out;
+
+ if (!(ip_hdr(skb)->frag_off & htons(IP_DF)) || skb->ignore_df)
+ goto out;
+
+ mtu = dst_mtu(skb_dst(skb));
+ if ((!skb_is_gso(skb) && skb->len > mtu) ||
+ (skb_is_gso(skb) &&
+ !skb_gso_validate_network_len(skb, ip_skb_dst_mtu(skb->sk, skb)))) {
+ skb->protocol = htons(ETH_P_IP);
+
+ if (skb->sk)
+ xfrm_local_error(skb, mtu);
+ else
+ icmp_send(skb, ICMP_DEST_UNREACH,
+ ICMP_FRAG_NEEDED, htonl(mtu));
+ ret = -EMSGSIZE;
+ }
+out:
+ return ret;
+}
+
+static int xfrm4_extract_output(struct xfrm_state *x, struct sk_buff *skb)
+{
+ int err;
+
+ if (x->outer_mode.encap == XFRM_MODE_BEET &&
+ ip_is_fragment(ip_hdr(skb))) {
+ net_warn_ratelimited("BEET mode doesn't support inner IPv4 fragments\n");
+ return -EAFNOSUPPORT;
+ }
+
+ err = xfrm4_tunnel_check_size(skb);
+ if (err)
+ return err;
+
+ XFRM_MODE_SKB_CB(skb)->protocol = ip_hdr(skb)->protocol;
+
+ xfrm4_extract_header(skb);
+ return 0;
+}
+
+#if IS_ENABLED(CONFIG_IPV6)
+static int xfrm6_tunnel_check_size(struct sk_buff *skb)
+{
+ int mtu, ret = 0;
+ struct dst_entry *dst = skb_dst(skb);
+
+ if (skb->ignore_df)
+ goto out;
+
+ mtu = dst_mtu(dst);
+ if (mtu < IPV6_MIN_MTU)
+ mtu = IPV6_MIN_MTU;
+
+ if ((!skb_is_gso(skb) && skb->len > mtu) ||
+ (skb_is_gso(skb) &&
+ !skb_gso_validate_network_len(skb, ip6_skb_dst_mtu(skb)))) {
+ skb->dev = dst->dev;
+ skb->protocol = htons(ETH_P_IPV6);
+
+ if (xfrm6_local_dontfrag(skb->sk))
+ ipv6_stub->xfrm6_local_rxpmtu(skb, mtu);
+ else if (skb->sk)
+ xfrm_local_error(skb, mtu);
+ else
+ icmpv6_send(skb, ICMPV6_PKT_TOOBIG, 0, mtu);
+ ret = -EMSGSIZE;
+ }
+out:
+ return ret;
+}
+#endif
+
+static int xfrm6_extract_output(struct xfrm_state *x, struct sk_buff *skb)
+{
+#if IS_ENABLED(CONFIG_IPV6)
+ int err;
+
+ err = xfrm6_tunnel_check_size(skb);
+ if (err)
+ return err;
+
+ XFRM_MODE_SKB_CB(skb)->protocol = ipv6_hdr(skb)->nexthdr;
+
+ xfrm6_extract_header(skb);
+ return 0;
+#else
+ WARN_ON_ONCE(1);
+ return -EAFNOSUPPORT;
+#endif
+}
+
+static int xfrm_inner_extract_output(struct xfrm_state *x, struct sk_buff *skb)
+{
+ const struct xfrm_mode *inner_mode;
+
+ if (x->sel.family == AF_UNSPEC)
+ inner_mode = xfrm_ip2inner_mode(x,
+ xfrm_af2proto(skb_dst(skb)->ops->family));
+ else
+ inner_mode = &x->inner_mode;
+
+ if (inner_mode == NULL)
+ return -EAFNOSUPPORT;
+
+ switch (inner_mode->family) {
+ case AF_INET:
+ return xfrm4_extract_output(x, skb);
+ case AF_INET6:
+ return xfrm6_extract_output(x, skb);
+ }
+
+ return -EAFNOSUPPORT;
+}
+
+void xfrm_local_error(struct sk_buff *skb, int mtu)
+{
+ unsigned int proto;
+ struct xfrm_state_afinfo *afinfo;
+
+ if (skb->protocol == htons(ETH_P_IP))
+ proto = AF_INET;
+ else if (skb->protocol == htons(ETH_P_IPV6) &&
+ skb->sk->sk_family == AF_INET6)
+ proto = AF_INET6;
+ else
+ return;
+
+ afinfo = xfrm_state_get_afinfo(proto);
+ if (afinfo) {
+ afinfo->local_error(skb, mtu);
+ rcu_read_unlock();
+ }
+}
+EXPORT_SYMBOL_GPL(xfrm_local_error);
diff --git a/net/xfrm/xfrm_policy.c b/net/xfrm/xfrm_policy.c
new file mode 100644
index 000000000..e47c670c7
--- /dev/null
+++ b/net/xfrm/xfrm_policy.c
@@ -0,0 +1,4490 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * xfrm_policy.c
+ *
+ * Changes:
+ * Mitsuru KANDA @USAGI
+ * Kazunori MIYAZAWA @USAGI
+ * Kunihiro Ishiguro <kunihiro@ipinfusion.com>
+ * IPv6 support
+ * Kazunori MIYAZAWA @USAGI
+ * YOSHIFUJI Hideaki
+ * Split up af-specific portion
+ * Derek Atkins <derek@ihtfp.com> Add the post_input processor
+ *
+ */
+
+#include <linux/err.h>
+#include <linux/slab.h>
+#include <linux/kmod.h>
+#include <linux/list.h>
+#include <linux/spinlock.h>
+#include <linux/workqueue.h>
+#include <linux/notifier.h>
+#include <linux/netdevice.h>
+#include <linux/netfilter.h>
+#include <linux/module.h>
+#include <linux/cache.h>
+#include <linux/cpu.h>
+#include <linux/audit.h>
+#include <linux/rhashtable.h>
+#include <linux/if_tunnel.h>
+#include <net/dst.h>
+#include <net/flow.h>
+#include <net/inet_ecn.h>
+#include <net/xfrm.h>
+#include <net/ip.h>
+#include <net/gre.h>
+#if IS_ENABLED(CONFIG_IPV6_MIP6)
+#include <net/mip6.h>
+#endif
+#ifdef CONFIG_XFRM_STATISTICS
+#include <net/snmp.h>
+#endif
+#ifdef CONFIG_XFRM_ESPINTCP
+#include <net/espintcp.h>
+#endif
+
+#include "xfrm_hash.h"
+
+#define XFRM_QUEUE_TMO_MIN ((unsigned)(HZ/10))
+#define XFRM_QUEUE_TMO_MAX ((unsigned)(60*HZ))
+#define XFRM_MAX_QUEUE_LEN 100
+
+struct xfrm_flo {
+ struct dst_entry *dst_orig;
+ u8 flags;
+};
+
+/* prefixes smaller than this are stored in lists, not trees. */
+#define INEXACT_PREFIXLEN_IPV4 16
+#define INEXACT_PREFIXLEN_IPV6 48
+
+struct xfrm_pol_inexact_node {
+ struct rb_node node;
+ union {
+ xfrm_address_t addr;
+ struct rcu_head rcu;
+ };
+ u8 prefixlen;
+
+ struct rb_root root;
+
+ /* the policies matching this node, can be empty list */
+ struct hlist_head hhead;
+};
+
+/* xfrm inexact policy search tree:
+ * xfrm_pol_inexact_bin = hash(dir,type,family,if_id);
+ * |
+ * +---- root_d: sorted by daddr:prefix
+ * | |
+ * | xfrm_pol_inexact_node
+ * | |
+ * | +- root: sorted by saddr/prefix
+ * | | |
+ * | | xfrm_pol_inexact_node
+ * | | |
+ * | | + root: unused
+ * | | |
+ * | | + hhead: saddr:daddr policies
+ * | |
+ * | +- coarse policies and all any:daddr policies
+ * |
+ * +---- root_s: sorted by saddr:prefix
+ * | |
+ * | xfrm_pol_inexact_node
+ * | |
+ * | + root: unused
+ * | |
+ * | + hhead: saddr:any policies
+ * |
+ * +---- coarse policies and all any:any policies
+ *
+ * Lookups return four candidate lists:
+ * 1. any:any list from top-level xfrm_pol_inexact_bin
+ * 2. any:daddr list from daddr tree
+ * 3. saddr:daddr list from 2nd level daddr tree
+ * 4. saddr:any list from saddr tree
+ *
+ * This result set then needs to be searched for the policy with
+ * the lowest priority. If two results have same prio, youngest one wins.
+ */
+
+struct xfrm_pol_inexact_key {
+ possible_net_t net;
+ u32 if_id;
+ u16 family;
+ u8 dir, type;
+};
+
+struct xfrm_pol_inexact_bin {
+ struct xfrm_pol_inexact_key k;
+ struct rhash_head head;
+ /* list containing '*:*' policies */
+ struct hlist_head hhead;
+
+ seqcount_spinlock_t count;
+ /* tree sorted by daddr/prefix */
+ struct rb_root root_d;
+
+ /* tree sorted by saddr/prefix */
+ struct rb_root root_s;
+
+ /* slow path below */
+ struct list_head inexact_bins;
+ struct rcu_head rcu;
+};
+
+enum xfrm_pol_inexact_candidate_type {
+ XFRM_POL_CAND_BOTH,
+ XFRM_POL_CAND_SADDR,
+ XFRM_POL_CAND_DADDR,
+ XFRM_POL_CAND_ANY,
+
+ XFRM_POL_CAND_MAX,
+};
+
+struct xfrm_pol_inexact_candidates {
+ struct hlist_head *res[XFRM_POL_CAND_MAX];
+};
+
+static DEFINE_SPINLOCK(xfrm_if_cb_lock);
+static struct xfrm_if_cb const __rcu *xfrm_if_cb __read_mostly;
+
+static DEFINE_SPINLOCK(xfrm_policy_afinfo_lock);
+static struct xfrm_policy_afinfo const __rcu *xfrm_policy_afinfo[AF_INET6 + 1]
+ __read_mostly;
+
+static struct kmem_cache *xfrm_dst_cache __ro_after_init;
+
+static struct rhashtable xfrm_policy_inexact_table;
+static const struct rhashtable_params xfrm_pol_inexact_params;
+
+static void xfrm_init_pmtu(struct xfrm_dst **bundle, int nr);
+static int stale_bundle(struct dst_entry *dst);
+static int xfrm_bundle_ok(struct xfrm_dst *xdst);
+static void xfrm_policy_queue_process(struct timer_list *t);
+
+static void __xfrm_policy_link(struct xfrm_policy *pol, int dir);
+static struct xfrm_policy *__xfrm_policy_unlink(struct xfrm_policy *pol,
+ int dir);
+
+static struct xfrm_pol_inexact_bin *
+xfrm_policy_inexact_lookup(struct net *net, u8 type, u16 family, u8 dir,
+ u32 if_id);
+
+static struct xfrm_pol_inexact_bin *
+xfrm_policy_inexact_lookup_rcu(struct net *net,
+ u8 type, u16 family, u8 dir, u32 if_id);
+static struct xfrm_policy *
+xfrm_policy_insert_list(struct hlist_head *chain, struct xfrm_policy *policy,
+ bool excl);
+static void xfrm_policy_insert_inexact_list(struct hlist_head *chain,
+ struct xfrm_policy *policy);
+
+static bool
+xfrm_policy_find_inexact_candidates(struct xfrm_pol_inexact_candidates *cand,
+ struct xfrm_pol_inexact_bin *b,
+ const xfrm_address_t *saddr,
+ const xfrm_address_t *daddr);
+
+static inline bool xfrm_pol_hold_rcu(struct xfrm_policy *policy)
+{
+ return refcount_inc_not_zero(&policy->refcnt);
+}
+
+static inline bool
+__xfrm4_selector_match(const struct xfrm_selector *sel, const struct flowi *fl)
+{
+ const struct flowi4 *fl4 = &fl->u.ip4;
+
+ return addr4_match(fl4->daddr, sel->daddr.a4, sel->prefixlen_d) &&
+ addr4_match(fl4->saddr, sel->saddr.a4, sel->prefixlen_s) &&
+ !((xfrm_flowi_dport(fl, &fl4->uli) ^ sel->dport) & sel->dport_mask) &&
+ !((xfrm_flowi_sport(fl, &fl4->uli) ^ sel->sport) & sel->sport_mask) &&
+ (fl4->flowi4_proto == sel->proto || !sel->proto) &&
+ (fl4->flowi4_oif == sel->ifindex || !sel->ifindex);
+}
+
+static inline bool
+__xfrm6_selector_match(const struct xfrm_selector *sel, const struct flowi *fl)
+{
+ const struct flowi6 *fl6 = &fl->u.ip6;
+
+ return addr_match(&fl6->daddr, &sel->daddr, sel->prefixlen_d) &&
+ addr_match(&fl6->saddr, &sel->saddr, sel->prefixlen_s) &&
+ !((xfrm_flowi_dport(fl, &fl6->uli) ^ sel->dport) & sel->dport_mask) &&
+ !((xfrm_flowi_sport(fl, &fl6->uli) ^ sel->sport) & sel->sport_mask) &&
+ (fl6->flowi6_proto == sel->proto || !sel->proto) &&
+ (fl6->flowi6_oif == sel->ifindex || !sel->ifindex);
+}
+
+bool xfrm_selector_match(const struct xfrm_selector *sel, const struct flowi *fl,
+ unsigned short family)
+{
+ switch (family) {
+ case AF_INET:
+ return __xfrm4_selector_match(sel, fl);
+ case AF_INET6:
+ return __xfrm6_selector_match(sel, fl);
+ }
+ return false;
+}
+
+static const struct xfrm_policy_afinfo *xfrm_policy_get_afinfo(unsigned short family)
+{
+ const struct xfrm_policy_afinfo *afinfo;
+
+ if (unlikely(family >= ARRAY_SIZE(xfrm_policy_afinfo)))
+ return NULL;
+ rcu_read_lock();
+ afinfo = rcu_dereference(xfrm_policy_afinfo[family]);
+ if (unlikely(!afinfo))
+ rcu_read_unlock();
+ return afinfo;
+}
+
+/* Called with rcu_read_lock(). */
+static const struct xfrm_if_cb *xfrm_if_get_cb(void)
+{
+ return rcu_dereference(xfrm_if_cb);
+}
+
+struct dst_entry *__xfrm_dst_lookup(struct net *net, int tos, int oif,
+ const xfrm_address_t *saddr,
+ const xfrm_address_t *daddr,
+ int family, u32 mark)
+{
+ const struct xfrm_policy_afinfo *afinfo;
+ struct dst_entry *dst;
+
+ afinfo = xfrm_policy_get_afinfo(family);
+ if (unlikely(afinfo == NULL))
+ return ERR_PTR(-EAFNOSUPPORT);
+
+ dst = afinfo->dst_lookup(net, tos, oif, saddr, daddr, mark);
+
+ rcu_read_unlock();
+
+ return dst;
+}
+EXPORT_SYMBOL(__xfrm_dst_lookup);
+
+static inline struct dst_entry *xfrm_dst_lookup(struct xfrm_state *x,
+ int tos, int oif,
+ xfrm_address_t *prev_saddr,
+ xfrm_address_t *prev_daddr,
+ int family, u32 mark)
+{
+ struct net *net = xs_net(x);
+ xfrm_address_t *saddr = &x->props.saddr;
+ xfrm_address_t *daddr = &x->id.daddr;
+ struct dst_entry *dst;
+
+ if (x->type->flags & XFRM_TYPE_LOCAL_COADDR) {
+ saddr = x->coaddr;
+ daddr = prev_daddr;
+ }
+ if (x->type->flags & XFRM_TYPE_REMOTE_COADDR) {
+ saddr = prev_saddr;
+ daddr = x->coaddr;
+ }
+
+ dst = __xfrm_dst_lookup(net, tos, oif, saddr, daddr, family, mark);
+
+ if (!IS_ERR(dst)) {
+ if (prev_saddr != saddr)
+ memcpy(prev_saddr, saddr, sizeof(*prev_saddr));
+ if (prev_daddr != daddr)
+ memcpy(prev_daddr, daddr, sizeof(*prev_daddr));
+ }
+
+ return dst;
+}
+
+static inline unsigned long make_jiffies(long secs)
+{
+ if (secs >= (MAX_SCHEDULE_TIMEOUT-1)/HZ)
+ return MAX_SCHEDULE_TIMEOUT-1;
+ else
+ return secs*HZ;
+}
+
+static void xfrm_policy_timer(struct timer_list *t)
+{
+ struct xfrm_policy *xp = from_timer(xp, t, timer);
+ time64_t now = ktime_get_real_seconds();
+ time64_t next = TIME64_MAX;
+ int warn = 0;
+ int dir;
+
+ read_lock(&xp->lock);
+
+ if (unlikely(xp->walk.dead))
+ goto out;
+
+ dir = xfrm_policy_id2dir(xp->index);
+
+ if (xp->lft.hard_add_expires_seconds) {
+ time64_t tmo = xp->lft.hard_add_expires_seconds +
+ xp->curlft.add_time - now;
+ if (tmo <= 0)
+ goto expired;
+ if (tmo < next)
+ next = tmo;
+ }
+ if (xp->lft.hard_use_expires_seconds) {
+ time64_t tmo = xp->lft.hard_use_expires_seconds +
+ (READ_ONCE(xp->curlft.use_time) ? : xp->curlft.add_time) - now;
+ if (tmo <= 0)
+ goto expired;
+ if (tmo < next)
+ next = tmo;
+ }
+ if (xp->lft.soft_add_expires_seconds) {
+ time64_t tmo = xp->lft.soft_add_expires_seconds +
+ xp->curlft.add_time - now;
+ if (tmo <= 0) {
+ warn = 1;
+ tmo = XFRM_KM_TIMEOUT;
+ }
+ if (tmo < next)
+ next = tmo;
+ }
+ if (xp->lft.soft_use_expires_seconds) {
+ time64_t tmo = xp->lft.soft_use_expires_seconds +
+ (READ_ONCE(xp->curlft.use_time) ? : xp->curlft.add_time) - now;
+ if (tmo <= 0) {
+ warn = 1;
+ tmo = XFRM_KM_TIMEOUT;
+ }
+ if (tmo < next)
+ next = tmo;
+ }
+
+ if (warn)
+ km_policy_expired(xp, dir, 0, 0);
+ if (next != TIME64_MAX &&
+ !mod_timer(&xp->timer, jiffies + make_jiffies(next)))
+ xfrm_pol_hold(xp);
+
+out:
+ read_unlock(&xp->lock);
+ xfrm_pol_put(xp);
+ return;
+
+expired:
+ read_unlock(&xp->lock);
+ if (!xfrm_policy_delete(xp, dir))
+ km_policy_expired(xp, dir, 1, 0);
+ xfrm_pol_put(xp);
+}
+
+/* Allocate xfrm_policy. Not used here, it is supposed to be used by pfkeyv2
+ * SPD calls.
+ */
+
+struct xfrm_policy *xfrm_policy_alloc(struct net *net, gfp_t gfp)
+{
+ struct xfrm_policy *policy;
+
+ policy = kzalloc(sizeof(struct xfrm_policy), gfp);
+
+ if (policy) {
+ write_pnet(&policy->xp_net, net);
+ INIT_LIST_HEAD(&policy->walk.all);
+ INIT_HLIST_NODE(&policy->bydst_inexact_list);
+ INIT_HLIST_NODE(&policy->bydst);
+ INIT_HLIST_NODE(&policy->byidx);
+ rwlock_init(&policy->lock);
+ refcount_set(&policy->refcnt, 1);
+ skb_queue_head_init(&policy->polq.hold_queue);
+ timer_setup(&policy->timer, xfrm_policy_timer, 0);
+ timer_setup(&policy->polq.hold_timer,
+ xfrm_policy_queue_process, 0);
+ }
+ return policy;
+}
+EXPORT_SYMBOL(xfrm_policy_alloc);
+
+static void xfrm_policy_destroy_rcu(struct rcu_head *head)
+{
+ struct xfrm_policy *policy = container_of(head, struct xfrm_policy, rcu);
+
+ security_xfrm_policy_free(policy->security);
+ kfree(policy);
+}
+
+/* Destroy xfrm_policy: descendant resources must be released to this moment. */
+
+void xfrm_policy_destroy(struct xfrm_policy *policy)
+{
+ BUG_ON(!policy->walk.dead);
+
+ if (del_timer(&policy->timer) || del_timer(&policy->polq.hold_timer))
+ BUG();
+
+ call_rcu(&policy->rcu, xfrm_policy_destroy_rcu);
+}
+EXPORT_SYMBOL(xfrm_policy_destroy);
+
+/* Rule must be locked. Release descendant resources, announce
+ * entry dead. The rule must be unlinked from lists to the moment.
+ */
+
+static void xfrm_policy_kill(struct xfrm_policy *policy)
+{
+ write_lock_bh(&policy->lock);
+ policy->walk.dead = 1;
+ write_unlock_bh(&policy->lock);
+
+ atomic_inc(&policy->genid);
+
+ if (del_timer(&policy->polq.hold_timer))
+ xfrm_pol_put(policy);
+ skb_queue_purge(&policy->polq.hold_queue);
+
+ if (del_timer(&policy->timer))
+ xfrm_pol_put(policy);
+
+ xfrm_pol_put(policy);
+}
+
+static unsigned int xfrm_policy_hashmax __read_mostly = 1 * 1024 * 1024;
+
+static inline unsigned int idx_hash(struct net *net, u32 index)
+{
+ return __idx_hash(index, net->xfrm.policy_idx_hmask);
+}
+
+/* calculate policy hash thresholds */
+static void __get_hash_thresh(struct net *net,
+ unsigned short family, int dir,
+ u8 *dbits, u8 *sbits)
+{
+ switch (family) {
+ case AF_INET:
+ *dbits = net->xfrm.policy_bydst[dir].dbits4;
+ *sbits = net->xfrm.policy_bydst[dir].sbits4;
+ break;
+
+ case AF_INET6:
+ *dbits = net->xfrm.policy_bydst[dir].dbits6;
+ *sbits = net->xfrm.policy_bydst[dir].sbits6;
+ break;
+
+ default:
+ *dbits = 0;
+ *sbits = 0;
+ }
+}
+
+static struct hlist_head *policy_hash_bysel(struct net *net,
+ const struct xfrm_selector *sel,
+ unsigned short family, int dir)
+{
+ unsigned int hmask = net->xfrm.policy_bydst[dir].hmask;
+ unsigned int hash;
+ u8 dbits;
+ u8 sbits;
+
+ __get_hash_thresh(net, family, dir, &dbits, &sbits);
+ hash = __sel_hash(sel, family, hmask, dbits, sbits);
+
+ if (hash == hmask + 1)
+ return NULL;
+
+ return rcu_dereference_check(net->xfrm.policy_bydst[dir].table,
+ lockdep_is_held(&net->xfrm.xfrm_policy_lock)) + hash;
+}
+
+static struct hlist_head *policy_hash_direct(struct net *net,
+ const xfrm_address_t *daddr,
+ const xfrm_address_t *saddr,
+ unsigned short family, int dir)
+{
+ unsigned int hmask = net->xfrm.policy_bydst[dir].hmask;
+ unsigned int hash;
+ u8 dbits;
+ u8 sbits;
+
+ __get_hash_thresh(net, family, dir, &dbits, &sbits);
+ hash = __addr_hash(daddr, saddr, family, hmask, dbits, sbits);
+
+ return rcu_dereference_check(net->xfrm.policy_bydst[dir].table,
+ lockdep_is_held(&net->xfrm.xfrm_policy_lock)) + hash;
+}
+
+static void xfrm_dst_hash_transfer(struct net *net,
+ struct hlist_head *list,
+ struct hlist_head *ndsttable,
+ unsigned int nhashmask,
+ int dir)
+{
+ struct hlist_node *tmp, *entry0 = NULL;
+ struct xfrm_policy *pol;
+ unsigned int h0 = 0;
+ u8 dbits;
+ u8 sbits;
+
+redo:
+ hlist_for_each_entry_safe(pol, tmp, list, bydst) {
+ unsigned int h;
+
+ __get_hash_thresh(net, pol->family, dir, &dbits, &sbits);
+ h = __addr_hash(&pol->selector.daddr, &pol->selector.saddr,
+ pol->family, nhashmask, dbits, sbits);
+ if (!entry0) {
+ hlist_del_rcu(&pol->bydst);
+ hlist_add_head_rcu(&pol->bydst, ndsttable + h);
+ h0 = h;
+ } else {
+ if (h != h0)
+ continue;
+ hlist_del_rcu(&pol->bydst);
+ hlist_add_behind_rcu(&pol->bydst, entry0);
+ }
+ entry0 = &pol->bydst;
+ }
+ if (!hlist_empty(list)) {
+ entry0 = NULL;
+ goto redo;
+ }
+}
+
+static void xfrm_idx_hash_transfer(struct hlist_head *list,
+ struct hlist_head *nidxtable,
+ unsigned int nhashmask)
+{
+ struct hlist_node *tmp;
+ struct xfrm_policy *pol;
+
+ hlist_for_each_entry_safe(pol, tmp, list, byidx) {
+ unsigned int h;
+
+ h = __idx_hash(pol->index, nhashmask);
+ hlist_add_head(&pol->byidx, nidxtable+h);
+ }
+}
+
+static unsigned long xfrm_new_hash_mask(unsigned int old_hmask)
+{
+ return ((old_hmask + 1) << 1) - 1;
+}
+
+static void xfrm_bydst_resize(struct net *net, int dir)
+{
+ unsigned int hmask = net->xfrm.policy_bydst[dir].hmask;
+ unsigned int nhashmask = xfrm_new_hash_mask(hmask);
+ unsigned int nsize = (nhashmask + 1) * sizeof(struct hlist_head);
+ struct hlist_head *ndst = xfrm_hash_alloc(nsize);
+ struct hlist_head *odst;
+ int i;
+
+ if (!ndst)
+ return;
+
+ spin_lock_bh(&net->xfrm.xfrm_policy_lock);
+ write_seqcount_begin(&net->xfrm.xfrm_policy_hash_generation);
+
+ odst = rcu_dereference_protected(net->xfrm.policy_bydst[dir].table,
+ lockdep_is_held(&net->xfrm.xfrm_policy_lock));
+
+ for (i = hmask; i >= 0; i--)
+ xfrm_dst_hash_transfer(net, odst + i, ndst, nhashmask, dir);
+
+ rcu_assign_pointer(net->xfrm.policy_bydst[dir].table, ndst);
+ net->xfrm.policy_bydst[dir].hmask = nhashmask;
+
+ write_seqcount_end(&net->xfrm.xfrm_policy_hash_generation);
+ spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
+
+ synchronize_rcu();
+
+ xfrm_hash_free(odst, (hmask + 1) * sizeof(struct hlist_head));
+}
+
+static void xfrm_byidx_resize(struct net *net, int total)
+{
+ unsigned int hmask = net->xfrm.policy_idx_hmask;
+ unsigned int nhashmask = xfrm_new_hash_mask(hmask);
+ unsigned int nsize = (nhashmask + 1) * sizeof(struct hlist_head);
+ struct hlist_head *oidx = net->xfrm.policy_byidx;
+ struct hlist_head *nidx = xfrm_hash_alloc(nsize);
+ int i;
+
+ if (!nidx)
+ return;
+
+ spin_lock_bh(&net->xfrm.xfrm_policy_lock);
+
+ for (i = hmask; i >= 0; i--)
+ xfrm_idx_hash_transfer(oidx + i, nidx, nhashmask);
+
+ net->xfrm.policy_byidx = nidx;
+ net->xfrm.policy_idx_hmask = nhashmask;
+
+ spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
+
+ xfrm_hash_free(oidx, (hmask + 1) * sizeof(struct hlist_head));
+}
+
+static inline int xfrm_bydst_should_resize(struct net *net, int dir, int *total)
+{
+ unsigned int cnt = net->xfrm.policy_count[dir];
+ unsigned int hmask = net->xfrm.policy_bydst[dir].hmask;
+
+ if (total)
+ *total += cnt;
+
+ if ((hmask + 1) < xfrm_policy_hashmax &&
+ cnt > hmask)
+ return 1;
+
+ return 0;
+}
+
+static inline int xfrm_byidx_should_resize(struct net *net, int total)
+{
+ unsigned int hmask = net->xfrm.policy_idx_hmask;
+
+ if ((hmask + 1) < xfrm_policy_hashmax &&
+ total > hmask)
+ return 1;
+
+ return 0;
+}
+
+void xfrm_spd_getinfo(struct net *net, struct xfrmk_spdinfo *si)
+{
+ si->incnt = net->xfrm.policy_count[XFRM_POLICY_IN];
+ si->outcnt = net->xfrm.policy_count[XFRM_POLICY_OUT];
+ si->fwdcnt = net->xfrm.policy_count[XFRM_POLICY_FWD];
+ si->inscnt = net->xfrm.policy_count[XFRM_POLICY_IN+XFRM_POLICY_MAX];
+ si->outscnt = net->xfrm.policy_count[XFRM_POLICY_OUT+XFRM_POLICY_MAX];
+ si->fwdscnt = net->xfrm.policy_count[XFRM_POLICY_FWD+XFRM_POLICY_MAX];
+ si->spdhcnt = net->xfrm.policy_idx_hmask;
+ si->spdhmcnt = xfrm_policy_hashmax;
+}
+EXPORT_SYMBOL(xfrm_spd_getinfo);
+
+static DEFINE_MUTEX(hash_resize_mutex);
+static void xfrm_hash_resize(struct work_struct *work)
+{
+ struct net *net = container_of(work, struct net, xfrm.policy_hash_work);
+ int dir, total;
+
+ mutex_lock(&hash_resize_mutex);
+
+ total = 0;
+ for (dir = 0; dir < XFRM_POLICY_MAX; dir++) {
+ if (xfrm_bydst_should_resize(net, dir, &total))
+ xfrm_bydst_resize(net, dir);
+ }
+ if (xfrm_byidx_should_resize(net, total))
+ xfrm_byidx_resize(net, total);
+
+ mutex_unlock(&hash_resize_mutex);
+}
+
+/* Make sure *pol can be inserted into fastbin.
+ * Useful to check that later insert requests will be successful
+ * (provided xfrm_policy_lock is held throughout).
+ */
+static struct xfrm_pol_inexact_bin *
+xfrm_policy_inexact_alloc_bin(const struct xfrm_policy *pol, u8 dir)
+{
+ struct xfrm_pol_inexact_bin *bin, *prev;
+ struct xfrm_pol_inexact_key k = {
+ .family = pol->family,
+ .type = pol->type,
+ .dir = dir,
+ .if_id = pol->if_id,
+ };
+ struct net *net = xp_net(pol);
+
+ lockdep_assert_held(&net->xfrm.xfrm_policy_lock);
+
+ write_pnet(&k.net, net);
+ bin = rhashtable_lookup_fast(&xfrm_policy_inexact_table, &k,
+ xfrm_pol_inexact_params);
+ if (bin)
+ return bin;
+
+ bin = kzalloc(sizeof(*bin), GFP_ATOMIC);
+ if (!bin)
+ return NULL;
+
+ bin->k = k;
+ INIT_HLIST_HEAD(&bin->hhead);
+ bin->root_d = RB_ROOT;
+ bin->root_s = RB_ROOT;
+ seqcount_spinlock_init(&bin->count, &net->xfrm.xfrm_policy_lock);
+
+ prev = rhashtable_lookup_get_insert_key(&xfrm_policy_inexact_table,
+ &bin->k, &bin->head,
+ xfrm_pol_inexact_params);
+ if (!prev) {
+ list_add(&bin->inexact_bins, &net->xfrm.inexact_bins);
+ return bin;
+ }
+
+ kfree(bin);
+
+ return IS_ERR(prev) ? NULL : prev;
+}
+
+static bool xfrm_pol_inexact_addr_use_any_list(const xfrm_address_t *addr,
+ int family, u8 prefixlen)
+{
+ if (xfrm_addr_any(addr, family))
+ return true;
+
+ if (family == AF_INET6 && prefixlen < INEXACT_PREFIXLEN_IPV6)
+ return true;
+
+ if (family == AF_INET && prefixlen < INEXACT_PREFIXLEN_IPV4)
+ return true;
+
+ return false;
+}
+
+static bool
+xfrm_policy_inexact_insert_use_any_list(const struct xfrm_policy *policy)
+{
+ const xfrm_address_t *addr;
+ bool saddr_any, daddr_any;
+ u8 prefixlen;
+
+ addr = &policy->selector.saddr;
+ prefixlen = policy->selector.prefixlen_s;
+
+ saddr_any = xfrm_pol_inexact_addr_use_any_list(addr,
+ policy->family,
+ prefixlen);
+ addr = &policy->selector.daddr;
+ prefixlen = policy->selector.prefixlen_d;
+ daddr_any = xfrm_pol_inexact_addr_use_any_list(addr,
+ policy->family,
+ prefixlen);
+ return saddr_any && daddr_any;
+}
+
+static void xfrm_pol_inexact_node_init(struct xfrm_pol_inexact_node *node,
+ const xfrm_address_t *addr, u8 prefixlen)
+{
+ node->addr = *addr;
+ node->prefixlen = prefixlen;
+}
+
+static struct xfrm_pol_inexact_node *
+xfrm_pol_inexact_node_alloc(const xfrm_address_t *addr, u8 prefixlen)
+{
+ struct xfrm_pol_inexact_node *node;
+
+ node = kzalloc(sizeof(*node), GFP_ATOMIC);
+ if (node)
+ xfrm_pol_inexact_node_init(node, addr, prefixlen);
+
+ return node;
+}
+
+static int xfrm_policy_addr_delta(const xfrm_address_t *a,
+ const xfrm_address_t *b,
+ u8 prefixlen, u16 family)
+{
+ u32 ma, mb, mask;
+ unsigned int pdw, pbi;
+ int delta = 0;
+
+ switch (family) {
+ case AF_INET:
+ if (prefixlen == 0)
+ return 0;
+ mask = ~0U << (32 - prefixlen);
+ ma = ntohl(a->a4) & mask;
+ mb = ntohl(b->a4) & mask;
+ if (ma < mb)
+ delta = -1;
+ else if (ma > mb)
+ delta = 1;
+ break;
+ case AF_INET6:
+ pdw = prefixlen >> 5;
+ pbi = prefixlen & 0x1f;
+
+ if (pdw) {
+ delta = memcmp(a->a6, b->a6, pdw << 2);
+ if (delta)
+ return delta;
+ }
+ if (pbi) {
+ mask = ~0U << (32 - pbi);
+ ma = ntohl(a->a6[pdw]) & mask;
+ mb = ntohl(b->a6[pdw]) & mask;
+ if (ma < mb)
+ delta = -1;
+ else if (ma > mb)
+ delta = 1;
+ }
+ break;
+ default:
+ break;
+ }
+
+ return delta;
+}
+
+static void xfrm_policy_inexact_list_reinsert(struct net *net,
+ struct xfrm_pol_inexact_node *n,
+ u16 family)
+{
+ unsigned int matched_s, matched_d;
+ struct xfrm_policy *policy, *p;
+
+ matched_s = 0;
+ matched_d = 0;
+
+ list_for_each_entry_reverse(policy, &net->xfrm.policy_all, walk.all) {
+ struct hlist_node *newpos = NULL;
+ bool matches_s, matches_d;
+
+ if (policy->walk.dead || !policy->bydst_reinsert)
+ continue;
+
+ WARN_ON_ONCE(policy->family != family);
+
+ policy->bydst_reinsert = false;
+ hlist_for_each_entry(p, &n->hhead, bydst) {
+ if (policy->priority > p->priority)
+ newpos = &p->bydst;
+ else if (policy->priority == p->priority &&
+ policy->pos > p->pos)
+ newpos = &p->bydst;
+ else
+ break;
+ }
+
+ if (newpos)
+ hlist_add_behind_rcu(&policy->bydst, newpos);
+ else
+ hlist_add_head_rcu(&policy->bydst, &n->hhead);
+
+ /* paranoia checks follow.
+ * Check that the reinserted policy matches at least
+ * saddr or daddr for current node prefix.
+ *
+ * Matching both is fine, matching saddr in one policy
+ * (but not daddr) and then matching only daddr in another
+ * is a bug.
+ */
+ matches_s = xfrm_policy_addr_delta(&policy->selector.saddr,
+ &n->addr,
+ n->prefixlen,
+ family) == 0;
+ matches_d = xfrm_policy_addr_delta(&policy->selector.daddr,
+ &n->addr,
+ n->prefixlen,
+ family) == 0;
+ if (matches_s && matches_d)
+ continue;
+
+ WARN_ON_ONCE(!matches_s && !matches_d);
+ if (matches_s)
+ matched_s++;
+ if (matches_d)
+ matched_d++;
+ WARN_ON_ONCE(matched_s && matched_d);
+ }
+}
+
+static void xfrm_policy_inexact_node_reinsert(struct net *net,
+ struct xfrm_pol_inexact_node *n,
+ struct rb_root *new,
+ u16 family)
+{
+ struct xfrm_pol_inexact_node *node;
+ struct rb_node **p, *parent;
+
+ /* we should not have another subtree here */
+ WARN_ON_ONCE(!RB_EMPTY_ROOT(&n->root));
+restart:
+ parent = NULL;
+ p = &new->rb_node;
+ while (*p) {
+ u8 prefixlen;
+ int delta;
+
+ parent = *p;
+ node = rb_entry(*p, struct xfrm_pol_inexact_node, node);
+
+ prefixlen = min(node->prefixlen, n->prefixlen);
+
+ delta = xfrm_policy_addr_delta(&n->addr, &node->addr,
+ prefixlen, family);
+ if (delta < 0) {
+ p = &parent->rb_left;
+ } else if (delta > 0) {
+ p = &parent->rb_right;
+ } else {
+ bool same_prefixlen = node->prefixlen == n->prefixlen;
+ struct xfrm_policy *tmp;
+
+ hlist_for_each_entry(tmp, &n->hhead, bydst) {
+ tmp->bydst_reinsert = true;
+ hlist_del_rcu(&tmp->bydst);
+ }
+
+ node->prefixlen = prefixlen;
+
+ xfrm_policy_inexact_list_reinsert(net, node, family);
+
+ if (same_prefixlen) {
+ kfree_rcu(n, rcu);
+ return;
+ }
+
+ rb_erase(*p, new);
+ kfree_rcu(n, rcu);
+ n = node;
+ goto restart;
+ }
+ }
+
+ rb_link_node_rcu(&n->node, parent, p);
+ rb_insert_color(&n->node, new);
+}
+
+/* merge nodes v and n */
+static void xfrm_policy_inexact_node_merge(struct net *net,
+ struct xfrm_pol_inexact_node *v,
+ struct xfrm_pol_inexact_node *n,
+ u16 family)
+{
+ struct xfrm_pol_inexact_node *node;
+ struct xfrm_policy *tmp;
+ struct rb_node *rnode;
+
+ /* To-be-merged node v has a subtree.
+ *
+ * Dismantle it and insert its nodes to n->root.
+ */
+ while ((rnode = rb_first(&v->root)) != NULL) {
+ node = rb_entry(rnode, struct xfrm_pol_inexact_node, node);
+ rb_erase(&node->node, &v->root);
+ xfrm_policy_inexact_node_reinsert(net, node, &n->root,
+ family);
+ }
+
+ hlist_for_each_entry(tmp, &v->hhead, bydst) {
+ tmp->bydst_reinsert = true;
+ hlist_del_rcu(&tmp->bydst);
+ }
+
+ xfrm_policy_inexact_list_reinsert(net, n, family);
+}
+
+static struct xfrm_pol_inexact_node *
+xfrm_policy_inexact_insert_node(struct net *net,
+ struct rb_root *root,
+ xfrm_address_t *addr,
+ u16 family, u8 prefixlen, u8 dir)
+{
+ struct xfrm_pol_inexact_node *cached = NULL;
+ struct rb_node **p, *parent = NULL;
+ struct xfrm_pol_inexact_node *node;
+
+ p = &root->rb_node;
+ while (*p) {
+ int delta;
+
+ parent = *p;
+ node = rb_entry(*p, struct xfrm_pol_inexact_node, node);
+
+ delta = xfrm_policy_addr_delta(addr, &node->addr,
+ node->prefixlen,
+ family);
+ if (delta == 0 && prefixlen >= node->prefixlen) {
+ WARN_ON_ONCE(cached); /* ipsec policies got lost */
+ return node;
+ }
+
+ if (delta < 0)
+ p = &parent->rb_left;
+ else
+ p = &parent->rb_right;
+
+ if (prefixlen < node->prefixlen) {
+ delta = xfrm_policy_addr_delta(addr, &node->addr,
+ prefixlen,
+ family);
+ if (delta)
+ continue;
+
+ /* This node is a subnet of the new prefix. It needs
+ * to be removed and re-inserted with the smaller
+ * prefix and all nodes that are now also covered
+ * by the reduced prefixlen.
+ */
+ rb_erase(&node->node, root);
+
+ if (!cached) {
+ xfrm_pol_inexact_node_init(node, addr,
+ prefixlen);
+ cached = node;
+ } else {
+ /* This node also falls within the new
+ * prefixlen. Merge the to-be-reinserted
+ * node and this one.
+ */
+ xfrm_policy_inexact_node_merge(net, node,
+ cached, family);
+ kfree_rcu(node, rcu);
+ }
+
+ /* restart */
+ p = &root->rb_node;
+ parent = NULL;
+ }
+ }
+
+ node = cached;
+ if (!node) {
+ node = xfrm_pol_inexact_node_alloc(addr, prefixlen);
+ if (!node)
+ return NULL;
+ }
+
+ rb_link_node_rcu(&node->node, parent, p);
+ rb_insert_color(&node->node, root);
+
+ return node;
+}
+
+static void xfrm_policy_inexact_gc_tree(struct rb_root *r, bool rm)
+{
+ struct xfrm_pol_inexact_node *node;
+ struct rb_node *rn = rb_first(r);
+
+ while (rn) {
+ node = rb_entry(rn, struct xfrm_pol_inexact_node, node);
+
+ xfrm_policy_inexact_gc_tree(&node->root, rm);
+ rn = rb_next(rn);
+
+ if (!hlist_empty(&node->hhead) || !RB_EMPTY_ROOT(&node->root)) {
+ WARN_ON_ONCE(rm);
+ continue;
+ }
+
+ rb_erase(&node->node, r);
+ kfree_rcu(node, rcu);
+ }
+}
+
+static void __xfrm_policy_inexact_prune_bin(struct xfrm_pol_inexact_bin *b, bool net_exit)
+{
+ write_seqcount_begin(&b->count);
+ xfrm_policy_inexact_gc_tree(&b->root_d, net_exit);
+ xfrm_policy_inexact_gc_tree(&b->root_s, net_exit);
+ write_seqcount_end(&b->count);
+
+ if (!RB_EMPTY_ROOT(&b->root_d) || !RB_EMPTY_ROOT(&b->root_s) ||
+ !hlist_empty(&b->hhead)) {
+ WARN_ON_ONCE(net_exit);
+ return;
+ }
+
+ if (rhashtable_remove_fast(&xfrm_policy_inexact_table, &b->head,
+ xfrm_pol_inexact_params) == 0) {
+ list_del(&b->inexact_bins);
+ kfree_rcu(b, rcu);
+ }
+}
+
+static void xfrm_policy_inexact_prune_bin(struct xfrm_pol_inexact_bin *b)
+{
+ struct net *net = read_pnet(&b->k.net);
+
+ spin_lock_bh(&net->xfrm.xfrm_policy_lock);
+ __xfrm_policy_inexact_prune_bin(b, false);
+ spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
+}
+
+static void __xfrm_policy_inexact_flush(struct net *net)
+{
+ struct xfrm_pol_inexact_bin *bin, *t;
+
+ lockdep_assert_held(&net->xfrm.xfrm_policy_lock);
+
+ list_for_each_entry_safe(bin, t, &net->xfrm.inexact_bins, inexact_bins)
+ __xfrm_policy_inexact_prune_bin(bin, false);
+}
+
+static struct hlist_head *
+xfrm_policy_inexact_alloc_chain(struct xfrm_pol_inexact_bin *bin,
+ struct xfrm_policy *policy, u8 dir)
+{
+ struct xfrm_pol_inexact_node *n;
+ struct net *net;
+
+ net = xp_net(policy);
+ lockdep_assert_held(&net->xfrm.xfrm_policy_lock);
+
+ if (xfrm_policy_inexact_insert_use_any_list(policy))
+ return &bin->hhead;
+
+ if (xfrm_pol_inexact_addr_use_any_list(&policy->selector.daddr,
+ policy->family,
+ policy->selector.prefixlen_d)) {
+ write_seqcount_begin(&bin->count);
+ n = xfrm_policy_inexact_insert_node(net,
+ &bin->root_s,
+ &policy->selector.saddr,
+ policy->family,
+ policy->selector.prefixlen_s,
+ dir);
+ write_seqcount_end(&bin->count);
+ if (!n)
+ return NULL;
+
+ return &n->hhead;
+ }
+
+ /* daddr is fixed */
+ write_seqcount_begin(&bin->count);
+ n = xfrm_policy_inexact_insert_node(net,
+ &bin->root_d,
+ &policy->selector.daddr,
+ policy->family,
+ policy->selector.prefixlen_d, dir);
+ write_seqcount_end(&bin->count);
+ if (!n)
+ return NULL;
+
+ /* saddr is wildcard */
+ if (xfrm_pol_inexact_addr_use_any_list(&policy->selector.saddr,
+ policy->family,
+ policy->selector.prefixlen_s))
+ return &n->hhead;
+
+ write_seqcount_begin(&bin->count);
+ n = xfrm_policy_inexact_insert_node(net,
+ &n->root,
+ &policy->selector.saddr,
+ policy->family,
+ policy->selector.prefixlen_s, dir);
+ write_seqcount_end(&bin->count);
+ if (!n)
+ return NULL;
+
+ return &n->hhead;
+}
+
+static struct xfrm_policy *
+xfrm_policy_inexact_insert(struct xfrm_policy *policy, u8 dir, int excl)
+{
+ struct xfrm_pol_inexact_bin *bin;
+ struct xfrm_policy *delpol;
+ struct hlist_head *chain;
+ struct net *net;
+
+ bin = xfrm_policy_inexact_alloc_bin(policy, dir);
+ if (!bin)
+ return ERR_PTR(-ENOMEM);
+
+ net = xp_net(policy);
+ lockdep_assert_held(&net->xfrm.xfrm_policy_lock);
+
+ chain = xfrm_policy_inexact_alloc_chain(bin, policy, dir);
+ if (!chain) {
+ __xfrm_policy_inexact_prune_bin(bin, false);
+ return ERR_PTR(-ENOMEM);
+ }
+
+ delpol = xfrm_policy_insert_list(chain, policy, excl);
+ if (delpol && excl) {
+ __xfrm_policy_inexact_prune_bin(bin, false);
+ return ERR_PTR(-EEXIST);
+ }
+
+ chain = &net->xfrm.policy_inexact[dir];
+ xfrm_policy_insert_inexact_list(chain, policy);
+
+ if (delpol)
+ __xfrm_policy_inexact_prune_bin(bin, false);
+
+ return delpol;
+}
+
+static void xfrm_hash_rebuild(struct work_struct *work)
+{
+ struct net *net = container_of(work, struct net,
+ xfrm.policy_hthresh.work);
+ unsigned int hmask;
+ struct xfrm_policy *pol;
+ struct xfrm_policy *policy;
+ struct hlist_head *chain;
+ struct hlist_head *odst;
+ struct hlist_node *newpos;
+ int i;
+ int dir;
+ unsigned seq;
+ u8 lbits4, rbits4, lbits6, rbits6;
+
+ mutex_lock(&hash_resize_mutex);
+
+ /* read selector prefixlen thresholds */
+ do {
+ seq = read_seqbegin(&net->xfrm.policy_hthresh.lock);
+
+ lbits4 = net->xfrm.policy_hthresh.lbits4;
+ rbits4 = net->xfrm.policy_hthresh.rbits4;
+ lbits6 = net->xfrm.policy_hthresh.lbits6;
+ rbits6 = net->xfrm.policy_hthresh.rbits6;
+ } while (read_seqretry(&net->xfrm.policy_hthresh.lock, seq));
+
+ spin_lock_bh(&net->xfrm.xfrm_policy_lock);
+ write_seqcount_begin(&net->xfrm.xfrm_policy_hash_generation);
+
+ /* make sure that we can insert the indirect policies again before
+ * we start with destructive action.
+ */
+ list_for_each_entry(policy, &net->xfrm.policy_all, walk.all) {
+ struct xfrm_pol_inexact_bin *bin;
+ u8 dbits, sbits;
+
+ if (policy->walk.dead)
+ continue;
+
+ dir = xfrm_policy_id2dir(policy->index);
+ if (dir >= XFRM_POLICY_MAX)
+ continue;
+
+ if ((dir & XFRM_POLICY_MASK) == XFRM_POLICY_OUT) {
+ if (policy->family == AF_INET) {
+ dbits = rbits4;
+ sbits = lbits4;
+ } else {
+ dbits = rbits6;
+ sbits = lbits6;
+ }
+ } else {
+ if (policy->family == AF_INET) {
+ dbits = lbits4;
+ sbits = rbits4;
+ } else {
+ dbits = lbits6;
+ sbits = rbits6;
+ }
+ }
+
+ if (policy->selector.prefixlen_d < dbits ||
+ policy->selector.prefixlen_s < sbits)
+ continue;
+
+ bin = xfrm_policy_inexact_alloc_bin(policy, dir);
+ if (!bin)
+ goto out_unlock;
+
+ if (!xfrm_policy_inexact_alloc_chain(bin, policy, dir))
+ goto out_unlock;
+ }
+
+ /* reset the bydst and inexact table in all directions */
+ for (dir = 0; dir < XFRM_POLICY_MAX; dir++) {
+ struct hlist_node *n;
+
+ hlist_for_each_entry_safe(policy, n,
+ &net->xfrm.policy_inexact[dir],
+ bydst_inexact_list) {
+ hlist_del_rcu(&policy->bydst);
+ hlist_del_init(&policy->bydst_inexact_list);
+ }
+
+ hmask = net->xfrm.policy_bydst[dir].hmask;
+ odst = net->xfrm.policy_bydst[dir].table;
+ for (i = hmask; i >= 0; i--) {
+ hlist_for_each_entry_safe(policy, n, odst + i, bydst)
+ hlist_del_rcu(&policy->bydst);
+ }
+ if ((dir & XFRM_POLICY_MASK) == XFRM_POLICY_OUT) {
+ /* dir out => dst = remote, src = local */
+ net->xfrm.policy_bydst[dir].dbits4 = rbits4;
+ net->xfrm.policy_bydst[dir].sbits4 = lbits4;
+ net->xfrm.policy_bydst[dir].dbits6 = rbits6;
+ net->xfrm.policy_bydst[dir].sbits6 = lbits6;
+ } else {
+ /* dir in/fwd => dst = local, src = remote */
+ net->xfrm.policy_bydst[dir].dbits4 = lbits4;
+ net->xfrm.policy_bydst[dir].sbits4 = rbits4;
+ net->xfrm.policy_bydst[dir].dbits6 = lbits6;
+ net->xfrm.policy_bydst[dir].sbits6 = rbits6;
+ }
+ }
+
+ /* re-insert all policies by order of creation */
+ list_for_each_entry_reverse(policy, &net->xfrm.policy_all, walk.all) {
+ if (policy->walk.dead)
+ continue;
+ dir = xfrm_policy_id2dir(policy->index);
+ if (dir >= XFRM_POLICY_MAX) {
+ /* skip socket policies */
+ continue;
+ }
+ newpos = NULL;
+ chain = policy_hash_bysel(net, &policy->selector,
+ policy->family, dir);
+
+ if (!chain) {
+ void *p = xfrm_policy_inexact_insert(policy, dir, 0);
+
+ WARN_ONCE(IS_ERR(p), "reinsert: %ld\n", PTR_ERR(p));
+ continue;
+ }
+
+ hlist_for_each_entry(pol, chain, bydst) {
+ if (policy->priority >= pol->priority)
+ newpos = &pol->bydst;
+ else
+ break;
+ }
+ if (newpos)
+ hlist_add_behind_rcu(&policy->bydst, newpos);
+ else
+ hlist_add_head_rcu(&policy->bydst, chain);
+ }
+
+out_unlock:
+ __xfrm_policy_inexact_flush(net);
+ write_seqcount_end(&net->xfrm.xfrm_policy_hash_generation);
+ spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
+
+ mutex_unlock(&hash_resize_mutex);
+}
+
+void xfrm_policy_hash_rebuild(struct net *net)
+{
+ schedule_work(&net->xfrm.policy_hthresh.work);
+}
+EXPORT_SYMBOL(xfrm_policy_hash_rebuild);
+
+/* Generate new index... KAME seems to generate them ordered by cost
+ * of an absolute inpredictability of ordering of rules. This will not pass. */
+static u32 xfrm_gen_index(struct net *net, int dir, u32 index)
+{
+ for (;;) {
+ struct hlist_head *list;
+ struct xfrm_policy *p;
+ u32 idx;
+ int found;
+
+ if (!index) {
+ idx = (net->xfrm.idx_generator | dir);
+ net->xfrm.idx_generator += 8;
+ } else {
+ idx = index;
+ index = 0;
+ }
+
+ if (idx == 0)
+ idx = 8;
+ list = net->xfrm.policy_byidx + idx_hash(net, idx);
+ found = 0;
+ hlist_for_each_entry(p, list, byidx) {
+ if (p->index == idx) {
+ found = 1;
+ break;
+ }
+ }
+ if (!found)
+ return idx;
+ }
+}
+
+static inline int selector_cmp(struct xfrm_selector *s1, struct xfrm_selector *s2)
+{
+ u32 *p1 = (u32 *) s1;
+ u32 *p2 = (u32 *) s2;
+ int len = sizeof(struct xfrm_selector) / sizeof(u32);
+ int i;
+
+ for (i = 0; i < len; i++) {
+ if (p1[i] != p2[i])
+ return 1;
+ }
+
+ return 0;
+}
+
+static void xfrm_policy_requeue(struct xfrm_policy *old,
+ struct xfrm_policy *new)
+{
+ struct xfrm_policy_queue *pq = &old->polq;
+ struct sk_buff_head list;
+
+ if (skb_queue_empty(&pq->hold_queue))
+ return;
+
+ __skb_queue_head_init(&list);
+
+ spin_lock_bh(&pq->hold_queue.lock);
+ skb_queue_splice_init(&pq->hold_queue, &list);
+ if (del_timer(&pq->hold_timer))
+ xfrm_pol_put(old);
+ spin_unlock_bh(&pq->hold_queue.lock);
+
+ pq = &new->polq;
+
+ spin_lock_bh(&pq->hold_queue.lock);
+ skb_queue_splice(&list, &pq->hold_queue);
+ pq->timeout = XFRM_QUEUE_TMO_MIN;
+ if (!mod_timer(&pq->hold_timer, jiffies))
+ xfrm_pol_hold(new);
+ spin_unlock_bh(&pq->hold_queue.lock);
+}
+
+static inline bool xfrm_policy_mark_match(const struct xfrm_mark *mark,
+ struct xfrm_policy *pol)
+{
+ return mark->v == pol->mark.v && mark->m == pol->mark.m;
+}
+
+static u32 xfrm_pol_bin_key(const void *data, u32 len, u32 seed)
+{
+ const struct xfrm_pol_inexact_key *k = data;
+ u32 a = k->type << 24 | k->dir << 16 | k->family;
+
+ return jhash_3words(a, k->if_id, net_hash_mix(read_pnet(&k->net)),
+ seed);
+}
+
+static u32 xfrm_pol_bin_obj(const void *data, u32 len, u32 seed)
+{
+ const struct xfrm_pol_inexact_bin *b = data;
+
+ return xfrm_pol_bin_key(&b->k, 0, seed);
+}
+
+static int xfrm_pol_bin_cmp(struct rhashtable_compare_arg *arg,
+ const void *ptr)
+{
+ const struct xfrm_pol_inexact_key *key = arg->key;
+ const struct xfrm_pol_inexact_bin *b = ptr;
+ int ret;
+
+ if (!net_eq(read_pnet(&b->k.net), read_pnet(&key->net)))
+ return -1;
+
+ ret = b->k.dir ^ key->dir;
+ if (ret)
+ return ret;
+
+ ret = b->k.type ^ key->type;
+ if (ret)
+ return ret;
+
+ ret = b->k.family ^ key->family;
+ if (ret)
+ return ret;
+
+ return b->k.if_id ^ key->if_id;
+}
+
+static const struct rhashtable_params xfrm_pol_inexact_params = {
+ .head_offset = offsetof(struct xfrm_pol_inexact_bin, head),
+ .hashfn = xfrm_pol_bin_key,
+ .obj_hashfn = xfrm_pol_bin_obj,
+ .obj_cmpfn = xfrm_pol_bin_cmp,
+ .automatic_shrinking = true,
+};
+
+static void xfrm_policy_insert_inexact_list(struct hlist_head *chain,
+ struct xfrm_policy *policy)
+{
+ struct xfrm_policy *pol, *delpol = NULL;
+ struct hlist_node *newpos = NULL;
+ int i = 0;
+
+ hlist_for_each_entry(pol, chain, bydst_inexact_list) {
+ if (pol->type == policy->type &&
+ pol->if_id == policy->if_id &&
+ !selector_cmp(&pol->selector, &policy->selector) &&
+ xfrm_policy_mark_match(&policy->mark, pol) &&
+ xfrm_sec_ctx_match(pol->security, policy->security) &&
+ !WARN_ON(delpol)) {
+ delpol = pol;
+ if (policy->priority > pol->priority)
+ continue;
+ } else if (policy->priority >= pol->priority) {
+ newpos = &pol->bydst_inexact_list;
+ continue;
+ }
+ if (delpol)
+ break;
+ }
+
+ if (newpos)
+ hlist_add_behind_rcu(&policy->bydst_inexact_list, newpos);
+ else
+ hlist_add_head_rcu(&policy->bydst_inexact_list, chain);
+
+ hlist_for_each_entry(pol, chain, bydst_inexact_list) {
+ pol->pos = i;
+ i++;
+ }
+}
+
+static struct xfrm_policy *xfrm_policy_insert_list(struct hlist_head *chain,
+ struct xfrm_policy *policy,
+ bool excl)
+{
+ struct xfrm_policy *pol, *newpos = NULL, *delpol = NULL;
+
+ hlist_for_each_entry(pol, chain, bydst) {
+ if (pol->type == policy->type &&
+ pol->if_id == policy->if_id &&
+ !selector_cmp(&pol->selector, &policy->selector) &&
+ xfrm_policy_mark_match(&policy->mark, pol) &&
+ xfrm_sec_ctx_match(pol->security, policy->security) &&
+ !WARN_ON(delpol)) {
+ if (excl)
+ return ERR_PTR(-EEXIST);
+ delpol = pol;
+ if (policy->priority > pol->priority)
+ continue;
+ } else if (policy->priority >= pol->priority) {
+ newpos = pol;
+ continue;
+ }
+ if (delpol)
+ break;
+ }
+
+ if (newpos)
+ hlist_add_behind_rcu(&policy->bydst, &newpos->bydst);
+ else
+ hlist_add_head_rcu(&policy->bydst, chain);
+
+ return delpol;
+}
+
+int xfrm_policy_insert(int dir, struct xfrm_policy *policy, int excl)
+{
+ struct net *net = xp_net(policy);
+ struct xfrm_policy *delpol;
+ struct hlist_head *chain;
+
+ spin_lock_bh(&net->xfrm.xfrm_policy_lock);
+ chain = policy_hash_bysel(net, &policy->selector, policy->family, dir);
+ if (chain)
+ delpol = xfrm_policy_insert_list(chain, policy, excl);
+ else
+ delpol = xfrm_policy_inexact_insert(policy, dir, excl);
+
+ if (IS_ERR(delpol)) {
+ spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
+ return PTR_ERR(delpol);
+ }
+
+ __xfrm_policy_link(policy, dir);
+
+ /* After previous checking, family can either be AF_INET or AF_INET6 */
+ if (policy->family == AF_INET)
+ rt_genid_bump_ipv4(net);
+ else
+ rt_genid_bump_ipv6(net);
+
+ if (delpol) {
+ xfrm_policy_requeue(delpol, policy);
+ __xfrm_policy_unlink(delpol, dir);
+ }
+ policy->index = delpol ? delpol->index : xfrm_gen_index(net, dir, policy->index);
+ hlist_add_head(&policy->byidx, net->xfrm.policy_byidx+idx_hash(net, policy->index));
+ policy->curlft.add_time = ktime_get_real_seconds();
+ policy->curlft.use_time = 0;
+ if (!mod_timer(&policy->timer, jiffies + HZ))
+ xfrm_pol_hold(policy);
+ spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
+
+ if (delpol)
+ xfrm_policy_kill(delpol);
+ else if (xfrm_bydst_should_resize(net, dir, NULL))
+ schedule_work(&net->xfrm.policy_hash_work);
+
+ return 0;
+}
+EXPORT_SYMBOL(xfrm_policy_insert);
+
+static struct xfrm_policy *
+__xfrm_policy_bysel_ctx(struct hlist_head *chain, const struct xfrm_mark *mark,
+ u32 if_id, u8 type, int dir, struct xfrm_selector *sel,
+ struct xfrm_sec_ctx *ctx)
+{
+ struct xfrm_policy *pol;
+
+ if (!chain)
+ return NULL;
+
+ hlist_for_each_entry(pol, chain, bydst) {
+ if (pol->type == type &&
+ pol->if_id == if_id &&
+ xfrm_policy_mark_match(mark, pol) &&
+ !selector_cmp(sel, &pol->selector) &&
+ xfrm_sec_ctx_match(ctx, pol->security))
+ return pol;
+ }
+
+ return NULL;
+}
+
+struct xfrm_policy *
+xfrm_policy_bysel_ctx(struct net *net, const struct xfrm_mark *mark, u32 if_id,
+ u8 type, int dir, struct xfrm_selector *sel,
+ struct xfrm_sec_ctx *ctx, int delete, int *err)
+{
+ struct xfrm_pol_inexact_bin *bin = NULL;
+ struct xfrm_policy *pol, *ret = NULL;
+ struct hlist_head *chain;
+
+ *err = 0;
+ spin_lock_bh(&net->xfrm.xfrm_policy_lock);
+ chain = policy_hash_bysel(net, sel, sel->family, dir);
+ if (!chain) {
+ struct xfrm_pol_inexact_candidates cand;
+ int i;
+
+ bin = xfrm_policy_inexact_lookup(net, type,
+ sel->family, dir, if_id);
+ if (!bin) {
+ spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
+ return NULL;
+ }
+
+ if (!xfrm_policy_find_inexact_candidates(&cand, bin,
+ &sel->saddr,
+ &sel->daddr)) {
+ spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
+ return NULL;
+ }
+
+ pol = NULL;
+ for (i = 0; i < ARRAY_SIZE(cand.res); i++) {
+ struct xfrm_policy *tmp;
+
+ tmp = __xfrm_policy_bysel_ctx(cand.res[i], mark,
+ if_id, type, dir,
+ sel, ctx);
+ if (!tmp)
+ continue;
+
+ if (!pol || tmp->pos < pol->pos)
+ pol = tmp;
+ }
+ } else {
+ pol = __xfrm_policy_bysel_ctx(chain, mark, if_id, type, dir,
+ sel, ctx);
+ }
+
+ if (pol) {
+ xfrm_pol_hold(pol);
+ if (delete) {
+ *err = security_xfrm_policy_delete(pol->security);
+ if (*err) {
+ spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
+ return pol;
+ }
+ __xfrm_policy_unlink(pol, dir);
+ }
+ ret = pol;
+ }
+ spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
+
+ if (ret && delete)
+ xfrm_policy_kill(ret);
+ if (bin && delete)
+ xfrm_policy_inexact_prune_bin(bin);
+ return ret;
+}
+EXPORT_SYMBOL(xfrm_policy_bysel_ctx);
+
+struct xfrm_policy *
+xfrm_policy_byid(struct net *net, const struct xfrm_mark *mark, u32 if_id,
+ u8 type, int dir, u32 id, int delete, int *err)
+{
+ struct xfrm_policy *pol, *ret;
+ struct hlist_head *chain;
+
+ *err = -ENOENT;
+ if (xfrm_policy_id2dir(id) != dir)
+ return NULL;
+
+ *err = 0;
+ spin_lock_bh(&net->xfrm.xfrm_policy_lock);
+ chain = net->xfrm.policy_byidx + idx_hash(net, id);
+ ret = NULL;
+ hlist_for_each_entry(pol, chain, byidx) {
+ if (pol->type == type && pol->index == id &&
+ pol->if_id == if_id && xfrm_policy_mark_match(mark, pol)) {
+ xfrm_pol_hold(pol);
+ if (delete) {
+ *err = security_xfrm_policy_delete(
+ pol->security);
+ if (*err) {
+ spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
+ return pol;
+ }
+ __xfrm_policy_unlink(pol, dir);
+ }
+ ret = pol;
+ break;
+ }
+ }
+ spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
+
+ if (ret && delete)
+ xfrm_policy_kill(ret);
+ return ret;
+}
+EXPORT_SYMBOL(xfrm_policy_byid);
+
+#ifdef CONFIG_SECURITY_NETWORK_XFRM
+static inline int
+xfrm_policy_flush_secctx_check(struct net *net, u8 type, bool task_valid)
+{
+ struct xfrm_policy *pol;
+ int err = 0;
+
+ list_for_each_entry(pol, &net->xfrm.policy_all, walk.all) {
+ if (pol->walk.dead ||
+ xfrm_policy_id2dir(pol->index) >= XFRM_POLICY_MAX ||
+ pol->type != type)
+ continue;
+
+ err = security_xfrm_policy_delete(pol->security);
+ if (err) {
+ xfrm_audit_policy_delete(pol, 0, task_valid);
+ return err;
+ }
+ }
+ return err;
+}
+#else
+static inline int
+xfrm_policy_flush_secctx_check(struct net *net, u8 type, bool task_valid)
+{
+ return 0;
+}
+#endif
+
+int xfrm_policy_flush(struct net *net, u8 type, bool task_valid)
+{
+ int dir, err = 0, cnt = 0;
+ struct xfrm_policy *pol;
+
+ spin_lock_bh(&net->xfrm.xfrm_policy_lock);
+
+ err = xfrm_policy_flush_secctx_check(net, type, task_valid);
+ if (err)
+ goto out;
+
+again:
+ list_for_each_entry(pol, &net->xfrm.policy_all, walk.all) {
+ if (pol->walk.dead)
+ continue;
+
+ dir = xfrm_policy_id2dir(pol->index);
+ if (dir >= XFRM_POLICY_MAX ||
+ pol->type != type)
+ continue;
+
+ __xfrm_policy_unlink(pol, dir);
+ spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
+ cnt++;
+ xfrm_audit_policy_delete(pol, 1, task_valid);
+ xfrm_policy_kill(pol);
+ spin_lock_bh(&net->xfrm.xfrm_policy_lock);
+ goto again;
+ }
+ if (cnt)
+ __xfrm_policy_inexact_flush(net);
+ else
+ err = -ESRCH;
+out:
+ spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
+ return err;
+}
+EXPORT_SYMBOL(xfrm_policy_flush);
+
+int xfrm_policy_walk(struct net *net, struct xfrm_policy_walk *walk,
+ int (*func)(struct xfrm_policy *, int, int, void*),
+ void *data)
+{
+ struct xfrm_policy *pol;
+ struct xfrm_policy_walk_entry *x;
+ int error = 0;
+
+ if (walk->type >= XFRM_POLICY_TYPE_MAX &&
+ walk->type != XFRM_POLICY_TYPE_ANY)
+ return -EINVAL;
+
+ if (list_empty(&walk->walk.all) && walk->seq != 0)
+ return 0;
+
+ spin_lock_bh(&net->xfrm.xfrm_policy_lock);
+ if (list_empty(&walk->walk.all))
+ x = list_first_entry(&net->xfrm.policy_all, struct xfrm_policy_walk_entry, all);
+ else
+ x = list_first_entry(&walk->walk.all,
+ struct xfrm_policy_walk_entry, all);
+
+ list_for_each_entry_from(x, &net->xfrm.policy_all, all) {
+ if (x->dead)
+ continue;
+ pol = container_of(x, struct xfrm_policy, walk);
+ if (walk->type != XFRM_POLICY_TYPE_ANY &&
+ walk->type != pol->type)
+ continue;
+ error = func(pol, xfrm_policy_id2dir(pol->index),
+ walk->seq, data);
+ if (error) {
+ list_move_tail(&walk->walk.all, &x->all);
+ goto out;
+ }
+ walk->seq++;
+ }
+ if (walk->seq == 0) {
+ error = -ENOENT;
+ goto out;
+ }
+ list_del_init(&walk->walk.all);
+out:
+ spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
+ return error;
+}
+EXPORT_SYMBOL(xfrm_policy_walk);
+
+void xfrm_policy_walk_init(struct xfrm_policy_walk *walk, u8 type)
+{
+ INIT_LIST_HEAD(&walk->walk.all);
+ walk->walk.dead = 1;
+ walk->type = type;
+ walk->seq = 0;
+}
+EXPORT_SYMBOL(xfrm_policy_walk_init);
+
+void xfrm_policy_walk_done(struct xfrm_policy_walk *walk, struct net *net)
+{
+ if (list_empty(&walk->walk.all))
+ return;
+
+ spin_lock_bh(&net->xfrm.xfrm_policy_lock); /*FIXME where is net? */
+ list_del(&walk->walk.all);
+ spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
+}
+EXPORT_SYMBOL(xfrm_policy_walk_done);
+
+/*
+ * Find policy to apply to this flow.
+ *
+ * Returns 0 if policy found, else an -errno.
+ */
+static int xfrm_policy_match(const struct xfrm_policy *pol,
+ const struct flowi *fl,
+ u8 type, u16 family, u32 if_id)
+{
+ const struct xfrm_selector *sel = &pol->selector;
+ int ret = -ESRCH;
+ bool match;
+
+ if (pol->family != family ||
+ pol->if_id != if_id ||
+ (fl->flowi_mark & pol->mark.m) != pol->mark.v ||
+ pol->type != type)
+ return ret;
+
+ match = xfrm_selector_match(sel, fl, family);
+ if (match)
+ ret = security_xfrm_policy_lookup(pol->security, fl->flowi_secid);
+ return ret;
+}
+
+static struct xfrm_pol_inexact_node *
+xfrm_policy_lookup_inexact_addr(const struct rb_root *r,
+ seqcount_spinlock_t *count,
+ const xfrm_address_t *addr, u16 family)
+{
+ const struct rb_node *parent;
+ int seq;
+
+again:
+ seq = read_seqcount_begin(count);
+
+ parent = rcu_dereference_raw(r->rb_node);
+ while (parent) {
+ struct xfrm_pol_inexact_node *node;
+ int delta;
+
+ node = rb_entry(parent, struct xfrm_pol_inexact_node, node);
+
+ delta = xfrm_policy_addr_delta(addr, &node->addr,
+ node->prefixlen, family);
+ if (delta < 0) {
+ parent = rcu_dereference_raw(parent->rb_left);
+ continue;
+ } else if (delta > 0) {
+ parent = rcu_dereference_raw(parent->rb_right);
+ continue;
+ }
+
+ return node;
+ }
+
+ if (read_seqcount_retry(count, seq))
+ goto again;
+
+ return NULL;
+}
+
+static bool
+xfrm_policy_find_inexact_candidates(struct xfrm_pol_inexact_candidates *cand,
+ struct xfrm_pol_inexact_bin *b,
+ const xfrm_address_t *saddr,
+ const xfrm_address_t *daddr)
+{
+ struct xfrm_pol_inexact_node *n;
+ u16 family;
+
+ if (!b)
+ return false;
+
+ family = b->k.family;
+ memset(cand, 0, sizeof(*cand));
+ cand->res[XFRM_POL_CAND_ANY] = &b->hhead;
+
+ n = xfrm_policy_lookup_inexact_addr(&b->root_d, &b->count, daddr,
+ family);
+ if (n) {
+ cand->res[XFRM_POL_CAND_DADDR] = &n->hhead;
+ n = xfrm_policy_lookup_inexact_addr(&n->root, &b->count, saddr,
+ family);
+ if (n)
+ cand->res[XFRM_POL_CAND_BOTH] = &n->hhead;
+ }
+
+ n = xfrm_policy_lookup_inexact_addr(&b->root_s, &b->count, saddr,
+ family);
+ if (n)
+ cand->res[XFRM_POL_CAND_SADDR] = &n->hhead;
+
+ return true;
+}
+
+static struct xfrm_pol_inexact_bin *
+xfrm_policy_inexact_lookup_rcu(struct net *net, u8 type, u16 family,
+ u8 dir, u32 if_id)
+{
+ struct xfrm_pol_inexact_key k = {
+ .family = family,
+ .type = type,
+ .dir = dir,
+ .if_id = if_id,
+ };
+
+ write_pnet(&k.net, net);
+
+ return rhashtable_lookup(&xfrm_policy_inexact_table, &k,
+ xfrm_pol_inexact_params);
+}
+
+static struct xfrm_pol_inexact_bin *
+xfrm_policy_inexact_lookup(struct net *net, u8 type, u16 family,
+ u8 dir, u32 if_id)
+{
+ struct xfrm_pol_inexact_bin *bin;
+
+ lockdep_assert_held(&net->xfrm.xfrm_policy_lock);
+
+ rcu_read_lock();
+ bin = xfrm_policy_inexact_lookup_rcu(net, type, family, dir, if_id);
+ rcu_read_unlock();
+
+ return bin;
+}
+
+static struct xfrm_policy *
+__xfrm_policy_eval_candidates(struct hlist_head *chain,
+ struct xfrm_policy *prefer,
+ const struct flowi *fl,
+ u8 type, u16 family, u32 if_id)
+{
+ u32 priority = prefer ? prefer->priority : ~0u;
+ struct xfrm_policy *pol;
+
+ if (!chain)
+ return NULL;
+
+ hlist_for_each_entry_rcu(pol, chain, bydst) {
+ int err;
+
+ if (pol->priority > priority)
+ break;
+
+ err = xfrm_policy_match(pol, fl, type, family, if_id);
+ if (err) {
+ if (err != -ESRCH)
+ return ERR_PTR(err);
+
+ continue;
+ }
+
+ if (prefer) {
+ /* matches. Is it older than *prefer? */
+ if (pol->priority == priority &&
+ prefer->pos < pol->pos)
+ return prefer;
+ }
+
+ return pol;
+ }
+
+ return NULL;
+}
+
+static struct xfrm_policy *
+xfrm_policy_eval_candidates(struct xfrm_pol_inexact_candidates *cand,
+ struct xfrm_policy *prefer,
+ const struct flowi *fl,
+ u8 type, u16 family, u32 if_id)
+{
+ struct xfrm_policy *tmp;
+ int i;
+
+ for (i = 0; i < ARRAY_SIZE(cand->res); i++) {
+ tmp = __xfrm_policy_eval_candidates(cand->res[i],
+ prefer,
+ fl, type, family, if_id);
+ if (!tmp)
+ continue;
+
+ if (IS_ERR(tmp))
+ return tmp;
+ prefer = tmp;
+ }
+
+ return prefer;
+}
+
+static struct xfrm_policy *xfrm_policy_lookup_bytype(struct net *net, u8 type,
+ const struct flowi *fl,
+ u16 family, u8 dir,
+ u32 if_id)
+{
+ struct xfrm_pol_inexact_candidates cand;
+ const xfrm_address_t *daddr, *saddr;
+ struct xfrm_pol_inexact_bin *bin;
+ struct xfrm_policy *pol, *ret;
+ struct hlist_head *chain;
+ unsigned int sequence;
+ int err;
+
+ daddr = xfrm_flowi_daddr(fl, family);
+ saddr = xfrm_flowi_saddr(fl, family);
+ if (unlikely(!daddr || !saddr))
+ return NULL;
+
+ rcu_read_lock();
+ retry:
+ do {
+ sequence = read_seqcount_begin(&net->xfrm.xfrm_policy_hash_generation);
+ chain = policy_hash_direct(net, daddr, saddr, family, dir);
+ } while (read_seqcount_retry(&net->xfrm.xfrm_policy_hash_generation, sequence));
+
+ ret = NULL;
+ hlist_for_each_entry_rcu(pol, chain, bydst) {
+ err = xfrm_policy_match(pol, fl, type, family, if_id);
+ if (err) {
+ if (err == -ESRCH)
+ continue;
+ else {
+ ret = ERR_PTR(err);
+ goto fail;
+ }
+ } else {
+ ret = pol;
+ break;
+ }
+ }
+ bin = xfrm_policy_inexact_lookup_rcu(net, type, family, dir, if_id);
+ if (!bin || !xfrm_policy_find_inexact_candidates(&cand, bin, saddr,
+ daddr))
+ goto skip_inexact;
+
+ pol = xfrm_policy_eval_candidates(&cand, ret, fl, type,
+ family, if_id);
+ if (pol) {
+ ret = pol;
+ if (IS_ERR(pol))
+ goto fail;
+ }
+
+skip_inexact:
+ if (read_seqcount_retry(&net->xfrm.xfrm_policy_hash_generation, sequence))
+ goto retry;
+
+ if (ret && !xfrm_pol_hold_rcu(ret))
+ goto retry;
+fail:
+ rcu_read_unlock();
+
+ return ret;
+}
+
+static struct xfrm_policy *xfrm_policy_lookup(struct net *net,
+ const struct flowi *fl,
+ u16 family, u8 dir, u32 if_id)
+{
+#ifdef CONFIG_XFRM_SUB_POLICY
+ struct xfrm_policy *pol;
+
+ pol = xfrm_policy_lookup_bytype(net, XFRM_POLICY_TYPE_SUB, fl, family,
+ dir, if_id);
+ if (pol != NULL)
+ return pol;
+#endif
+ return xfrm_policy_lookup_bytype(net, XFRM_POLICY_TYPE_MAIN, fl, family,
+ dir, if_id);
+}
+
+static struct xfrm_policy *xfrm_sk_policy_lookup(const struct sock *sk, int dir,
+ const struct flowi *fl,
+ u16 family, u32 if_id)
+{
+ struct xfrm_policy *pol;
+
+ rcu_read_lock();
+ again:
+ pol = rcu_dereference(sk->sk_policy[dir]);
+ if (pol != NULL) {
+ bool match;
+ int err = 0;
+
+ if (pol->family != family) {
+ pol = NULL;
+ goto out;
+ }
+
+ match = xfrm_selector_match(&pol->selector, fl, family);
+ if (match) {
+ if ((READ_ONCE(sk->sk_mark) & pol->mark.m) != pol->mark.v ||
+ pol->if_id != if_id) {
+ pol = NULL;
+ goto out;
+ }
+ err = security_xfrm_policy_lookup(pol->security,
+ fl->flowi_secid);
+ if (!err) {
+ if (!xfrm_pol_hold_rcu(pol))
+ goto again;
+ } else if (err == -ESRCH) {
+ pol = NULL;
+ } else {
+ pol = ERR_PTR(err);
+ }
+ } else
+ pol = NULL;
+ }
+out:
+ rcu_read_unlock();
+ return pol;
+}
+
+static void __xfrm_policy_link(struct xfrm_policy *pol, int dir)
+{
+ struct net *net = xp_net(pol);
+
+ list_add(&pol->walk.all, &net->xfrm.policy_all);
+ net->xfrm.policy_count[dir]++;
+ xfrm_pol_hold(pol);
+}
+
+static struct xfrm_policy *__xfrm_policy_unlink(struct xfrm_policy *pol,
+ int dir)
+{
+ struct net *net = xp_net(pol);
+
+ if (list_empty(&pol->walk.all))
+ return NULL;
+
+ /* Socket policies are not hashed. */
+ if (!hlist_unhashed(&pol->bydst)) {
+ hlist_del_rcu(&pol->bydst);
+ hlist_del_init(&pol->bydst_inexact_list);
+ hlist_del(&pol->byidx);
+ }
+
+ list_del_init(&pol->walk.all);
+ net->xfrm.policy_count[dir]--;
+
+ return pol;
+}
+
+static void xfrm_sk_policy_link(struct xfrm_policy *pol, int dir)
+{
+ __xfrm_policy_link(pol, XFRM_POLICY_MAX + dir);
+}
+
+static void xfrm_sk_policy_unlink(struct xfrm_policy *pol, int dir)
+{
+ __xfrm_policy_unlink(pol, XFRM_POLICY_MAX + dir);
+}
+
+int xfrm_policy_delete(struct xfrm_policy *pol, int dir)
+{
+ struct net *net = xp_net(pol);
+
+ spin_lock_bh(&net->xfrm.xfrm_policy_lock);
+ pol = __xfrm_policy_unlink(pol, dir);
+ spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
+ if (pol) {
+ xfrm_policy_kill(pol);
+ return 0;
+ }
+ return -ENOENT;
+}
+EXPORT_SYMBOL(xfrm_policy_delete);
+
+int xfrm_sk_policy_insert(struct sock *sk, int dir, struct xfrm_policy *pol)
+{
+ struct net *net = sock_net(sk);
+ struct xfrm_policy *old_pol;
+
+#ifdef CONFIG_XFRM_SUB_POLICY
+ if (pol && pol->type != XFRM_POLICY_TYPE_MAIN)
+ return -EINVAL;
+#endif
+
+ spin_lock_bh(&net->xfrm.xfrm_policy_lock);
+ old_pol = rcu_dereference_protected(sk->sk_policy[dir],
+ lockdep_is_held(&net->xfrm.xfrm_policy_lock));
+ if (pol) {
+ pol->curlft.add_time = ktime_get_real_seconds();
+ pol->index = xfrm_gen_index(net, XFRM_POLICY_MAX+dir, 0);
+ xfrm_sk_policy_link(pol, dir);
+ }
+ rcu_assign_pointer(sk->sk_policy[dir], pol);
+ if (old_pol) {
+ if (pol)
+ xfrm_policy_requeue(old_pol, pol);
+
+ /* Unlinking succeeds always. This is the only function
+ * allowed to delete or replace socket policy.
+ */
+ xfrm_sk_policy_unlink(old_pol, dir);
+ }
+ spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
+
+ if (old_pol) {
+ xfrm_policy_kill(old_pol);
+ }
+ return 0;
+}
+
+static struct xfrm_policy *clone_policy(const struct xfrm_policy *old, int dir)
+{
+ struct xfrm_policy *newp = xfrm_policy_alloc(xp_net(old), GFP_ATOMIC);
+ struct net *net = xp_net(old);
+
+ if (newp) {
+ newp->selector = old->selector;
+ if (security_xfrm_policy_clone(old->security,
+ &newp->security)) {
+ kfree(newp);
+ return NULL; /* ENOMEM */
+ }
+ newp->lft = old->lft;
+ newp->curlft = old->curlft;
+ newp->mark = old->mark;
+ newp->if_id = old->if_id;
+ newp->action = old->action;
+ newp->flags = old->flags;
+ newp->xfrm_nr = old->xfrm_nr;
+ newp->index = old->index;
+ newp->type = old->type;
+ newp->family = old->family;
+ memcpy(newp->xfrm_vec, old->xfrm_vec,
+ newp->xfrm_nr*sizeof(struct xfrm_tmpl));
+ spin_lock_bh(&net->xfrm.xfrm_policy_lock);
+ xfrm_sk_policy_link(newp, dir);
+ spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
+ xfrm_pol_put(newp);
+ }
+ return newp;
+}
+
+int __xfrm_sk_clone_policy(struct sock *sk, const struct sock *osk)
+{
+ const struct xfrm_policy *p;
+ struct xfrm_policy *np;
+ int i, ret = 0;
+
+ rcu_read_lock();
+ for (i = 0; i < 2; i++) {
+ p = rcu_dereference(osk->sk_policy[i]);
+ if (p) {
+ np = clone_policy(p, i);
+ if (unlikely(!np)) {
+ ret = -ENOMEM;
+ break;
+ }
+ rcu_assign_pointer(sk->sk_policy[i], np);
+ }
+ }
+ rcu_read_unlock();
+ return ret;
+}
+
+static int
+xfrm_get_saddr(struct net *net, int oif, xfrm_address_t *local,
+ xfrm_address_t *remote, unsigned short family, u32 mark)
+{
+ int err;
+ const struct xfrm_policy_afinfo *afinfo = xfrm_policy_get_afinfo(family);
+
+ if (unlikely(afinfo == NULL))
+ return -EINVAL;
+ err = afinfo->get_saddr(net, oif, local, remote, mark);
+ rcu_read_unlock();
+ return err;
+}
+
+/* Resolve list of templates for the flow, given policy. */
+
+static int
+xfrm_tmpl_resolve_one(struct xfrm_policy *policy, const struct flowi *fl,
+ struct xfrm_state **xfrm, unsigned short family)
+{
+ struct net *net = xp_net(policy);
+ int nx;
+ int i, error;
+ xfrm_address_t *daddr = xfrm_flowi_daddr(fl, family);
+ xfrm_address_t *saddr = xfrm_flowi_saddr(fl, family);
+ xfrm_address_t tmp;
+
+ for (nx = 0, i = 0; i < policy->xfrm_nr; i++) {
+ struct xfrm_state *x;
+ xfrm_address_t *remote = daddr;
+ xfrm_address_t *local = saddr;
+ struct xfrm_tmpl *tmpl = &policy->xfrm_vec[i];
+
+ if (tmpl->mode == XFRM_MODE_TUNNEL ||
+ tmpl->mode == XFRM_MODE_BEET) {
+ remote = &tmpl->id.daddr;
+ local = &tmpl->saddr;
+ if (xfrm_addr_any(local, tmpl->encap_family)) {
+ error = xfrm_get_saddr(net, fl->flowi_oif,
+ &tmp, remote,
+ tmpl->encap_family, 0);
+ if (error)
+ goto fail;
+ local = &tmp;
+ }
+ }
+
+ x = xfrm_state_find(remote, local, fl, tmpl, policy, &error,
+ family, policy->if_id);
+
+ if (x && x->km.state == XFRM_STATE_VALID) {
+ xfrm[nx++] = x;
+ daddr = remote;
+ saddr = local;
+ continue;
+ }
+ if (x) {
+ error = (x->km.state == XFRM_STATE_ERROR ?
+ -EINVAL : -EAGAIN);
+ xfrm_state_put(x);
+ } else if (error == -ESRCH) {
+ error = -EAGAIN;
+ }
+
+ if (!tmpl->optional)
+ goto fail;
+ }
+ return nx;
+
+fail:
+ for (nx--; nx >= 0; nx--)
+ xfrm_state_put(xfrm[nx]);
+ return error;
+}
+
+static int
+xfrm_tmpl_resolve(struct xfrm_policy **pols, int npols, const struct flowi *fl,
+ struct xfrm_state **xfrm, unsigned short family)
+{
+ struct xfrm_state *tp[XFRM_MAX_DEPTH];
+ struct xfrm_state **tpp = (npols > 1) ? tp : xfrm;
+ int cnx = 0;
+ int error;
+ int ret;
+ int i;
+
+ for (i = 0; i < npols; i++) {
+ if (cnx + pols[i]->xfrm_nr >= XFRM_MAX_DEPTH) {
+ error = -ENOBUFS;
+ goto fail;
+ }
+
+ ret = xfrm_tmpl_resolve_one(pols[i], fl, &tpp[cnx], family);
+ if (ret < 0) {
+ error = ret;
+ goto fail;
+ } else
+ cnx += ret;
+ }
+
+ /* found states are sorted for outbound processing */
+ if (npols > 1)
+ xfrm_state_sort(xfrm, tpp, cnx, family);
+
+ return cnx;
+
+ fail:
+ for (cnx--; cnx >= 0; cnx--)
+ xfrm_state_put(tpp[cnx]);
+ return error;
+
+}
+
+static int xfrm_get_tos(const struct flowi *fl, int family)
+{
+ if (family == AF_INET)
+ return IPTOS_RT_MASK & fl->u.ip4.flowi4_tos;
+
+ return 0;
+}
+
+static inline struct xfrm_dst *xfrm_alloc_dst(struct net *net, int family)
+{
+ const struct xfrm_policy_afinfo *afinfo = xfrm_policy_get_afinfo(family);
+ struct dst_ops *dst_ops;
+ struct xfrm_dst *xdst;
+
+ if (!afinfo)
+ return ERR_PTR(-EINVAL);
+
+ switch (family) {
+ case AF_INET:
+ dst_ops = &net->xfrm.xfrm4_dst_ops;
+ break;
+#if IS_ENABLED(CONFIG_IPV6)
+ case AF_INET6:
+ dst_ops = &net->xfrm.xfrm6_dst_ops;
+ break;
+#endif
+ default:
+ BUG();
+ }
+ xdst = dst_alloc(dst_ops, NULL, 1, DST_OBSOLETE_NONE, 0);
+
+ if (likely(xdst)) {
+ memset_after(xdst, 0, u.dst);
+ } else
+ xdst = ERR_PTR(-ENOBUFS);
+
+ rcu_read_unlock();
+
+ return xdst;
+}
+
+static void xfrm_init_path(struct xfrm_dst *path, struct dst_entry *dst,
+ int nfheader_len)
+{
+ if (dst->ops->family == AF_INET6) {
+ struct rt6_info *rt = (struct rt6_info *)dst;
+ path->path_cookie = rt6_get_cookie(rt);
+ path->u.rt6.rt6i_nfheader_len = nfheader_len;
+ }
+}
+
+static inline int xfrm_fill_dst(struct xfrm_dst *xdst, struct net_device *dev,
+ const struct flowi *fl)
+{
+ const struct xfrm_policy_afinfo *afinfo =
+ xfrm_policy_get_afinfo(xdst->u.dst.ops->family);
+ int err;
+
+ if (!afinfo)
+ return -EINVAL;
+
+ err = afinfo->fill_dst(xdst, dev, fl);
+
+ rcu_read_unlock();
+
+ return err;
+}
+
+
+/* Allocate chain of dst_entry's, attach known xfrm's, calculate
+ * all the metrics... Shortly, bundle a bundle.
+ */
+
+static struct dst_entry *xfrm_bundle_create(struct xfrm_policy *policy,
+ struct xfrm_state **xfrm,
+ struct xfrm_dst **bundle,
+ int nx,
+ const struct flowi *fl,
+ struct dst_entry *dst)
+{
+ const struct xfrm_state_afinfo *afinfo;
+ const struct xfrm_mode *inner_mode;
+ struct net *net = xp_net(policy);
+ unsigned long now = jiffies;
+ struct net_device *dev;
+ struct xfrm_dst *xdst_prev = NULL;
+ struct xfrm_dst *xdst0 = NULL;
+ int i = 0;
+ int err;
+ int header_len = 0;
+ int nfheader_len = 0;
+ int trailer_len = 0;
+ int tos;
+ int family = policy->selector.family;
+ xfrm_address_t saddr, daddr;
+
+ xfrm_flowi_addr_get(fl, &saddr, &daddr, family);
+
+ tos = xfrm_get_tos(fl, family);
+
+ dst_hold(dst);
+
+ for (; i < nx; i++) {
+ struct xfrm_dst *xdst = xfrm_alloc_dst(net, family);
+ struct dst_entry *dst1 = &xdst->u.dst;
+
+ err = PTR_ERR(xdst);
+ if (IS_ERR(xdst)) {
+ dst_release(dst);
+ goto put_states;
+ }
+
+ bundle[i] = xdst;
+ if (!xdst_prev)
+ xdst0 = xdst;
+ else
+ /* Ref count is taken during xfrm_alloc_dst()
+ * No need to do dst_clone() on dst1
+ */
+ xfrm_dst_set_child(xdst_prev, &xdst->u.dst);
+
+ if (xfrm[i]->sel.family == AF_UNSPEC) {
+ inner_mode = xfrm_ip2inner_mode(xfrm[i],
+ xfrm_af2proto(family));
+ if (!inner_mode) {
+ err = -EAFNOSUPPORT;
+ dst_release(dst);
+ goto put_states;
+ }
+ } else
+ inner_mode = &xfrm[i]->inner_mode;
+
+ xdst->route = dst;
+ dst_copy_metrics(dst1, dst);
+
+ if (xfrm[i]->props.mode != XFRM_MODE_TRANSPORT) {
+ __u32 mark = 0;
+ int oif;
+
+ if (xfrm[i]->props.smark.v || xfrm[i]->props.smark.m)
+ mark = xfrm_smark_get(fl->flowi_mark, xfrm[i]);
+
+ family = xfrm[i]->props.family;
+ oif = fl->flowi_oif ? : fl->flowi_l3mdev;
+ dst = xfrm_dst_lookup(xfrm[i], tos, oif,
+ &saddr, &daddr, family, mark);
+ err = PTR_ERR(dst);
+ if (IS_ERR(dst))
+ goto put_states;
+ } else
+ dst_hold(dst);
+
+ dst1->xfrm = xfrm[i];
+ xdst->xfrm_genid = xfrm[i]->genid;
+
+ dst1->obsolete = DST_OBSOLETE_FORCE_CHK;
+ dst1->lastuse = now;
+
+ dst1->input = dst_discard;
+
+ rcu_read_lock();
+ afinfo = xfrm_state_afinfo_get_rcu(inner_mode->family);
+ if (likely(afinfo))
+ dst1->output = afinfo->output;
+ else
+ dst1->output = dst_discard_out;
+ rcu_read_unlock();
+
+ xdst_prev = xdst;
+
+ header_len += xfrm[i]->props.header_len;
+ if (xfrm[i]->type->flags & XFRM_TYPE_NON_FRAGMENT)
+ nfheader_len += xfrm[i]->props.header_len;
+ trailer_len += xfrm[i]->props.trailer_len;
+ }
+
+ xfrm_dst_set_child(xdst_prev, dst);
+ xdst0->path = dst;
+
+ err = -ENODEV;
+ dev = dst->dev;
+ if (!dev)
+ goto free_dst;
+
+ xfrm_init_path(xdst0, dst, nfheader_len);
+ xfrm_init_pmtu(bundle, nx);
+
+ for (xdst_prev = xdst0; xdst_prev != (struct xfrm_dst *)dst;
+ xdst_prev = (struct xfrm_dst *) xfrm_dst_child(&xdst_prev->u.dst)) {
+ err = xfrm_fill_dst(xdst_prev, dev, fl);
+ if (err)
+ goto free_dst;
+
+ xdst_prev->u.dst.header_len = header_len;
+ xdst_prev->u.dst.trailer_len = trailer_len;
+ header_len -= xdst_prev->u.dst.xfrm->props.header_len;
+ trailer_len -= xdst_prev->u.dst.xfrm->props.trailer_len;
+ }
+
+ return &xdst0->u.dst;
+
+put_states:
+ for (; i < nx; i++)
+ xfrm_state_put(xfrm[i]);
+free_dst:
+ if (xdst0)
+ dst_release_immediate(&xdst0->u.dst);
+
+ return ERR_PTR(err);
+}
+
+static int xfrm_expand_policies(const struct flowi *fl, u16 family,
+ struct xfrm_policy **pols,
+ int *num_pols, int *num_xfrms)
+{
+ int i;
+
+ if (*num_pols == 0 || !pols[0]) {
+ *num_pols = 0;
+ *num_xfrms = 0;
+ return 0;
+ }
+ if (IS_ERR(pols[0])) {
+ *num_pols = 0;
+ return PTR_ERR(pols[0]);
+ }
+
+ *num_xfrms = pols[0]->xfrm_nr;
+
+#ifdef CONFIG_XFRM_SUB_POLICY
+ if (pols[0]->action == XFRM_POLICY_ALLOW &&
+ pols[0]->type != XFRM_POLICY_TYPE_MAIN) {
+ pols[1] = xfrm_policy_lookup_bytype(xp_net(pols[0]),
+ XFRM_POLICY_TYPE_MAIN,
+ fl, family,
+ XFRM_POLICY_OUT,
+ pols[0]->if_id);
+ if (pols[1]) {
+ if (IS_ERR(pols[1])) {
+ xfrm_pols_put(pols, *num_pols);
+ *num_pols = 0;
+ return PTR_ERR(pols[1]);
+ }
+ (*num_pols)++;
+ (*num_xfrms) += pols[1]->xfrm_nr;
+ }
+ }
+#endif
+ for (i = 0; i < *num_pols; i++) {
+ if (pols[i]->action != XFRM_POLICY_ALLOW) {
+ *num_xfrms = -1;
+ break;
+ }
+ }
+
+ return 0;
+
+}
+
+static struct xfrm_dst *
+xfrm_resolve_and_create_bundle(struct xfrm_policy **pols, int num_pols,
+ const struct flowi *fl, u16 family,
+ struct dst_entry *dst_orig)
+{
+ struct net *net = xp_net(pols[0]);
+ struct xfrm_state *xfrm[XFRM_MAX_DEPTH];
+ struct xfrm_dst *bundle[XFRM_MAX_DEPTH];
+ struct xfrm_dst *xdst;
+ struct dst_entry *dst;
+ int err;
+
+ /* Try to instantiate a bundle */
+ err = xfrm_tmpl_resolve(pols, num_pols, fl, xfrm, family);
+ if (err <= 0) {
+ if (err == 0)
+ return NULL;
+
+ if (err != -EAGAIN)
+ XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTPOLERROR);
+ return ERR_PTR(err);
+ }
+
+ dst = xfrm_bundle_create(pols[0], xfrm, bundle, err, fl, dst_orig);
+ if (IS_ERR(dst)) {
+ XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTBUNDLEGENERROR);
+ return ERR_CAST(dst);
+ }
+
+ xdst = (struct xfrm_dst *)dst;
+ xdst->num_xfrms = err;
+ xdst->num_pols = num_pols;
+ memcpy(xdst->pols, pols, sizeof(struct xfrm_policy *) * num_pols);
+ xdst->policy_genid = atomic_read(&pols[0]->genid);
+
+ return xdst;
+}
+
+static void xfrm_policy_queue_process(struct timer_list *t)
+{
+ struct sk_buff *skb;
+ struct sock *sk;
+ struct dst_entry *dst;
+ struct xfrm_policy *pol = from_timer(pol, t, polq.hold_timer);
+ struct net *net = xp_net(pol);
+ struct xfrm_policy_queue *pq = &pol->polq;
+ struct flowi fl;
+ struct sk_buff_head list;
+ __u32 skb_mark;
+
+ spin_lock(&pq->hold_queue.lock);
+ skb = skb_peek(&pq->hold_queue);
+ if (!skb) {
+ spin_unlock(&pq->hold_queue.lock);
+ goto out;
+ }
+ dst = skb_dst(skb);
+ sk = skb->sk;
+
+ /* Fixup the mark to support VTI. */
+ skb_mark = skb->mark;
+ skb->mark = pol->mark.v;
+ xfrm_decode_session(skb, &fl, dst->ops->family);
+ skb->mark = skb_mark;
+ spin_unlock(&pq->hold_queue.lock);
+
+ dst_hold(xfrm_dst_path(dst));
+ dst = xfrm_lookup(net, xfrm_dst_path(dst), &fl, sk, XFRM_LOOKUP_QUEUE);
+ if (IS_ERR(dst))
+ goto purge_queue;
+
+ if (dst->flags & DST_XFRM_QUEUE) {
+ dst_release(dst);
+
+ if (pq->timeout >= XFRM_QUEUE_TMO_MAX)
+ goto purge_queue;
+
+ pq->timeout = pq->timeout << 1;
+ if (!mod_timer(&pq->hold_timer, jiffies + pq->timeout))
+ xfrm_pol_hold(pol);
+ goto out;
+ }
+
+ dst_release(dst);
+
+ __skb_queue_head_init(&list);
+
+ spin_lock(&pq->hold_queue.lock);
+ pq->timeout = 0;
+ skb_queue_splice_init(&pq->hold_queue, &list);
+ spin_unlock(&pq->hold_queue.lock);
+
+ while (!skb_queue_empty(&list)) {
+ skb = __skb_dequeue(&list);
+
+ /* Fixup the mark to support VTI. */
+ skb_mark = skb->mark;
+ skb->mark = pol->mark.v;
+ xfrm_decode_session(skb, &fl, skb_dst(skb)->ops->family);
+ skb->mark = skb_mark;
+
+ dst_hold(xfrm_dst_path(skb_dst(skb)));
+ dst = xfrm_lookup(net, xfrm_dst_path(skb_dst(skb)), &fl, skb->sk, 0);
+ if (IS_ERR(dst)) {
+ kfree_skb(skb);
+ continue;
+ }
+
+ nf_reset_ct(skb);
+ skb_dst_drop(skb);
+ skb_dst_set(skb, dst);
+
+ dst_output(net, skb->sk, skb);
+ }
+
+out:
+ xfrm_pol_put(pol);
+ return;
+
+purge_queue:
+ pq->timeout = 0;
+ skb_queue_purge(&pq->hold_queue);
+ xfrm_pol_put(pol);
+}
+
+static int xdst_queue_output(struct net *net, struct sock *sk, struct sk_buff *skb)
+{
+ unsigned long sched_next;
+ struct dst_entry *dst = skb_dst(skb);
+ struct xfrm_dst *xdst = (struct xfrm_dst *) dst;
+ struct xfrm_policy *pol = xdst->pols[0];
+ struct xfrm_policy_queue *pq = &pol->polq;
+
+ if (unlikely(skb_fclone_busy(sk, skb))) {
+ kfree_skb(skb);
+ return 0;
+ }
+
+ if (pq->hold_queue.qlen > XFRM_MAX_QUEUE_LEN) {
+ kfree_skb(skb);
+ return -EAGAIN;
+ }
+
+ skb_dst_force(skb);
+
+ spin_lock_bh(&pq->hold_queue.lock);
+
+ if (!pq->timeout)
+ pq->timeout = XFRM_QUEUE_TMO_MIN;
+
+ sched_next = jiffies + pq->timeout;
+
+ if (del_timer(&pq->hold_timer)) {
+ if (time_before(pq->hold_timer.expires, sched_next))
+ sched_next = pq->hold_timer.expires;
+ xfrm_pol_put(pol);
+ }
+
+ __skb_queue_tail(&pq->hold_queue, skb);
+ if (!mod_timer(&pq->hold_timer, sched_next))
+ xfrm_pol_hold(pol);
+
+ spin_unlock_bh(&pq->hold_queue.lock);
+
+ return 0;
+}
+
+static struct xfrm_dst *xfrm_create_dummy_bundle(struct net *net,
+ struct xfrm_flo *xflo,
+ const struct flowi *fl,
+ int num_xfrms,
+ u16 family)
+{
+ int err;
+ struct net_device *dev;
+ struct dst_entry *dst;
+ struct dst_entry *dst1;
+ struct xfrm_dst *xdst;
+
+ xdst = xfrm_alloc_dst(net, family);
+ if (IS_ERR(xdst))
+ return xdst;
+
+ if (!(xflo->flags & XFRM_LOOKUP_QUEUE) ||
+ net->xfrm.sysctl_larval_drop ||
+ num_xfrms <= 0)
+ return xdst;
+
+ dst = xflo->dst_orig;
+ dst1 = &xdst->u.dst;
+ dst_hold(dst);
+ xdst->route = dst;
+
+ dst_copy_metrics(dst1, dst);
+
+ dst1->obsolete = DST_OBSOLETE_FORCE_CHK;
+ dst1->flags |= DST_XFRM_QUEUE;
+ dst1->lastuse = jiffies;
+
+ dst1->input = dst_discard;
+ dst1->output = xdst_queue_output;
+
+ dst_hold(dst);
+ xfrm_dst_set_child(xdst, dst);
+ xdst->path = dst;
+
+ xfrm_init_path((struct xfrm_dst *)dst1, dst, 0);
+
+ err = -ENODEV;
+ dev = dst->dev;
+ if (!dev)
+ goto free_dst;
+
+ err = xfrm_fill_dst(xdst, dev, fl);
+ if (err)
+ goto free_dst;
+
+out:
+ return xdst;
+
+free_dst:
+ dst_release(dst1);
+ xdst = ERR_PTR(err);
+ goto out;
+}
+
+static struct xfrm_dst *xfrm_bundle_lookup(struct net *net,
+ const struct flowi *fl,
+ u16 family, u8 dir,
+ struct xfrm_flo *xflo, u32 if_id)
+{
+ struct xfrm_policy *pols[XFRM_POLICY_TYPE_MAX];
+ int num_pols = 0, num_xfrms = 0, err;
+ struct xfrm_dst *xdst;
+
+ /* Resolve policies to use if we couldn't get them from
+ * previous cache entry */
+ num_pols = 1;
+ pols[0] = xfrm_policy_lookup(net, fl, family, dir, if_id);
+ err = xfrm_expand_policies(fl, family, pols,
+ &num_pols, &num_xfrms);
+ if (err < 0)
+ goto inc_error;
+ if (num_pols == 0)
+ return NULL;
+ if (num_xfrms <= 0)
+ goto make_dummy_bundle;
+
+ xdst = xfrm_resolve_and_create_bundle(pols, num_pols, fl, family,
+ xflo->dst_orig);
+ if (IS_ERR(xdst)) {
+ err = PTR_ERR(xdst);
+ if (err == -EREMOTE) {
+ xfrm_pols_put(pols, num_pols);
+ return NULL;
+ }
+
+ if (err != -EAGAIN)
+ goto error;
+ goto make_dummy_bundle;
+ } else if (xdst == NULL) {
+ num_xfrms = 0;
+ goto make_dummy_bundle;
+ }
+
+ return xdst;
+
+make_dummy_bundle:
+ /* We found policies, but there's no bundles to instantiate:
+ * either because the policy blocks, has no transformations or
+ * we could not build template (no xfrm_states).*/
+ xdst = xfrm_create_dummy_bundle(net, xflo, fl, num_xfrms, family);
+ if (IS_ERR(xdst)) {
+ xfrm_pols_put(pols, num_pols);
+ return ERR_CAST(xdst);
+ }
+ xdst->num_pols = num_pols;
+ xdst->num_xfrms = num_xfrms;
+ memcpy(xdst->pols, pols, sizeof(struct xfrm_policy *) * num_pols);
+
+ return xdst;
+
+inc_error:
+ XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTPOLERROR);
+error:
+ xfrm_pols_put(pols, num_pols);
+ return ERR_PTR(err);
+}
+
+static struct dst_entry *make_blackhole(struct net *net, u16 family,
+ struct dst_entry *dst_orig)
+{
+ const struct xfrm_policy_afinfo *afinfo = xfrm_policy_get_afinfo(family);
+ struct dst_entry *ret;
+
+ if (!afinfo) {
+ dst_release(dst_orig);
+ return ERR_PTR(-EINVAL);
+ } else {
+ ret = afinfo->blackhole_route(net, dst_orig);
+ }
+ rcu_read_unlock();
+
+ return ret;
+}
+
+/* Finds/creates a bundle for given flow and if_id
+ *
+ * At the moment we eat a raw IP route. Mostly to speed up lookups
+ * on interfaces with disabled IPsec.
+ *
+ * xfrm_lookup uses an if_id of 0 by default, and is provided for
+ * compatibility
+ */
+struct dst_entry *xfrm_lookup_with_ifid(struct net *net,
+ struct dst_entry *dst_orig,
+ const struct flowi *fl,
+ const struct sock *sk,
+ int flags, u32 if_id)
+{
+ struct xfrm_policy *pols[XFRM_POLICY_TYPE_MAX];
+ struct xfrm_dst *xdst;
+ struct dst_entry *dst, *route;
+ u16 family = dst_orig->ops->family;
+ u8 dir = XFRM_POLICY_OUT;
+ int i, err, num_pols, num_xfrms = 0, drop_pols = 0;
+
+ dst = NULL;
+ xdst = NULL;
+ route = NULL;
+
+ sk = sk_const_to_full_sk(sk);
+ if (sk && sk->sk_policy[XFRM_POLICY_OUT]) {
+ num_pols = 1;
+ pols[0] = xfrm_sk_policy_lookup(sk, XFRM_POLICY_OUT, fl, family,
+ if_id);
+ err = xfrm_expand_policies(fl, family, pols,
+ &num_pols, &num_xfrms);
+ if (err < 0)
+ goto dropdst;
+
+ if (num_pols) {
+ if (num_xfrms <= 0) {
+ drop_pols = num_pols;
+ goto no_transform;
+ }
+
+ xdst = xfrm_resolve_and_create_bundle(
+ pols, num_pols, fl,
+ family, dst_orig);
+
+ if (IS_ERR(xdst)) {
+ xfrm_pols_put(pols, num_pols);
+ err = PTR_ERR(xdst);
+ if (err == -EREMOTE)
+ goto nopol;
+
+ goto dropdst;
+ } else if (xdst == NULL) {
+ num_xfrms = 0;
+ drop_pols = num_pols;
+ goto no_transform;
+ }
+
+ route = xdst->route;
+ }
+ }
+
+ if (xdst == NULL) {
+ struct xfrm_flo xflo;
+
+ xflo.dst_orig = dst_orig;
+ xflo.flags = flags;
+
+ /* To accelerate a bit... */
+ if (!if_id && ((dst_orig->flags & DST_NOXFRM) ||
+ !net->xfrm.policy_count[XFRM_POLICY_OUT]))
+ goto nopol;
+
+ xdst = xfrm_bundle_lookup(net, fl, family, dir, &xflo, if_id);
+ if (xdst == NULL)
+ goto nopol;
+ if (IS_ERR(xdst)) {
+ err = PTR_ERR(xdst);
+ goto dropdst;
+ }
+
+ num_pols = xdst->num_pols;
+ num_xfrms = xdst->num_xfrms;
+ memcpy(pols, xdst->pols, sizeof(struct xfrm_policy *) * num_pols);
+ route = xdst->route;
+ }
+
+ dst = &xdst->u.dst;
+ if (route == NULL && num_xfrms > 0) {
+ /* The only case when xfrm_bundle_lookup() returns a
+ * bundle with null route, is when the template could
+ * not be resolved. It means policies are there, but
+ * bundle could not be created, since we don't yet
+ * have the xfrm_state's. We need to wait for KM to
+ * negotiate new SA's or bail out with error.*/
+ if (net->xfrm.sysctl_larval_drop) {
+ XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTNOSTATES);
+ err = -EREMOTE;
+ goto error;
+ }
+
+ err = -EAGAIN;
+
+ XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTNOSTATES);
+ goto error;
+ }
+
+no_transform:
+ if (num_pols == 0)
+ goto nopol;
+
+ if ((flags & XFRM_LOOKUP_ICMP) &&
+ !(pols[0]->flags & XFRM_POLICY_ICMP)) {
+ err = -ENOENT;
+ goto error;
+ }
+
+ for (i = 0; i < num_pols; i++)
+ WRITE_ONCE(pols[i]->curlft.use_time, ktime_get_real_seconds());
+
+ if (num_xfrms < 0) {
+ /* Prohibit the flow */
+ XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTPOLBLOCK);
+ err = -EPERM;
+ goto error;
+ } else if (num_xfrms > 0) {
+ /* Flow transformed */
+ dst_release(dst_orig);
+ } else {
+ /* Flow passes untransformed */
+ dst_release(dst);
+ dst = dst_orig;
+ }
+ok:
+ xfrm_pols_put(pols, drop_pols);
+ if (dst && dst->xfrm &&
+ dst->xfrm->props.mode == XFRM_MODE_TUNNEL)
+ dst->flags |= DST_XFRM_TUNNEL;
+ return dst;
+
+nopol:
+ if ((!dst_orig->dev || !(dst_orig->dev->flags & IFF_LOOPBACK)) &&
+ net->xfrm.policy_default[dir] == XFRM_USERPOLICY_BLOCK) {
+ err = -EPERM;
+ goto error;
+ }
+ if (!(flags & XFRM_LOOKUP_ICMP)) {
+ dst = dst_orig;
+ goto ok;
+ }
+ err = -ENOENT;
+error:
+ dst_release(dst);
+dropdst:
+ if (!(flags & XFRM_LOOKUP_KEEP_DST_REF))
+ dst_release(dst_orig);
+ xfrm_pols_put(pols, drop_pols);
+ return ERR_PTR(err);
+}
+EXPORT_SYMBOL(xfrm_lookup_with_ifid);
+
+/* Main function: finds/creates a bundle for given flow.
+ *
+ * At the moment we eat a raw IP route. Mostly to speed up lookups
+ * on interfaces with disabled IPsec.
+ */
+struct dst_entry *xfrm_lookup(struct net *net, struct dst_entry *dst_orig,
+ const struct flowi *fl, const struct sock *sk,
+ int flags)
+{
+ return xfrm_lookup_with_ifid(net, dst_orig, fl, sk, flags, 0);
+}
+EXPORT_SYMBOL(xfrm_lookup);
+
+/* Callers of xfrm_lookup_route() must ensure a call to dst_output().
+ * Otherwise we may send out blackholed packets.
+ */
+struct dst_entry *xfrm_lookup_route(struct net *net, struct dst_entry *dst_orig,
+ const struct flowi *fl,
+ const struct sock *sk, int flags)
+{
+ struct dst_entry *dst = xfrm_lookup(net, dst_orig, fl, sk,
+ flags | XFRM_LOOKUP_QUEUE |
+ XFRM_LOOKUP_KEEP_DST_REF);
+
+ if (PTR_ERR(dst) == -EREMOTE)
+ return make_blackhole(net, dst_orig->ops->family, dst_orig);
+
+ if (IS_ERR(dst))
+ dst_release(dst_orig);
+
+ return dst;
+}
+EXPORT_SYMBOL(xfrm_lookup_route);
+
+static inline int
+xfrm_secpath_reject(int idx, struct sk_buff *skb, const struct flowi *fl)
+{
+ struct sec_path *sp = skb_sec_path(skb);
+ struct xfrm_state *x;
+
+ if (!sp || idx < 0 || idx >= sp->len)
+ return 0;
+ x = sp->xvec[idx];
+ if (!x->type->reject)
+ return 0;
+ return x->type->reject(x, skb, fl);
+}
+
+/* When skb is transformed back to its "native" form, we have to
+ * check policy restrictions. At the moment we make this in maximally
+ * stupid way. Shame on me. :-) Of course, connected sockets must
+ * have policy cached at them.
+ */
+
+static inline int
+xfrm_state_ok(const struct xfrm_tmpl *tmpl, const struct xfrm_state *x,
+ unsigned short family, u32 if_id)
+{
+ if (xfrm_state_kern(x))
+ return tmpl->optional && !xfrm_state_addr_cmp(tmpl, x, tmpl->encap_family);
+ return x->id.proto == tmpl->id.proto &&
+ (x->id.spi == tmpl->id.spi || !tmpl->id.spi) &&
+ (x->props.reqid == tmpl->reqid || !tmpl->reqid) &&
+ x->props.mode == tmpl->mode &&
+ (tmpl->allalgs || (tmpl->aalgos & (1<<x->props.aalgo)) ||
+ !(xfrm_id_proto_match(tmpl->id.proto, IPSEC_PROTO_ANY))) &&
+ !(x->props.mode != XFRM_MODE_TRANSPORT &&
+ xfrm_state_addr_cmp(tmpl, x, family)) &&
+ (if_id == 0 || if_id == x->if_id);
+}
+
+/*
+ * 0 or more than 0 is returned when validation is succeeded (either bypass
+ * because of optional transport mode, or next index of the matched secpath
+ * state with the template.
+ * -1 is returned when no matching template is found.
+ * Otherwise "-2 - errored_index" is returned.
+ */
+static inline int
+xfrm_policy_ok(const struct xfrm_tmpl *tmpl, const struct sec_path *sp, int start,
+ unsigned short family, u32 if_id)
+{
+ int idx = start;
+
+ if (tmpl->optional) {
+ if (tmpl->mode == XFRM_MODE_TRANSPORT)
+ return start;
+ } else
+ start = -1;
+ for (; idx < sp->len; idx++) {
+ if (xfrm_state_ok(tmpl, sp->xvec[idx], family, if_id))
+ return ++idx;
+ if (sp->xvec[idx]->props.mode != XFRM_MODE_TRANSPORT) {
+ if (idx < sp->verified_cnt) {
+ /* Secpath entry previously verified, consider optional and
+ * continue searching
+ */
+ continue;
+ }
+
+ if (start == -1)
+ start = -2-idx;
+ break;
+ }
+ }
+ return start;
+}
+
+static void
+decode_session4(struct sk_buff *skb, struct flowi *fl, bool reverse)
+{
+ const struct iphdr *iph = ip_hdr(skb);
+ int ihl = iph->ihl;
+ u8 *xprth = skb_network_header(skb) + ihl * 4;
+ struct flowi4 *fl4 = &fl->u.ip4;
+ int oif = 0;
+
+ if (skb_dst(skb) && skb_dst(skb)->dev)
+ oif = skb_dst(skb)->dev->ifindex;
+
+ memset(fl4, 0, sizeof(struct flowi4));
+ fl4->flowi4_mark = skb->mark;
+ fl4->flowi4_oif = reverse ? skb->skb_iif : oif;
+
+ fl4->flowi4_proto = iph->protocol;
+ fl4->daddr = reverse ? iph->saddr : iph->daddr;
+ fl4->saddr = reverse ? iph->daddr : iph->saddr;
+ fl4->flowi4_tos = iph->tos & ~INET_ECN_MASK;
+
+ if (!ip_is_fragment(iph)) {
+ switch (iph->protocol) {
+ case IPPROTO_UDP:
+ case IPPROTO_UDPLITE:
+ case IPPROTO_TCP:
+ case IPPROTO_SCTP:
+ case IPPROTO_DCCP:
+ if (xprth + 4 < skb->data ||
+ pskb_may_pull(skb, xprth + 4 - skb->data)) {
+ __be16 *ports;
+
+ xprth = skb_network_header(skb) + ihl * 4;
+ ports = (__be16 *)xprth;
+
+ fl4->fl4_sport = ports[!!reverse];
+ fl4->fl4_dport = ports[!reverse];
+ }
+ break;
+ case IPPROTO_ICMP:
+ if (xprth + 2 < skb->data ||
+ pskb_may_pull(skb, xprth + 2 - skb->data)) {
+ u8 *icmp;
+
+ xprth = skb_network_header(skb) + ihl * 4;
+ icmp = xprth;
+
+ fl4->fl4_icmp_type = icmp[0];
+ fl4->fl4_icmp_code = icmp[1];
+ }
+ break;
+ case IPPROTO_GRE:
+ if (xprth + 12 < skb->data ||
+ pskb_may_pull(skb, xprth + 12 - skb->data)) {
+ __be16 *greflags;
+ __be32 *gre_hdr;
+
+ xprth = skb_network_header(skb) + ihl * 4;
+ greflags = (__be16 *)xprth;
+ gre_hdr = (__be32 *)xprth;
+
+ if (greflags[0] & GRE_KEY) {
+ if (greflags[0] & GRE_CSUM)
+ gre_hdr++;
+ fl4->fl4_gre_key = gre_hdr[1];
+ }
+ }
+ break;
+ default:
+ break;
+ }
+ }
+}
+
+#if IS_ENABLED(CONFIG_IPV6)
+static void
+decode_session6(struct sk_buff *skb, struct flowi *fl, bool reverse)
+{
+ struct flowi6 *fl6 = &fl->u.ip6;
+ int onlyproto = 0;
+ const struct ipv6hdr *hdr = ipv6_hdr(skb);
+ u32 offset = sizeof(*hdr);
+ struct ipv6_opt_hdr *exthdr;
+ const unsigned char *nh = skb_network_header(skb);
+ u16 nhoff = IP6CB(skb)->nhoff;
+ int oif = 0;
+ u8 nexthdr;
+
+ if (!nhoff)
+ nhoff = offsetof(struct ipv6hdr, nexthdr);
+
+ nexthdr = nh[nhoff];
+
+ if (skb_dst(skb) && skb_dst(skb)->dev)
+ oif = skb_dst(skb)->dev->ifindex;
+
+ memset(fl6, 0, sizeof(struct flowi6));
+ fl6->flowi6_mark = skb->mark;
+ fl6->flowi6_oif = reverse ? skb->skb_iif : oif;
+
+ fl6->daddr = reverse ? hdr->saddr : hdr->daddr;
+ fl6->saddr = reverse ? hdr->daddr : hdr->saddr;
+
+ while (nh + offset + sizeof(*exthdr) < skb->data ||
+ pskb_may_pull(skb, nh + offset + sizeof(*exthdr) - skb->data)) {
+ nh = skb_network_header(skb);
+ exthdr = (struct ipv6_opt_hdr *)(nh + offset);
+
+ switch (nexthdr) {
+ case NEXTHDR_FRAGMENT:
+ onlyproto = 1;
+ fallthrough;
+ case NEXTHDR_ROUTING:
+ case NEXTHDR_HOP:
+ case NEXTHDR_DEST:
+ offset += ipv6_optlen(exthdr);
+ nexthdr = exthdr->nexthdr;
+ break;
+ case IPPROTO_UDP:
+ case IPPROTO_UDPLITE:
+ case IPPROTO_TCP:
+ case IPPROTO_SCTP:
+ case IPPROTO_DCCP:
+ if (!onlyproto && (nh + offset + 4 < skb->data ||
+ pskb_may_pull(skb, nh + offset + 4 - skb->data))) {
+ __be16 *ports;
+
+ nh = skb_network_header(skb);
+ ports = (__be16 *)(nh + offset);
+ fl6->fl6_sport = ports[!!reverse];
+ fl6->fl6_dport = ports[!reverse];
+ }
+ fl6->flowi6_proto = nexthdr;
+ return;
+ case IPPROTO_ICMPV6:
+ if (!onlyproto && (nh + offset + 2 < skb->data ||
+ pskb_may_pull(skb, nh + offset + 2 - skb->data))) {
+ u8 *icmp;
+
+ nh = skb_network_header(skb);
+ icmp = (u8 *)(nh + offset);
+ fl6->fl6_icmp_type = icmp[0];
+ fl6->fl6_icmp_code = icmp[1];
+ }
+ fl6->flowi6_proto = nexthdr;
+ return;
+ case IPPROTO_GRE:
+ if (!onlyproto &&
+ (nh + offset + 12 < skb->data ||
+ pskb_may_pull(skb, nh + offset + 12 - skb->data))) {
+ struct gre_base_hdr *gre_hdr;
+ __be32 *gre_key;
+
+ nh = skb_network_header(skb);
+ gre_hdr = (struct gre_base_hdr *)(nh + offset);
+ gre_key = (__be32 *)(gre_hdr + 1);
+
+ if (gre_hdr->flags & GRE_KEY) {
+ if (gre_hdr->flags & GRE_CSUM)
+ gre_key++;
+ fl6->fl6_gre_key = *gre_key;
+ }
+ }
+ fl6->flowi6_proto = nexthdr;
+ return;
+
+#if IS_ENABLED(CONFIG_IPV6_MIP6)
+ case IPPROTO_MH:
+ offset += ipv6_optlen(exthdr);
+ if (!onlyproto && (nh + offset + 3 < skb->data ||
+ pskb_may_pull(skb, nh + offset + 3 - skb->data))) {
+ struct ip6_mh *mh;
+
+ nh = skb_network_header(skb);
+ mh = (struct ip6_mh *)(nh + offset);
+ fl6->fl6_mh_type = mh->ip6mh_type;
+ }
+ fl6->flowi6_proto = nexthdr;
+ return;
+#endif
+ default:
+ fl6->flowi6_proto = nexthdr;
+ return;
+ }
+ }
+}
+#endif
+
+int __xfrm_decode_session(struct sk_buff *skb, struct flowi *fl,
+ unsigned int family, int reverse)
+{
+ switch (family) {
+ case AF_INET:
+ decode_session4(skb, fl, reverse);
+ break;
+#if IS_ENABLED(CONFIG_IPV6)
+ case AF_INET6:
+ decode_session6(skb, fl, reverse);
+ break;
+#endif
+ default:
+ return -EAFNOSUPPORT;
+ }
+
+ return security_xfrm_decode_session(skb, &fl->flowi_secid);
+}
+EXPORT_SYMBOL(__xfrm_decode_session);
+
+static inline int secpath_has_nontransport(const struct sec_path *sp, int k, int *idxp)
+{
+ for (; k < sp->len; k++) {
+ if (sp->xvec[k]->props.mode != XFRM_MODE_TRANSPORT) {
+ *idxp = k;
+ return 1;
+ }
+ }
+
+ return 0;
+}
+
+int __xfrm_policy_check(struct sock *sk, int dir, struct sk_buff *skb,
+ unsigned short family)
+{
+ struct net *net = dev_net(skb->dev);
+ struct xfrm_policy *pol;
+ struct xfrm_policy *pols[XFRM_POLICY_TYPE_MAX];
+ int npols = 0;
+ int xfrm_nr;
+ int pi;
+ int reverse;
+ struct flowi fl;
+ int xerr_idx = -1;
+ const struct xfrm_if_cb *ifcb;
+ struct sec_path *sp;
+ u32 if_id = 0;
+
+ rcu_read_lock();
+ ifcb = xfrm_if_get_cb();
+
+ if (ifcb) {
+ struct xfrm_if_decode_session_result r;
+
+ if (ifcb->decode_session(skb, family, &r)) {
+ if_id = r.if_id;
+ net = r.net;
+ }
+ }
+ rcu_read_unlock();
+
+ reverse = dir & ~XFRM_POLICY_MASK;
+ dir &= XFRM_POLICY_MASK;
+
+ if (__xfrm_decode_session(skb, &fl, family, reverse) < 0) {
+ XFRM_INC_STATS(net, LINUX_MIB_XFRMINHDRERROR);
+ return 0;
+ }
+
+ nf_nat_decode_session(skb, &fl, family);
+
+ /* First, check used SA against their selectors. */
+ sp = skb_sec_path(skb);
+ if (sp) {
+ int i;
+
+ for (i = sp->len - 1; i >= 0; i--) {
+ struct xfrm_state *x = sp->xvec[i];
+ if (!xfrm_selector_match(&x->sel, &fl, family)) {
+ XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATEMISMATCH);
+ return 0;
+ }
+ }
+ }
+
+ pol = NULL;
+ sk = sk_to_full_sk(sk);
+ if (sk && sk->sk_policy[dir]) {
+ pol = xfrm_sk_policy_lookup(sk, dir, &fl, family, if_id);
+ if (IS_ERR(pol)) {
+ XFRM_INC_STATS(net, LINUX_MIB_XFRMINPOLERROR);
+ return 0;
+ }
+ }
+
+ if (!pol)
+ pol = xfrm_policy_lookup(net, &fl, family, dir, if_id);
+
+ if (IS_ERR(pol)) {
+ XFRM_INC_STATS(net, LINUX_MIB_XFRMINPOLERROR);
+ return 0;
+ }
+
+ if (!pol) {
+ if (net->xfrm.policy_default[dir] == XFRM_USERPOLICY_BLOCK) {
+ XFRM_INC_STATS(net, LINUX_MIB_XFRMINNOPOLS);
+ return 0;
+ }
+
+ if (sp && secpath_has_nontransport(sp, 0, &xerr_idx)) {
+ xfrm_secpath_reject(xerr_idx, skb, &fl);
+ XFRM_INC_STATS(net, LINUX_MIB_XFRMINNOPOLS);
+ return 0;
+ }
+ return 1;
+ }
+
+ /* This lockless write can happen from different cpus. */
+ WRITE_ONCE(pol->curlft.use_time, ktime_get_real_seconds());
+
+ pols[0] = pol;
+ npols++;
+#ifdef CONFIG_XFRM_SUB_POLICY
+ if (pols[0]->type != XFRM_POLICY_TYPE_MAIN) {
+ pols[1] = xfrm_policy_lookup_bytype(net, XFRM_POLICY_TYPE_MAIN,
+ &fl, family,
+ XFRM_POLICY_IN, if_id);
+ if (pols[1]) {
+ if (IS_ERR(pols[1])) {
+ XFRM_INC_STATS(net, LINUX_MIB_XFRMINPOLERROR);
+ xfrm_pol_put(pols[0]);
+ return 0;
+ }
+ /* This write can happen from different cpus. */
+ WRITE_ONCE(pols[1]->curlft.use_time,
+ ktime_get_real_seconds());
+ npols++;
+ }
+ }
+#endif
+
+ if (pol->action == XFRM_POLICY_ALLOW) {
+ static struct sec_path dummy;
+ struct xfrm_tmpl *tp[XFRM_MAX_DEPTH];
+ struct xfrm_tmpl *stp[XFRM_MAX_DEPTH];
+ struct xfrm_tmpl **tpp = tp;
+ int ti = 0;
+ int i, k;
+
+ sp = skb_sec_path(skb);
+ if (!sp)
+ sp = &dummy;
+
+ for (pi = 0; pi < npols; pi++) {
+ if (pols[pi] != pol &&
+ pols[pi]->action != XFRM_POLICY_ALLOW) {
+ XFRM_INC_STATS(net, LINUX_MIB_XFRMINPOLBLOCK);
+ goto reject;
+ }
+ if (ti + pols[pi]->xfrm_nr >= XFRM_MAX_DEPTH) {
+ XFRM_INC_STATS(net, LINUX_MIB_XFRMINBUFFERERROR);
+ goto reject_error;
+ }
+ for (i = 0; i < pols[pi]->xfrm_nr; i++)
+ tpp[ti++] = &pols[pi]->xfrm_vec[i];
+ }
+ xfrm_nr = ti;
+
+ if (npols > 1) {
+ xfrm_tmpl_sort(stp, tpp, xfrm_nr, family);
+ tpp = stp;
+ }
+
+ /* For each tunnel xfrm, find the first matching tmpl.
+ * For each tmpl before that, find corresponding xfrm.
+ * Order is _important_. Later we will implement
+ * some barriers, but at the moment barriers
+ * are implied between each two transformations.
+ * Upon success, marks secpath entries as having been
+ * verified to allow them to be skipped in future policy
+ * checks (e.g. nested tunnels).
+ */
+ for (i = xfrm_nr-1, k = 0; i >= 0; i--) {
+ k = xfrm_policy_ok(tpp[i], sp, k, family, if_id);
+ if (k < 0) {
+ if (k < -1)
+ /* "-2 - errored_index" returned */
+ xerr_idx = -(2+k);
+ XFRM_INC_STATS(net, LINUX_MIB_XFRMINTMPLMISMATCH);
+ goto reject;
+ }
+ }
+
+ if (secpath_has_nontransport(sp, k, &xerr_idx)) {
+ XFRM_INC_STATS(net, LINUX_MIB_XFRMINTMPLMISMATCH);
+ goto reject;
+ }
+
+ xfrm_pols_put(pols, npols);
+ sp->verified_cnt = k;
+
+ return 1;
+ }
+ XFRM_INC_STATS(net, LINUX_MIB_XFRMINPOLBLOCK);
+
+reject:
+ xfrm_secpath_reject(xerr_idx, skb, &fl);
+reject_error:
+ xfrm_pols_put(pols, npols);
+ return 0;
+}
+EXPORT_SYMBOL(__xfrm_policy_check);
+
+int __xfrm_route_forward(struct sk_buff *skb, unsigned short family)
+{
+ struct net *net = dev_net(skb->dev);
+ struct flowi fl;
+ struct dst_entry *dst;
+ int res = 1;
+
+ if (xfrm_decode_session(skb, &fl, family) < 0) {
+ XFRM_INC_STATS(net, LINUX_MIB_XFRMFWDHDRERROR);
+ return 0;
+ }
+
+ skb_dst_force(skb);
+ if (!skb_dst(skb)) {
+ XFRM_INC_STATS(net, LINUX_MIB_XFRMFWDHDRERROR);
+ return 0;
+ }
+
+ dst = xfrm_lookup(net, skb_dst(skb), &fl, NULL, XFRM_LOOKUP_QUEUE);
+ if (IS_ERR(dst)) {
+ res = 0;
+ dst = NULL;
+ }
+ skb_dst_set(skb, dst);
+ return res;
+}
+EXPORT_SYMBOL(__xfrm_route_forward);
+
+/* Optimize later using cookies and generation ids. */
+
+static struct dst_entry *xfrm_dst_check(struct dst_entry *dst, u32 cookie)
+{
+ /* Code (such as __xfrm4_bundle_create()) sets dst->obsolete
+ * to DST_OBSOLETE_FORCE_CHK to force all XFRM destinations to
+ * get validated by dst_ops->check on every use. We do this
+ * because when a normal route referenced by an XFRM dst is
+ * obsoleted we do not go looking around for all parent
+ * referencing XFRM dsts so that we can invalidate them. It
+ * is just too much work. Instead we make the checks here on
+ * every use. For example:
+ *
+ * XFRM dst A --> IPv4 dst X
+ *
+ * X is the "xdst->route" of A (X is also the "dst->path" of A
+ * in this example). If X is marked obsolete, "A" will not
+ * notice. That's what we are validating here via the
+ * stale_bundle() check.
+ *
+ * When a dst is removed from the fib tree, DST_OBSOLETE_DEAD will
+ * be marked on it.
+ * This will force stale_bundle() to fail on any xdst bundle with
+ * this dst linked in it.
+ */
+ if (dst->obsolete < 0 && !stale_bundle(dst))
+ return dst;
+
+ return NULL;
+}
+
+static int stale_bundle(struct dst_entry *dst)
+{
+ return !xfrm_bundle_ok((struct xfrm_dst *)dst);
+}
+
+void xfrm_dst_ifdown(struct dst_entry *dst, struct net_device *dev)
+{
+ while ((dst = xfrm_dst_child(dst)) && dst->xfrm && dst->dev == dev) {
+ dst->dev = blackhole_netdev;
+ dev_hold(dst->dev);
+ dev_put(dev);
+ }
+}
+EXPORT_SYMBOL(xfrm_dst_ifdown);
+
+static void xfrm_link_failure(struct sk_buff *skb)
+{
+ /* Impossible. Such dst must be popped before reaches point of failure. */
+}
+
+static struct dst_entry *xfrm_negative_advice(struct dst_entry *dst)
+{
+ if (dst) {
+ if (dst->obsolete) {
+ dst_release(dst);
+ dst = NULL;
+ }
+ }
+ return dst;
+}
+
+static void xfrm_init_pmtu(struct xfrm_dst **bundle, int nr)
+{
+ while (nr--) {
+ struct xfrm_dst *xdst = bundle[nr];
+ u32 pmtu, route_mtu_cached;
+ struct dst_entry *dst;
+
+ dst = &xdst->u.dst;
+ pmtu = dst_mtu(xfrm_dst_child(dst));
+ xdst->child_mtu_cached = pmtu;
+
+ pmtu = xfrm_state_mtu(dst->xfrm, pmtu);
+
+ route_mtu_cached = dst_mtu(xdst->route);
+ xdst->route_mtu_cached = route_mtu_cached;
+
+ if (pmtu > route_mtu_cached)
+ pmtu = route_mtu_cached;
+
+ dst_metric_set(dst, RTAX_MTU, pmtu);
+ }
+}
+
+/* Check that the bundle accepts the flow and its components are
+ * still valid.
+ */
+
+static int xfrm_bundle_ok(struct xfrm_dst *first)
+{
+ struct xfrm_dst *bundle[XFRM_MAX_DEPTH];
+ struct dst_entry *dst = &first->u.dst;
+ struct xfrm_dst *xdst;
+ int start_from, nr;
+ u32 mtu;
+
+ if (!dst_check(xfrm_dst_path(dst), ((struct xfrm_dst *)dst)->path_cookie) ||
+ (dst->dev && !netif_running(dst->dev)))
+ return 0;
+
+ if (dst->flags & DST_XFRM_QUEUE)
+ return 1;
+
+ start_from = nr = 0;
+ do {
+ struct xfrm_dst *xdst = (struct xfrm_dst *)dst;
+
+ if (dst->xfrm->km.state != XFRM_STATE_VALID)
+ return 0;
+ if (xdst->xfrm_genid != dst->xfrm->genid)
+ return 0;
+ if (xdst->num_pols > 0 &&
+ xdst->policy_genid != atomic_read(&xdst->pols[0]->genid))
+ return 0;
+
+ bundle[nr++] = xdst;
+
+ mtu = dst_mtu(xfrm_dst_child(dst));
+ if (xdst->child_mtu_cached != mtu) {
+ start_from = nr;
+ xdst->child_mtu_cached = mtu;
+ }
+
+ if (!dst_check(xdst->route, xdst->route_cookie))
+ return 0;
+ mtu = dst_mtu(xdst->route);
+ if (xdst->route_mtu_cached != mtu) {
+ start_from = nr;
+ xdst->route_mtu_cached = mtu;
+ }
+
+ dst = xfrm_dst_child(dst);
+ } while (dst->xfrm);
+
+ if (likely(!start_from))
+ return 1;
+
+ xdst = bundle[start_from - 1];
+ mtu = xdst->child_mtu_cached;
+ while (start_from--) {
+ dst = &xdst->u.dst;
+
+ mtu = xfrm_state_mtu(dst->xfrm, mtu);
+ if (mtu > xdst->route_mtu_cached)
+ mtu = xdst->route_mtu_cached;
+ dst_metric_set(dst, RTAX_MTU, mtu);
+ if (!start_from)
+ break;
+
+ xdst = bundle[start_from - 1];
+ xdst->child_mtu_cached = mtu;
+ }
+
+ return 1;
+}
+
+static unsigned int xfrm_default_advmss(const struct dst_entry *dst)
+{
+ return dst_metric_advmss(xfrm_dst_path(dst));
+}
+
+static unsigned int xfrm_mtu(const struct dst_entry *dst)
+{
+ unsigned int mtu = dst_metric_raw(dst, RTAX_MTU);
+
+ return mtu ? : dst_mtu(xfrm_dst_path(dst));
+}
+
+static const void *xfrm_get_dst_nexthop(const struct dst_entry *dst,
+ const void *daddr)
+{
+ while (dst->xfrm) {
+ const struct xfrm_state *xfrm = dst->xfrm;
+
+ dst = xfrm_dst_child(dst);
+
+ if (xfrm->props.mode == XFRM_MODE_TRANSPORT)
+ continue;
+ if (xfrm->type->flags & XFRM_TYPE_REMOTE_COADDR)
+ daddr = xfrm->coaddr;
+ else if (!(xfrm->type->flags & XFRM_TYPE_LOCAL_COADDR))
+ daddr = &xfrm->id.daddr;
+ }
+ return daddr;
+}
+
+static struct neighbour *xfrm_neigh_lookup(const struct dst_entry *dst,
+ struct sk_buff *skb,
+ const void *daddr)
+{
+ const struct dst_entry *path = xfrm_dst_path(dst);
+
+ if (!skb)
+ daddr = xfrm_get_dst_nexthop(dst, daddr);
+ return path->ops->neigh_lookup(path, skb, daddr);
+}
+
+static void xfrm_confirm_neigh(const struct dst_entry *dst, const void *daddr)
+{
+ const struct dst_entry *path = xfrm_dst_path(dst);
+
+ daddr = xfrm_get_dst_nexthop(dst, daddr);
+ path->ops->confirm_neigh(path, daddr);
+}
+
+int xfrm_policy_register_afinfo(const struct xfrm_policy_afinfo *afinfo, int family)
+{
+ int err = 0;
+
+ if (WARN_ON(family >= ARRAY_SIZE(xfrm_policy_afinfo)))
+ return -EAFNOSUPPORT;
+
+ spin_lock(&xfrm_policy_afinfo_lock);
+ if (unlikely(xfrm_policy_afinfo[family] != NULL))
+ err = -EEXIST;
+ else {
+ struct dst_ops *dst_ops = afinfo->dst_ops;
+ if (likely(dst_ops->kmem_cachep == NULL))
+ dst_ops->kmem_cachep = xfrm_dst_cache;
+ if (likely(dst_ops->check == NULL))
+ dst_ops->check = xfrm_dst_check;
+ if (likely(dst_ops->default_advmss == NULL))
+ dst_ops->default_advmss = xfrm_default_advmss;
+ if (likely(dst_ops->mtu == NULL))
+ dst_ops->mtu = xfrm_mtu;
+ if (likely(dst_ops->negative_advice == NULL))
+ dst_ops->negative_advice = xfrm_negative_advice;
+ if (likely(dst_ops->link_failure == NULL))
+ dst_ops->link_failure = xfrm_link_failure;
+ if (likely(dst_ops->neigh_lookup == NULL))
+ dst_ops->neigh_lookup = xfrm_neigh_lookup;
+ if (likely(!dst_ops->confirm_neigh))
+ dst_ops->confirm_neigh = xfrm_confirm_neigh;
+ rcu_assign_pointer(xfrm_policy_afinfo[family], afinfo);
+ }
+ spin_unlock(&xfrm_policy_afinfo_lock);
+
+ return err;
+}
+EXPORT_SYMBOL(xfrm_policy_register_afinfo);
+
+void xfrm_policy_unregister_afinfo(const struct xfrm_policy_afinfo *afinfo)
+{
+ struct dst_ops *dst_ops = afinfo->dst_ops;
+ int i;
+
+ for (i = 0; i < ARRAY_SIZE(xfrm_policy_afinfo); i++) {
+ if (xfrm_policy_afinfo[i] != afinfo)
+ continue;
+ RCU_INIT_POINTER(xfrm_policy_afinfo[i], NULL);
+ break;
+ }
+
+ synchronize_rcu();
+
+ dst_ops->kmem_cachep = NULL;
+ dst_ops->check = NULL;
+ dst_ops->negative_advice = NULL;
+ dst_ops->link_failure = NULL;
+}
+EXPORT_SYMBOL(xfrm_policy_unregister_afinfo);
+
+void xfrm_if_register_cb(const struct xfrm_if_cb *ifcb)
+{
+ spin_lock(&xfrm_if_cb_lock);
+ rcu_assign_pointer(xfrm_if_cb, ifcb);
+ spin_unlock(&xfrm_if_cb_lock);
+}
+EXPORT_SYMBOL(xfrm_if_register_cb);
+
+void xfrm_if_unregister_cb(void)
+{
+ RCU_INIT_POINTER(xfrm_if_cb, NULL);
+ synchronize_rcu();
+}
+EXPORT_SYMBOL(xfrm_if_unregister_cb);
+
+#ifdef CONFIG_XFRM_STATISTICS
+static int __net_init xfrm_statistics_init(struct net *net)
+{
+ int rv;
+ net->mib.xfrm_statistics = alloc_percpu(struct linux_xfrm_mib);
+ if (!net->mib.xfrm_statistics)
+ return -ENOMEM;
+ rv = xfrm_proc_init(net);
+ if (rv < 0)
+ free_percpu(net->mib.xfrm_statistics);
+ return rv;
+}
+
+static void xfrm_statistics_fini(struct net *net)
+{
+ xfrm_proc_fini(net);
+ free_percpu(net->mib.xfrm_statistics);
+}
+#else
+static int __net_init xfrm_statistics_init(struct net *net)
+{
+ return 0;
+}
+
+static void xfrm_statistics_fini(struct net *net)
+{
+}
+#endif
+
+static int __net_init xfrm_policy_init(struct net *net)
+{
+ unsigned int hmask, sz;
+ int dir, err;
+
+ if (net_eq(net, &init_net)) {
+ xfrm_dst_cache = kmem_cache_create("xfrm_dst_cache",
+ sizeof(struct xfrm_dst),
+ 0, SLAB_HWCACHE_ALIGN|SLAB_PANIC,
+ NULL);
+ err = rhashtable_init(&xfrm_policy_inexact_table,
+ &xfrm_pol_inexact_params);
+ BUG_ON(err);
+ }
+
+ hmask = 8 - 1;
+ sz = (hmask+1) * sizeof(struct hlist_head);
+
+ net->xfrm.policy_byidx = xfrm_hash_alloc(sz);
+ if (!net->xfrm.policy_byidx)
+ goto out_byidx;
+ net->xfrm.policy_idx_hmask = hmask;
+
+ for (dir = 0; dir < XFRM_POLICY_MAX; dir++) {
+ struct xfrm_policy_hash *htab;
+
+ net->xfrm.policy_count[dir] = 0;
+ net->xfrm.policy_count[XFRM_POLICY_MAX + dir] = 0;
+ INIT_HLIST_HEAD(&net->xfrm.policy_inexact[dir]);
+
+ htab = &net->xfrm.policy_bydst[dir];
+ htab->table = xfrm_hash_alloc(sz);
+ if (!htab->table)
+ goto out_bydst;
+ htab->hmask = hmask;
+ htab->dbits4 = 32;
+ htab->sbits4 = 32;
+ htab->dbits6 = 128;
+ htab->sbits6 = 128;
+ }
+ net->xfrm.policy_hthresh.lbits4 = 32;
+ net->xfrm.policy_hthresh.rbits4 = 32;
+ net->xfrm.policy_hthresh.lbits6 = 128;
+ net->xfrm.policy_hthresh.rbits6 = 128;
+
+ seqlock_init(&net->xfrm.policy_hthresh.lock);
+
+ INIT_LIST_HEAD(&net->xfrm.policy_all);
+ INIT_LIST_HEAD(&net->xfrm.inexact_bins);
+ INIT_WORK(&net->xfrm.policy_hash_work, xfrm_hash_resize);
+ INIT_WORK(&net->xfrm.policy_hthresh.work, xfrm_hash_rebuild);
+ return 0;
+
+out_bydst:
+ for (dir--; dir >= 0; dir--) {
+ struct xfrm_policy_hash *htab;
+
+ htab = &net->xfrm.policy_bydst[dir];
+ xfrm_hash_free(htab->table, sz);
+ }
+ xfrm_hash_free(net->xfrm.policy_byidx, sz);
+out_byidx:
+ return -ENOMEM;
+}
+
+static void xfrm_policy_fini(struct net *net)
+{
+ struct xfrm_pol_inexact_bin *b, *t;
+ unsigned int sz;
+ int dir;
+
+ flush_work(&net->xfrm.policy_hash_work);
+#ifdef CONFIG_XFRM_SUB_POLICY
+ xfrm_policy_flush(net, XFRM_POLICY_TYPE_SUB, false);
+#endif
+ xfrm_policy_flush(net, XFRM_POLICY_TYPE_MAIN, false);
+
+ WARN_ON(!list_empty(&net->xfrm.policy_all));
+
+ for (dir = 0; dir < XFRM_POLICY_MAX; dir++) {
+ struct xfrm_policy_hash *htab;
+
+ WARN_ON(!hlist_empty(&net->xfrm.policy_inexact[dir]));
+
+ htab = &net->xfrm.policy_bydst[dir];
+ sz = (htab->hmask + 1) * sizeof(struct hlist_head);
+ WARN_ON(!hlist_empty(htab->table));
+ xfrm_hash_free(htab->table, sz);
+ }
+
+ sz = (net->xfrm.policy_idx_hmask + 1) * sizeof(struct hlist_head);
+ WARN_ON(!hlist_empty(net->xfrm.policy_byidx));
+ xfrm_hash_free(net->xfrm.policy_byidx, sz);
+
+ spin_lock_bh(&net->xfrm.xfrm_policy_lock);
+ list_for_each_entry_safe(b, t, &net->xfrm.inexact_bins, inexact_bins)
+ __xfrm_policy_inexact_prune_bin(b, true);
+ spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
+}
+
+static int __net_init xfrm_net_init(struct net *net)
+{
+ int rv;
+
+ /* Initialize the per-net locks here */
+ spin_lock_init(&net->xfrm.xfrm_state_lock);
+ spin_lock_init(&net->xfrm.xfrm_policy_lock);
+ seqcount_spinlock_init(&net->xfrm.xfrm_policy_hash_generation, &net->xfrm.xfrm_policy_lock);
+ mutex_init(&net->xfrm.xfrm_cfg_mutex);
+ net->xfrm.policy_default[XFRM_POLICY_IN] = XFRM_USERPOLICY_ACCEPT;
+ net->xfrm.policy_default[XFRM_POLICY_FWD] = XFRM_USERPOLICY_ACCEPT;
+ net->xfrm.policy_default[XFRM_POLICY_OUT] = XFRM_USERPOLICY_ACCEPT;
+
+ rv = xfrm_statistics_init(net);
+ if (rv < 0)
+ goto out_statistics;
+ rv = xfrm_state_init(net);
+ if (rv < 0)
+ goto out_state;
+ rv = xfrm_policy_init(net);
+ if (rv < 0)
+ goto out_policy;
+ rv = xfrm_sysctl_init(net);
+ if (rv < 0)
+ goto out_sysctl;
+
+ return 0;
+
+out_sysctl:
+ xfrm_policy_fini(net);
+out_policy:
+ xfrm_state_fini(net);
+out_state:
+ xfrm_statistics_fini(net);
+out_statistics:
+ return rv;
+}
+
+static void __net_exit xfrm_net_exit(struct net *net)
+{
+ xfrm_sysctl_fini(net);
+ xfrm_policy_fini(net);
+ xfrm_state_fini(net);
+ xfrm_statistics_fini(net);
+}
+
+static struct pernet_operations __net_initdata xfrm_net_ops = {
+ .init = xfrm_net_init,
+ .exit = xfrm_net_exit,
+};
+
+void __init xfrm_init(void)
+{
+ register_pernet_subsys(&xfrm_net_ops);
+ xfrm_dev_init();
+ xfrm_input_init();
+
+#ifdef CONFIG_XFRM_ESPINTCP
+ espintcp_init();
+#endif
+}
+
+#ifdef CONFIG_AUDITSYSCALL
+static void xfrm_audit_common_policyinfo(struct xfrm_policy *xp,
+ struct audit_buffer *audit_buf)
+{
+ struct xfrm_sec_ctx *ctx = xp->security;
+ struct xfrm_selector *sel = &xp->selector;
+
+ if (ctx)
+ audit_log_format(audit_buf, " sec_alg=%u sec_doi=%u sec_obj=%s",
+ ctx->ctx_alg, ctx->ctx_doi, ctx->ctx_str);
+
+ switch (sel->family) {
+ case AF_INET:
+ audit_log_format(audit_buf, " src=%pI4", &sel->saddr.a4);
+ if (sel->prefixlen_s != 32)
+ audit_log_format(audit_buf, " src_prefixlen=%d",
+ sel->prefixlen_s);
+ audit_log_format(audit_buf, " dst=%pI4", &sel->daddr.a4);
+ if (sel->prefixlen_d != 32)
+ audit_log_format(audit_buf, " dst_prefixlen=%d",
+ sel->prefixlen_d);
+ break;
+ case AF_INET6:
+ audit_log_format(audit_buf, " src=%pI6", sel->saddr.a6);
+ if (sel->prefixlen_s != 128)
+ audit_log_format(audit_buf, " src_prefixlen=%d",
+ sel->prefixlen_s);
+ audit_log_format(audit_buf, " dst=%pI6", sel->daddr.a6);
+ if (sel->prefixlen_d != 128)
+ audit_log_format(audit_buf, " dst_prefixlen=%d",
+ sel->prefixlen_d);
+ break;
+ }
+}
+
+void xfrm_audit_policy_add(struct xfrm_policy *xp, int result, bool task_valid)
+{
+ struct audit_buffer *audit_buf;
+
+ audit_buf = xfrm_audit_start("SPD-add");
+ if (audit_buf == NULL)
+ return;
+ xfrm_audit_helper_usrinfo(task_valid, audit_buf);
+ audit_log_format(audit_buf, " res=%u", result);
+ xfrm_audit_common_policyinfo(xp, audit_buf);
+ audit_log_end(audit_buf);
+}
+EXPORT_SYMBOL_GPL(xfrm_audit_policy_add);
+
+void xfrm_audit_policy_delete(struct xfrm_policy *xp, int result,
+ bool task_valid)
+{
+ struct audit_buffer *audit_buf;
+
+ audit_buf = xfrm_audit_start("SPD-delete");
+ if (audit_buf == NULL)
+ return;
+ xfrm_audit_helper_usrinfo(task_valid, audit_buf);
+ audit_log_format(audit_buf, " res=%u", result);
+ xfrm_audit_common_policyinfo(xp, audit_buf);
+ audit_log_end(audit_buf);
+}
+EXPORT_SYMBOL_GPL(xfrm_audit_policy_delete);
+#endif
+
+#ifdef CONFIG_XFRM_MIGRATE
+static bool xfrm_migrate_selector_match(const struct xfrm_selector *sel_cmp,
+ const struct xfrm_selector *sel_tgt)
+{
+ if (sel_cmp->proto == IPSEC_ULPROTO_ANY) {
+ if (sel_tgt->family == sel_cmp->family &&
+ xfrm_addr_equal(&sel_tgt->daddr, &sel_cmp->daddr,
+ sel_cmp->family) &&
+ xfrm_addr_equal(&sel_tgt->saddr, &sel_cmp->saddr,
+ sel_cmp->family) &&
+ sel_tgt->prefixlen_d == sel_cmp->prefixlen_d &&
+ sel_tgt->prefixlen_s == sel_cmp->prefixlen_s) {
+ return true;
+ }
+ } else {
+ if (memcmp(sel_tgt, sel_cmp, sizeof(*sel_tgt)) == 0) {
+ return true;
+ }
+ }
+ return false;
+}
+
+static struct xfrm_policy *xfrm_migrate_policy_find(const struct xfrm_selector *sel,
+ u8 dir, u8 type, struct net *net, u32 if_id)
+{
+ struct xfrm_policy *pol, *ret = NULL;
+ struct hlist_head *chain;
+ u32 priority = ~0U;
+
+ spin_lock_bh(&net->xfrm.xfrm_policy_lock);
+ chain = policy_hash_direct(net, &sel->daddr, &sel->saddr, sel->family, dir);
+ hlist_for_each_entry(pol, chain, bydst) {
+ if ((if_id == 0 || pol->if_id == if_id) &&
+ xfrm_migrate_selector_match(sel, &pol->selector) &&
+ pol->type == type) {
+ ret = pol;
+ priority = ret->priority;
+ break;
+ }
+ }
+ chain = &net->xfrm.policy_inexact[dir];
+ hlist_for_each_entry(pol, chain, bydst_inexact_list) {
+ if ((pol->priority >= priority) && ret)
+ break;
+
+ if ((if_id == 0 || pol->if_id == if_id) &&
+ xfrm_migrate_selector_match(sel, &pol->selector) &&
+ pol->type == type) {
+ ret = pol;
+ break;
+ }
+ }
+
+ xfrm_pol_hold(ret);
+
+ spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
+
+ return ret;
+}
+
+static int migrate_tmpl_match(const struct xfrm_migrate *m, const struct xfrm_tmpl *t)
+{
+ int match = 0;
+
+ if (t->mode == m->mode && t->id.proto == m->proto &&
+ (m->reqid == 0 || t->reqid == m->reqid)) {
+ switch (t->mode) {
+ case XFRM_MODE_TUNNEL:
+ case XFRM_MODE_BEET:
+ if (xfrm_addr_equal(&t->id.daddr, &m->old_daddr,
+ m->old_family) &&
+ xfrm_addr_equal(&t->saddr, &m->old_saddr,
+ m->old_family)) {
+ match = 1;
+ }
+ break;
+ case XFRM_MODE_TRANSPORT:
+ /* in case of transport mode, template does not store
+ any IP addresses, hence we just compare mode and
+ protocol */
+ match = 1;
+ break;
+ default:
+ break;
+ }
+ }
+ return match;
+}
+
+/* update endpoint address(es) of template(s) */
+static int xfrm_policy_migrate(struct xfrm_policy *pol,
+ struct xfrm_migrate *m, int num_migrate)
+{
+ struct xfrm_migrate *mp;
+ int i, j, n = 0;
+
+ write_lock_bh(&pol->lock);
+ if (unlikely(pol->walk.dead)) {
+ /* target policy has been deleted */
+ write_unlock_bh(&pol->lock);
+ return -ENOENT;
+ }
+
+ for (i = 0; i < pol->xfrm_nr; i++) {
+ for (j = 0, mp = m; j < num_migrate; j++, mp++) {
+ if (!migrate_tmpl_match(mp, &pol->xfrm_vec[i]))
+ continue;
+ n++;
+ if (pol->xfrm_vec[i].mode != XFRM_MODE_TUNNEL &&
+ pol->xfrm_vec[i].mode != XFRM_MODE_BEET)
+ continue;
+ /* update endpoints */
+ memcpy(&pol->xfrm_vec[i].id.daddr, &mp->new_daddr,
+ sizeof(pol->xfrm_vec[i].id.daddr));
+ memcpy(&pol->xfrm_vec[i].saddr, &mp->new_saddr,
+ sizeof(pol->xfrm_vec[i].saddr));
+ pol->xfrm_vec[i].encap_family = mp->new_family;
+ /* flush bundles */
+ atomic_inc(&pol->genid);
+ }
+ }
+
+ write_unlock_bh(&pol->lock);
+
+ if (!n)
+ return -ENODATA;
+
+ return 0;
+}
+
+static int xfrm_migrate_check(const struct xfrm_migrate *m, int num_migrate)
+{
+ int i, j;
+
+ if (num_migrate < 1 || num_migrate > XFRM_MAX_DEPTH)
+ return -EINVAL;
+
+ for (i = 0; i < num_migrate; i++) {
+ if (xfrm_addr_any(&m[i].new_daddr, m[i].new_family) ||
+ xfrm_addr_any(&m[i].new_saddr, m[i].new_family))
+ return -EINVAL;
+
+ /* check if there is any duplicated entry */
+ for (j = i + 1; j < num_migrate; j++) {
+ if (!memcmp(&m[i].old_daddr, &m[j].old_daddr,
+ sizeof(m[i].old_daddr)) &&
+ !memcmp(&m[i].old_saddr, &m[j].old_saddr,
+ sizeof(m[i].old_saddr)) &&
+ m[i].proto == m[j].proto &&
+ m[i].mode == m[j].mode &&
+ m[i].reqid == m[j].reqid &&
+ m[i].old_family == m[j].old_family)
+ return -EINVAL;
+ }
+ }
+
+ return 0;
+}
+
+int xfrm_migrate(const struct xfrm_selector *sel, u8 dir, u8 type,
+ struct xfrm_migrate *m, int num_migrate,
+ struct xfrm_kmaddress *k, struct net *net,
+ struct xfrm_encap_tmpl *encap, u32 if_id)
+{
+ int i, err, nx_cur = 0, nx_new = 0;
+ struct xfrm_policy *pol = NULL;
+ struct xfrm_state *x, *xc;
+ struct xfrm_state *x_cur[XFRM_MAX_DEPTH];
+ struct xfrm_state *x_new[XFRM_MAX_DEPTH];
+ struct xfrm_migrate *mp;
+
+ /* Stage 0 - sanity checks */
+ if ((err = xfrm_migrate_check(m, num_migrate)) < 0)
+ goto out;
+
+ if (dir >= XFRM_POLICY_MAX) {
+ err = -EINVAL;
+ goto out;
+ }
+
+ /* Stage 1 - find policy */
+ if ((pol = xfrm_migrate_policy_find(sel, dir, type, net, if_id)) == NULL) {
+ err = -ENOENT;
+ goto out;
+ }
+
+ /* Stage 2 - find and update state(s) */
+ for (i = 0, mp = m; i < num_migrate; i++, mp++) {
+ if ((x = xfrm_migrate_state_find(mp, net, if_id))) {
+ x_cur[nx_cur] = x;
+ nx_cur++;
+ xc = xfrm_state_migrate(x, mp, encap);
+ if (xc) {
+ x_new[nx_new] = xc;
+ nx_new++;
+ } else {
+ err = -ENODATA;
+ goto restore_state;
+ }
+ }
+ }
+
+ /* Stage 3 - update policy */
+ if ((err = xfrm_policy_migrate(pol, m, num_migrate)) < 0)
+ goto restore_state;
+
+ /* Stage 4 - delete old state(s) */
+ if (nx_cur) {
+ xfrm_states_put(x_cur, nx_cur);
+ xfrm_states_delete(x_cur, nx_cur);
+ }
+
+ /* Stage 5 - announce */
+ km_migrate(sel, dir, type, m, num_migrate, k, encap);
+
+ xfrm_pol_put(pol);
+
+ return 0;
+out:
+ return err;
+
+restore_state:
+ if (pol)
+ xfrm_pol_put(pol);
+ if (nx_cur)
+ xfrm_states_put(x_cur, nx_cur);
+ if (nx_new)
+ xfrm_states_delete(x_new, nx_new);
+
+ return err;
+}
+EXPORT_SYMBOL(xfrm_migrate);
+#endif
diff --git a/net/xfrm/xfrm_proc.c b/net/xfrm/xfrm_proc.c
new file mode 100644
index 000000000..fee9b5cf3
--- /dev/null
+++ b/net/xfrm/xfrm_proc.c
@@ -0,0 +1,75 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * xfrm_proc.c
+ *
+ * Copyright (C)2006-2007 USAGI/WIDE Project
+ *
+ * Authors: Masahide NAKAMURA <nakam@linux-ipv6.org>
+ */
+#include <linux/proc_fs.h>
+#include <linux/seq_file.h>
+#include <linux/export.h>
+#include <net/snmp.h>
+#include <net/xfrm.h>
+
+static const struct snmp_mib xfrm_mib_list[] = {
+ SNMP_MIB_ITEM("XfrmInError", LINUX_MIB_XFRMINERROR),
+ SNMP_MIB_ITEM("XfrmInBufferError", LINUX_MIB_XFRMINBUFFERERROR),
+ SNMP_MIB_ITEM("XfrmInHdrError", LINUX_MIB_XFRMINHDRERROR),
+ SNMP_MIB_ITEM("XfrmInNoStates", LINUX_MIB_XFRMINNOSTATES),
+ SNMP_MIB_ITEM("XfrmInStateProtoError", LINUX_MIB_XFRMINSTATEPROTOERROR),
+ SNMP_MIB_ITEM("XfrmInStateModeError", LINUX_MIB_XFRMINSTATEMODEERROR),
+ SNMP_MIB_ITEM("XfrmInStateSeqError", LINUX_MIB_XFRMINSTATESEQERROR),
+ SNMP_MIB_ITEM("XfrmInStateExpired", LINUX_MIB_XFRMINSTATEEXPIRED),
+ SNMP_MIB_ITEM("XfrmInStateMismatch", LINUX_MIB_XFRMINSTATEMISMATCH),
+ SNMP_MIB_ITEM("XfrmInStateInvalid", LINUX_MIB_XFRMINSTATEINVALID),
+ SNMP_MIB_ITEM("XfrmInTmplMismatch", LINUX_MIB_XFRMINTMPLMISMATCH),
+ SNMP_MIB_ITEM("XfrmInNoPols", LINUX_MIB_XFRMINNOPOLS),
+ SNMP_MIB_ITEM("XfrmInPolBlock", LINUX_MIB_XFRMINPOLBLOCK),
+ SNMP_MIB_ITEM("XfrmInPolError", LINUX_MIB_XFRMINPOLERROR),
+ SNMP_MIB_ITEM("XfrmOutError", LINUX_MIB_XFRMOUTERROR),
+ SNMP_MIB_ITEM("XfrmOutBundleGenError", LINUX_MIB_XFRMOUTBUNDLEGENERROR),
+ SNMP_MIB_ITEM("XfrmOutBundleCheckError", LINUX_MIB_XFRMOUTBUNDLECHECKERROR),
+ SNMP_MIB_ITEM("XfrmOutNoStates", LINUX_MIB_XFRMOUTNOSTATES),
+ SNMP_MIB_ITEM("XfrmOutStateProtoError", LINUX_MIB_XFRMOUTSTATEPROTOERROR),
+ SNMP_MIB_ITEM("XfrmOutStateModeError", LINUX_MIB_XFRMOUTSTATEMODEERROR),
+ SNMP_MIB_ITEM("XfrmOutStateSeqError", LINUX_MIB_XFRMOUTSTATESEQERROR),
+ SNMP_MIB_ITEM("XfrmOutStateExpired", LINUX_MIB_XFRMOUTSTATEEXPIRED),
+ SNMP_MIB_ITEM("XfrmOutPolBlock", LINUX_MIB_XFRMOUTPOLBLOCK),
+ SNMP_MIB_ITEM("XfrmOutPolDead", LINUX_MIB_XFRMOUTPOLDEAD),
+ SNMP_MIB_ITEM("XfrmOutPolError", LINUX_MIB_XFRMOUTPOLERROR),
+ SNMP_MIB_ITEM("XfrmFwdHdrError", LINUX_MIB_XFRMFWDHDRERROR),
+ SNMP_MIB_ITEM("XfrmOutStateInvalid", LINUX_MIB_XFRMOUTSTATEINVALID),
+ SNMP_MIB_ITEM("XfrmAcquireError", LINUX_MIB_XFRMACQUIREERROR),
+ SNMP_MIB_SENTINEL
+};
+
+static int xfrm_statistics_seq_show(struct seq_file *seq, void *v)
+{
+ unsigned long buff[LINUX_MIB_XFRMMAX];
+ struct net *net = seq->private;
+ int i;
+
+ memset(buff, 0, sizeof(unsigned long) * LINUX_MIB_XFRMMAX);
+
+ snmp_get_cpu_field_batch(buff, xfrm_mib_list,
+ net->mib.xfrm_statistics);
+ for (i = 0; xfrm_mib_list[i].name; i++)
+ seq_printf(seq, "%-24s\t%lu\n", xfrm_mib_list[i].name,
+ buff[i]);
+
+ return 0;
+}
+
+int __net_init xfrm_proc_init(struct net *net)
+{
+ if (!proc_create_net_single("xfrm_stat", 0444, net->proc_net,
+ xfrm_statistics_seq_show, NULL))
+ return -ENOMEM;
+ return 0;
+}
+
+void xfrm_proc_fini(struct net *net)
+{
+ remove_proc_entry("xfrm_stat", net->proc_net);
+}
diff --git a/net/xfrm/xfrm_replay.c b/net/xfrm/xfrm_replay.c
new file mode 100644
index 000000000..ce56d659c
--- /dev/null
+++ b/net/xfrm/xfrm_replay.c
@@ -0,0 +1,795 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * xfrm_replay.c - xfrm replay detection, derived from xfrm_state.c.
+ *
+ * Copyright (C) 2010 secunet Security Networks AG
+ * Copyright (C) 2010 Steffen Klassert <steffen.klassert@secunet.com>
+ */
+
+#include <linux/export.h>
+#include <net/xfrm.h>
+
+u32 xfrm_replay_seqhi(struct xfrm_state *x, __be32 net_seq)
+{
+ u32 seq, seq_hi, bottom;
+ struct xfrm_replay_state_esn *replay_esn = x->replay_esn;
+
+ if (!(x->props.flags & XFRM_STATE_ESN))
+ return 0;
+
+ seq = ntohl(net_seq);
+ seq_hi = replay_esn->seq_hi;
+ bottom = replay_esn->seq - replay_esn->replay_window + 1;
+
+ if (likely(replay_esn->seq >= replay_esn->replay_window - 1)) {
+ /* A. same subspace */
+ if (unlikely(seq < bottom))
+ seq_hi++;
+ } else {
+ /* B. window spans two subspaces */
+ if (unlikely(seq >= bottom))
+ seq_hi--;
+ }
+
+ return seq_hi;
+}
+EXPORT_SYMBOL(xfrm_replay_seqhi);
+
+static void xfrm_replay_notify_bmp(struct xfrm_state *x, int event);
+static void xfrm_replay_notify_esn(struct xfrm_state *x, int event);
+
+void xfrm_replay_notify(struct xfrm_state *x, int event)
+{
+ struct km_event c;
+ /* we send notify messages in case
+ * 1. we updated on of the sequence numbers, and the seqno difference
+ * is at least x->replay_maxdiff, in this case we also update the
+ * timeout of our timer function
+ * 2. if x->replay_maxage has elapsed since last update,
+ * and there were changes
+ *
+ * The state structure must be locked!
+ */
+
+ switch (x->repl_mode) {
+ case XFRM_REPLAY_MODE_LEGACY:
+ break;
+ case XFRM_REPLAY_MODE_BMP:
+ xfrm_replay_notify_bmp(x, event);
+ return;
+ case XFRM_REPLAY_MODE_ESN:
+ xfrm_replay_notify_esn(x, event);
+ return;
+ }
+
+ switch (event) {
+ case XFRM_REPLAY_UPDATE:
+ if (!x->replay_maxdiff ||
+ ((x->replay.seq - x->preplay.seq < x->replay_maxdiff) &&
+ (x->replay.oseq - x->preplay.oseq < x->replay_maxdiff))) {
+ if (x->xflags & XFRM_TIME_DEFER)
+ event = XFRM_REPLAY_TIMEOUT;
+ else
+ return;
+ }
+
+ break;
+
+ case XFRM_REPLAY_TIMEOUT:
+ if (memcmp(&x->replay, &x->preplay,
+ sizeof(struct xfrm_replay_state)) == 0) {
+ x->xflags |= XFRM_TIME_DEFER;
+ return;
+ }
+
+ break;
+ }
+
+ memcpy(&x->preplay, &x->replay, sizeof(struct xfrm_replay_state));
+ c.event = XFRM_MSG_NEWAE;
+ c.data.aevent = event;
+ km_state_notify(x, &c);
+
+ if (x->replay_maxage &&
+ !mod_timer(&x->rtimer, jiffies + x->replay_maxage))
+ x->xflags &= ~XFRM_TIME_DEFER;
+}
+
+static int __xfrm_replay_overflow(struct xfrm_state *x, struct sk_buff *skb)
+{
+ int err = 0;
+ struct net *net = xs_net(x);
+
+ if (x->type->flags & XFRM_TYPE_REPLAY_PROT) {
+ XFRM_SKB_CB(skb)->seq.output.low = ++x->replay.oseq;
+ XFRM_SKB_CB(skb)->seq.output.hi = 0;
+ if (unlikely(x->replay.oseq == 0) &&
+ !(x->props.extra_flags & XFRM_SA_XFLAG_OSEQ_MAY_WRAP)) {
+ x->replay.oseq--;
+ xfrm_audit_state_replay_overflow(x, skb);
+ err = -EOVERFLOW;
+
+ return err;
+ }
+ if (xfrm_aevent_is_on(net))
+ xfrm_replay_notify(x, XFRM_REPLAY_UPDATE);
+ }
+
+ return err;
+}
+
+static int xfrm_replay_check_legacy(struct xfrm_state *x,
+ struct sk_buff *skb, __be32 net_seq)
+{
+ u32 diff;
+ u32 seq = ntohl(net_seq);
+
+ if (!x->props.replay_window)
+ return 0;
+
+ if (unlikely(seq == 0))
+ goto err;
+
+ if (likely(seq > x->replay.seq))
+ return 0;
+
+ diff = x->replay.seq - seq;
+ if (diff >= x->props.replay_window) {
+ x->stats.replay_window++;
+ goto err;
+ }
+
+ if (x->replay.bitmap & (1U << diff)) {
+ x->stats.replay++;
+ goto err;
+ }
+ return 0;
+
+err:
+ xfrm_audit_state_replay(x, skb, net_seq);
+ return -EINVAL;
+}
+
+static void xfrm_replay_advance_bmp(struct xfrm_state *x, __be32 net_seq);
+static void xfrm_replay_advance_esn(struct xfrm_state *x, __be32 net_seq);
+
+void xfrm_replay_advance(struct xfrm_state *x, __be32 net_seq)
+{
+ u32 diff, seq;
+
+ switch (x->repl_mode) {
+ case XFRM_REPLAY_MODE_LEGACY:
+ break;
+ case XFRM_REPLAY_MODE_BMP:
+ return xfrm_replay_advance_bmp(x, net_seq);
+ case XFRM_REPLAY_MODE_ESN:
+ return xfrm_replay_advance_esn(x, net_seq);
+ }
+
+ if (!x->props.replay_window)
+ return;
+
+ seq = ntohl(net_seq);
+ if (seq > x->replay.seq) {
+ diff = seq - x->replay.seq;
+ if (diff < x->props.replay_window)
+ x->replay.bitmap = ((x->replay.bitmap) << diff) | 1;
+ else
+ x->replay.bitmap = 1;
+ x->replay.seq = seq;
+ } else {
+ diff = x->replay.seq - seq;
+ x->replay.bitmap |= (1U << diff);
+ }
+
+ if (xfrm_aevent_is_on(xs_net(x)))
+ xfrm_replay_notify(x, XFRM_REPLAY_UPDATE);
+}
+
+static int xfrm_replay_overflow_bmp(struct xfrm_state *x, struct sk_buff *skb)
+{
+ int err = 0;
+ struct xfrm_replay_state_esn *replay_esn = x->replay_esn;
+ struct net *net = xs_net(x);
+
+ if (x->type->flags & XFRM_TYPE_REPLAY_PROT) {
+ XFRM_SKB_CB(skb)->seq.output.low = ++replay_esn->oseq;
+ XFRM_SKB_CB(skb)->seq.output.hi = 0;
+ if (unlikely(replay_esn->oseq == 0) &&
+ !(x->props.extra_flags & XFRM_SA_XFLAG_OSEQ_MAY_WRAP)) {
+ replay_esn->oseq--;
+ xfrm_audit_state_replay_overflow(x, skb);
+ err = -EOVERFLOW;
+
+ return err;
+ }
+ if (xfrm_aevent_is_on(net))
+ xfrm_replay_notify(x, XFRM_REPLAY_UPDATE);
+ }
+
+ return err;
+}
+
+static int xfrm_replay_check_bmp(struct xfrm_state *x,
+ struct sk_buff *skb, __be32 net_seq)
+{
+ unsigned int bitnr, nr;
+ struct xfrm_replay_state_esn *replay_esn = x->replay_esn;
+ u32 pos;
+ u32 seq = ntohl(net_seq);
+ u32 diff = replay_esn->seq - seq;
+
+ if (!replay_esn->replay_window)
+ return 0;
+
+ if (unlikely(seq == 0))
+ goto err;
+
+ if (likely(seq > replay_esn->seq))
+ return 0;
+
+ if (diff >= replay_esn->replay_window) {
+ x->stats.replay_window++;
+ goto err;
+ }
+
+ pos = (replay_esn->seq - 1) % replay_esn->replay_window;
+
+ if (pos >= diff)
+ bitnr = (pos - diff) % replay_esn->replay_window;
+ else
+ bitnr = replay_esn->replay_window - (diff - pos);
+
+ nr = bitnr >> 5;
+ bitnr = bitnr & 0x1F;
+ if (replay_esn->bmp[nr] & (1U << bitnr))
+ goto err_replay;
+
+ return 0;
+
+err_replay:
+ x->stats.replay++;
+err:
+ xfrm_audit_state_replay(x, skb, net_seq);
+ return -EINVAL;
+}
+
+static void xfrm_replay_advance_bmp(struct xfrm_state *x, __be32 net_seq)
+{
+ unsigned int bitnr, nr, i;
+ u32 diff;
+ struct xfrm_replay_state_esn *replay_esn = x->replay_esn;
+ u32 seq = ntohl(net_seq);
+ u32 pos;
+
+ if (!replay_esn->replay_window)
+ return;
+
+ pos = (replay_esn->seq - 1) % replay_esn->replay_window;
+
+ if (seq > replay_esn->seq) {
+ diff = seq - replay_esn->seq;
+
+ if (diff < replay_esn->replay_window) {
+ for (i = 1; i < diff; i++) {
+ bitnr = (pos + i) % replay_esn->replay_window;
+ nr = bitnr >> 5;
+ bitnr = bitnr & 0x1F;
+ replay_esn->bmp[nr] &= ~(1U << bitnr);
+ }
+ } else {
+ nr = (replay_esn->replay_window - 1) >> 5;
+ for (i = 0; i <= nr; i++)
+ replay_esn->bmp[i] = 0;
+ }
+
+ bitnr = (pos + diff) % replay_esn->replay_window;
+ replay_esn->seq = seq;
+ } else {
+ diff = replay_esn->seq - seq;
+
+ if (pos >= diff)
+ bitnr = (pos - diff) % replay_esn->replay_window;
+ else
+ bitnr = replay_esn->replay_window - (diff - pos);
+ }
+
+ nr = bitnr >> 5;
+ bitnr = bitnr & 0x1F;
+ replay_esn->bmp[nr] |= (1U << bitnr);
+
+ if (xfrm_aevent_is_on(xs_net(x)))
+ xfrm_replay_notify(x, XFRM_REPLAY_UPDATE);
+}
+
+static void xfrm_replay_notify_bmp(struct xfrm_state *x, int event)
+{
+ struct km_event c;
+ struct xfrm_replay_state_esn *replay_esn = x->replay_esn;
+ struct xfrm_replay_state_esn *preplay_esn = x->preplay_esn;
+
+ /* we send notify messages in case
+ * 1. we updated on of the sequence numbers, and the seqno difference
+ * is at least x->replay_maxdiff, in this case we also update the
+ * timeout of our timer function
+ * 2. if x->replay_maxage has elapsed since last update,
+ * and there were changes
+ *
+ * The state structure must be locked!
+ */
+
+ switch (event) {
+ case XFRM_REPLAY_UPDATE:
+ if (!x->replay_maxdiff ||
+ ((replay_esn->seq - preplay_esn->seq < x->replay_maxdiff) &&
+ (replay_esn->oseq - preplay_esn->oseq
+ < x->replay_maxdiff))) {
+ if (x->xflags & XFRM_TIME_DEFER)
+ event = XFRM_REPLAY_TIMEOUT;
+ else
+ return;
+ }
+
+ break;
+
+ case XFRM_REPLAY_TIMEOUT:
+ if (memcmp(x->replay_esn, x->preplay_esn,
+ xfrm_replay_state_esn_len(replay_esn)) == 0) {
+ x->xflags |= XFRM_TIME_DEFER;
+ return;
+ }
+
+ break;
+ }
+
+ memcpy(x->preplay_esn, x->replay_esn,
+ xfrm_replay_state_esn_len(replay_esn));
+ c.event = XFRM_MSG_NEWAE;
+ c.data.aevent = event;
+ km_state_notify(x, &c);
+
+ if (x->replay_maxage &&
+ !mod_timer(&x->rtimer, jiffies + x->replay_maxage))
+ x->xflags &= ~XFRM_TIME_DEFER;
+}
+
+static void xfrm_replay_notify_esn(struct xfrm_state *x, int event)
+{
+ u32 seq_diff, oseq_diff;
+ struct km_event c;
+ struct xfrm_replay_state_esn *replay_esn = x->replay_esn;
+ struct xfrm_replay_state_esn *preplay_esn = x->preplay_esn;
+
+ /* we send notify messages in case
+ * 1. we updated on of the sequence numbers, and the seqno difference
+ * is at least x->replay_maxdiff, in this case we also update the
+ * timeout of our timer function
+ * 2. if x->replay_maxage has elapsed since last update,
+ * and there were changes
+ *
+ * The state structure must be locked!
+ */
+
+ switch (event) {
+ case XFRM_REPLAY_UPDATE:
+ if (x->replay_maxdiff) {
+ if (replay_esn->seq_hi == preplay_esn->seq_hi)
+ seq_diff = replay_esn->seq - preplay_esn->seq;
+ else
+ seq_diff = ~preplay_esn->seq + replay_esn->seq
+ + 1;
+
+ if (replay_esn->oseq_hi == preplay_esn->oseq_hi)
+ oseq_diff = replay_esn->oseq
+ - preplay_esn->oseq;
+ else
+ oseq_diff = ~preplay_esn->oseq
+ + replay_esn->oseq + 1;
+
+ if (seq_diff >= x->replay_maxdiff ||
+ oseq_diff >= x->replay_maxdiff)
+ break;
+ }
+
+ if (x->xflags & XFRM_TIME_DEFER)
+ event = XFRM_REPLAY_TIMEOUT;
+ else
+ return;
+
+ break;
+
+ case XFRM_REPLAY_TIMEOUT:
+ if (memcmp(x->replay_esn, x->preplay_esn,
+ xfrm_replay_state_esn_len(replay_esn)) == 0) {
+ x->xflags |= XFRM_TIME_DEFER;
+ return;
+ }
+
+ break;
+ }
+
+ memcpy(x->preplay_esn, x->replay_esn,
+ xfrm_replay_state_esn_len(replay_esn));
+ c.event = XFRM_MSG_NEWAE;
+ c.data.aevent = event;
+ km_state_notify(x, &c);
+
+ if (x->replay_maxage &&
+ !mod_timer(&x->rtimer, jiffies + x->replay_maxage))
+ x->xflags &= ~XFRM_TIME_DEFER;
+}
+
+static int xfrm_replay_overflow_esn(struct xfrm_state *x, struct sk_buff *skb)
+{
+ int err = 0;
+ struct xfrm_replay_state_esn *replay_esn = x->replay_esn;
+ struct net *net = xs_net(x);
+
+ if (x->type->flags & XFRM_TYPE_REPLAY_PROT) {
+ XFRM_SKB_CB(skb)->seq.output.low = ++replay_esn->oseq;
+ XFRM_SKB_CB(skb)->seq.output.hi = replay_esn->oseq_hi;
+
+ if (unlikely(replay_esn->oseq == 0)) {
+ XFRM_SKB_CB(skb)->seq.output.hi = ++replay_esn->oseq_hi;
+
+ if (replay_esn->oseq_hi == 0) {
+ replay_esn->oseq--;
+ replay_esn->oseq_hi--;
+ xfrm_audit_state_replay_overflow(x, skb);
+ err = -EOVERFLOW;
+
+ return err;
+ }
+ }
+ if (xfrm_aevent_is_on(net))
+ xfrm_replay_notify(x, XFRM_REPLAY_UPDATE);
+ }
+
+ return err;
+}
+
+static int xfrm_replay_check_esn(struct xfrm_state *x,
+ struct sk_buff *skb, __be32 net_seq)
+{
+ unsigned int bitnr, nr;
+ u32 diff;
+ struct xfrm_replay_state_esn *replay_esn = x->replay_esn;
+ u32 pos;
+ u32 seq = ntohl(net_seq);
+ u32 wsize = replay_esn->replay_window;
+ u32 top = replay_esn->seq;
+ u32 bottom = top - wsize + 1;
+
+ if (!wsize)
+ return 0;
+
+ if (unlikely(seq == 0 && replay_esn->seq_hi == 0 &&
+ (replay_esn->seq < replay_esn->replay_window - 1)))
+ goto err;
+
+ diff = top - seq;
+
+ if (likely(top >= wsize - 1)) {
+ /* A. same subspace */
+ if (likely(seq > top) || seq < bottom)
+ return 0;
+ } else {
+ /* B. window spans two subspaces */
+ if (likely(seq > top && seq < bottom))
+ return 0;
+ if (seq >= bottom)
+ diff = ~seq + top + 1;
+ }
+
+ if (diff >= replay_esn->replay_window) {
+ x->stats.replay_window++;
+ goto err;
+ }
+
+ pos = (replay_esn->seq - 1) % replay_esn->replay_window;
+
+ if (pos >= diff)
+ bitnr = (pos - diff) % replay_esn->replay_window;
+ else
+ bitnr = replay_esn->replay_window - (diff - pos);
+
+ nr = bitnr >> 5;
+ bitnr = bitnr & 0x1F;
+ if (replay_esn->bmp[nr] & (1U << bitnr))
+ goto err_replay;
+
+ return 0;
+
+err_replay:
+ x->stats.replay++;
+err:
+ xfrm_audit_state_replay(x, skb, net_seq);
+ return -EINVAL;
+}
+
+int xfrm_replay_check(struct xfrm_state *x,
+ struct sk_buff *skb, __be32 net_seq)
+{
+ switch (x->repl_mode) {
+ case XFRM_REPLAY_MODE_LEGACY:
+ break;
+ case XFRM_REPLAY_MODE_BMP:
+ return xfrm_replay_check_bmp(x, skb, net_seq);
+ case XFRM_REPLAY_MODE_ESN:
+ return xfrm_replay_check_esn(x, skb, net_seq);
+ }
+
+ return xfrm_replay_check_legacy(x, skb, net_seq);
+}
+
+static int xfrm_replay_recheck_esn(struct xfrm_state *x,
+ struct sk_buff *skb, __be32 net_seq)
+{
+ if (unlikely(XFRM_SKB_CB(skb)->seq.input.hi !=
+ htonl(xfrm_replay_seqhi(x, net_seq)))) {
+ x->stats.replay_window++;
+ return -EINVAL;
+ }
+
+ return xfrm_replay_check_esn(x, skb, net_seq);
+}
+
+int xfrm_replay_recheck(struct xfrm_state *x,
+ struct sk_buff *skb, __be32 net_seq)
+{
+ switch (x->repl_mode) {
+ case XFRM_REPLAY_MODE_LEGACY:
+ break;
+ case XFRM_REPLAY_MODE_BMP:
+ /* no special recheck treatment */
+ return xfrm_replay_check_bmp(x, skb, net_seq);
+ case XFRM_REPLAY_MODE_ESN:
+ return xfrm_replay_recheck_esn(x, skb, net_seq);
+ }
+
+ return xfrm_replay_check_legacy(x, skb, net_seq);
+}
+
+static void xfrm_replay_advance_esn(struct xfrm_state *x, __be32 net_seq)
+{
+ unsigned int bitnr, nr, i;
+ int wrap;
+ u32 diff, pos, seq, seq_hi;
+ struct xfrm_replay_state_esn *replay_esn = x->replay_esn;
+
+ if (!replay_esn->replay_window)
+ return;
+
+ seq = ntohl(net_seq);
+ pos = (replay_esn->seq - 1) % replay_esn->replay_window;
+ seq_hi = xfrm_replay_seqhi(x, net_seq);
+ wrap = seq_hi - replay_esn->seq_hi;
+
+ if ((!wrap && seq > replay_esn->seq) || wrap > 0) {
+ if (likely(!wrap))
+ diff = seq - replay_esn->seq;
+ else
+ diff = ~replay_esn->seq + seq + 1;
+
+ if (diff < replay_esn->replay_window) {
+ for (i = 1; i < diff; i++) {
+ bitnr = (pos + i) % replay_esn->replay_window;
+ nr = bitnr >> 5;
+ bitnr = bitnr & 0x1F;
+ replay_esn->bmp[nr] &= ~(1U << bitnr);
+ }
+ } else {
+ nr = (replay_esn->replay_window - 1) >> 5;
+ for (i = 0; i <= nr; i++)
+ replay_esn->bmp[i] = 0;
+ }
+
+ bitnr = (pos + diff) % replay_esn->replay_window;
+ replay_esn->seq = seq;
+
+ if (unlikely(wrap > 0))
+ replay_esn->seq_hi++;
+ } else {
+ diff = replay_esn->seq - seq;
+
+ if (pos >= diff)
+ bitnr = (pos - diff) % replay_esn->replay_window;
+ else
+ bitnr = replay_esn->replay_window - (diff - pos);
+ }
+
+ xfrm_dev_state_advance_esn(x);
+
+ nr = bitnr >> 5;
+ bitnr = bitnr & 0x1F;
+ replay_esn->bmp[nr] |= (1U << bitnr);
+
+ if (xfrm_aevent_is_on(xs_net(x)))
+ xfrm_replay_notify(x, XFRM_REPLAY_UPDATE);
+}
+
+#ifdef CONFIG_XFRM_OFFLOAD
+static int xfrm_replay_overflow_offload(struct xfrm_state *x, struct sk_buff *skb)
+{
+ int err = 0;
+ struct net *net = xs_net(x);
+ struct xfrm_offload *xo = xfrm_offload(skb);
+ __u32 oseq = x->replay.oseq;
+
+ if (!xo)
+ return __xfrm_replay_overflow(x, skb);
+
+ if (x->type->flags & XFRM_TYPE_REPLAY_PROT) {
+ if (!skb_is_gso(skb)) {
+ XFRM_SKB_CB(skb)->seq.output.low = ++oseq;
+ xo->seq.low = oseq;
+ } else {
+ XFRM_SKB_CB(skb)->seq.output.low = oseq + 1;
+ xo->seq.low = oseq + 1;
+ oseq += skb_shinfo(skb)->gso_segs;
+ }
+
+ XFRM_SKB_CB(skb)->seq.output.hi = 0;
+ xo->seq.hi = 0;
+ if (unlikely(oseq < x->replay.oseq) &&
+ !(x->props.extra_flags & XFRM_SA_XFLAG_OSEQ_MAY_WRAP)) {
+ xfrm_audit_state_replay_overflow(x, skb);
+ err = -EOVERFLOW;
+
+ return err;
+ }
+
+ x->replay.oseq = oseq;
+
+ if (xfrm_aevent_is_on(net))
+ xfrm_replay_notify(x, XFRM_REPLAY_UPDATE);
+ }
+
+ return err;
+}
+
+static int xfrm_replay_overflow_offload_bmp(struct xfrm_state *x, struct sk_buff *skb)
+{
+ int err = 0;
+ struct xfrm_offload *xo = xfrm_offload(skb);
+ struct xfrm_replay_state_esn *replay_esn = x->replay_esn;
+ struct net *net = xs_net(x);
+ __u32 oseq = replay_esn->oseq;
+
+ if (!xo)
+ return xfrm_replay_overflow_bmp(x, skb);
+
+ if (x->type->flags & XFRM_TYPE_REPLAY_PROT) {
+ if (!skb_is_gso(skb)) {
+ XFRM_SKB_CB(skb)->seq.output.low = ++oseq;
+ xo->seq.low = oseq;
+ } else {
+ XFRM_SKB_CB(skb)->seq.output.low = oseq + 1;
+ xo->seq.low = oseq + 1;
+ oseq += skb_shinfo(skb)->gso_segs;
+ }
+
+ XFRM_SKB_CB(skb)->seq.output.hi = 0;
+ xo->seq.hi = 0;
+ if (unlikely(oseq < replay_esn->oseq) &&
+ !(x->props.extra_flags & XFRM_SA_XFLAG_OSEQ_MAY_WRAP)) {
+ xfrm_audit_state_replay_overflow(x, skb);
+ err = -EOVERFLOW;
+
+ return err;
+ } else {
+ replay_esn->oseq = oseq;
+ }
+
+ if (xfrm_aevent_is_on(net))
+ xfrm_replay_notify(x, XFRM_REPLAY_UPDATE);
+ }
+
+ return err;
+}
+
+static int xfrm_replay_overflow_offload_esn(struct xfrm_state *x, struct sk_buff *skb)
+{
+ int err = 0;
+ struct xfrm_offload *xo = xfrm_offload(skb);
+ struct xfrm_replay_state_esn *replay_esn = x->replay_esn;
+ struct net *net = xs_net(x);
+ __u32 oseq = replay_esn->oseq;
+ __u32 oseq_hi = replay_esn->oseq_hi;
+
+ if (!xo)
+ return xfrm_replay_overflow_esn(x, skb);
+
+ if (x->type->flags & XFRM_TYPE_REPLAY_PROT) {
+ if (!skb_is_gso(skb)) {
+ XFRM_SKB_CB(skb)->seq.output.low = ++oseq;
+ XFRM_SKB_CB(skb)->seq.output.hi = oseq_hi;
+ xo->seq.low = oseq;
+ xo->seq.hi = oseq_hi;
+ } else {
+ XFRM_SKB_CB(skb)->seq.output.low = oseq + 1;
+ XFRM_SKB_CB(skb)->seq.output.hi = oseq_hi;
+ xo->seq.low = oseq + 1;
+ xo->seq.hi = oseq_hi;
+ oseq += skb_shinfo(skb)->gso_segs;
+ }
+
+ if (unlikely(xo->seq.low < replay_esn->oseq)) {
+ XFRM_SKB_CB(skb)->seq.output.hi = ++oseq_hi;
+ xo->seq.hi = oseq_hi;
+ replay_esn->oseq_hi = oseq_hi;
+ if (replay_esn->oseq_hi == 0) {
+ replay_esn->oseq--;
+ replay_esn->oseq_hi--;
+ xfrm_audit_state_replay_overflow(x, skb);
+ err = -EOVERFLOW;
+
+ return err;
+ }
+ }
+
+ replay_esn->oseq = oseq;
+
+ if (xfrm_aevent_is_on(net))
+ xfrm_replay_notify(x, XFRM_REPLAY_UPDATE);
+ }
+
+ return err;
+}
+
+int xfrm_replay_overflow(struct xfrm_state *x, struct sk_buff *skb)
+{
+ switch (x->repl_mode) {
+ case XFRM_REPLAY_MODE_LEGACY:
+ break;
+ case XFRM_REPLAY_MODE_BMP:
+ return xfrm_replay_overflow_offload_bmp(x, skb);
+ case XFRM_REPLAY_MODE_ESN:
+ return xfrm_replay_overflow_offload_esn(x, skb);
+ }
+
+ return xfrm_replay_overflow_offload(x, skb);
+}
+#else
+int xfrm_replay_overflow(struct xfrm_state *x, struct sk_buff *skb)
+{
+ switch (x->repl_mode) {
+ case XFRM_REPLAY_MODE_LEGACY:
+ break;
+ case XFRM_REPLAY_MODE_BMP:
+ return xfrm_replay_overflow_bmp(x, skb);
+ case XFRM_REPLAY_MODE_ESN:
+ return xfrm_replay_overflow_esn(x, skb);
+ }
+
+ return __xfrm_replay_overflow(x, skb);
+}
+#endif
+
+int xfrm_init_replay(struct xfrm_state *x, struct netlink_ext_ack *extack)
+{
+ struct xfrm_replay_state_esn *replay_esn = x->replay_esn;
+
+ if (replay_esn) {
+ if (replay_esn->replay_window >
+ replay_esn->bmp_len * sizeof(__u32) * 8) {
+ NL_SET_ERR_MSG(extack, "ESN replay window is too large for the chosen bitmap size");
+ return -EINVAL;
+ }
+
+ if (x->props.flags & XFRM_STATE_ESN) {
+ if (replay_esn->replay_window == 0) {
+ NL_SET_ERR_MSG(extack, "ESN replay window must be > 0");
+ return -EINVAL;
+ }
+ x->repl_mode = XFRM_REPLAY_MODE_ESN;
+ } else {
+ x->repl_mode = XFRM_REPLAY_MODE_BMP;
+ }
+ } else {
+ x->repl_mode = XFRM_REPLAY_MODE_LEGACY;
+ }
+
+ return 0;
+}
+EXPORT_SYMBOL(xfrm_init_replay);
diff --git a/net/xfrm/xfrm_state.c b/net/xfrm/xfrm_state.c
new file mode 100644
index 000000000..2f4cf976b
--- /dev/null
+++ b/net/xfrm/xfrm_state.c
@@ -0,0 +1,2933 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * xfrm_state.c
+ *
+ * Changes:
+ * Mitsuru KANDA @USAGI
+ * Kazunori MIYAZAWA @USAGI
+ * Kunihiro Ishiguro <kunihiro@ipinfusion.com>
+ * IPv6 support
+ * YOSHIFUJI Hideaki @USAGI
+ * Split up af-specific functions
+ * Derek Atkins <derek@ihtfp.com>
+ * Add UDP Encapsulation
+ *
+ */
+
+#include <linux/compat.h>
+#include <linux/workqueue.h>
+#include <net/xfrm.h>
+#include <linux/pfkeyv2.h>
+#include <linux/ipsec.h>
+#include <linux/module.h>
+#include <linux/cache.h>
+#include <linux/audit.h>
+#include <linux/uaccess.h>
+#include <linux/ktime.h>
+#include <linux/slab.h>
+#include <linux/interrupt.h>
+#include <linux/kernel.h>
+
+#include <crypto/aead.h>
+
+#include "xfrm_hash.h"
+
+#define xfrm_state_deref_prot(table, net) \
+ rcu_dereference_protected((table), lockdep_is_held(&(net)->xfrm.xfrm_state_lock))
+
+static void xfrm_state_gc_task(struct work_struct *work);
+
+/* Each xfrm_state may be linked to two tables:
+
+ 1. Hash table by (spi,daddr,ah/esp) to find SA by SPI. (input,ctl)
+ 2. Hash table by (daddr,family,reqid) to find what SAs exist for given
+ destination/tunnel endpoint. (output)
+ */
+
+static unsigned int xfrm_state_hashmax __read_mostly = 1 * 1024 * 1024;
+static struct kmem_cache *xfrm_state_cache __ro_after_init;
+
+static DECLARE_WORK(xfrm_state_gc_work, xfrm_state_gc_task);
+static HLIST_HEAD(xfrm_state_gc_list);
+
+static inline bool xfrm_state_hold_rcu(struct xfrm_state __rcu *x)
+{
+ return refcount_inc_not_zero(&x->refcnt);
+}
+
+static inline unsigned int xfrm_dst_hash(struct net *net,
+ const xfrm_address_t *daddr,
+ const xfrm_address_t *saddr,
+ u32 reqid,
+ unsigned short family)
+{
+ return __xfrm_dst_hash(daddr, saddr, reqid, family, net->xfrm.state_hmask);
+}
+
+static inline unsigned int xfrm_src_hash(struct net *net,
+ const xfrm_address_t *daddr,
+ const xfrm_address_t *saddr,
+ unsigned short family)
+{
+ return __xfrm_src_hash(daddr, saddr, family, net->xfrm.state_hmask);
+}
+
+static inline unsigned int
+xfrm_spi_hash(struct net *net, const xfrm_address_t *daddr,
+ __be32 spi, u8 proto, unsigned short family)
+{
+ return __xfrm_spi_hash(daddr, spi, proto, family, net->xfrm.state_hmask);
+}
+
+static unsigned int xfrm_seq_hash(struct net *net, u32 seq)
+{
+ return __xfrm_seq_hash(seq, net->xfrm.state_hmask);
+}
+
+static void xfrm_hash_transfer(struct hlist_head *list,
+ struct hlist_head *ndsttable,
+ struct hlist_head *nsrctable,
+ struct hlist_head *nspitable,
+ struct hlist_head *nseqtable,
+ unsigned int nhashmask)
+{
+ struct hlist_node *tmp;
+ struct xfrm_state *x;
+
+ hlist_for_each_entry_safe(x, tmp, list, bydst) {
+ unsigned int h;
+
+ h = __xfrm_dst_hash(&x->id.daddr, &x->props.saddr,
+ x->props.reqid, x->props.family,
+ nhashmask);
+ hlist_add_head_rcu(&x->bydst, ndsttable + h);
+
+ h = __xfrm_src_hash(&x->id.daddr, &x->props.saddr,
+ x->props.family,
+ nhashmask);
+ hlist_add_head_rcu(&x->bysrc, nsrctable + h);
+
+ if (x->id.spi) {
+ h = __xfrm_spi_hash(&x->id.daddr, x->id.spi,
+ x->id.proto, x->props.family,
+ nhashmask);
+ hlist_add_head_rcu(&x->byspi, nspitable + h);
+ }
+
+ if (x->km.seq) {
+ h = __xfrm_seq_hash(x->km.seq, nhashmask);
+ hlist_add_head_rcu(&x->byseq, nseqtable + h);
+ }
+ }
+}
+
+static unsigned long xfrm_hash_new_size(unsigned int state_hmask)
+{
+ return ((state_hmask + 1) << 1) * sizeof(struct hlist_head);
+}
+
+static void xfrm_hash_resize(struct work_struct *work)
+{
+ struct net *net = container_of(work, struct net, xfrm.state_hash_work);
+ struct hlist_head *ndst, *nsrc, *nspi, *nseq, *odst, *osrc, *ospi, *oseq;
+ unsigned long nsize, osize;
+ unsigned int nhashmask, ohashmask;
+ int i;
+
+ nsize = xfrm_hash_new_size(net->xfrm.state_hmask);
+ ndst = xfrm_hash_alloc(nsize);
+ if (!ndst)
+ return;
+ nsrc = xfrm_hash_alloc(nsize);
+ if (!nsrc) {
+ xfrm_hash_free(ndst, nsize);
+ return;
+ }
+ nspi = xfrm_hash_alloc(nsize);
+ if (!nspi) {
+ xfrm_hash_free(ndst, nsize);
+ xfrm_hash_free(nsrc, nsize);
+ return;
+ }
+ nseq = xfrm_hash_alloc(nsize);
+ if (!nseq) {
+ xfrm_hash_free(ndst, nsize);
+ xfrm_hash_free(nsrc, nsize);
+ xfrm_hash_free(nspi, nsize);
+ return;
+ }
+
+ spin_lock_bh(&net->xfrm.xfrm_state_lock);
+ write_seqcount_begin(&net->xfrm.xfrm_state_hash_generation);
+
+ nhashmask = (nsize / sizeof(struct hlist_head)) - 1U;
+ odst = xfrm_state_deref_prot(net->xfrm.state_bydst, net);
+ for (i = net->xfrm.state_hmask; i >= 0; i--)
+ xfrm_hash_transfer(odst + i, ndst, nsrc, nspi, nseq, nhashmask);
+
+ osrc = xfrm_state_deref_prot(net->xfrm.state_bysrc, net);
+ ospi = xfrm_state_deref_prot(net->xfrm.state_byspi, net);
+ oseq = xfrm_state_deref_prot(net->xfrm.state_byseq, net);
+ ohashmask = net->xfrm.state_hmask;
+
+ rcu_assign_pointer(net->xfrm.state_bydst, ndst);
+ rcu_assign_pointer(net->xfrm.state_bysrc, nsrc);
+ rcu_assign_pointer(net->xfrm.state_byspi, nspi);
+ rcu_assign_pointer(net->xfrm.state_byseq, nseq);
+ net->xfrm.state_hmask = nhashmask;
+
+ write_seqcount_end(&net->xfrm.xfrm_state_hash_generation);
+ spin_unlock_bh(&net->xfrm.xfrm_state_lock);
+
+ osize = (ohashmask + 1) * sizeof(struct hlist_head);
+
+ synchronize_rcu();
+
+ xfrm_hash_free(odst, osize);
+ xfrm_hash_free(osrc, osize);
+ xfrm_hash_free(ospi, osize);
+ xfrm_hash_free(oseq, osize);
+}
+
+static DEFINE_SPINLOCK(xfrm_state_afinfo_lock);
+static struct xfrm_state_afinfo __rcu *xfrm_state_afinfo[NPROTO];
+
+static DEFINE_SPINLOCK(xfrm_state_gc_lock);
+
+int __xfrm_state_delete(struct xfrm_state *x);
+
+int km_query(struct xfrm_state *x, struct xfrm_tmpl *t, struct xfrm_policy *pol);
+static bool km_is_alive(const struct km_event *c);
+void km_state_expired(struct xfrm_state *x, int hard, u32 portid);
+
+int xfrm_register_type(const struct xfrm_type *type, unsigned short family)
+{
+ struct xfrm_state_afinfo *afinfo = xfrm_state_get_afinfo(family);
+ int err = 0;
+
+ if (!afinfo)
+ return -EAFNOSUPPORT;
+
+#define X(afi, T, name) do { \
+ WARN_ON((afi)->type_ ## name); \
+ (afi)->type_ ## name = (T); \
+ } while (0)
+
+ switch (type->proto) {
+ case IPPROTO_COMP:
+ X(afinfo, type, comp);
+ break;
+ case IPPROTO_AH:
+ X(afinfo, type, ah);
+ break;
+ case IPPROTO_ESP:
+ X(afinfo, type, esp);
+ break;
+ case IPPROTO_IPIP:
+ X(afinfo, type, ipip);
+ break;
+ case IPPROTO_DSTOPTS:
+ X(afinfo, type, dstopts);
+ break;
+ case IPPROTO_ROUTING:
+ X(afinfo, type, routing);
+ break;
+ case IPPROTO_IPV6:
+ X(afinfo, type, ipip6);
+ break;
+ default:
+ WARN_ON(1);
+ err = -EPROTONOSUPPORT;
+ break;
+ }
+#undef X
+ rcu_read_unlock();
+ return err;
+}
+EXPORT_SYMBOL(xfrm_register_type);
+
+void xfrm_unregister_type(const struct xfrm_type *type, unsigned short family)
+{
+ struct xfrm_state_afinfo *afinfo = xfrm_state_get_afinfo(family);
+
+ if (unlikely(afinfo == NULL))
+ return;
+
+#define X(afi, T, name) do { \
+ WARN_ON((afi)->type_ ## name != (T)); \
+ (afi)->type_ ## name = NULL; \
+ } while (0)
+
+ switch (type->proto) {
+ case IPPROTO_COMP:
+ X(afinfo, type, comp);
+ break;
+ case IPPROTO_AH:
+ X(afinfo, type, ah);
+ break;
+ case IPPROTO_ESP:
+ X(afinfo, type, esp);
+ break;
+ case IPPROTO_IPIP:
+ X(afinfo, type, ipip);
+ break;
+ case IPPROTO_DSTOPTS:
+ X(afinfo, type, dstopts);
+ break;
+ case IPPROTO_ROUTING:
+ X(afinfo, type, routing);
+ break;
+ case IPPROTO_IPV6:
+ X(afinfo, type, ipip6);
+ break;
+ default:
+ WARN_ON(1);
+ break;
+ }
+#undef X
+ rcu_read_unlock();
+}
+EXPORT_SYMBOL(xfrm_unregister_type);
+
+static const struct xfrm_type *xfrm_get_type(u8 proto, unsigned short family)
+{
+ const struct xfrm_type *type = NULL;
+ struct xfrm_state_afinfo *afinfo;
+ int modload_attempted = 0;
+
+retry:
+ afinfo = xfrm_state_get_afinfo(family);
+ if (unlikely(afinfo == NULL))
+ return NULL;
+
+ switch (proto) {
+ case IPPROTO_COMP:
+ type = afinfo->type_comp;
+ break;
+ case IPPROTO_AH:
+ type = afinfo->type_ah;
+ break;
+ case IPPROTO_ESP:
+ type = afinfo->type_esp;
+ break;
+ case IPPROTO_IPIP:
+ type = afinfo->type_ipip;
+ break;
+ case IPPROTO_DSTOPTS:
+ type = afinfo->type_dstopts;
+ break;
+ case IPPROTO_ROUTING:
+ type = afinfo->type_routing;
+ break;
+ case IPPROTO_IPV6:
+ type = afinfo->type_ipip6;
+ break;
+ default:
+ break;
+ }
+
+ if (unlikely(type && !try_module_get(type->owner)))
+ type = NULL;
+
+ rcu_read_unlock();
+
+ if (!type && !modload_attempted) {
+ request_module("xfrm-type-%d-%d", family, proto);
+ modload_attempted = 1;
+ goto retry;
+ }
+
+ return type;
+}
+
+static void xfrm_put_type(const struct xfrm_type *type)
+{
+ module_put(type->owner);
+}
+
+int xfrm_register_type_offload(const struct xfrm_type_offload *type,
+ unsigned short family)
+{
+ struct xfrm_state_afinfo *afinfo = xfrm_state_get_afinfo(family);
+ int err = 0;
+
+ if (unlikely(afinfo == NULL))
+ return -EAFNOSUPPORT;
+
+ switch (type->proto) {
+ case IPPROTO_ESP:
+ WARN_ON(afinfo->type_offload_esp);
+ afinfo->type_offload_esp = type;
+ break;
+ default:
+ WARN_ON(1);
+ err = -EPROTONOSUPPORT;
+ break;
+ }
+
+ rcu_read_unlock();
+ return err;
+}
+EXPORT_SYMBOL(xfrm_register_type_offload);
+
+void xfrm_unregister_type_offload(const struct xfrm_type_offload *type,
+ unsigned short family)
+{
+ struct xfrm_state_afinfo *afinfo = xfrm_state_get_afinfo(family);
+
+ if (unlikely(afinfo == NULL))
+ return;
+
+ switch (type->proto) {
+ case IPPROTO_ESP:
+ WARN_ON(afinfo->type_offload_esp != type);
+ afinfo->type_offload_esp = NULL;
+ break;
+ default:
+ WARN_ON(1);
+ break;
+ }
+ rcu_read_unlock();
+}
+EXPORT_SYMBOL(xfrm_unregister_type_offload);
+
+static const struct xfrm_type_offload *
+xfrm_get_type_offload(u8 proto, unsigned short family, bool try_load)
+{
+ const struct xfrm_type_offload *type = NULL;
+ struct xfrm_state_afinfo *afinfo;
+
+retry:
+ afinfo = xfrm_state_get_afinfo(family);
+ if (unlikely(afinfo == NULL))
+ return NULL;
+
+ switch (proto) {
+ case IPPROTO_ESP:
+ type = afinfo->type_offload_esp;
+ break;
+ default:
+ break;
+ }
+
+ if ((type && !try_module_get(type->owner)))
+ type = NULL;
+
+ rcu_read_unlock();
+
+ if (!type && try_load) {
+ request_module("xfrm-offload-%d-%d", family, proto);
+ try_load = false;
+ goto retry;
+ }
+
+ return type;
+}
+
+static void xfrm_put_type_offload(const struct xfrm_type_offload *type)
+{
+ module_put(type->owner);
+}
+
+static const struct xfrm_mode xfrm4_mode_map[XFRM_MODE_MAX] = {
+ [XFRM_MODE_BEET] = {
+ .encap = XFRM_MODE_BEET,
+ .flags = XFRM_MODE_FLAG_TUNNEL,
+ .family = AF_INET,
+ },
+ [XFRM_MODE_TRANSPORT] = {
+ .encap = XFRM_MODE_TRANSPORT,
+ .family = AF_INET,
+ },
+ [XFRM_MODE_TUNNEL] = {
+ .encap = XFRM_MODE_TUNNEL,
+ .flags = XFRM_MODE_FLAG_TUNNEL,
+ .family = AF_INET,
+ },
+};
+
+static const struct xfrm_mode xfrm6_mode_map[XFRM_MODE_MAX] = {
+ [XFRM_MODE_BEET] = {
+ .encap = XFRM_MODE_BEET,
+ .flags = XFRM_MODE_FLAG_TUNNEL,
+ .family = AF_INET6,
+ },
+ [XFRM_MODE_ROUTEOPTIMIZATION] = {
+ .encap = XFRM_MODE_ROUTEOPTIMIZATION,
+ .family = AF_INET6,
+ },
+ [XFRM_MODE_TRANSPORT] = {
+ .encap = XFRM_MODE_TRANSPORT,
+ .family = AF_INET6,
+ },
+ [XFRM_MODE_TUNNEL] = {
+ .encap = XFRM_MODE_TUNNEL,
+ .flags = XFRM_MODE_FLAG_TUNNEL,
+ .family = AF_INET6,
+ },
+};
+
+static const struct xfrm_mode *xfrm_get_mode(unsigned int encap, int family)
+{
+ const struct xfrm_mode *mode;
+
+ if (unlikely(encap >= XFRM_MODE_MAX))
+ return NULL;
+
+ switch (family) {
+ case AF_INET:
+ mode = &xfrm4_mode_map[encap];
+ if (mode->family == family)
+ return mode;
+ break;
+ case AF_INET6:
+ mode = &xfrm6_mode_map[encap];
+ if (mode->family == family)
+ return mode;
+ break;
+ default:
+ break;
+ }
+
+ return NULL;
+}
+
+void xfrm_state_free(struct xfrm_state *x)
+{
+ kmem_cache_free(xfrm_state_cache, x);
+}
+EXPORT_SYMBOL(xfrm_state_free);
+
+static void ___xfrm_state_destroy(struct xfrm_state *x)
+{
+ hrtimer_cancel(&x->mtimer);
+ del_timer_sync(&x->rtimer);
+ kfree(x->aead);
+ kfree(x->aalg);
+ kfree(x->ealg);
+ kfree(x->calg);
+ kfree(x->encap);
+ kfree(x->coaddr);
+ kfree(x->replay_esn);
+ kfree(x->preplay_esn);
+ if (x->type_offload)
+ xfrm_put_type_offload(x->type_offload);
+ if (x->type) {
+ x->type->destructor(x);
+ xfrm_put_type(x->type);
+ }
+ if (x->xfrag.page)
+ put_page(x->xfrag.page);
+ xfrm_dev_state_free(x);
+ security_xfrm_state_free(x);
+ xfrm_state_free(x);
+}
+
+static void xfrm_state_gc_task(struct work_struct *work)
+{
+ struct xfrm_state *x;
+ struct hlist_node *tmp;
+ struct hlist_head gc_list;
+
+ spin_lock_bh(&xfrm_state_gc_lock);
+ hlist_move_list(&xfrm_state_gc_list, &gc_list);
+ spin_unlock_bh(&xfrm_state_gc_lock);
+
+ synchronize_rcu();
+
+ hlist_for_each_entry_safe(x, tmp, &gc_list, gclist)
+ ___xfrm_state_destroy(x);
+}
+
+static enum hrtimer_restart xfrm_timer_handler(struct hrtimer *me)
+{
+ struct xfrm_state *x = container_of(me, struct xfrm_state, mtimer);
+ enum hrtimer_restart ret = HRTIMER_NORESTART;
+ time64_t now = ktime_get_real_seconds();
+ time64_t next = TIME64_MAX;
+ int warn = 0;
+ int err = 0;
+
+ spin_lock(&x->lock);
+ if (x->km.state == XFRM_STATE_DEAD)
+ goto out;
+ if (x->km.state == XFRM_STATE_EXPIRED)
+ goto expired;
+ if (x->lft.hard_add_expires_seconds) {
+ long tmo = x->lft.hard_add_expires_seconds +
+ x->curlft.add_time - now;
+ if (tmo <= 0) {
+ if (x->xflags & XFRM_SOFT_EXPIRE) {
+ /* enter hard expire without soft expire first?!
+ * setting a new date could trigger this.
+ * workaround: fix x->curflt.add_time by below:
+ */
+ x->curlft.add_time = now - x->saved_tmo - 1;
+ tmo = x->lft.hard_add_expires_seconds - x->saved_tmo;
+ } else
+ goto expired;
+ }
+ if (tmo < next)
+ next = tmo;
+ }
+ if (x->lft.hard_use_expires_seconds) {
+ long tmo = x->lft.hard_use_expires_seconds +
+ (READ_ONCE(x->curlft.use_time) ? : now) - now;
+ if (tmo <= 0)
+ goto expired;
+ if (tmo < next)
+ next = tmo;
+ }
+ if (x->km.dying)
+ goto resched;
+ if (x->lft.soft_add_expires_seconds) {
+ long tmo = x->lft.soft_add_expires_seconds +
+ x->curlft.add_time - now;
+ if (tmo <= 0) {
+ warn = 1;
+ x->xflags &= ~XFRM_SOFT_EXPIRE;
+ } else if (tmo < next) {
+ next = tmo;
+ x->xflags |= XFRM_SOFT_EXPIRE;
+ x->saved_tmo = tmo;
+ }
+ }
+ if (x->lft.soft_use_expires_seconds) {
+ long tmo = x->lft.soft_use_expires_seconds +
+ (READ_ONCE(x->curlft.use_time) ? : now) - now;
+ if (tmo <= 0)
+ warn = 1;
+ else if (tmo < next)
+ next = tmo;
+ }
+
+ x->km.dying = warn;
+ if (warn)
+ km_state_expired(x, 0, 0);
+resched:
+ if (next != TIME64_MAX) {
+ hrtimer_forward_now(&x->mtimer, ktime_set(next, 0));
+ ret = HRTIMER_RESTART;
+ }
+
+ goto out;
+
+expired:
+ if (x->km.state == XFRM_STATE_ACQ && x->id.spi == 0)
+ x->km.state = XFRM_STATE_EXPIRED;
+
+ err = __xfrm_state_delete(x);
+ if (!err)
+ km_state_expired(x, 1, 0);
+
+ xfrm_audit_state_delete(x, err ? 0 : 1, true);
+
+out:
+ spin_unlock(&x->lock);
+ return ret;
+}
+
+static void xfrm_replay_timer_handler(struct timer_list *t);
+
+struct xfrm_state *xfrm_state_alloc(struct net *net)
+{
+ struct xfrm_state *x;
+
+ x = kmem_cache_zalloc(xfrm_state_cache, GFP_ATOMIC);
+
+ if (x) {
+ write_pnet(&x->xs_net, net);
+ refcount_set(&x->refcnt, 1);
+ atomic_set(&x->tunnel_users, 0);
+ INIT_LIST_HEAD(&x->km.all);
+ INIT_HLIST_NODE(&x->bydst);
+ INIT_HLIST_NODE(&x->bysrc);
+ INIT_HLIST_NODE(&x->byspi);
+ INIT_HLIST_NODE(&x->byseq);
+ hrtimer_init(&x->mtimer, CLOCK_BOOTTIME, HRTIMER_MODE_ABS_SOFT);
+ x->mtimer.function = xfrm_timer_handler;
+ timer_setup(&x->rtimer, xfrm_replay_timer_handler, 0);
+ x->curlft.add_time = ktime_get_real_seconds();
+ x->lft.soft_byte_limit = XFRM_INF;
+ x->lft.soft_packet_limit = XFRM_INF;
+ x->lft.hard_byte_limit = XFRM_INF;
+ x->lft.hard_packet_limit = XFRM_INF;
+ x->replay_maxage = 0;
+ x->replay_maxdiff = 0;
+ spin_lock_init(&x->lock);
+ }
+ return x;
+}
+EXPORT_SYMBOL(xfrm_state_alloc);
+
+void __xfrm_state_destroy(struct xfrm_state *x, bool sync)
+{
+ WARN_ON(x->km.state != XFRM_STATE_DEAD);
+
+ if (sync) {
+ synchronize_rcu();
+ ___xfrm_state_destroy(x);
+ } else {
+ spin_lock_bh(&xfrm_state_gc_lock);
+ hlist_add_head(&x->gclist, &xfrm_state_gc_list);
+ spin_unlock_bh(&xfrm_state_gc_lock);
+ schedule_work(&xfrm_state_gc_work);
+ }
+}
+EXPORT_SYMBOL(__xfrm_state_destroy);
+
+int __xfrm_state_delete(struct xfrm_state *x)
+{
+ struct net *net = xs_net(x);
+ int err = -ESRCH;
+
+ if (x->km.state != XFRM_STATE_DEAD) {
+ x->km.state = XFRM_STATE_DEAD;
+ spin_lock(&net->xfrm.xfrm_state_lock);
+ list_del(&x->km.all);
+ hlist_del_rcu(&x->bydst);
+ hlist_del_rcu(&x->bysrc);
+ if (x->km.seq)
+ hlist_del_rcu(&x->byseq);
+ if (x->id.spi)
+ hlist_del_rcu(&x->byspi);
+ net->xfrm.state_num--;
+ spin_unlock(&net->xfrm.xfrm_state_lock);
+
+ if (x->encap_sk)
+ sock_put(rcu_dereference_raw(x->encap_sk));
+
+ xfrm_dev_state_delete(x);
+
+ /* All xfrm_state objects are created by xfrm_state_alloc.
+ * The xfrm_state_alloc call gives a reference, and that
+ * is what we are dropping here.
+ */
+ xfrm_state_put(x);
+ err = 0;
+ }
+
+ return err;
+}
+EXPORT_SYMBOL(__xfrm_state_delete);
+
+int xfrm_state_delete(struct xfrm_state *x)
+{
+ int err;
+
+ spin_lock_bh(&x->lock);
+ err = __xfrm_state_delete(x);
+ spin_unlock_bh(&x->lock);
+
+ return err;
+}
+EXPORT_SYMBOL(xfrm_state_delete);
+
+#ifdef CONFIG_SECURITY_NETWORK_XFRM
+static inline int
+xfrm_state_flush_secctx_check(struct net *net, u8 proto, bool task_valid)
+{
+ int i, err = 0;
+
+ for (i = 0; i <= net->xfrm.state_hmask; i++) {
+ struct xfrm_state *x;
+
+ hlist_for_each_entry(x, net->xfrm.state_bydst+i, bydst) {
+ if (xfrm_id_proto_match(x->id.proto, proto) &&
+ (err = security_xfrm_state_delete(x)) != 0) {
+ xfrm_audit_state_delete(x, 0, task_valid);
+ return err;
+ }
+ }
+ }
+
+ return err;
+}
+
+static inline int
+xfrm_dev_state_flush_secctx_check(struct net *net, struct net_device *dev, bool task_valid)
+{
+ int i, err = 0;
+
+ for (i = 0; i <= net->xfrm.state_hmask; i++) {
+ struct xfrm_state *x;
+ struct xfrm_dev_offload *xso;
+
+ hlist_for_each_entry(x, net->xfrm.state_bydst+i, bydst) {
+ xso = &x->xso;
+
+ if (xso->dev == dev &&
+ (err = security_xfrm_state_delete(x)) != 0) {
+ xfrm_audit_state_delete(x, 0, task_valid);
+ return err;
+ }
+ }
+ }
+
+ return err;
+}
+#else
+static inline int
+xfrm_state_flush_secctx_check(struct net *net, u8 proto, bool task_valid)
+{
+ return 0;
+}
+
+static inline int
+xfrm_dev_state_flush_secctx_check(struct net *net, struct net_device *dev, bool task_valid)
+{
+ return 0;
+}
+#endif
+
+int xfrm_state_flush(struct net *net, u8 proto, bool task_valid, bool sync)
+{
+ int i, err = 0, cnt = 0;
+
+ spin_lock_bh(&net->xfrm.xfrm_state_lock);
+ err = xfrm_state_flush_secctx_check(net, proto, task_valid);
+ if (err)
+ goto out;
+
+ err = -ESRCH;
+ for (i = 0; i <= net->xfrm.state_hmask; i++) {
+ struct xfrm_state *x;
+restart:
+ hlist_for_each_entry(x, net->xfrm.state_bydst+i, bydst) {
+ if (!xfrm_state_kern(x) &&
+ xfrm_id_proto_match(x->id.proto, proto)) {
+ xfrm_state_hold(x);
+ spin_unlock_bh(&net->xfrm.xfrm_state_lock);
+
+ err = xfrm_state_delete(x);
+ xfrm_audit_state_delete(x, err ? 0 : 1,
+ task_valid);
+ if (sync)
+ xfrm_state_put_sync(x);
+ else
+ xfrm_state_put(x);
+ if (!err)
+ cnt++;
+
+ spin_lock_bh(&net->xfrm.xfrm_state_lock);
+ goto restart;
+ }
+ }
+ }
+out:
+ spin_unlock_bh(&net->xfrm.xfrm_state_lock);
+ if (cnt)
+ err = 0;
+
+ return err;
+}
+EXPORT_SYMBOL(xfrm_state_flush);
+
+int xfrm_dev_state_flush(struct net *net, struct net_device *dev, bool task_valid)
+{
+ int i, err = 0, cnt = 0;
+
+ spin_lock_bh(&net->xfrm.xfrm_state_lock);
+ err = xfrm_dev_state_flush_secctx_check(net, dev, task_valid);
+ if (err)
+ goto out;
+
+ err = -ESRCH;
+ for (i = 0; i <= net->xfrm.state_hmask; i++) {
+ struct xfrm_state *x;
+ struct xfrm_dev_offload *xso;
+restart:
+ hlist_for_each_entry(x, net->xfrm.state_bydst+i, bydst) {
+ xso = &x->xso;
+
+ if (!xfrm_state_kern(x) && xso->dev == dev) {
+ xfrm_state_hold(x);
+ spin_unlock_bh(&net->xfrm.xfrm_state_lock);
+
+ err = xfrm_state_delete(x);
+ xfrm_audit_state_delete(x, err ? 0 : 1,
+ task_valid);
+ xfrm_state_put(x);
+ if (!err)
+ cnt++;
+
+ spin_lock_bh(&net->xfrm.xfrm_state_lock);
+ goto restart;
+ }
+ }
+ }
+ if (cnt)
+ err = 0;
+
+out:
+ spin_unlock_bh(&net->xfrm.xfrm_state_lock);
+ return err;
+}
+EXPORT_SYMBOL(xfrm_dev_state_flush);
+
+void xfrm_sad_getinfo(struct net *net, struct xfrmk_sadinfo *si)
+{
+ spin_lock_bh(&net->xfrm.xfrm_state_lock);
+ si->sadcnt = net->xfrm.state_num;
+ si->sadhcnt = net->xfrm.state_hmask + 1;
+ si->sadhmcnt = xfrm_state_hashmax;
+ spin_unlock_bh(&net->xfrm.xfrm_state_lock);
+}
+EXPORT_SYMBOL(xfrm_sad_getinfo);
+
+static void
+__xfrm4_init_tempsel(struct xfrm_selector *sel, const struct flowi *fl)
+{
+ const struct flowi4 *fl4 = &fl->u.ip4;
+
+ sel->daddr.a4 = fl4->daddr;
+ sel->saddr.a4 = fl4->saddr;
+ sel->dport = xfrm_flowi_dport(fl, &fl4->uli);
+ sel->dport_mask = htons(0xffff);
+ sel->sport = xfrm_flowi_sport(fl, &fl4->uli);
+ sel->sport_mask = htons(0xffff);
+ sel->family = AF_INET;
+ sel->prefixlen_d = 32;
+ sel->prefixlen_s = 32;
+ sel->proto = fl4->flowi4_proto;
+ sel->ifindex = fl4->flowi4_oif;
+}
+
+static void
+__xfrm6_init_tempsel(struct xfrm_selector *sel, const struct flowi *fl)
+{
+ const struct flowi6 *fl6 = &fl->u.ip6;
+
+ /* Initialize temporary selector matching only to current session. */
+ *(struct in6_addr *)&sel->daddr = fl6->daddr;
+ *(struct in6_addr *)&sel->saddr = fl6->saddr;
+ sel->dport = xfrm_flowi_dport(fl, &fl6->uli);
+ sel->dport_mask = htons(0xffff);
+ sel->sport = xfrm_flowi_sport(fl, &fl6->uli);
+ sel->sport_mask = htons(0xffff);
+ sel->family = AF_INET6;
+ sel->prefixlen_d = 128;
+ sel->prefixlen_s = 128;
+ sel->proto = fl6->flowi6_proto;
+ sel->ifindex = fl6->flowi6_oif;
+}
+
+static void
+xfrm_init_tempstate(struct xfrm_state *x, const struct flowi *fl,
+ const struct xfrm_tmpl *tmpl,
+ const xfrm_address_t *daddr, const xfrm_address_t *saddr,
+ unsigned short family)
+{
+ switch (family) {
+ case AF_INET:
+ __xfrm4_init_tempsel(&x->sel, fl);
+ break;
+ case AF_INET6:
+ __xfrm6_init_tempsel(&x->sel, fl);
+ break;
+ }
+
+ x->id = tmpl->id;
+
+ switch (tmpl->encap_family) {
+ case AF_INET:
+ if (x->id.daddr.a4 == 0)
+ x->id.daddr.a4 = daddr->a4;
+ x->props.saddr = tmpl->saddr;
+ if (x->props.saddr.a4 == 0)
+ x->props.saddr.a4 = saddr->a4;
+ break;
+ case AF_INET6:
+ if (ipv6_addr_any((struct in6_addr *)&x->id.daddr))
+ memcpy(&x->id.daddr, daddr, sizeof(x->sel.daddr));
+ memcpy(&x->props.saddr, &tmpl->saddr, sizeof(x->props.saddr));
+ if (ipv6_addr_any((struct in6_addr *)&x->props.saddr))
+ memcpy(&x->props.saddr, saddr, sizeof(x->props.saddr));
+ break;
+ }
+
+ x->props.mode = tmpl->mode;
+ x->props.reqid = tmpl->reqid;
+ x->props.family = tmpl->encap_family;
+}
+
+static struct xfrm_state *__xfrm_state_lookup(struct net *net, u32 mark,
+ const xfrm_address_t *daddr,
+ __be32 spi, u8 proto,
+ unsigned short family)
+{
+ unsigned int h = xfrm_spi_hash(net, daddr, spi, proto, family);
+ struct xfrm_state *x;
+
+ hlist_for_each_entry_rcu(x, net->xfrm.state_byspi + h, byspi) {
+ if (x->props.family != family ||
+ x->id.spi != spi ||
+ x->id.proto != proto ||
+ !xfrm_addr_equal(&x->id.daddr, daddr, family))
+ continue;
+
+ if ((mark & x->mark.m) != x->mark.v)
+ continue;
+ if (!xfrm_state_hold_rcu(x))
+ continue;
+ return x;
+ }
+
+ return NULL;
+}
+
+static struct xfrm_state *__xfrm_state_lookup_byaddr(struct net *net, u32 mark,
+ const xfrm_address_t *daddr,
+ const xfrm_address_t *saddr,
+ u8 proto, unsigned short family)
+{
+ unsigned int h = xfrm_src_hash(net, daddr, saddr, family);
+ struct xfrm_state *x;
+
+ hlist_for_each_entry_rcu(x, net->xfrm.state_bysrc + h, bysrc) {
+ if (x->props.family != family ||
+ x->id.proto != proto ||
+ !xfrm_addr_equal(&x->id.daddr, daddr, family) ||
+ !xfrm_addr_equal(&x->props.saddr, saddr, family))
+ continue;
+
+ if ((mark & x->mark.m) != x->mark.v)
+ continue;
+ if (!xfrm_state_hold_rcu(x))
+ continue;
+ return x;
+ }
+
+ return NULL;
+}
+
+static inline struct xfrm_state *
+__xfrm_state_locate(struct xfrm_state *x, int use_spi, int family)
+{
+ struct net *net = xs_net(x);
+ u32 mark = x->mark.v & x->mark.m;
+
+ if (use_spi)
+ return __xfrm_state_lookup(net, mark, &x->id.daddr,
+ x->id.spi, x->id.proto, family);
+ else
+ return __xfrm_state_lookup_byaddr(net, mark,
+ &x->id.daddr,
+ &x->props.saddr,
+ x->id.proto, family);
+}
+
+static void xfrm_hash_grow_check(struct net *net, int have_hash_collision)
+{
+ if (have_hash_collision &&
+ (net->xfrm.state_hmask + 1) < xfrm_state_hashmax &&
+ net->xfrm.state_num > net->xfrm.state_hmask)
+ schedule_work(&net->xfrm.state_hash_work);
+}
+
+static void xfrm_state_look_at(struct xfrm_policy *pol, struct xfrm_state *x,
+ const struct flowi *fl, unsigned short family,
+ struct xfrm_state **best, int *acq_in_progress,
+ int *error)
+{
+ /* Resolution logic:
+ * 1. There is a valid state with matching selector. Done.
+ * 2. Valid state with inappropriate selector. Skip.
+ *
+ * Entering area of "sysdeps".
+ *
+ * 3. If state is not valid, selector is temporary, it selects
+ * only session which triggered previous resolution. Key
+ * manager will do something to install a state with proper
+ * selector.
+ */
+ if (x->km.state == XFRM_STATE_VALID) {
+ if ((x->sel.family &&
+ (x->sel.family != family ||
+ !xfrm_selector_match(&x->sel, fl, family))) ||
+ !security_xfrm_state_pol_flow_match(x, pol,
+ &fl->u.__fl_common))
+ return;
+
+ if (!*best ||
+ (*best)->km.dying > x->km.dying ||
+ ((*best)->km.dying == x->km.dying &&
+ (*best)->curlft.add_time < x->curlft.add_time))
+ *best = x;
+ } else if (x->km.state == XFRM_STATE_ACQ) {
+ *acq_in_progress = 1;
+ } else if (x->km.state == XFRM_STATE_ERROR ||
+ x->km.state == XFRM_STATE_EXPIRED) {
+ if ((!x->sel.family ||
+ (x->sel.family == family &&
+ xfrm_selector_match(&x->sel, fl, family))) &&
+ security_xfrm_state_pol_flow_match(x, pol,
+ &fl->u.__fl_common))
+ *error = -ESRCH;
+ }
+}
+
+struct xfrm_state *
+xfrm_state_find(const xfrm_address_t *daddr, const xfrm_address_t *saddr,
+ const struct flowi *fl, struct xfrm_tmpl *tmpl,
+ struct xfrm_policy *pol, int *err,
+ unsigned short family, u32 if_id)
+{
+ static xfrm_address_t saddr_wildcard = { };
+ struct net *net = xp_net(pol);
+ unsigned int h, h_wildcard;
+ struct xfrm_state *x, *x0, *to_put;
+ int acquire_in_progress = 0;
+ int error = 0;
+ struct xfrm_state *best = NULL;
+ u32 mark = pol->mark.v & pol->mark.m;
+ unsigned short encap_family = tmpl->encap_family;
+ unsigned int sequence;
+ struct km_event c;
+
+ to_put = NULL;
+
+ sequence = read_seqcount_begin(&net->xfrm.xfrm_state_hash_generation);
+
+ rcu_read_lock();
+ h = xfrm_dst_hash(net, daddr, saddr, tmpl->reqid, encap_family);
+ hlist_for_each_entry_rcu(x, net->xfrm.state_bydst + h, bydst) {
+ if (x->props.family == encap_family &&
+ x->props.reqid == tmpl->reqid &&
+ (mark & x->mark.m) == x->mark.v &&
+ x->if_id == if_id &&
+ !(x->props.flags & XFRM_STATE_WILDRECV) &&
+ xfrm_state_addr_check(x, daddr, saddr, encap_family) &&
+ tmpl->mode == x->props.mode &&
+ tmpl->id.proto == x->id.proto &&
+ (tmpl->id.spi == x->id.spi || !tmpl->id.spi))
+ xfrm_state_look_at(pol, x, fl, family,
+ &best, &acquire_in_progress, &error);
+ }
+ if (best || acquire_in_progress)
+ goto found;
+
+ h_wildcard = xfrm_dst_hash(net, daddr, &saddr_wildcard, tmpl->reqid, encap_family);
+ hlist_for_each_entry_rcu(x, net->xfrm.state_bydst + h_wildcard, bydst) {
+ if (x->props.family == encap_family &&
+ x->props.reqid == tmpl->reqid &&
+ (mark & x->mark.m) == x->mark.v &&
+ x->if_id == if_id &&
+ !(x->props.flags & XFRM_STATE_WILDRECV) &&
+ xfrm_addr_equal(&x->id.daddr, daddr, encap_family) &&
+ tmpl->mode == x->props.mode &&
+ tmpl->id.proto == x->id.proto &&
+ (tmpl->id.spi == x->id.spi || !tmpl->id.spi))
+ xfrm_state_look_at(pol, x, fl, family,
+ &best, &acquire_in_progress, &error);
+ }
+
+found:
+ x = best;
+ if (!x && !error && !acquire_in_progress) {
+ if (tmpl->id.spi &&
+ (x0 = __xfrm_state_lookup(net, mark, daddr, tmpl->id.spi,
+ tmpl->id.proto, encap_family)) != NULL) {
+ to_put = x0;
+ error = -EEXIST;
+ goto out;
+ }
+
+ c.net = net;
+ /* If the KMs have no listeners (yet...), avoid allocating an SA
+ * for each and every packet - garbage collection might not
+ * handle the flood.
+ */
+ if (!km_is_alive(&c)) {
+ error = -ESRCH;
+ goto out;
+ }
+
+ x = xfrm_state_alloc(net);
+ if (x == NULL) {
+ error = -ENOMEM;
+ goto out;
+ }
+ /* Initialize temporary state matching only
+ * to current session. */
+ xfrm_init_tempstate(x, fl, tmpl, daddr, saddr, family);
+ memcpy(&x->mark, &pol->mark, sizeof(x->mark));
+ x->if_id = if_id;
+
+ error = security_xfrm_state_alloc_acquire(x, pol->security, fl->flowi_secid);
+ if (error) {
+ x->km.state = XFRM_STATE_DEAD;
+ to_put = x;
+ x = NULL;
+ goto out;
+ }
+
+ if (km_query(x, tmpl, pol) == 0) {
+ spin_lock_bh(&net->xfrm.xfrm_state_lock);
+ x->km.state = XFRM_STATE_ACQ;
+ list_add(&x->km.all, &net->xfrm.state_all);
+ hlist_add_head_rcu(&x->bydst, net->xfrm.state_bydst + h);
+ h = xfrm_src_hash(net, daddr, saddr, encap_family);
+ hlist_add_head_rcu(&x->bysrc, net->xfrm.state_bysrc + h);
+ if (x->id.spi) {
+ h = xfrm_spi_hash(net, &x->id.daddr, x->id.spi, x->id.proto, encap_family);
+ hlist_add_head_rcu(&x->byspi, net->xfrm.state_byspi + h);
+ }
+ if (x->km.seq) {
+ h = xfrm_seq_hash(net, x->km.seq);
+ hlist_add_head_rcu(&x->byseq, net->xfrm.state_byseq + h);
+ }
+ x->lft.hard_add_expires_seconds = net->xfrm.sysctl_acq_expires;
+ hrtimer_start(&x->mtimer,
+ ktime_set(net->xfrm.sysctl_acq_expires, 0),
+ HRTIMER_MODE_REL_SOFT);
+ net->xfrm.state_num++;
+ xfrm_hash_grow_check(net, x->bydst.next != NULL);
+ spin_unlock_bh(&net->xfrm.xfrm_state_lock);
+ } else {
+ x->km.state = XFRM_STATE_DEAD;
+ to_put = x;
+ x = NULL;
+ error = -ESRCH;
+ }
+ }
+out:
+ if (x) {
+ if (!xfrm_state_hold_rcu(x)) {
+ *err = -EAGAIN;
+ x = NULL;
+ }
+ } else {
+ *err = acquire_in_progress ? -EAGAIN : error;
+ }
+ rcu_read_unlock();
+ if (to_put)
+ xfrm_state_put(to_put);
+
+ if (read_seqcount_retry(&net->xfrm.xfrm_state_hash_generation, sequence)) {
+ *err = -EAGAIN;
+ if (x) {
+ xfrm_state_put(x);
+ x = NULL;
+ }
+ }
+
+ return x;
+}
+
+struct xfrm_state *
+xfrm_stateonly_find(struct net *net, u32 mark, u32 if_id,
+ xfrm_address_t *daddr, xfrm_address_t *saddr,
+ unsigned short family, u8 mode, u8 proto, u32 reqid)
+{
+ unsigned int h;
+ struct xfrm_state *rx = NULL, *x = NULL;
+
+ spin_lock_bh(&net->xfrm.xfrm_state_lock);
+ h = xfrm_dst_hash(net, daddr, saddr, reqid, family);
+ hlist_for_each_entry(x, net->xfrm.state_bydst+h, bydst) {
+ if (x->props.family == family &&
+ x->props.reqid == reqid &&
+ (mark & x->mark.m) == x->mark.v &&
+ x->if_id == if_id &&
+ !(x->props.flags & XFRM_STATE_WILDRECV) &&
+ xfrm_state_addr_check(x, daddr, saddr, family) &&
+ mode == x->props.mode &&
+ proto == x->id.proto &&
+ x->km.state == XFRM_STATE_VALID) {
+ rx = x;
+ break;
+ }
+ }
+
+ if (rx)
+ xfrm_state_hold(rx);
+ spin_unlock_bh(&net->xfrm.xfrm_state_lock);
+
+
+ return rx;
+}
+EXPORT_SYMBOL(xfrm_stateonly_find);
+
+struct xfrm_state *xfrm_state_lookup_byspi(struct net *net, __be32 spi,
+ unsigned short family)
+{
+ struct xfrm_state *x;
+ struct xfrm_state_walk *w;
+
+ spin_lock_bh(&net->xfrm.xfrm_state_lock);
+ list_for_each_entry(w, &net->xfrm.state_all, all) {
+ x = container_of(w, struct xfrm_state, km);
+ if (x->props.family != family ||
+ x->id.spi != spi)
+ continue;
+
+ xfrm_state_hold(x);
+ spin_unlock_bh(&net->xfrm.xfrm_state_lock);
+ return x;
+ }
+ spin_unlock_bh(&net->xfrm.xfrm_state_lock);
+ return NULL;
+}
+EXPORT_SYMBOL(xfrm_state_lookup_byspi);
+
+static void __xfrm_state_insert(struct xfrm_state *x)
+{
+ struct net *net = xs_net(x);
+ unsigned int h;
+
+ list_add(&x->km.all, &net->xfrm.state_all);
+
+ h = xfrm_dst_hash(net, &x->id.daddr, &x->props.saddr,
+ x->props.reqid, x->props.family);
+ hlist_add_head_rcu(&x->bydst, net->xfrm.state_bydst + h);
+
+ h = xfrm_src_hash(net, &x->id.daddr, &x->props.saddr, x->props.family);
+ hlist_add_head_rcu(&x->bysrc, net->xfrm.state_bysrc + h);
+
+ if (x->id.spi) {
+ h = xfrm_spi_hash(net, &x->id.daddr, x->id.spi, x->id.proto,
+ x->props.family);
+
+ hlist_add_head_rcu(&x->byspi, net->xfrm.state_byspi + h);
+ }
+
+ if (x->km.seq) {
+ h = xfrm_seq_hash(net, x->km.seq);
+
+ hlist_add_head_rcu(&x->byseq, net->xfrm.state_byseq + h);
+ }
+
+ hrtimer_start(&x->mtimer, ktime_set(1, 0), HRTIMER_MODE_REL_SOFT);
+ if (x->replay_maxage)
+ mod_timer(&x->rtimer, jiffies + x->replay_maxage);
+
+ net->xfrm.state_num++;
+
+ xfrm_hash_grow_check(net, x->bydst.next != NULL);
+}
+
+/* net->xfrm.xfrm_state_lock is held */
+static void __xfrm_state_bump_genids(struct xfrm_state *xnew)
+{
+ struct net *net = xs_net(xnew);
+ unsigned short family = xnew->props.family;
+ u32 reqid = xnew->props.reqid;
+ struct xfrm_state *x;
+ unsigned int h;
+ u32 mark = xnew->mark.v & xnew->mark.m;
+ u32 if_id = xnew->if_id;
+
+ h = xfrm_dst_hash(net, &xnew->id.daddr, &xnew->props.saddr, reqid, family);
+ hlist_for_each_entry(x, net->xfrm.state_bydst+h, bydst) {
+ if (x->props.family == family &&
+ x->props.reqid == reqid &&
+ x->if_id == if_id &&
+ (mark & x->mark.m) == x->mark.v &&
+ xfrm_addr_equal(&x->id.daddr, &xnew->id.daddr, family) &&
+ xfrm_addr_equal(&x->props.saddr, &xnew->props.saddr, family))
+ x->genid++;
+ }
+}
+
+void xfrm_state_insert(struct xfrm_state *x)
+{
+ struct net *net = xs_net(x);
+
+ spin_lock_bh(&net->xfrm.xfrm_state_lock);
+ __xfrm_state_bump_genids(x);
+ __xfrm_state_insert(x);
+ spin_unlock_bh(&net->xfrm.xfrm_state_lock);
+}
+EXPORT_SYMBOL(xfrm_state_insert);
+
+/* net->xfrm.xfrm_state_lock is held */
+static struct xfrm_state *__find_acq_core(struct net *net,
+ const struct xfrm_mark *m,
+ unsigned short family, u8 mode,
+ u32 reqid, u32 if_id, u8 proto,
+ const xfrm_address_t *daddr,
+ const xfrm_address_t *saddr,
+ int create)
+{
+ unsigned int h = xfrm_dst_hash(net, daddr, saddr, reqid, family);
+ struct xfrm_state *x;
+ u32 mark = m->v & m->m;
+
+ hlist_for_each_entry(x, net->xfrm.state_bydst+h, bydst) {
+ if (x->props.reqid != reqid ||
+ x->props.mode != mode ||
+ x->props.family != family ||
+ x->km.state != XFRM_STATE_ACQ ||
+ x->id.spi != 0 ||
+ x->id.proto != proto ||
+ (mark & x->mark.m) != x->mark.v ||
+ !xfrm_addr_equal(&x->id.daddr, daddr, family) ||
+ !xfrm_addr_equal(&x->props.saddr, saddr, family))
+ continue;
+
+ xfrm_state_hold(x);
+ return x;
+ }
+
+ if (!create)
+ return NULL;
+
+ x = xfrm_state_alloc(net);
+ if (likely(x)) {
+ switch (family) {
+ case AF_INET:
+ x->sel.daddr.a4 = daddr->a4;
+ x->sel.saddr.a4 = saddr->a4;
+ x->sel.prefixlen_d = 32;
+ x->sel.prefixlen_s = 32;
+ x->props.saddr.a4 = saddr->a4;
+ x->id.daddr.a4 = daddr->a4;
+ break;
+
+ case AF_INET6:
+ x->sel.daddr.in6 = daddr->in6;
+ x->sel.saddr.in6 = saddr->in6;
+ x->sel.prefixlen_d = 128;
+ x->sel.prefixlen_s = 128;
+ x->props.saddr.in6 = saddr->in6;
+ x->id.daddr.in6 = daddr->in6;
+ break;
+ }
+
+ x->km.state = XFRM_STATE_ACQ;
+ x->id.proto = proto;
+ x->props.family = family;
+ x->props.mode = mode;
+ x->props.reqid = reqid;
+ x->if_id = if_id;
+ x->mark.v = m->v;
+ x->mark.m = m->m;
+ x->lft.hard_add_expires_seconds = net->xfrm.sysctl_acq_expires;
+ xfrm_state_hold(x);
+ hrtimer_start(&x->mtimer,
+ ktime_set(net->xfrm.sysctl_acq_expires, 0),
+ HRTIMER_MODE_REL_SOFT);
+ list_add(&x->km.all, &net->xfrm.state_all);
+ hlist_add_head_rcu(&x->bydst, net->xfrm.state_bydst + h);
+ h = xfrm_src_hash(net, daddr, saddr, family);
+ hlist_add_head_rcu(&x->bysrc, net->xfrm.state_bysrc + h);
+
+ net->xfrm.state_num++;
+
+ xfrm_hash_grow_check(net, x->bydst.next != NULL);
+ }
+
+ return x;
+}
+
+static struct xfrm_state *__xfrm_find_acq_byseq(struct net *net, u32 mark, u32 seq);
+
+int xfrm_state_add(struct xfrm_state *x)
+{
+ struct net *net = xs_net(x);
+ struct xfrm_state *x1, *to_put;
+ int family;
+ int err;
+ u32 mark = x->mark.v & x->mark.m;
+ int use_spi = xfrm_id_proto_match(x->id.proto, IPSEC_PROTO_ANY);
+
+ family = x->props.family;
+
+ to_put = NULL;
+
+ spin_lock_bh(&net->xfrm.xfrm_state_lock);
+
+ x1 = __xfrm_state_locate(x, use_spi, family);
+ if (x1) {
+ to_put = x1;
+ x1 = NULL;
+ err = -EEXIST;
+ goto out;
+ }
+
+ if (use_spi && x->km.seq) {
+ x1 = __xfrm_find_acq_byseq(net, mark, x->km.seq);
+ if (x1 && ((x1->id.proto != x->id.proto) ||
+ !xfrm_addr_equal(&x1->id.daddr, &x->id.daddr, family))) {
+ to_put = x1;
+ x1 = NULL;
+ }
+ }
+
+ if (use_spi && !x1)
+ x1 = __find_acq_core(net, &x->mark, family, x->props.mode,
+ x->props.reqid, x->if_id, x->id.proto,
+ &x->id.daddr, &x->props.saddr, 0);
+
+ __xfrm_state_bump_genids(x);
+ __xfrm_state_insert(x);
+ err = 0;
+
+out:
+ spin_unlock_bh(&net->xfrm.xfrm_state_lock);
+
+ if (x1) {
+ xfrm_state_delete(x1);
+ xfrm_state_put(x1);
+ }
+
+ if (to_put)
+ xfrm_state_put(to_put);
+
+ return err;
+}
+EXPORT_SYMBOL(xfrm_state_add);
+
+#ifdef CONFIG_XFRM_MIGRATE
+static inline int clone_security(struct xfrm_state *x, struct xfrm_sec_ctx *security)
+{
+ struct xfrm_user_sec_ctx *uctx;
+ int size = sizeof(*uctx) + security->ctx_len;
+ int err;
+
+ uctx = kmalloc(size, GFP_KERNEL);
+ if (!uctx)
+ return -ENOMEM;
+
+ uctx->exttype = XFRMA_SEC_CTX;
+ uctx->len = size;
+ uctx->ctx_doi = security->ctx_doi;
+ uctx->ctx_alg = security->ctx_alg;
+ uctx->ctx_len = security->ctx_len;
+ memcpy(uctx + 1, security->ctx_str, security->ctx_len);
+ err = security_xfrm_state_alloc(x, uctx);
+ kfree(uctx);
+ if (err)
+ return err;
+
+ return 0;
+}
+
+static struct xfrm_state *xfrm_state_clone(struct xfrm_state *orig,
+ struct xfrm_encap_tmpl *encap)
+{
+ struct net *net = xs_net(orig);
+ struct xfrm_state *x = xfrm_state_alloc(net);
+ if (!x)
+ goto out;
+
+ memcpy(&x->id, &orig->id, sizeof(x->id));
+ memcpy(&x->sel, &orig->sel, sizeof(x->sel));
+ memcpy(&x->lft, &orig->lft, sizeof(x->lft));
+ x->props.mode = orig->props.mode;
+ x->props.replay_window = orig->props.replay_window;
+ x->props.reqid = orig->props.reqid;
+ x->props.family = orig->props.family;
+ x->props.saddr = orig->props.saddr;
+
+ if (orig->aalg) {
+ x->aalg = xfrm_algo_auth_clone(orig->aalg);
+ if (!x->aalg)
+ goto error;
+ }
+ x->props.aalgo = orig->props.aalgo;
+
+ if (orig->aead) {
+ x->aead = xfrm_algo_aead_clone(orig->aead);
+ x->geniv = orig->geniv;
+ if (!x->aead)
+ goto error;
+ }
+ if (orig->ealg) {
+ x->ealg = xfrm_algo_clone(orig->ealg);
+ if (!x->ealg)
+ goto error;
+ }
+ x->props.ealgo = orig->props.ealgo;
+
+ if (orig->calg) {
+ x->calg = xfrm_algo_clone(orig->calg);
+ if (!x->calg)
+ goto error;
+ }
+ x->props.calgo = orig->props.calgo;
+
+ if (encap || orig->encap) {
+ if (encap)
+ x->encap = kmemdup(encap, sizeof(*x->encap),
+ GFP_KERNEL);
+ else
+ x->encap = kmemdup(orig->encap, sizeof(*x->encap),
+ GFP_KERNEL);
+
+ if (!x->encap)
+ goto error;
+ }
+
+ if (orig->security)
+ if (clone_security(x, orig->security))
+ goto error;
+
+ if (orig->coaddr) {
+ x->coaddr = kmemdup(orig->coaddr, sizeof(*x->coaddr),
+ GFP_KERNEL);
+ if (!x->coaddr)
+ goto error;
+ }
+
+ if (orig->replay_esn) {
+ if (xfrm_replay_clone(x, orig))
+ goto error;
+ }
+
+ memcpy(&x->mark, &orig->mark, sizeof(x->mark));
+ memcpy(&x->props.smark, &orig->props.smark, sizeof(x->props.smark));
+
+ x->props.flags = orig->props.flags;
+ x->props.extra_flags = orig->props.extra_flags;
+
+ x->if_id = orig->if_id;
+ x->tfcpad = orig->tfcpad;
+ x->replay_maxdiff = orig->replay_maxdiff;
+ x->replay_maxage = orig->replay_maxage;
+ memcpy(&x->curlft, &orig->curlft, sizeof(x->curlft));
+ x->km.state = orig->km.state;
+ x->km.seq = orig->km.seq;
+ x->replay = orig->replay;
+ x->preplay = orig->preplay;
+ x->mapping_maxage = orig->mapping_maxage;
+ x->lastused = orig->lastused;
+ x->new_mapping = 0;
+ x->new_mapping_sport = 0;
+
+ return x;
+
+ error:
+ xfrm_state_put(x);
+out:
+ return NULL;
+}
+
+struct xfrm_state *xfrm_migrate_state_find(struct xfrm_migrate *m, struct net *net,
+ u32 if_id)
+{
+ unsigned int h;
+ struct xfrm_state *x = NULL;
+
+ spin_lock_bh(&net->xfrm.xfrm_state_lock);
+
+ if (m->reqid) {
+ h = xfrm_dst_hash(net, &m->old_daddr, &m->old_saddr,
+ m->reqid, m->old_family);
+ hlist_for_each_entry(x, net->xfrm.state_bydst+h, bydst) {
+ if (x->props.mode != m->mode ||
+ x->id.proto != m->proto)
+ continue;
+ if (m->reqid && x->props.reqid != m->reqid)
+ continue;
+ if (if_id != 0 && x->if_id != if_id)
+ continue;
+ if (!xfrm_addr_equal(&x->id.daddr, &m->old_daddr,
+ m->old_family) ||
+ !xfrm_addr_equal(&x->props.saddr, &m->old_saddr,
+ m->old_family))
+ continue;
+ xfrm_state_hold(x);
+ break;
+ }
+ } else {
+ h = xfrm_src_hash(net, &m->old_daddr, &m->old_saddr,
+ m->old_family);
+ hlist_for_each_entry(x, net->xfrm.state_bysrc+h, bysrc) {
+ if (x->props.mode != m->mode ||
+ x->id.proto != m->proto)
+ continue;
+ if (if_id != 0 && x->if_id != if_id)
+ continue;
+ if (!xfrm_addr_equal(&x->id.daddr, &m->old_daddr,
+ m->old_family) ||
+ !xfrm_addr_equal(&x->props.saddr, &m->old_saddr,
+ m->old_family))
+ continue;
+ xfrm_state_hold(x);
+ break;
+ }
+ }
+
+ spin_unlock_bh(&net->xfrm.xfrm_state_lock);
+
+ return x;
+}
+EXPORT_SYMBOL(xfrm_migrate_state_find);
+
+struct xfrm_state *xfrm_state_migrate(struct xfrm_state *x,
+ struct xfrm_migrate *m,
+ struct xfrm_encap_tmpl *encap)
+{
+ struct xfrm_state *xc;
+
+ xc = xfrm_state_clone(x, encap);
+ if (!xc)
+ return NULL;
+
+ xc->props.family = m->new_family;
+
+ if (xfrm_init_state(xc) < 0)
+ goto error;
+
+ memcpy(&xc->id.daddr, &m->new_daddr, sizeof(xc->id.daddr));
+ memcpy(&xc->props.saddr, &m->new_saddr, sizeof(xc->props.saddr));
+
+ /* add state */
+ if (xfrm_addr_equal(&x->id.daddr, &m->new_daddr, m->new_family)) {
+ /* a care is needed when the destination address of the
+ state is to be updated as it is a part of triplet */
+ xfrm_state_insert(xc);
+ } else {
+ if (xfrm_state_add(xc) < 0)
+ goto error;
+ }
+
+ return xc;
+error:
+ xfrm_state_put(xc);
+ return NULL;
+}
+EXPORT_SYMBOL(xfrm_state_migrate);
+#endif
+
+int xfrm_state_update(struct xfrm_state *x)
+{
+ struct xfrm_state *x1, *to_put;
+ int err;
+ int use_spi = xfrm_id_proto_match(x->id.proto, IPSEC_PROTO_ANY);
+ struct net *net = xs_net(x);
+
+ to_put = NULL;
+
+ spin_lock_bh(&net->xfrm.xfrm_state_lock);
+ x1 = __xfrm_state_locate(x, use_spi, x->props.family);
+
+ err = -ESRCH;
+ if (!x1)
+ goto out;
+
+ if (xfrm_state_kern(x1)) {
+ to_put = x1;
+ err = -EEXIST;
+ goto out;
+ }
+
+ if (x1->km.state == XFRM_STATE_ACQ) {
+ __xfrm_state_insert(x);
+ x = NULL;
+ }
+ err = 0;
+
+out:
+ spin_unlock_bh(&net->xfrm.xfrm_state_lock);
+
+ if (to_put)
+ xfrm_state_put(to_put);
+
+ if (err)
+ return err;
+
+ if (!x) {
+ xfrm_state_delete(x1);
+ xfrm_state_put(x1);
+ return 0;
+ }
+
+ err = -EINVAL;
+ spin_lock_bh(&x1->lock);
+ if (likely(x1->km.state == XFRM_STATE_VALID)) {
+ if (x->encap && x1->encap &&
+ x->encap->encap_type == x1->encap->encap_type)
+ memcpy(x1->encap, x->encap, sizeof(*x1->encap));
+ else if (x->encap || x1->encap)
+ goto fail;
+
+ if (x->coaddr && x1->coaddr) {
+ memcpy(x1->coaddr, x->coaddr, sizeof(*x1->coaddr));
+ }
+ if (!use_spi && memcmp(&x1->sel, &x->sel, sizeof(x1->sel)))
+ memcpy(&x1->sel, &x->sel, sizeof(x1->sel));
+ memcpy(&x1->lft, &x->lft, sizeof(x1->lft));
+ x1->km.dying = 0;
+
+ hrtimer_start(&x1->mtimer, ktime_set(1, 0),
+ HRTIMER_MODE_REL_SOFT);
+ if (READ_ONCE(x1->curlft.use_time))
+ xfrm_state_check_expire(x1);
+
+ if (x->props.smark.m || x->props.smark.v || x->if_id) {
+ spin_lock_bh(&net->xfrm.xfrm_state_lock);
+
+ if (x->props.smark.m || x->props.smark.v)
+ x1->props.smark = x->props.smark;
+
+ if (x->if_id)
+ x1->if_id = x->if_id;
+
+ __xfrm_state_bump_genids(x1);
+ spin_unlock_bh(&net->xfrm.xfrm_state_lock);
+ }
+
+ err = 0;
+ x->km.state = XFRM_STATE_DEAD;
+ __xfrm_state_put(x);
+ }
+
+fail:
+ spin_unlock_bh(&x1->lock);
+
+ xfrm_state_put(x1);
+
+ return err;
+}
+EXPORT_SYMBOL(xfrm_state_update);
+
+int xfrm_state_check_expire(struct xfrm_state *x)
+{
+ if (!READ_ONCE(x->curlft.use_time))
+ WRITE_ONCE(x->curlft.use_time, ktime_get_real_seconds());
+
+ if (x->curlft.bytes >= x->lft.hard_byte_limit ||
+ x->curlft.packets >= x->lft.hard_packet_limit) {
+ x->km.state = XFRM_STATE_EXPIRED;
+ hrtimer_start(&x->mtimer, 0, HRTIMER_MODE_REL_SOFT);
+ return -EINVAL;
+ }
+
+ if (!x->km.dying &&
+ (x->curlft.bytes >= x->lft.soft_byte_limit ||
+ x->curlft.packets >= x->lft.soft_packet_limit)) {
+ x->km.dying = 1;
+ km_state_expired(x, 0, 0);
+ }
+ return 0;
+}
+EXPORT_SYMBOL(xfrm_state_check_expire);
+
+struct xfrm_state *
+xfrm_state_lookup(struct net *net, u32 mark, const xfrm_address_t *daddr, __be32 spi,
+ u8 proto, unsigned short family)
+{
+ struct xfrm_state *x;
+
+ rcu_read_lock();
+ x = __xfrm_state_lookup(net, mark, daddr, spi, proto, family);
+ rcu_read_unlock();
+ return x;
+}
+EXPORT_SYMBOL(xfrm_state_lookup);
+
+struct xfrm_state *
+xfrm_state_lookup_byaddr(struct net *net, u32 mark,
+ const xfrm_address_t *daddr, const xfrm_address_t *saddr,
+ u8 proto, unsigned short family)
+{
+ struct xfrm_state *x;
+
+ spin_lock_bh(&net->xfrm.xfrm_state_lock);
+ x = __xfrm_state_lookup_byaddr(net, mark, daddr, saddr, proto, family);
+ spin_unlock_bh(&net->xfrm.xfrm_state_lock);
+ return x;
+}
+EXPORT_SYMBOL(xfrm_state_lookup_byaddr);
+
+struct xfrm_state *
+xfrm_find_acq(struct net *net, const struct xfrm_mark *mark, u8 mode, u32 reqid,
+ u32 if_id, u8 proto, const xfrm_address_t *daddr,
+ const xfrm_address_t *saddr, int create, unsigned short family)
+{
+ struct xfrm_state *x;
+
+ spin_lock_bh(&net->xfrm.xfrm_state_lock);
+ x = __find_acq_core(net, mark, family, mode, reqid, if_id, proto, daddr, saddr, create);
+ spin_unlock_bh(&net->xfrm.xfrm_state_lock);
+
+ return x;
+}
+EXPORT_SYMBOL(xfrm_find_acq);
+
+#ifdef CONFIG_XFRM_SUB_POLICY
+#if IS_ENABLED(CONFIG_IPV6)
+/* distribution counting sort function for xfrm_state and xfrm_tmpl */
+static void
+__xfrm6_sort(void **dst, void **src, int n,
+ int (*cmp)(const void *p), int maxclass)
+{
+ int count[XFRM_MAX_DEPTH] = { };
+ int class[XFRM_MAX_DEPTH];
+ int i;
+
+ for (i = 0; i < n; i++) {
+ int c = cmp(src[i]);
+
+ class[i] = c;
+ count[c]++;
+ }
+
+ for (i = 2; i < maxclass; i++)
+ count[i] += count[i - 1];
+
+ for (i = 0; i < n; i++) {
+ dst[count[class[i] - 1]++] = src[i];
+ src[i] = NULL;
+ }
+}
+
+/* Rule for xfrm_state:
+ *
+ * rule 1: select IPsec transport except AH
+ * rule 2: select MIPv6 RO or inbound trigger
+ * rule 3: select IPsec transport AH
+ * rule 4: select IPsec tunnel
+ * rule 5: others
+ */
+static int __xfrm6_state_sort_cmp(const void *p)
+{
+ const struct xfrm_state *v = p;
+
+ switch (v->props.mode) {
+ case XFRM_MODE_TRANSPORT:
+ if (v->id.proto != IPPROTO_AH)
+ return 1;
+ else
+ return 3;
+#if IS_ENABLED(CONFIG_IPV6_MIP6)
+ case XFRM_MODE_ROUTEOPTIMIZATION:
+ case XFRM_MODE_IN_TRIGGER:
+ return 2;
+#endif
+ case XFRM_MODE_TUNNEL:
+ case XFRM_MODE_BEET:
+ return 4;
+ }
+ return 5;
+}
+
+/* Rule for xfrm_tmpl:
+ *
+ * rule 1: select IPsec transport
+ * rule 2: select MIPv6 RO or inbound trigger
+ * rule 3: select IPsec tunnel
+ * rule 4: others
+ */
+static int __xfrm6_tmpl_sort_cmp(const void *p)
+{
+ const struct xfrm_tmpl *v = p;
+
+ switch (v->mode) {
+ case XFRM_MODE_TRANSPORT:
+ return 1;
+#if IS_ENABLED(CONFIG_IPV6_MIP6)
+ case XFRM_MODE_ROUTEOPTIMIZATION:
+ case XFRM_MODE_IN_TRIGGER:
+ return 2;
+#endif
+ case XFRM_MODE_TUNNEL:
+ case XFRM_MODE_BEET:
+ return 3;
+ }
+ return 4;
+}
+#else
+static inline int __xfrm6_state_sort_cmp(const void *p) { return 5; }
+static inline int __xfrm6_tmpl_sort_cmp(const void *p) { return 4; }
+
+static inline void
+__xfrm6_sort(void **dst, void **src, int n,
+ int (*cmp)(const void *p), int maxclass)
+{
+ int i;
+
+ for (i = 0; i < n; i++)
+ dst[i] = src[i];
+}
+#endif /* CONFIG_IPV6 */
+
+void
+xfrm_tmpl_sort(struct xfrm_tmpl **dst, struct xfrm_tmpl **src, int n,
+ unsigned short family)
+{
+ int i;
+
+ if (family == AF_INET6)
+ __xfrm6_sort((void **)dst, (void **)src, n,
+ __xfrm6_tmpl_sort_cmp, 5);
+ else
+ for (i = 0; i < n; i++)
+ dst[i] = src[i];
+}
+
+void
+xfrm_state_sort(struct xfrm_state **dst, struct xfrm_state **src, int n,
+ unsigned short family)
+{
+ int i;
+
+ if (family == AF_INET6)
+ __xfrm6_sort((void **)dst, (void **)src, n,
+ __xfrm6_state_sort_cmp, 6);
+ else
+ for (i = 0; i < n; i++)
+ dst[i] = src[i];
+}
+#endif
+
+/* Silly enough, but I'm lazy to build resolution list */
+
+static struct xfrm_state *__xfrm_find_acq_byseq(struct net *net, u32 mark, u32 seq)
+{
+ unsigned int h = xfrm_seq_hash(net, seq);
+ struct xfrm_state *x;
+
+ hlist_for_each_entry_rcu(x, net->xfrm.state_byseq + h, byseq) {
+ if (x->km.seq == seq &&
+ (mark & x->mark.m) == x->mark.v &&
+ x->km.state == XFRM_STATE_ACQ) {
+ xfrm_state_hold(x);
+ return x;
+ }
+ }
+
+ return NULL;
+}
+
+struct xfrm_state *xfrm_find_acq_byseq(struct net *net, u32 mark, u32 seq)
+{
+ struct xfrm_state *x;
+
+ spin_lock_bh(&net->xfrm.xfrm_state_lock);
+ x = __xfrm_find_acq_byseq(net, mark, seq);
+ spin_unlock_bh(&net->xfrm.xfrm_state_lock);
+ return x;
+}
+EXPORT_SYMBOL(xfrm_find_acq_byseq);
+
+u32 xfrm_get_acqseq(void)
+{
+ u32 res;
+ static atomic_t acqseq;
+
+ do {
+ res = atomic_inc_return(&acqseq);
+ } while (!res);
+
+ return res;
+}
+EXPORT_SYMBOL(xfrm_get_acqseq);
+
+int verify_spi_info(u8 proto, u32 min, u32 max)
+{
+ switch (proto) {
+ case IPPROTO_AH:
+ case IPPROTO_ESP:
+ break;
+
+ case IPPROTO_COMP:
+ /* IPCOMP spi is 16-bits. */
+ if (max >= 0x10000)
+ return -EINVAL;
+ break;
+
+ default:
+ return -EINVAL;
+ }
+
+ if (min > max)
+ return -EINVAL;
+
+ return 0;
+}
+EXPORT_SYMBOL(verify_spi_info);
+
+int xfrm_alloc_spi(struct xfrm_state *x, u32 low, u32 high)
+{
+ struct net *net = xs_net(x);
+ unsigned int h;
+ struct xfrm_state *x0;
+ int err = -ENOENT;
+ __be32 minspi = htonl(low);
+ __be32 maxspi = htonl(high);
+ __be32 newspi = 0;
+ u32 mark = x->mark.v & x->mark.m;
+
+ spin_lock_bh(&x->lock);
+ if (x->km.state == XFRM_STATE_DEAD)
+ goto unlock;
+
+ err = 0;
+ if (x->id.spi)
+ goto unlock;
+
+ err = -ENOENT;
+
+ if (minspi == maxspi) {
+ x0 = xfrm_state_lookup(net, mark, &x->id.daddr, minspi, x->id.proto, x->props.family);
+ if (x0) {
+ xfrm_state_put(x0);
+ goto unlock;
+ }
+ newspi = minspi;
+ } else {
+ u32 spi = 0;
+ for (h = 0; h < high-low+1; h++) {
+ spi = low + prandom_u32_max(high - low + 1);
+ x0 = xfrm_state_lookup(net, mark, &x->id.daddr, htonl(spi), x->id.proto, x->props.family);
+ if (x0 == NULL) {
+ newspi = htonl(spi);
+ break;
+ }
+ xfrm_state_put(x0);
+ }
+ }
+ if (newspi) {
+ spin_lock_bh(&net->xfrm.xfrm_state_lock);
+ x->id.spi = newspi;
+ h = xfrm_spi_hash(net, &x->id.daddr, x->id.spi, x->id.proto, x->props.family);
+ hlist_add_head_rcu(&x->byspi, net->xfrm.state_byspi + h);
+ spin_unlock_bh(&net->xfrm.xfrm_state_lock);
+
+ err = 0;
+ }
+
+unlock:
+ spin_unlock_bh(&x->lock);
+
+ return err;
+}
+EXPORT_SYMBOL(xfrm_alloc_spi);
+
+static bool __xfrm_state_filter_match(struct xfrm_state *x,
+ struct xfrm_address_filter *filter)
+{
+ if (filter) {
+ if ((filter->family == AF_INET ||
+ filter->family == AF_INET6) &&
+ x->props.family != filter->family)
+ return false;
+
+ return addr_match(&x->props.saddr, &filter->saddr,
+ filter->splen) &&
+ addr_match(&x->id.daddr, &filter->daddr,
+ filter->dplen);
+ }
+ return true;
+}
+
+int xfrm_state_walk(struct net *net, struct xfrm_state_walk *walk,
+ int (*func)(struct xfrm_state *, int, void*),
+ void *data)
+{
+ struct xfrm_state *state;
+ struct xfrm_state_walk *x;
+ int err = 0;
+
+ if (walk->seq != 0 && list_empty(&walk->all))
+ return 0;
+
+ spin_lock_bh(&net->xfrm.xfrm_state_lock);
+ if (list_empty(&walk->all))
+ x = list_first_entry(&net->xfrm.state_all, struct xfrm_state_walk, all);
+ else
+ x = list_first_entry(&walk->all, struct xfrm_state_walk, all);
+ list_for_each_entry_from(x, &net->xfrm.state_all, all) {
+ if (x->state == XFRM_STATE_DEAD)
+ continue;
+ state = container_of(x, struct xfrm_state, km);
+ if (!xfrm_id_proto_match(state->id.proto, walk->proto))
+ continue;
+ if (!__xfrm_state_filter_match(state, walk->filter))
+ continue;
+ err = func(state, walk->seq, data);
+ if (err) {
+ list_move_tail(&walk->all, &x->all);
+ goto out;
+ }
+ walk->seq++;
+ }
+ if (walk->seq == 0) {
+ err = -ENOENT;
+ goto out;
+ }
+ list_del_init(&walk->all);
+out:
+ spin_unlock_bh(&net->xfrm.xfrm_state_lock);
+ return err;
+}
+EXPORT_SYMBOL(xfrm_state_walk);
+
+void xfrm_state_walk_init(struct xfrm_state_walk *walk, u8 proto,
+ struct xfrm_address_filter *filter)
+{
+ INIT_LIST_HEAD(&walk->all);
+ walk->proto = proto;
+ walk->state = XFRM_STATE_DEAD;
+ walk->seq = 0;
+ walk->filter = filter;
+}
+EXPORT_SYMBOL(xfrm_state_walk_init);
+
+void xfrm_state_walk_done(struct xfrm_state_walk *walk, struct net *net)
+{
+ kfree(walk->filter);
+
+ if (list_empty(&walk->all))
+ return;
+
+ spin_lock_bh(&net->xfrm.xfrm_state_lock);
+ list_del(&walk->all);
+ spin_unlock_bh(&net->xfrm.xfrm_state_lock);
+}
+EXPORT_SYMBOL(xfrm_state_walk_done);
+
+static void xfrm_replay_timer_handler(struct timer_list *t)
+{
+ struct xfrm_state *x = from_timer(x, t, rtimer);
+
+ spin_lock(&x->lock);
+
+ if (x->km.state == XFRM_STATE_VALID) {
+ if (xfrm_aevent_is_on(xs_net(x)))
+ xfrm_replay_notify(x, XFRM_REPLAY_TIMEOUT);
+ else
+ x->xflags |= XFRM_TIME_DEFER;
+ }
+
+ spin_unlock(&x->lock);
+}
+
+static LIST_HEAD(xfrm_km_list);
+
+void km_policy_notify(struct xfrm_policy *xp, int dir, const struct km_event *c)
+{
+ struct xfrm_mgr *km;
+
+ rcu_read_lock();
+ list_for_each_entry_rcu(km, &xfrm_km_list, list)
+ if (km->notify_policy)
+ km->notify_policy(xp, dir, c);
+ rcu_read_unlock();
+}
+
+void km_state_notify(struct xfrm_state *x, const struct km_event *c)
+{
+ struct xfrm_mgr *km;
+ rcu_read_lock();
+ list_for_each_entry_rcu(km, &xfrm_km_list, list)
+ if (km->notify)
+ km->notify(x, c);
+ rcu_read_unlock();
+}
+
+EXPORT_SYMBOL(km_policy_notify);
+EXPORT_SYMBOL(km_state_notify);
+
+void km_state_expired(struct xfrm_state *x, int hard, u32 portid)
+{
+ struct km_event c;
+
+ c.data.hard = hard;
+ c.portid = portid;
+ c.event = XFRM_MSG_EXPIRE;
+ km_state_notify(x, &c);
+}
+
+EXPORT_SYMBOL(km_state_expired);
+/*
+ * We send to all registered managers regardless of failure
+ * We are happy with one success
+*/
+int km_query(struct xfrm_state *x, struct xfrm_tmpl *t, struct xfrm_policy *pol)
+{
+ int err = -EINVAL, acqret;
+ struct xfrm_mgr *km;
+
+ rcu_read_lock();
+ list_for_each_entry_rcu(km, &xfrm_km_list, list) {
+ acqret = km->acquire(x, t, pol);
+ if (!acqret)
+ err = acqret;
+ }
+ rcu_read_unlock();
+ return err;
+}
+EXPORT_SYMBOL(km_query);
+
+static int __km_new_mapping(struct xfrm_state *x, xfrm_address_t *ipaddr, __be16 sport)
+{
+ int err = -EINVAL;
+ struct xfrm_mgr *km;
+
+ rcu_read_lock();
+ list_for_each_entry_rcu(km, &xfrm_km_list, list) {
+ if (km->new_mapping)
+ err = km->new_mapping(x, ipaddr, sport);
+ if (!err)
+ break;
+ }
+ rcu_read_unlock();
+ return err;
+}
+
+int km_new_mapping(struct xfrm_state *x, xfrm_address_t *ipaddr, __be16 sport)
+{
+ int ret = 0;
+
+ if (x->mapping_maxage) {
+ if ((jiffies / HZ - x->new_mapping) > x->mapping_maxage ||
+ x->new_mapping_sport != sport) {
+ x->new_mapping_sport = sport;
+ x->new_mapping = jiffies / HZ;
+ ret = __km_new_mapping(x, ipaddr, sport);
+ }
+ } else {
+ ret = __km_new_mapping(x, ipaddr, sport);
+ }
+
+ return ret;
+}
+EXPORT_SYMBOL(km_new_mapping);
+
+void km_policy_expired(struct xfrm_policy *pol, int dir, int hard, u32 portid)
+{
+ struct km_event c;
+
+ c.data.hard = hard;
+ c.portid = portid;
+ c.event = XFRM_MSG_POLEXPIRE;
+ km_policy_notify(pol, dir, &c);
+}
+EXPORT_SYMBOL(km_policy_expired);
+
+#ifdef CONFIG_XFRM_MIGRATE
+int km_migrate(const struct xfrm_selector *sel, u8 dir, u8 type,
+ const struct xfrm_migrate *m, int num_migrate,
+ const struct xfrm_kmaddress *k,
+ const struct xfrm_encap_tmpl *encap)
+{
+ int err = -EINVAL;
+ int ret;
+ struct xfrm_mgr *km;
+
+ rcu_read_lock();
+ list_for_each_entry_rcu(km, &xfrm_km_list, list) {
+ if (km->migrate) {
+ ret = km->migrate(sel, dir, type, m, num_migrate, k,
+ encap);
+ if (!ret)
+ err = ret;
+ }
+ }
+ rcu_read_unlock();
+ return err;
+}
+EXPORT_SYMBOL(km_migrate);
+#endif
+
+int km_report(struct net *net, u8 proto, struct xfrm_selector *sel, xfrm_address_t *addr)
+{
+ int err = -EINVAL;
+ int ret;
+ struct xfrm_mgr *km;
+
+ rcu_read_lock();
+ list_for_each_entry_rcu(km, &xfrm_km_list, list) {
+ if (km->report) {
+ ret = km->report(net, proto, sel, addr);
+ if (!ret)
+ err = ret;
+ }
+ }
+ rcu_read_unlock();
+ return err;
+}
+EXPORT_SYMBOL(km_report);
+
+static bool km_is_alive(const struct km_event *c)
+{
+ struct xfrm_mgr *km;
+ bool is_alive = false;
+
+ rcu_read_lock();
+ list_for_each_entry_rcu(km, &xfrm_km_list, list) {
+ if (km->is_alive && km->is_alive(c)) {
+ is_alive = true;
+ break;
+ }
+ }
+ rcu_read_unlock();
+
+ return is_alive;
+}
+
+#if IS_ENABLED(CONFIG_XFRM_USER_COMPAT)
+static DEFINE_SPINLOCK(xfrm_translator_lock);
+static struct xfrm_translator __rcu *xfrm_translator;
+
+struct xfrm_translator *xfrm_get_translator(void)
+{
+ struct xfrm_translator *xtr;
+
+ rcu_read_lock();
+ xtr = rcu_dereference(xfrm_translator);
+ if (unlikely(!xtr))
+ goto out;
+ if (!try_module_get(xtr->owner))
+ xtr = NULL;
+out:
+ rcu_read_unlock();
+ return xtr;
+}
+EXPORT_SYMBOL_GPL(xfrm_get_translator);
+
+void xfrm_put_translator(struct xfrm_translator *xtr)
+{
+ module_put(xtr->owner);
+}
+EXPORT_SYMBOL_GPL(xfrm_put_translator);
+
+int xfrm_register_translator(struct xfrm_translator *xtr)
+{
+ int err = 0;
+
+ spin_lock_bh(&xfrm_translator_lock);
+ if (unlikely(xfrm_translator != NULL))
+ err = -EEXIST;
+ else
+ rcu_assign_pointer(xfrm_translator, xtr);
+ spin_unlock_bh(&xfrm_translator_lock);
+
+ return err;
+}
+EXPORT_SYMBOL_GPL(xfrm_register_translator);
+
+int xfrm_unregister_translator(struct xfrm_translator *xtr)
+{
+ int err = 0;
+
+ spin_lock_bh(&xfrm_translator_lock);
+ if (likely(xfrm_translator != NULL)) {
+ if (rcu_access_pointer(xfrm_translator) != xtr)
+ err = -EINVAL;
+ else
+ RCU_INIT_POINTER(xfrm_translator, NULL);
+ }
+ spin_unlock_bh(&xfrm_translator_lock);
+ synchronize_rcu();
+
+ return err;
+}
+EXPORT_SYMBOL_GPL(xfrm_unregister_translator);
+#endif
+
+int xfrm_user_policy(struct sock *sk, int optname, sockptr_t optval, int optlen)
+{
+ int err;
+ u8 *data;
+ struct xfrm_mgr *km;
+ struct xfrm_policy *pol = NULL;
+
+ if (sockptr_is_null(optval) && !optlen) {
+ xfrm_sk_policy_insert(sk, XFRM_POLICY_IN, NULL);
+ xfrm_sk_policy_insert(sk, XFRM_POLICY_OUT, NULL);
+ __sk_dst_reset(sk);
+ return 0;
+ }
+
+ if (optlen <= 0 || optlen > PAGE_SIZE)
+ return -EMSGSIZE;
+
+ data = memdup_sockptr(optval, optlen);
+ if (IS_ERR(data))
+ return PTR_ERR(data);
+
+ if (in_compat_syscall()) {
+ struct xfrm_translator *xtr = xfrm_get_translator();
+
+ if (!xtr) {
+ kfree(data);
+ return -EOPNOTSUPP;
+ }
+
+ err = xtr->xlate_user_policy_sockptr(&data, optlen);
+ xfrm_put_translator(xtr);
+ if (err) {
+ kfree(data);
+ return err;
+ }
+ }
+
+ err = -EINVAL;
+ rcu_read_lock();
+ list_for_each_entry_rcu(km, &xfrm_km_list, list) {
+ pol = km->compile_policy(sk, optname, data,
+ optlen, &err);
+ if (err >= 0)
+ break;
+ }
+ rcu_read_unlock();
+
+ if (err >= 0) {
+ xfrm_sk_policy_insert(sk, err, pol);
+ xfrm_pol_put(pol);
+ __sk_dst_reset(sk);
+ err = 0;
+ }
+
+ kfree(data);
+ return err;
+}
+EXPORT_SYMBOL(xfrm_user_policy);
+
+static DEFINE_SPINLOCK(xfrm_km_lock);
+
+void xfrm_register_km(struct xfrm_mgr *km)
+{
+ spin_lock_bh(&xfrm_km_lock);
+ list_add_tail_rcu(&km->list, &xfrm_km_list);
+ spin_unlock_bh(&xfrm_km_lock);
+}
+EXPORT_SYMBOL(xfrm_register_km);
+
+void xfrm_unregister_km(struct xfrm_mgr *km)
+{
+ spin_lock_bh(&xfrm_km_lock);
+ list_del_rcu(&km->list);
+ spin_unlock_bh(&xfrm_km_lock);
+ synchronize_rcu();
+}
+EXPORT_SYMBOL(xfrm_unregister_km);
+
+int xfrm_state_register_afinfo(struct xfrm_state_afinfo *afinfo)
+{
+ int err = 0;
+
+ if (WARN_ON(afinfo->family >= NPROTO))
+ return -EAFNOSUPPORT;
+
+ spin_lock_bh(&xfrm_state_afinfo_lock);
+ if (unlikely(xfrm_state_afinfo[afinfo->family] != NULL))
+ err = -EEXIST;
+ else
+ rcu_assign_pointer(xfrm_state_afinfo[afinfo->family], afinfo);
+ spin_unlock_bh(&xfrm_state_afinfo_lock);
+ return err;
+}
+EXPORT_SYMBOL(xfrm_state_register_afinfo);
+
+int xfrm_state_unregister_afinfo(struct xfrm_state_afinfo *afinfo)
+{
+ int err = 0, family = afinfo->family;
+
+ if (WARN_ON(family >= NPROTO))
+ return -EAFNOSUPPORT;
+
+ spin_lock_bh(&xfrm_state_afinfo_lock);
+ if (likely(xfrm_state_afinfo[afinfo->family] != NULL)) {
+ if (rcu_access_pointer(xfrm_state_afinfo[family]) != afinfo)
+ err = -EINVAL;
+ else
+ RCU_INIT_POINTER(xfrm_state_afinfo[afinfo->family], NULL);
+ }
+ spin_unlock_bh(&xfrm_state_afinfo_lock);
+ synchronize_rcu();
+ return err;
+}
+EXPORT_SYMBOL(xfrm_state_unregister_afinfo);
+
+struct xfrm_state_afinfo *xfrm_state_afinfo_get_rcu(unsigned int family)
+{
+ if (unlikely(family >= NPROTO))
+ return NULL;
+
+ return rcu_dereference(xfrm_state_afinfo[family]);
+}
+EXPORT_SYMBOL_GPL(xfrm_state_afinfo_get_rcu);
+
+struct xfrm_state_afinfo *xfrm_state_get_afinfo(unsigned int family)
+{
+ struct xfrm_state_afinfo *afinfo;
+ if (unlikely(family >= NPROTO))
+ return NULL;
+ rcu_read_lock();
+ afinfo = rcu_dereference(xfrm_state_afinfo[family]);
+ if (unlikely(!afinfo))
+ rcu_read_unlock();
+ return afinfo;
+}
+
+void xfrm_flush_gc(void)
+{
+ flush_work(&xfrm_state_gc_work);
+}
+EXPORT_SYMBOL(xfrm_flush_gc);
+
+/* Temporarily located here until net/xfrm/xfrm_tunnel.c is created */
+void xfrm_state_delete_tunnel(struct xfrm_state *x)
+{
+ if (x->tunnel) {
+ struct xfrm_state *t = x->tunnel;
+
+ if (atomic_read(&t->tunnel_users) == 2)
+ xfrm_state_delete(t);
+ atomic_dec(&t->tunnel_users);
+ xfrm_state_put_sync(t);
+ x->tunnel = NULL;
+ }
+}
+EXPORT_SYMBOL(xfrm_state_delete_tunnel);
+
+u32 xfrm_state_mtu(struct xfrm_state *x, int mtu)
+{
+ const struct xfrm_type *type = READ_ONCE(x->type);
+ struct crypto_aead *aead;
+ u32 blksize, net_adj = 0;
+
+ if (x->km.state != XFRM_STATE_VALID ||
+ !type || type->proto != IPPROTO_ESP)
+ return mtu - x->props.header_len;
+
+ aead = x->data;
+ blksize = ALIGN(crypto_aead_blocksize(aead), 4);
+
+ switch (x->props.mode) {
+ case XFRM_MODE_TRANSPORT:
+ case XFRM_MODE_BEET:
+ if (x->props.family == AF_INET)
+ net_adj = sizeof(struct iphdr);
+ else if (x->props.family == AF_INET6)
+ net_adj = sizeof(struct ipv6hdr);
+ break;
+ case XFRM_MODE_TUNNEL:
+ break;
+ default:
+ WARN_ON_ONCE(1);
+ break;
+ }
+
+ return ((mtu - x->props.header_len - crypto_aead_authsize(aead) -
+ net_adj) & ~(blksize - 1)) + net_adj - 2;
+}
+EXPORT_SYMBOL_GPL(xfrm_state_mtu);
+
+int __xfrm_init_state(struct xfrm_state *x, bool init_replay, bool offload,
+ struct netlink_ext_ack *extack)
+{
+ const struct xfrm_mode *inner_mode;
+ const struct xfrm_mode *outer_mode;
+ int family = x->props.family;
+ int err;
+
+ if (family == AF_INET &&
+ READ_ONCE(xs_net(x)->ipv4.sysctl_ip_no_pmtu_disc))
+ x->props.flags |= XFRM_STATE_NOPMTUDISC;
+
+ err = -EPROTONOSUPPORT;
+
+ if (x->sel.family != AF_UNSPEC) {
+ inner_mode = xfrm_get_mode(x->props.mode, x->sel.family);
+ if (inner_mode == NULL) {
+ NL_SET_ERR_MSG(extack, "Requested mode not found");
+ goto error;
+ }
+
+ if (!(inner_mode->flags & XFRM_MODE_FLAG_TUNNEL) &&
+ family != x->sel.family) {
+ NL_SET_ERR_MSG(extack, "Only tunnel modes can accommodate a change of family");
+ goto error;
+ }
+
+ x->inner_mode = *inner_mode;
+ } else {
+ const struct xfrm_mode *inner_mode_iaf;
+ int iafamily = AF_INET;
+
+ inner_mode = xfrm_get_mode(x->props.mode, x->props.family);
+ if (inner_mode == NULL) {
+ NL_SET_ERR_MSG(extack, "Requested mode not found");
+ goto error;
+ }
+
+ x->inner_mode = *inner_mode;
+
+ if (x->props.family == AF_INET)
+ iafamily = AF_INET6;
+
+ inner_mode_iaf = xfrm_get_mode(x->props.mode, iafamily);
+ if (inner_mode_iaf) {
+ if (inner_mode_iaf->flags & XFRM_MODE_FLAG_TUNNEL)
+ x->inner_mode_iaf = *inner_mode_iaf;
+ }
+ }
+
+ x->type = xfrm_get_type(x->id.proto, family);
+ if (x->type == NULL) {
+ NL_SET_ERR_MSG(extack, "Requested type not found");
+ goto error;
+ }
+
+ x->type_offload = xfrm_get_type_offload(x->id.proto, family, offload);
+
+ err = x->type->init_state(x, extack);
+ if (err)
+ goto error;
+
+ outer_mode = xfrm_get_mode(x->props.mode, family);
+ if (!outer_mode) {
+ NL_SET_ERR_MSG(extack, "Requested mode not found");
+ err = -EPROTONOSUPPORT;
+ goto error;
+ }
+
+ x->outer_mode = *outer_mode;
+ if (init_replay) {
+ err = xfrm_init_replay(x, extack);
+ if (err)
+ goto error;
+ }
+
+error:
+ return err;
+}
+
+EXPORT_SYMBOL(__xfrm_init_state);
+
+int xfrm_init_state(struct xfrm_state *x)
+{
+ int err;
+
+ err = __xfrm_init_state(x, true, false, NULL);
+ if (!err)
+ x->km.state = XFRM_STATE_VALID;
+
+ return err;
+}
+
+EXPORT_SYMBOL(xfrm_init_state);
+
+int __net_init xfrm_state_init(struct net *net)
+{
+ unsigned int sz;
+
+ if (net_eq(net, &init_net))
+ xfrm_state_cache = KMEM_CACHE(xfrm_state,
+ SLAB_HWCACHE_ALIGN | SLAB_PANIC);
+
+ INIT_LIST_HEAD(&net->xfrm.state_all);
+
+ sz = sizeof(struct hlist_head) * 8;
+
+ net->xfrm.state_bydst = xfrm_hash_alloc(sz);
+ if (!net->xfrm.state_bydst)
+ goto out_bydst;
+ net->xfrm.state_bysrc = xfrm_hash_alloc(sz);
+ if (!net->xfrm.state_bysrc)
+ goto out_bysrc;
+ net->xfrm.state_byspi = xfrm_hash_alloc(sz);
+ if (!net->xfrm.state_byspi)
+ goto out_byspi;
+ net->xfrm.state_byseq = xfrm_hash_alloc(sz);
+ if (!net->xfrm.state_byseq)
+ goto out_byseq;
+ net->xfrm.state_hmask = ((sz / sizeof(struct hlist_head)) - 1);
+
+ net->xfrm.state_num = 0;
+ INIT_WORK(&net->xfrm.state_hash_work, xfrm_hash_resize);
+ spin_lock_init(&net->xfrm.xfrm_state_lock);
+ seqcount_spinlock_init(&net->xfrm.xfrm_state_hash_generation,
+ &net->xfrm.xfrm_state_lock);
+ return 0;
+
+out_byseq:
+ xfrm_hash_free(net->xfrm.state_byspi, sz);
+out_byspi:
+ xfrm_hash_free(net->xfrm.state_bysrc, sz);
+out_bysrc:
+ xfrm_hash_free(net->xfrm.state_bydst, sz);
+out_bydst:
+ return -ENOMEM;
+}
+
+void xfrm_state_fini(struct net *net)
+{
+ unsigned int sz;
+
+ flush_work(&net->xfrm.state_hash_work);
+ flush_work(&xfrm_state_gc_work);
+ xfrm_state_flush(net, 0, false, true);
+
+ WARN_ON(!list_empty(&net->xfrm.state_all));
+
+ sz = (net->xfrm.state_hmask + 1) * sizeof(struct hlist_head);
+ WARN_ON(!hlist_empty(net->xfrm.state_byseq));
+ xfrm_hash_free(net->xfrm.state_byseq, sz);
+ WARN_ON(!hlist_empty(net->xfrm.state_byspi));
+ xfrm_hash_free(net->xfrm.state_byspi, sz);
+ WARN_ON(!hlist_empty(net->xfrm.state_bysrc));
+ xfrm_hash_free(net->xfrm.state_bysrc, sz);
+ WARN_ON(!hlist_empty(net->xfrm.state_bydst));
+ xfrm_hash_free(net->xfrm.state_bydst, sz);
+}
+
+#ifdef CONFIG_AUDITSYSCALL
+static void xfrm_audit_helper_sainfo(struct xfrm_state *x,
+ struct audit_buffer *audit_buf)
+{
+ struct xfrm_sec_ctx *ctx = x->security;
+ u32 spi = ntohl(x->id.spi);
+
+ if (ctx)
+ audit_log_format(audit_buf, " sec_alg=%u sec_doi=%u sec_obj=%s",
+ ctx->ctx_alg, ctx->ctx_doi, ctx->ctx_str);
+
+ switch (x->props.family) {
+ case AF_INET:
+ audit_log_format(audit_buf, " src=%pI4 dst=%pI4",
+ &x->props.saddr.a4, &x->id.daddr.a4);
+ break;
+ case AF_INET6:
+ audit_log_format(audit_buf, " src=%pI6 dst=%pI6",
+ x->props.saddr.a6, x->id.daddr.a6);
+ break;
+ }
+
+ audit_log_format(audit_buf, " spi=%u(0x%x)", spi, spi);
+}
+
+static void xfrm_audit_helper_pktinfo(struct sk_buff *skb, u16 family,
+ struct audit_buffer *audit_buf)
+{
+ const struct iphdr *iph4;
+ const struct ipv6hdr *iph6;
+
+ switch (family) {
+ case AF_INET:
+ iph4 = ip_hdr(skb);
+ audit_log_format(audit_buf, " src=%pI4 dst=%pI4",
+ &iph4->saddr, &iph4->daddr);
+ break;
+ case AF_INET6:
+ iph6 = ipv6_hdr(skb);
+ audit_log_format(audit_buf,
+ " src=%pI6 dst=%pI6 flowlbl=0x%x%02x%02x",
+ &iph6->saddr, &iph6->daddr,
+ iph6->flow_lbl[0] & 0x0f,
+ iph6->flow_lbl[1],
+ iph6->flow_lbl[2]);
+ break;
+ }
+}
+
+void xfrm_audit_state_add(struct xfrm_state *x, int result, bool task_valid)
+{
+ struct audit_buffer *audit_buf;
+
+ audit_buf = xfrm_audit_start("SAD-add");
+ if (audit_buf == NULL)
+ return;
+ xfrm_audit_helper_usrinfo(task_valid, audit_buf);
+ xfrm_audit_helper_sainfo(x, audit_buf);
+ audit_log_format(audit_buf, " res=%u", result);
+ audit_log_end(audit_buf);
+}
+EXPORT_SYMBOL_GPL(xfrm_audit_state_add);
+
+void xfrm_audit_state_delete(struct xfrm_state *x, int result, bool task_valid)
+{
+ struct audit_buffer *audit_buf;
+
+ audit_buf = xfrm_audit_start("SAD-delete");
+ if (audit_buf == NULL)
+ return;
+ xfrm_audit_helper_usrinfo(task_valid, audit_buf);
+ xfrm_audit_helper_sainfo(x, audit_buf);
+ audit_log_format(audit_buf, " res=%u", result);
+ audit_log_end(audit_buf);
+}
+EXPORT_SYMBOL_GPL(xfrm_audit_state_delete);
+
+void xfrm_audit_state_replay_overflow(struct xfrm_state *x,
+ struct sk_buff *skb)
+{
+ struct audit_buffer *audit_buf;
+ u32 spi;
+
+ audit_buf = xfrm_audit_start("SA-replay-overflow");
+ if (audit_buf == NULL)
+ return;
+ xfrm_audit_helper_pktinfo(skb, x->props.family, audit_buf);
+ /* don't record the sequence number because it's inherent in this kind
+ * of audit message */
+ spi = ntohl(x->id.spi);
+ audit_log_format(audit_buf, " spi=%u(0x%x)", spi, spi);
+ audit_log_end(audit_buf);
+}
+EXPORT_SYMBOL_GPL(xfrm_audit_state_replay_overflow);
+
+void xfrm_audit_state_replay(struct xfrm_state *x,
+ struct sk_buff *skb, __be32 net_seq)
+{
+ struct audit_buffer *audit_buf;
+ u32 spi;
+
+ audit_buf = xfrm_audit_start("SA-replayed-pkt");
+ if (audit_buf == NULL)
+ return;
+ xfrm_audit_helper_pktinfo(skb, x->props.family, audit_buf);
+ spi = ntohl(x->id.spi);
+ audit_log_format(audit_buf, " spi=%u(0x%x) seqno=%u",
+ spi, spi, ntohl(net_seq));
+ audit_log_end(audit_buf);
+}
+EXPORT_SYMBOL_GPL(xfrm_audit_state_replay);
+
+void xfrm_audit_state_notfound_simple(struct sk_buff *skb, u16 family)
+{
+ struct audit_buffer *audit_buf;
+
+ audit_buf = xfrm_audit_start("SA-notfound");
+ if (audit_buf == NULL)
+ return;
+ xfrm_audit_helper_pktinfo(skb, family, audit_buf);
+ audit_log_end(audit_buf);
+}
+EXPORT_SYMBOL_GPL(xfrm_audit_state_notfound_simple);
+
+void xfrm_audit_state_notfound(struct sk_buff *skb, u16 family,
+ __be32 net_spi, __be32 net_seq)
+{
+ struct audit_buffer *audit_buf;
+ u32 spi;
+
+ audit_buf = xfrm_audit_start("SA-notfound");
+ if (audit_buf == NULL)
+ return;
+ xfrm_audit_helper_pktinfo(skb, family, audit_buf);
+ spi = ntohl(net_spi);
+ audit_log_format(audit_buf, " spi=%u(0x%x) seqno=%u",
+ spi, spi, ntohl(net_seq));
+ audit_log_end(audit_buf);
+}
+EXPORT_SYMBOL_GPL(xfrm_audit_state_notfound);
+
+void xfrm_audit_state_icvfail(struct xfrm_state *x,
+ struct sk_buff *skb, u8 proto)
+{
+ struct audit_buffer *audit_buf;
+ __be32 net_spi;
+ __be32 net_seq;
+
+ audit_buf = xfrm_audit_start("SA-icv-failure");
+ if (audit_buf == NULL)
+ return;
+ xfrm_audit_helper_pktinfo(skb, x->props.family, audit_buf);
+ if (xfrm_parse_spi(skb, proto, &net_spi, &net_seq) == 0) {
+ u32 spi = ntohl(net_spi);
+ audit_log_format(audit_buf, " spi=%u(0x%x) seqno=%u",
+ spi, spi, ntohl(net_seq));
+ }
+ audit_log_end(audit_buf);
+}
+EXPORT_SYMBOL_GPL(xfrm_audit_state_icvfail);
+#endif /* CONFIG_AUDITSYSCALL */
diff --git a/net/xfrm/xfrm_sysctl.c b/net/xfrm/xfrm_sysctl.c
new file mode 100644
index 000000000..0c6c5ef65
--- /dev/null
+++ b/net/xfrm/xfrm_sysctl.c
@@ -0,0 +1,87 @@
+// SPDX-License-Identifier: GPL-2.0
+#include <linux/sysctl.h>
+#include <linux/slab.h>
+#include <net/net_namespace.h>
+#include <net/xfrm.h>
+
+static void __net_init __xfrm_sysctl_init(struct net *net)
+{
+ net->xfrm.sysctl_aevent_etime = XFRM_AE_ETIME;
+ net->xfrm.sysctl_aevent_rseqth = XFRM_AE_SEQT_SIZE;
+ net->xfrm.sysctl_larval_drop = 1;
+ net->xfrm.sysctl_acq_expires = 30;
+}
+
+#ifdef CONFIG_SYSCTL
+static struct ctl_table xfrm_table[] = {
+ {
+ .procname = "xfrm_aevent_etime",
+ .maxlen = sizeof(u32),
+ .mode = 0644,
+ .proc_handler = proc_douintvec
+ },
+ {
+ .procname = "xfrm_aevent_rseqth",
+ .maxlen = sizeof(u32),
+ .mode = 0644,
+ .proc_handler = proc_douintvec
+ },
+ {
+ .procname = "xfrm_larval_drop",
+ .maxlen = sizeof(int),
+ .mode = 0644,
+ .proc_handler = proc_dointvec
+ },
+ {
+ .procname = "xfrm_acq_expires",
+ .maxlen = sizeof(int),
+ .mode = 0644,
+ .proc_handler = proc_dointvec
+ },
+ {}
+};
+
+int __net_init xfrm_sysctl_init(struct net *net)
+{
+ struct ctl_table *table;
+
+ __xfrm_sysctl_init(net);
+
+ table = kmemdup(xfrm_table, sizeof(xfrm_table), GFP_KERNEL);
+ if (!table)
+ goto out_kmemdup;
+ table[0].data = &net->xfrm.sysctl_aevent_etime;
+ table[1].data = &net->xfrm.sysctl_aevent_rseqth;
+ table[2].data = &net->xfrm.sysctl_larval_drop;
+ table[3].data = &net->xfrm.sysctl_acq_expires;
+
+ /* Don't export sysctls to unprivileged users */
+ if (net->user_ns != &init_user_ns)
+ table[0].procname = NULL;
+
+ net->xfrm.sysctl_hdr = register_net_sysctl(net, "net/core", table);
+ if (!net->xfrm.sysctl_hdr)
+ goto out_register;
+ return 0;
+
+out_register:
+ kfree(table);
+out_kmemdup:
+ return -ENOMEM;
+}
+
+void __net_exit xfrm_sysctl_fini(struct net *net)
+{
+ struct ctl_table *table;
+
+ table = net->xfrm.sysctl_hdr->ctl_table_arg;
+ unregister_net_sysctl_table(net->xfrm.sysctl_hdr);
+ kfree(table);
+}
+#else
+int __net_init xfrm_sysctl_init(struct net *net)
+{
+ __xfrm_sysctl_init(net);
+ return 0;
+}
+#endif
diff --git a/net/xfrm/xfrm_user.c b/net/xfrm/xfrm_user.c
new file mode 100644
index 000000000..d042ca012
--- /dev/null
+++ b/net/xfrm/xfrm_user.c
@@ -0,0 +1,3830 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/* xfrm_user.c: User interface to configure xfrm engine.
+ *
+ * Copyright (C) 2002 David S. Miller (davem@redhat.com)
+ *
+ * Changes:
+ * Mitsuru KANDA @USAGI
+ * Kazunori MIYAZAWA @USAGI
+ * Kunihiro Ishiguro <kunihiro@ipinfusion.com>
+ * IPv6 support
+ *
+ */
+
+#include <linux/compat.h>
+#include <linux/crypto.h>
+#include <linux/module.h>
+#include <linux/kernel.h>
+#include <linux/types.h>
+#include <linux/slab.h>
+#include <linux/socket.h>
+#include <linux/string.h>
+#include <linux/net.h>
+#include <linux/skbuff.h>
+#include <linux/pfkeyv2.h>
+#include <linux/ipsec.h>
+#include <linux/init.h>
+#include <linux/security.h>
+#include <net/sock.h>
+#include <net/xfrm.h>
+#include <net/netlink.h>
+#include <net/ah.h>
+#include <linux/uaccess.h>
+#if IS_ENABLED(CONFIG_IPV6)
+#include <linux/in6.h>
+#endif
+#include <asm/unaligned.h>
+
+static int verify_one_alg(struct nlattr **attrs, enum xfrm_attr_type_t type,
+ struct netlink_ext_ack *extack)
+{
+ struct nlattr *rt = attrs[type];
+ struct xfrm_algo *algp;
+
+ if (!rt)
+ return 0;
+
+ algp = nla_data(rt);
+ if (nla_len(rt) < (int)xfrm_alg_len(algp)) {
+ NL_SET_ERR_MSG(extack, "Invalid AUTH/CRYPT/COMP attribute length");
+ return -EINVAL;
+ }
+
+ switch (type) {
+ case XFRMA_ALG_AUTH:
+ case XFRMA_ALG_CRYPT:
+ case XFRMA_ALG_COMP:
+ break;
+
+ default:
+ NL_SET_ERR_MSG(extack, "Invalid algorithm attribute type");
+ return -EINVAL;
+ }
+
+ algp->alg_name[sizeof(algp->alg_name) - 1] = '\0';
+ return 0;
+}
+
+static int verify_auth_trunc(struct nlattr **attrs,
+ struct netlink_ext_ack *extack)
+{
+ struct nlattr *rt = attrs[XFRMA_ALG_AUTH_TRUNC];
+ struct xfrm_algo_auth *algp;
+
+ if (!rt)
+ return 0;
+
+ algp = nla_data(rt);
+ if (nla_len(rt) < (int)xfrm_alg_auth_len(algp)) {
+ NL_SET_ERR_MSG(extack, "Invalid AUTH_TRUNC attribute length");
+ return -EINVAL;
+ }
+
+ algp->alg_name[sizeof(algp->alg_name) - 1] = '\0';
+ return 0;
+}
+
+static int verify_aead(struct nlattr **attrs, struct netlink_ext_ack *extack)
+{
+ struct nlattr *rt = attrs[XFRMA_ALG_AEAD];
+ struct xfrm_algo_aead *algp;
+
+ if (!rt)
+ return 0;
+
+ algp = nla_data(rt);
+ if (nla_len(rt) < (int)aead_len(algp)) {
+ NL_SET_ERR_MSG(extack, "Invalid AEAD attribute length");
+ return -EINVAL;
+ }
+
+ algp->alg_name[sizeof(algp->alg_name) - 1] = '\0';
+ return 0;
+}
+
+static void verify_one_addr(struct nlattr **attrs, enum xfrm_attr_type_t type,
+ xfrm_address_t **addrp)
+{
+ struct nlattr *rt = attrs[type];
+
+ if (rt && addrp)
+ *addrp = nla_data(rt);
+}
+
+static inline int verify_sec_ctx_len(struct nlattr **attrs, struct netlink_ext_ack *extack)
+{
+ struct nlattr *rt = attrs[XFRMA_SEC_CTX];
+ struct xfrm_user_sec_ctx *uctx;
+
+ if (!rt)
+ return 0;
+
+ uctx = nla_data(rt);
+ if (uctx->len > nla_len(rt) ||
+ uctx->len != (sizeof(struct xfrm_user_sec_ctx) + uctx->ctx_len)) {
+ NL_SET_ERR_MSG(extack, "Invalid security context length");
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
+static inline int verify_replay(struct xfrm_usersa_info *p,
+ struct nlattr **attrs,
+ struct netlink_ext_ack *extack)
+{
+ struct nlattr *rt = attrs[XFRMA_REPLAY_ESN_VAL];
+ struct xfrm_replay_state_esn *rs;
+
+ if (!rt) {
+ if (p->flags & XFRM_STATE_ESN) {
+ NL_SET_ERR_MSG(extack, "Missing required attribute for ESN");
+ return -EINVAL;
+ }
+ return 0;
+ }
+
+ rs = nla_data(rt);
+
+ if (rs->bmp_len > XFRMA_REPLAY_ESN_MAX / sizeof(rs->bmp[0]) / 8) {
+ NL_SET_ERR_MSG(extack, "ESN bitmap length must be <= 128");
+ return -EINVAL;
+ }
+
+ if (nla_len(rt) < (int)xfrm_replay_state_esn_len(rs) &&
+ nla_len(rt) != sizeof(*rs)) {
+ NL_SET_ERR_MSG(extack, "ESN attribute is too short to fit the full bitmap length");
+ return -EINVAL;
+ }
+
+ /* As only ESP and AH support ESN feature. */
+ if ((p->id.proto != IPPROTO_ESP) && (p->id.proto != IPPROTO_AH)) {
+ NL_SET_ERR_MSG(extack, "ESN only supported for ESP and AH");
+ return -EINVAL;
+ }
+
+ if (p->replay_window != 0) {
+ NL_SET_ERR_MSG(extack, "ESN not compatible with legacy replay_window");
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
+static int verify_newsa_info(struct xfrm_usersa_info *p,
+ struct nlattr **attrs,
+ struct netlink_ext_ack *extack)
+{
+ int err;
+
+ err = -EINVAL;
+ switch (p->family) {
+ case AF_INET:
+ break;
+
+ case AF_INET6:
+#if IS_ENABLED(CONFIG_IPV6)
+ break;
+#else
+ err = -EAFNOSUPPORT;
+ NL_SET_ERR_MSG(extack, "IPv6 support disabled");
+ goto out;
+#endif
+
+ default:
+ NL_SET_ERR_MSG(extack, "Invalid address family");
+ goto out;
+ }
+
+ switch (p->sel.family) {
+ case AF_UNSPEC:
+ break;
+
+ case AF_INET:
+ if (p->sel.prefixlen_d > 32 || p->sel.prefixlen_s > 32) {
+ NL_SET_ERR_MSG(extack, "Invalid prefix length in selector (must be <= 32 for IPv4)");
+ goto out;
+ }
+
+ break;
+
+ case AF_INET6:
+#if IS_ENABLED(CONFIG_IPV6)
+ if (p->sel.prefixlen_d > 128 || p->sel.prefixlen_s > 128) {
+ NL_SET_ERR_MSG(extack, "Invalid prefix length in selector (must be <= 128 for IPv6)");
+ goto out;
+ }
+
+ break;
+#else
+ NL_SET_ERR_MSG(extack, "IPv6 support disabled");
+ err = -EAFNOSUPPORT;
+ goto out;
+#endif
+
+ default:
+ NL_SET_ERR_MSG(extack, "Invalid address family in selector");
+ goto out;
+ }
+
+ err = -EINVAL;
+ switch (p->id.proto) {
+ case IPPROTO_AH:
+ if (!attrs[XFRMA_ALG_AUTH] &&
+ !attrs[XFRMA_ALG_AUTH_TRUNC]) {
+ NL_SET_ERR_MSG(extack, "Missing required attribute for AH: AUTH_TRUNC or AUTH");
+ goto out;
+ }
+
+ if (attrs[XFRMA_ALG_AEAD] ||
+ attrs[XFRMA_ALG_CRYPT] ||
+ attrs[XFRMA_ALG_COMP] ||
+ attrs[XFRMA_TFCPAD]) {
+ NL_SET_ERR_MSG(extack, "Invalid attributes for AH: AEAD, CRYPT, COMP, TFCPAD");
+ goto out;
+ }
+ break;
+
+ case IPPROTO_ESP:
+ if (attrs[XFRMA_ALG_COMP]) {
+ NL_SET_ERR_MSG(extack, "Invalid attribute for ESP: COMP");
+ goto out;
+ }
+
+ if (!attrs[XFRMA_ALG_AUTH] &&
+ !attrs[XFRMA_ALG_AUTH_TRUNC] &&
+ !attrs[XFRMA_ALG_CRYPT] &&
+ !attrs[XFRMA_ALG_AEAD]) {
+ NL_SET_ERR_MSG(extack, "Missing required attribute for ESP: at least one of AUTH, AUTH_TRUNC, CRYPT, AEAD");
+ goto out;
+ }
+
+ if ((attrs[XFRMA_ALG_AUTH] ||
+ attrs[XFRMA_ALG_AUTH_TRUNC] ||
+ attrs[XFRMA_ALG_CRYPT]) &&
+ attrs[XFRMA_ALG_AEAD]) {
+ NL_SET_ERR_MSG(extack, "Invalid attribute combination for ESP: AEAD can't be used with AUTH, AUTH_TRUNC, CRYPT");
+ goto out;
+ }
+
+ if (attrs[XFRMA_TFCPAD] &&
+ p->mode != XFRM_MODE_TUNNEL) {
+ NL_SET_ERR_MSG(extack, "TFC padding can only be used in tunnel mode");
+ goto out;
+ }
+ break;
+
+ case IPPROTO_COMP:
+ if (!attrs[XFRMA_ALG_COMP]) {
+ NL_SET_ERR_MSG(extack, "Missing required attribute for COMP: COMP");
+ goto out;
+ }
+
+ if (attrs[XFRMA_ALG_AEAD] ||
+ attrs[XFRMA_ALG_AUTH] ||
+ attrs[XFRMA_ALG_AUTH_TRUNC] ||
+ attrs[XFRMA_ALG_CRYPT] ||
+ attrs[XFRMA_TFCPAD]) {
+ NL_SET_ERR_MSG(extack, "Invalid attributes for COMP: AEAD, AUTH, AUTH_TRUNC, CRYPT, TFCPAD");
+ goto out;
+ }
+
+ if (ntohl(p->id.spi) >= 0x10000) {
+ NL_SET_ERR_MSG(extack, "SPI is too large for COMP (must be < 0x10000)");
+ goto out;
+ }
+ break;
+
+#if IS_ENABLED(CONFIG_IPV6)
+ case IPPROTO_DSTOPTS:
+ case IPPROTO_ROUTING:
+ if (attrs[XFRMA_ALG_COMP] ||
+ attrs[XFRMA_ALG_AUTH] ||
+ attrs[XFRMA_ALG_AUTH_TRUNC] ||
+ attrs[XFRMA_ALG_AEAD] ||
+ attrs[XFRMA_ALG_CRYPT] ||
+ attrs[XFRMA_ENCAP] ||
+ attrs[XFRMA_SEC_CTX] ||
+ attrs[XFRMA_TFCPAD]) {
+ NL_SET_ERR_MSG(extack, "Invalid attributes for DSTOPTS/ROUTING");
+ goto out;
+ }
+
+ if (!attrs[XFRMA_COADDR]) {
+ NL_SET_ERR_MSG(extack, "Missing required COADDR attribute for DSTOPTS/ROUTING");
+ goto out;
+ }
+ break;
+#endif
+
+ default:
+ NL_SET_ERR_MSG(extack, "Unsupported protocol");
+ goto out;
+ }
+
+ if ((err = verify_aead(attrs, extack)))
+ goto out;
+ if ((err = verify_auth_trunc(attrs, extack)))
+ goto out;
+ if ((err = verify_one_alg(attrs, XFRMA_ALG_AUTH, extack)))
+ goto out;
+ if ((err = verify_one_alg(attrs, XFRMA_ALG_CRYPT, extack)))
+ goto out;
+ if ((err = verify_one_alg(attrs, XFRMA_ALG_COMP, extack)))
+ goto out;
+ if ((err = verify_sec_ctx_len(attrs, extack)))
+ goto out;
+ if ((err = verify_replay(p, attrs, extack)))
+ goto out;
+
+ err = -EINVAL;
+ switch (p->mode) {
+ case XFRM_MODE_TRANSPORT:
+ case XFRM_MODE_TUNNEL:
+ case XFRM_MODE_ROUTEOPTIMIZATION:
+ case XFRM_MODE_BEET:
+ break;
+
+ default:
+ NL_SET_ERR_MSG(extack, "Unsupported mode");
+ goto out;
+ }
+
+ err = 0;
+
+ if (attrs[XFRMA_MTIMER_THRESH]) {
+ if (!attrs[XFRMA_ENCAP]) {
+ NL_SET_ERR_MSG(extack, "MTIMER_THRESH attribute can only be set on ENCAP states");
+ err = -EINVAL;
+ goto out;
+ }
+ }
+
+out:
+ return err;
+}
+
+static int attach_one_algo(struct xfrm_algo **algpp, u8 *props,
+ struct xfrm_algo_desc *(*get_byname)(const char *, int),
+ struct nlattr *rta, struct netlink_ext_ack *extack)
+{
+ struct xfrm_algo *p, *ualg;
+ struct xfrm_algo_desc *algo;
+
+ if (!rta)
+ return 0;
+
+ ualg = nla_data(rta);
+
+ algo = get_byname(ualg->alg_name, 1);
+ if (!algo) {
+ NL_SET_ERR_MSG(extack, "Requested COMP algorithm not found");
+ return -ENOSYS;
+ }
+ *props = algo->desc.sadb_alg_id;
+
+ p = kmemdup(ualg, xfrm_alg_len(ualg), GFP_KERNEL);
+ if (!p)
+ return -ENOMEM;
+
+ strcpy(p->alg_name, algo->name);
+ *algpp = p;
+ return 0;
+}
+
+static int attach_crypt(struct xfrm_state *x, struct nlattr *rta,
+ struct netlink_ext_ack *extack)
+{
+ struct xfrm_algo *p, *ualg;
+ struct xfrm_algo_desc *algo;
+
+ if (!rta)
+ return 0;
+
+ ualg = nla_data(rta);
+
+ algo = xfrm_ealg_get_byname(ualg->alg_name, 1);
+ if (!algo) {
+ NL_SET_ERR_MSG(extack, "Requested CRYPT algorithm not found");
+ return -ENOSYS;
+ }
+ x->props.ealgo = algo->desc.sadb_alg_id;
+
+ p = kmemdup(ualg, xfrm_alg_len(ualg), GFP_KERNEL);
+ if (!p)
+ return -ENOMEM;
+
+ strcpy(p->alg_name, algo->name);
+ x->ealg = p;
+ x->geniv = algo->uinfo.encr.geniv;
+ return 0;
+}
+
+static int attach_auth(struct xfrm_algo_auth **algpp, u8 *props,
+ struct nlattr *rta, struct netlink_ext_ack *extack)
+{
+ struct xfrm_algo *ualg;
+ struct xfrm_algo_auth *p;
+ struct xfrm_algo_desc *algo;
+
+ if (!rta)
+ return 0;
+
+ ualg = nla_data(rta);
+
+ algo = xfrm_aalg_get_byname(ualg->alg_name, 1);
+ if (!algo) {
+ NL_SET_ERR_MSG(extack, "Requested AUTH algorithm not found");
+ return -ENOSYS;
+ }
+ *props = algo->desc.sadb_alg_id;
+
+ p = kmalloc(sizeof(*p) + (ualg->alg_key_len + 7) / 8, GFP_KERNEL);
+ if (!p)
+ return -ENOMEM;
+
+ strcpy(p->alg_name, algo->name);
+ p->alg_key_len = ualg->alg_key_len;
+ p->alg_trunc_len = algo->uinfo.auth.icv_truncbits;
+ memcpy(p->alg_key, ualg->alg_key, (ualg->alg_key_len + 7) / 8);
+
+ *algpp = p;
+ return 0;
+}
+
+static int attach_auth_trunc(struct xfrm_algo_auth **algpp, u8 *props,
+ struct nlattr *rta, struct netlink_ext_ack *extack)
+{
+ struct xfrm_algo_auth *p, *ualg;
+ struct xfrm_algo_desc *algo;
+
+ if (!rta)
+ return 0;
+
+ ualg = nla_data(rta);
+
+ algo = xfrm_aalg_get_byname(ualg->alg_name, 1);
+ if (!algo) {
+ NL_SET_ERR_MSG(extack, "Requested AUTH_TRUNC algorithm not found");
+ return -ENOSYS;
+ }
+ if (ualg->alg_trunc_len > algo->uinfo.auth.icv_fullbits) {
+ NL_SET_ERR_MSG(extack, "Invalid length requested for truncated ICV");
+ return -EINVAL;
+ }
+ *props = algo->desc.sadb_alg_id;
+
+ p = kmemdup(ualg, xfrm_alg_auth_len(ualg), GFP_KERNEL);
+ if (!p)
+ return -ENOMEM;
+
+ strcpy(p->alg_name, algo->name);
+ if (!p->alg_trunc_len)
+ p->alg_trunc_len = algo->uinfo.auth.icv_truncbits;
+
+ *algpp = p;
+ return 0;
+}
+
+static int attach_aead(struct xfrm_state *x, struct nlattr *rta,
+ struct netlink_ext_ack *extack)
+{
+ struct xfrm_algo_aead *p, *ualg;
+ struct xfrm_algo_desc *algo;
+
+ if (!rta)
+ return 0;
+
+ ualg = nla_data(rta);
+
+ algo = xfrm_aead_get_byname(ualg->alg_name, ualg->alg_icv_len, 1);
+ if (!algo) {
+ NL_SET_ERR_MSG(extack, "Requested AEAD algorithm not found");
+ return -ENOSYS;
+ }
+ x->props.ealgo = algo->desc.sadb_alg_id;
+
+ p = kmemdup(ualg, aead_len(ualg), GFP_KERNEL);
+ if (!p)
+ return -ENOMEM;
+
+ strcpy(p->alg_name, algo->name);
+ x->aead = p;
+ x->geniv = algo->uinfo.aead.geniv;
+ return 0;
+}
+
+static inline int xfrm_replay_verify_len(struct xfrm_replay_state_esn *replay_esn,
+ struct nlattr *rp)
+{
+ struct xfrm_replay_state_esn *up;
+ unsigned int ulen;
+
+ if (!replay_esn || !rp)
+ return 0;
+
+ up = nla_data(rp);
+ ulen = xfrm_replay_state_esn_len(up);
+
+ /* Check the overall length and the internal bitmap length to avoid
+ * potential overflow. */
+ if (nla_len(rp) < (int)ulen ||
+ xfrm_replay_state_esn_len(replay_esn) != ulen ||
+ replay_esn->bmp_len != up->bmp_len)
+ return -EINVAL;
+
+ if (up->replay_window > up->bmp_len * sizeof(__u32) * 8)
+ return -EINVAL;
+
+ return 0;
+}
+
+static int xfrm_alloc_replay_state_esn(struct xfrm_replay_state_esn **replay_esn,
+ struct xfrm_replay_state_esn **preplay_esn,
+ struct nlattr *rta)
+{
+ struct xfrm_replay_state_esn *p, *pp, *up;
+ unsigned int klen, ulen;
+
+ if (!rta)
+ return 0;
+
+ up = nla_data(rta);
+ klen = xfrm_replay_state_esn_len(up);
+ ulen = nla_len(rta) >= (int)klen ? klen : sizeof(*up);
+
+ p = kzalloc(klen, GFP_KERNEL);
+ if (!p)
+ return -ENOMEM;
+
+ pp = kzalloc(klen, GFP_KERNEL);
+ if (!pp) {
+ kfree(p);
+ return -ENOMEM;
+ }
+
+ memcpy(p, up, ulen);
+ memcpy(pp, up, ulen);
+
+ *replay_esn = p;
+ *preplay_esn = pp;
+
+ return 0;
+}
+
+static inline unsigned int xfrm_user_sec_ctx_size(struct xfrm_sec_ctx *xfrm_ctx)
+{
+ unsigned int len = 0;
+
+ if (xfrm_ctx) {
+ len += sizeof(struct xfrm_user_sec_ctx);
+ len += xfrm_ctx->ctx_len;
+ }
+ return len;
+}
+
+static void copy_from_user_state(struct xfrm_state *x, struct xfrm_usersa_info *p)
+{
+ memcpy(&x->id, &p->id, sizeof(x->id));
+ memcpy(&x->sel, &p->sel, sizeof(x->sel));
+ memcpy(&x->lft, &p->lft, sizeof(x->lft));
+ x->props.mode = p->mode;
+ x->props.replay_window = min_t(unsigned int, p->replay_window,
+ sizeof(x->replay.bitmap) * 8);
+ x->props.reqid = p->reqid;
+ x->props.family = p->family;
+ memcpy(&x->props.saddr, &p->saddr, sizeof(x->props.saddr));
+ x->props.flags = p->flags;
+
+ if (!x->sel.family && !(p->flags & XFRM_STATE_AF_UNSPEC))
+ x->sel.family = p->family;
+}
+
+/*
+ * someday when pfkey also has support, we could have the code
+ * somehow made shareable and move it to xfrm_state.c - JHS
+ *
+*/
+static void xfrm_update_ae_params(struct xfrm_state *x, struct nlattr **attrs,
+ int update_esn)
+{
+ struct nlattr *rp = attrs[XFRMA_REPLAY_VAL];
+ struct nlattr *re = update_esn ? attrs[XFRMA_REPLAY_ESN_VAL] : NULL;
+ struct nlattr *lt = attrs[XFRMA_LTIME_VAL];
+ struct nlattr *et = attrs[XFRMA_ETIMER_THRESH];
+ struct nlattr *rt = attrs[XFRMA_REPLAY_THRESH];
+ struct nlattr *mt = attrs[XFRMA_MTIMER_THRESH];
+
+ if (re && x->replay_esn && x->preplay_esn) {
+ struct xfrm_replay_state_esn *replay_esn;
+ replay_esn = nla_data(re);
+ memcpy(x->replay_esn, replay_esn,
+ xfrm_replay_state_esn_len(replay_esn));
+ memcpy(x->preplay_esn, replay_esn,
+ xfrm_replay_state_esn_len(replay_esn));
+ }
+
+ if (rp) {
+ struct xfrm_replay_state *replay;
+ replay = nla_data(rp);
+ memcpy(&x->replay, replay, sizeof(*replay));
+ memcpy(&x->preplay, replay, sizeof(*replay));
+ }
+
+ if (lt) {
+ struct xfrm_lifetime_cur *ltime;
+ ltime = nla_data(lt);
+ x->curlft.bytes = ltime->bytes;
+ x->curlft.packets = ltime->packets;
+ x->curlft.add_time = ltime->add_time;
+ x->curlft.use_time = ltime->use_time;
+ }
+
+ if (et)
+ x->replay_maxage = nla_get_u32(et);
+
+ if (rt)
+ x->replay_maxdiff = nla_get_u32(rt);
+
+ if (mt)
+ x->mapping_maxage = nla_get_u32(mt);
+}
+
+static void xfrm_smark_init(struct nlattr **attrs, struct xfrm_mark *m)
+{
+ if (attrs[XFRMA_SET_MARK]) {
+ m->v = nla_get_u32(attrs[XFRMA_SET_MARK]);
+ if (attrs[XFRMA_SET_MARK_MASK])
+ m->m = nla_get_u32(attrs[XFRMA_SET_MARK_MASK]);
+ else
+ m->m = 0xffffffff;
+ } else {
+ m->v = m->m = 0;
+ }
+}
+
+static struct xfrm_state *xfrm_state_construct(struct net *net,
+ struct xfrm_usersa_info *p,
+ struct nlattr **attrs,
+ int *errp,
+ struct netlink_ext_ack *extack)
+{
+ struct xfrm_state *x = xfrm_state_alloc(net);
+ int err = -ENOMEM;
+
+ if (!x)
+ goto error_no_put;
+
+ copy_from_user_state(x, p);
+
+ if (attrs[XFRMA_ENCAP]) {
+ x->encap = kmemdup(nla_data(attrs[XFRMA_ENCAP]),
+ sizeof(*x->encap), GFP_KERNEL);
+ if (x->encap == NULL)
+ goto error;
+ }
+
+ if (attrs[XFRMA_COADDR]) {
+ x->coaddr = kmemdup(nla_data(attrs[XFRMA_COADDR]),
+ sizeof(*x->coaddr), GFP_KERNEL);
+ if (x->coaddr == NULL)
+ goto error;
+ }
+
+ if (attrs[XFRMA_SA_EXTRA_FLAGS])
+ x->props.extra_flags = nla_get_u32(attrs[XFRMA_SA_EXTRA_FLAGS]);
+
+ if ((err = attach_aead(x, attrs[XFRMA_ALG_AEAD], extack)))
+ goto error;
+ if ((err = attach_auth_trunc(&x->aalg, &x->props.aalgo,
+ attrs[XFRMA_ALG_AUTH_TRUNC], extack)))
+ goto error;
+ if (!x->props.aalgo) {
+ if ((err = attach_auth(&x->aalg, &x->props.aalgo,
+ attrs[XFRMA_ALG_AUTH], extack)))
+ goto error;
+ }
+ if ((err = attach_crypt(x, attrs[XFRMA_ALG_CRYPT], extack)))
+ goto error;
+ if ((err = attach_one_algo(&x->calg, &x->props.calgo,
+ xfrm_calg_get_byname,
+ attrs[XFRMA_ALG_COMP], extack)))
+ goto error;
+
+ if (attrs[XFRMA_TFCPAD])
+ x->tfcpad = nla_get_u32(attrs[XFRMA_TFCPAD]);
+
+ xfrm_mark_get(attrs, &x->mark);
+
+ xfrm_smark_init(attrs, &x->props.smark);
+
+ if (attrs[XFRMA_IF_ID])
+ x->if_id = nla_get_u32(attrs[XFRMA_IF_ID]);
+
+ err = __xfrm_init_state(x, false, attrs[XFRMA_OFFLOAD_DEV], extack);
+ if (err)
+ goto error;
+
+ if (attrs[XFRMA_SEC_CTX]) {
+ err = security_xfrm_state_alloc(x,
+ nla_data(attrs[XFRMA_SEC_CTX]));
+ if (err)
+ goto error;
+ }
+
+ if ((err = xfrm_alloc_replay_state_esn(&x->replay_esn, &x->preplay_esn,
+ attrs[XFRMA_REPLAY_ESN_VAL])))
+ goto error;
+
+ x->km.seq = p->seq;
+ x->replay_maxdiff = net->xfrm.sysctl_aevent_rseqth;
+ /* sysctl_xfrm_aevent_etime is in 100ms units */
+ x->replay_maxage = (net->xfrm.sysctl_aevent_etime*HZ)/XFRM_AE_ETH_M;
+
+ if ((err = xfrm_init_replay(x, extack)))
+ goto error;
+
+ /* override default values from above */
+ xfrm_update_ae_params(x, attrs, 0);
+
+ /* configure the hardware if offload is requested */
+ if (attrs[XFRMA_OFFLOAD_DEV]) {
+ err = xfrm_dev_state_add(net, x,
+ nla_data(attrs[XFRMA_OFFLOAD_DEV]),
+ extack);
+ if (err)
+ goto error;
+ }
+
+ return x;
+
+error:
+ x->km.state = XFRM_STATE_DEAD;
+ xfrm_state_put(x);
+error_no_put:
+ *errp = err;
+ return NULL;
+}
+
+static int xfrm_add_sa(struct sk_buff *skb, struct nlmsghdr *nlh,
+ struct nlattr **attrs, struct netlink_ext_ack *extack)
+{
+ struct net *net = sock_net(skb->sk);
+ struct xfrm_usersa_info *p = nlmsg_data(nlh);
+ struct xfrm_state *x;
+ int err;
+ struct km_event c;
+
+ err = verify_newsa_info(p, attrs, extack);
+ if (err)
+ return err;
+
+ x = xfrm_state_construct(net, p, attrs, &err, extack);
+ if (!x)
+ return err;
+
+ xfrm_state_hold(x);
+ if (nlh->nlmsg_type == XFRM_MSG_NEWSA)
+ err = xfrm_state_add(x);
+ else
+ err = xfrm_state_update(x);
+
+ xfrm_audit_state_add(x, err ? 0 : 1, true);
+
+ if (err < 0) {
+ x->km.state = XFRM_STATE_DEAD;
+ xfrm_dev_state_delete(x);
+ __xfrm_state_put(x);
+ goto out;
+ }
+
+ if (x->km.state == XFRM_STATE_VOID)
+ x->km.state = XFRM_STATE_VALID;
+
+ c.seq = nlh->nlmsg_seq;
+ c.portid = nlh->nlmsg_pid;
+ c.event = nlh->nlmsg_type;
+
+ km_state_notify(x, &c);
+out:
+ xfrm_state_put(x);
+ return err;
+}
+
+static struct xfrm_state *xfrm_user_state_lookup(struct net *net,
+ struct xfrm_usersa_id *p,
+ struct nlattr **attrs,
+ int *errp)
+{
+ struct xfrm_state *x = NULL;
+ struct xfrm_mark m;
+ int err;
+ u32 mark = xfrm_mark_get(attrs, &m);
+
+ if (xfrm_id_proto_match(p->proto, IPSEC_PROTO_ANY)) {
+ err = -ESRCH;
+ x = xfrm_state_lookup(net, mark, &p->daddr, p->spi, p->proto, p->family);
+ } else {
+ xfrm_address_t *saddr = NULL;
+
+ verify_one_addr(attrs, XFRMA_SRCADDR, &saddr);
+ if (!saddr) {
+ err = -EINVAL;
+ goto out;
+ }
+
+ err = -ESRCH;
+ x = xfrm_state_lookup_byaddr(net, mark,
+ &p->daddr, saddr,
+ p->proto, p->family);
+ }
+
+ out:
+ if (!x && errp)
+ *errp = err;
+ return x;
+}
+
+static int xfrm_del_sa(struct sk_buff *skb, struct nlmsghdr *nlh,
+ struct nlattr **attrs, struct netlink_ext_ack *extack)
+{
+ struct net *net = sock_net(skb->sk);
+ struct xfrm_state *x;
+ int err = -ESRCH;
+ struct km_event c;
+ struct xfrm_usersa_id *p = nlmsg_data(nlh);
+
+ x = xfrm_user_state_lookup(net, p, attrs, &err);
+ if (x == NULL)
+ return err;
+
+ if ((err = security_xfrm_state_delete(x)) != 0)
+ goto out;
+
+ if (xfrm_state_kern(x)) {
+ err = -EPERM;
+ goto out;
+ }
+
+ err = xfrm_state_delete(x);
+
+ if (err < 0)
+ goto out;
+
+ c.seq = nlh->nlmsg_seq;
+ c.portid = nlh->nlmsg_pid;
+ c.event = nlh->nlmsg_type;
+ km_state_notify(x, &c);
+
+out:
+ xfrm_audit_state_delete(x, err ? 0 : 1, true);
+ xfrm_state_put(x);
+ return err;
+}
+
+static void copy_to_user_state(struct xfrm_state *x, struct xfrm_usersa_info *p)
+{
+ memset(p, 0, sizeof(*p));
+ memcpy(&p->id, &x->id, sizeof(p->id));
+ memcpy(&p->sel, &x->sel, sizeof(p->sel));
+ memcpy(&p->lft, &x->lft, sizeof(p->lft));
+ memcpy(&p->curlft, &x->curlft, sizeof(p->curlft));
+ put_unaligned(x->stats.replay_window, &p->stats.replay_window);
+ put_unaligned(x->stats.replay, &p->stats.replay);
+ put_unaligned(x->stats.integrity_failed, &p->stats.integrity_failed);
+ memcpy(&p->saddr, &x->props.saddr, sizeof(p->saddr));
+ p->mode = x->props.mode;
+ p->replay_window = x->props.replay_window;
+ p->reqid = x->props.reqid;
+ p->family = x->props.family;
+ p->flags = x->props.flags;
+ p->seq = x->km.seq;
+}
+
+struct xfrm_dump_info {
+ struct sk_buff *in_skb;
+ struct sk_buff *out_skb;
+ u32 nlmsg_seq;
+ u16 nlmsg_flags;
+};
+
+static int copy_sec_ctx(struct xfrm_sec_ctx *s, struct sk_buff *skb)
+{
+ struct xfrm_user_sec_ctx *uctx;
+ struct nlattr *attr;
+ int ctx_size = sizeof(*uctx) + s->ctx_len;
+
+ attr = nla_reserve(skb, XFRMA_SEC_CTX, ctx_size);
+ if (attr == NULL)
+ return -EMSGSIZE;
+
+ uctx = nla_data(attr);
+ uctx->exttype = XFRMA_SEC_CTX;
+ uctx->len = ctx_size;
+ uctx->ctx_doi = s->ctx_doi;
+ uctx->ctx_alg = s->ctx_alg;
+ uctx->ctx_len = s->ctx_len;
+ memcpy(uctx + 1, s->ctx_str, s->ctx_len);
+
+ return 0;
+}
+
+static int copy_user_offload(struct xfrm_dev_offload *xso, struct sk_buff *skb)
+{
+ struct xfrm_user_offload *xuo;
+ struct nlattr *attr;
+
+ attr = nla_reserve(skb, XFRMA_OFFLOAD_DEV, sizeof(*xuo));
+ if (attr == NULL)
+ return -EMSGSIZE;
+
+ xuo = nla_data(attr);
+ memset(xuo, 0, sizeof(*xuo));
+ xuo->ifindex = xso->dev->ifindex;
+ if (xso->dir == XFRM_DEV_OFFLOAD_IN)
+ xuo->flags = XFRM_OFFLOAD_INBOUND;
+
+ return 0;
+}
+
+static bool xfrm_redact(void)
+{
+ return IS_ENABLED(CONFIG_SECURITY) &&
+ security_locked_down(LOCKDOWN_XFRM_SECRET);
+}
+
+static int copy_to_user_auth(struct xfrm_algo_auth *auth, struct sk_buff *skb)
+{
+ struct xfrm_algo *algo;
+ struct xfrm_algo_auth *ap;
+ struct nlattr *nla;
+ bool redact_secret = xfrm_redact();
+
+ nla = nla_reserve(skb, XFRMA_ALG_AUTH,
+ sizeof(*algo) + (auth->alg_key_len + 7) / 8);
+ if (!nla)
+ return -EMSGSIZE;
+ algo = nla_data(nla);
+ strncpy(algo->alg_name, auth->alg_name, sizeof(algo->alg_name));
+
+ if (redact_secret && auth->alg_key_len)
+ memset(algo->alg_key, 0, (auth->alg_key_len + 7) / 8);
+ else
+ memcpy(algo->alg_key, auth->alg_key,
+ (auth->alg_key_len + 7) / 8);
+ algo->alg_key_len = auth->alg_key_len;
+
+ nla = nla_reserve(skb, XFRMA_ALG_AUTH_TRUNC, xfrm_alg_auth_len(auth));
+ if (!nla)
+ return -EMSGSIZE;
+ ap = nla_data(nla);
+ memcpy(ap, auth, sizeof(struct xfrm_algo_auth));
+ if (redact_secret && auth->alg_key_len)
+ memset(ap->alg_key, 0, (auth->alg_key_len + 7) / 8);
+ else
+ memcpy(ap->alg_key, auth->alg_key,
+ (auth->alg_key_len + 7) / 8);
+ return 0;
+}
+
+static int copy_to_user_aead(struct xfrm_algo_aead *aead, struct sk_buff *skb)
+{
+ struct nlattr *nla = nla_reserve(skb, XFRMA_ALG_AEAD, aead_len(aead));
+ struct xfrm_algo_aead *ap;
+ bool redact_secret = xfrm_redact();
+
+ if (!nla)
+ return -EMSGSIZE;
+
+ ap = nla_data(nla);
+ strscpy_pad(ap->alg_name, aead->alg_name, sizeof(ap->alg_name));
+ ap->alg_key_len = aead->alg_key_len;
+ ap->alg_icv_len = aead->alg_icv_len;
+
+ if (redact_secret && aead->alg_key_len)
+ memset(ap->alg_key, 0, (aead->alg_key_len + 7) / 8);
+ else
+ memcpy(ap->alg_key, aead->alg_key,
+ (aead->alg_key_len + 7) / 8);
+ return 0;
+}
+
+static int copy_to_user_ealg(struct xfrm_algo *ealg, struct sk_buff *skb)
+{
+ struct xfrm_algo *ap;
+ bool redact_secret = xfrm_redact();
+ struct nlattr *nla = nla_reserve(skb, XFRMA_ALG_CRYPT,
+ xfrm_alg_len(ealg));
+ if (!nla)
+ return -EMSGSIZE;
+
+ ap = nla_data(nla);
+ strscpy_pad(ap->alg_name, ealg->alg_name, sizeof(ap->alg_name));
+ ap->alg_key_len = ealg->alg_key_len;
+
+ if (redact_secret && ealg->alg_key_len)
+ memset(ap->alg_key, 0, (ealg->alg_key_len + 7) / 8);
+ else
+ memcpy(ap->alg_key, ealg->alg_key,
+ (ealg->alg_key_len + 7) / 8);
+
+ return 0;
+}
+
+static int copy_to_user_calg(struct xfrm_algo *calg, struct sk_buff *skb)
+{
+ struct nlattr *nla = nla_reserve(skb, XFRMA_ALG_COMP, sizeof(*calg));
+ struct xfrm_algo *ap;
+
+ if (!nla)
+ return -EMSGSIZE;
+
+ ap = nla_data(nla);
+ strscpy_pad(ap->alg_name, calg->alg_name, sizeof(ap->alg_name));
+ ap->alg_key_len = 0;
+
+ return 0;
+}
+
+static int copy_to_user_encap(struct xfrm_encap_tmpl *ep, struct sk_buff *skb)
+{
+ struct nlattr *nla = nla_reserve(skb, XFRMA_ENCAP, sizeof(*ep));
+ struct xfrm_encap_tmpl *uep;
+
+ if (!nla)
+ return -EMSGSIZE;
+
+ uep = nla_data(nla);
+ memset(uep, 0, sizeof(*uep));
+
+ uep->encap_type = ep->encap_type;
+ uep->encap_sport = ep->encap_sport;
+ uep->encap_dport = ep->encap_dport;
+ uep->encap_oa = ep->encap_oa;
+
+ return 0;
+}
+
+static int xfrm_smark_put(struct sk_buff *skb, struct xfrm_mark *m)
+{
+ int ret = 0;
+
+ if (m->v | m->m) {
+ ret = nla_put_u32(skb, XFRMA_SET_MARK, m->v);
+ if (!ret)
+ ret = nla_put_u32(skb, XFRMA_SET_MARK_MASK, m->m);
+ }
+ return ret;
+}
+
+/* Don't change this without updating xfrm_sa_len! */
+static int copy_to_user_state_extra(struct xfrm_state *x,
+ struct xfrm_usersa_info *p,
+ struct sk_buff *skb)
+{
+ int ret = 0;
+
+ copy_to_user_state(x, p);
+
+ if (x->props.extra_flags) {
+ ret = nla_put_u32(skb, XFRMA_SA_EXTRA_FLAGS,
+ x->props.extra_flags);
+ if (ret)
+ goto out;
+ }
+
+ if (x->coaddr) {
+ ret = nla_put(skb, XFRMA_COADDR, sizeof(*x->coaddr), x->coaddr);
+ if (ret)
+ goto out;
+ }
+ if (x->lastused) {
+ ret = nla_put_u64_64bit(skb, XFRMA_LASTUSED, x->lastused,
+ XFRMA_PAD);
+ if (ret)
+ goto out;
+ }
+ if (x->aead) {
+ ret = copy_to_user_aead(x->aead, skb);
+ if (ret)
+ goto out;
+ }
+ if (x->aalg) {
+ ret = copy_to_user_auth(x->aalg, skb);
+ if (ret)
+ goto out;
+ }
+ if (x->ealg) {
+ ret = copy_to_user_ealg(x->ealg, skb);
+ if (ret)
+ goto out;
+ }
+ if (x->calg) {
+ ret = copy_to_user_calg(x->calg, skb);
+ if (ret)
+ goto out;
+ }
+ if (x->encap) {
+ ret = copy_to_user_encap(x->encap, skb);
+ if (ret)
+ goto out;
+ }
+ if (x->tfcpad) {
+ ret = nla_put_u32(skb, XFRMA_TFCPAD, x->tfcpad);
+ if (ret)
+ goto out;
+ }
+ ret = xfrm_mark_put(skb, &x->mark);
+ if (ret)
+ goto out;
+
+ ret = xfrm_smark_put(skb, &x->props.smark);
+ if (ret)
+ goto out;
+
+ if (x->replay_esn)
+ ret = nla_put(skb, XFRMA_REPLAY_ESN_VAL,
+ xfrm_replay_state_esn_len(x->replay_esn),
+ x->replay_esn);
+ else
+ ret = nla_put(skb, XFRMA_REPLAY_VAL, sizeof(x->replay),
+ &x->replay);
+ if (ret)
+ goto out;
+ if(x->xso.dev)
+ ret = copy_user_offload(&x->xso, skb);
+ if (ret)
+ goto out;
+ if (x->if_id) {
+ ret = nla_put_u32(skb, XFRMA_IF_ID, x->if_id);
+ if (ret)
+ goto out;
+ }
+ if (x->security) {
+ ret = copy_sec_ctx(x->security, skb);
+ if (ret)
+ goto out;
+ }
+ if (x->mapping_maxage)
+ ret = nla_put_u32(skb, XFRMA_MTIMER_THRESH, x->mapping_maxage);
+out:
+ return ret;
+}
+
+static int dump_one_state(struct xfrm_state *x, int count, void *ptr)
+{
+ struct xfrm_dump_info *sp = ptr;
+ struct sk_buff *in_skb = sp->in_skb;
+ struct sk_buff *skb = sp->out_skb;
+ struct xfrm_translator *xtr;
+ struct xfrm_usersa_info *p;
+ struct nlmsghdr *nlh;
+ int err;
+
+ nlh = nlmsg_put(skb, NETLINK_CB(in_skb).portid, sp->nlmsg_seq,
+ XFRM_MSG_NEWSA, sizeof(*p), sp->nlmsg_flags);
+ if (nlh == NULL)
+ return -EMSGSIZE;
+
+ p = nlmsg_data(nlh);
+
+ err = copy_to_user_state_extra(x, p, skb);
+ if (err) {
+ nlmsg_cancel(skb, nlh);
+ return err;
+ }
+ nlmsg_end(skb, nlh);
+
+ xtr = xfrm_get_translator();
+ if (xtr) {
+ err = xtr->alloc_compat(skb, nlh);
+
+ xfrm_put_translator(xtr);
+ if (err) {
+ nlmsg_cancel(skb, nlh);
+ return err;
+ }
+ }
+
+ return 0;
+}
+
+static int xfrm_dump_sa_done(struct netlink_callback *cb)
+{
+ struct xfrm_state_walk *walk = (struct xfrm_state_walk *) &cb->args[1];
+ struct sock *sk = cb->skb->sk;
+ struct net *net = sock_net(sk);
+
+ if (cb->args[0])
+ xfrm_state_walk_done(walk, net);
+ return 0;
+}
+
+static int xfrm_dump_sa(struct sk_buff *skb, struct netlink_callback *cb)
+{
+ struct net *net = sock_net(skb->sk);
+ struct xfrm_state_walk *walk = (struct xfrm_state_walk *) &cb->args[1];
+ struct xfrm_dump_info info;
+
+ BUILD_BUG_ON(sizeof(struct xfrm_state_walk) >
+ sizeof(cb->args) - sizeof(cb->args[0]));
+
+ info.in_skb = cb->skb;
+ info.out_skb = skb;
+ info.nlmsg_seq = cb->nlh->nlmsg_seq;
+ info.nlmsg_flags = NLM_F_MULTI;
+
+ if (!cb->args[0]) {
+ struct nlattr *attrs[XFRMA_MAX+1];
+ struct xfrm_address_filter *filter = NULL;
+ u8 proto = 0;
+ int err;
+
+ err = nlmsg_parse_deprecated(cb->nlh, 0, attrs, XFRMA_MAX,
+ xfrma_policy, cb->extack);
+ if (err < 0)
+ return err;
+
+ if (attrs[XFRMA_ADDRESS_FILTER]) {
+ filter = kmemdup(nla_data(attrs[XFRMA_ADDRESS_FILTER]),
+ sizeof(*filter), GFP_KERNEL);
+ if (filter == NULL)
+ return -ENOMEM;
+
+ /* see addr_match(), (prefix length >> 5) << 2
+ * will be used to compare xfrm_address_t
+ */
+ if (filter->splen > (sizeof(xfrm_address_t) << 3) ||
+ filter->dplen > (sizeof(xfrm_address_t) << 3)) {
+ kfree(filter);
+ return -EINVAL;
+ }
+ }
+
+ if (attrs[XFRMA_PROTO])
+ proto = nla_get_u8(attrs[XFRMA_PROTO]);
+
+ xfrm_state_walk_init(walk, proto, filter);
+ cb->args[0] = 1;
+ }
+
+ (void) xfrm_state_walk(net, walk, dump_one_state, &info);
+
+ return skb->len;
+}
+
+static struct sk_buff *xfrm_state_netlink(struct sk_buff *in_skb,
+ struct xfrm_state *x, u32 seq)
+{
+ struct xfrm_dump_info info;
+ struct sk_buff *skb;
+ int err;
+
+ skb = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_ATOMIC);
+ if (!skb)
+ return ERR_PTR(-ENOMEM);
+
+ info.in_skb = in_skb;
+ info.out_skb = skb;
+ info.nlmsg_seq = seq;
+ info.nlmsg_flags = 0;
+
+ err = dump_one_state(x, 0, &info);
+ if (err) {
+ kfree_skb(skb);
+ return ERR_PTR(err);
+ }
+
+ return skb;
+}
+
+/* A wrapper for nlmsg_multicast() checking that nlsk is still available.
+ * Must be called with RCU read lock.
+ */
+static inline int xfrm_nlmsg_multicast(struct net *net, struct sk_buff *skb,
+ u32 pid, unsigned int group)
+{
+ struct sock *nlsk = rcu_dereference(net->xfrm.nlsk);
+ struct xfrm_translator *xtr;
+
+ if (!nlsk) {
+ kfree_skb(skb);
+ return -EPIPE;
+ }
+
+ xtr = xfrm_get_translator();
+ if (xtr) {
+ int err = xtr->alloc_compat(skb, nlmsg_hdr(skb));
+
+ xfrm_put_translator(xtr);
+ if (err) {
+ kfree_skb(skb);
+ return err;
+ }
+ }
+
+ return nlmsg_multicast(nlsk, skb, pid, group, GFP_ATOMIC);
+}
+
+static inline unsigned int xfrm_spdinfo_msgsize(void)
+{
+ return NLMSG_ALIGN(4)
+ + nla_total_size(sizeof(struct xfrmu_spdinfo))
+ + nla_total_size(sizeof(struct xfrmu_spdhinfo))
+ + nla_total_size(sizeof(struct xfrmu_spdhthresh))
+ + nla_total_size(sizeof(struct xfrmu_spdhthresh));
+}
+
+static int build_spdinfo(struct sk_buff *skb, struct net *net,
+ u32 portid, u32 seq, u32 flags)
+{
+ struct xfrmk_spdinfo si;
+ struct xfrmu_spdinfo spc;
+ struct xfrmu_spdhinfo sph;
+ struct xfrmu_spdhthresh spt4, spt6;
+ struct nlmsghdr *nlh;
+ int err;
+ u32 *f;
+ unsigned lseq;
+
+ nlh = nlmsg_put(skb, portid, seq, XFRM_MSG_NEWSPDINFO, sizeof(u32), 0);
+ if (nlh == NULL) /* shouldn't really happen ... */
+ return -EMSGSIZE;
+
+ f = nlmsg_data(nlh);
+ *f = flags;
+ xfrm_spd_getinfo(net, &si);
+ spc.incnt = si.incnt;
+ spc.outcnt = si.outcnt;
+ spc.fwdcnt = si.fwdcnt;
+ spc.inscnt = si.inscnt;
+ spc.outscnt = si.outscnt;
+ spc.fwdscnt = si.fwdscnt;
+ sph.spdhcnt = si.spdhcnt;
+ sph.spdhmcnt = si.spdhmcnt;
+
+ do {
+ lseq = read_seqbegin(&net->xfrm.policy_hthresh.lock);
+
+ spt4.lbits = net->xfrm.policy_hthresh.lbits4;
+ spt4.rbits = net->xfrm.policy_hthresh.rbits4;
+ spt6.lbits = net->xfrm.policy_hthresh.lbits6;
+ spt6.rbits = net->xfrm.policy_hthresh.rbits6;
+ } while (read_seqretry(&net->xfrm.policy_hthresh.lock, lseq));
+
+ err = nla_put(skb, XFRMA_SPD_INFO, sizeof(spc), &spc);
+ if (!err)
+ err = nla_put(skb, XFRMA_SPD_HINFO, sizeof(sph), &sph);
+ if (!err)
+ err = nla_put(skb, XFRMA_SPD_IPV4_HTHRESH, sizeof(spt4), &spt4);
+ if (!err)
+ err = nla_put(skb, XFRMA_SPD_IPV6_HTHRESH, sizeof(spt6), &spt6);
+ if (err) {
+ nlmsg_cancel(skb, nlh);
+ return err;
+ }
+
+ nlmsg_end(skb, nlh);
+ return 0;
+}
+
+static int xfrm_set_spdinfo(struct sk_buff *skb, struct nlmsghdr *nlh,
+ struct nlattr **attrs,
+ struct netlink_ext_ack *extack)
+{
+ struct net *net = sock_net(skb->sk);
+ struct xfrmu_spdhthresh *thresh4 = NULL;
+ struct xfrmu_spdhthresh *thresh6 = NULL;
+
+ /* selector prefixlen thresholds to hash policies */
+ if (attrs[XFRMA_SPD_IPV4_HTHRESH]) {
+ struct nlattr *rta = attrs[XFRMA_SPD_IPV4_HTHRESH];
+
+ if (nla_len(rta) < sizeof(*thresh4))
+ return -EINVAL;
+ thresh4 = nla_data(rta);
+ if (thresh4->lbits > 32 || thresh4->rbits > 32)
+ return -EINVAL;
+ }
+ if (attrs[XFRMA_SPD_IPV6_HTHRESH]) {
+ struct nlattr *rta = attrs[XFRMA_SPD_IPV6_HTHRESH];
+
+ if (nla_len(rta) < sizeof(*thresh6))
+ return -EINVAL;
+ thresh6 = nla_data(rta);
+ if (thresh6->lbits > 128 || thresh6->rbits > 128)
+ return -EINVAL;
+ }
+
+ if (thresh4 || thresh6) {
+ write_seqlock(&net->xfrm.policy_hthresh.lock);
+ if (thresh4) {
+ net->xfrm.policy_hthresh.lbits4 = thresh4->lbits;
+ net->xfrm.policy_hthresh.rbits4 = thresh4->rbits;
+ }
+ if (thresh6) {
+ net->xfrm.policy_hthresh.lbits6 = thresh6->lbits;
+ net->xfrm.policy_hthresh.rbits6 = thresh6->rbits;
+ }
+ write_sequnlock(&net->xfrm.policy_hthresh.lock);
+
+ xfrm_policy_hash_rebuild(net);
+ }
+
+ return 0;
+}
+
+static int xfrm_get_spdinfo(struct sk_buff *skb, struct nlmsghdr *nlh,
+ struct nlattr **attrs,
+ struct netlink_ext_ack *extack)
+{
+ struct net *net = sock_net(skb->sk);
+ struct sk_buff *r_skb;
+ u32 *flags = nlmsg_data(nlh);
+ u32 sportid = NETLINK_CB(skb).portid;
+ u32 seq = nlh->nlmsg_seq;
+ int err;
+
+ r_skb = nlmsg_new(xfrm_spdinfo_msgsize(), GFP_ATOMIC);
+ if (r_skb == NULL)
+ return -ENOMEM;
+
+ err = build_spdinfo(r_skb, net, sportid, seq, *flags);
+ BUG_ON(err < 0);
+
+ return nlmsg_unicast(net->xfrm.nlsk, r_skb, sportid);
+}
+
+static inline unsigned int xfrm_sadinfo_msgsize(void)
+{
+ return NLMSG_ALIGN(4)
+ + nla_total_size(sizeof(struct xfrmu_sadhinfo))
+ + nla_total_size(4); /* XFRMA_SAD_CNT */
+}
+
+static int build_sadinfo(struct sk_buff *skb, struct net *net,
+ u32 portid, u32 seq, u32 flags)
+{
+ struct xfrmk_sadinfo si;
+ struct xfrmu_sadhinfo sh;
+ struct nlmsghdr *nlh;
+ int err;
+ u32 *f;
+
+ nlh = nlmsg_put(skb, portid, seq, XFRM_MSG_NEWSADINFO, sizeof(u32), 0);
+ if (nlh == NULL) /* shouldn't really happen ... */
+ return -EMSGSIZE;
+
+ f = nlmsg_data(nlh);
+ *f = flags;
+ xfrm_sad_getinfo(net, &si);
+
+ sh.sadhmcnt = si.sadhmcnt;
+ sh.sadhcnt = si.sadhcnt;
+
+ err = nla_put_u32(skb, XFRMA_SAD_CNT, si.sadcnt);
+ if (!err)
+ err = nla_put(skb, XFRMA_SAD_HINFO, sizeof(sh), &sh);
+ if (err) {
+ nlmsg_cancel(skb, nlh);
+ return err;
+ }
+
+ nlmsg_end(skb, nlh);
+ return 0;
+}
+
+static int xfrm_get_sadinfo(struct sk_buff *skb, struct nlmsghdr *nlh,
+ struct nlattr **attrs,
+ struct netlink_ext_ack *extack)
+{
+ struct net *net = sock_net(skb->sk);
+ struct sk_buff *r_skb;
+ u32 *flags = nlmsg_data(nlh);
+ u32 sportid = NETLINK_CB(skb).portid;
+ u32 seq = nlh->nlmsg_seq;
+ int err;
+
+ r_skb = nlmsg_new(xfrm_sadinfo_msgsize(), GFP_ATOMIC);
+ if (r_skb == NULL)
+ return -ENOMEM;
+
+ err = build_sadinfo(r_skb, net, sportid, seq, *flags);
+ BUG_ON(err < 0);
+
+ return nlmsg_unicast(net->xfrm.nlsk, r_skb, sportid);
+}
+
+static int xfrm_get_sa(struct sk_buff *skb, struct nlmsghdr *nlh,
+ struct nlattr **attrs, struct netlink_ext_ack *extack)
+{
+ struct net *net = sock_net(skb->sk);
+ struct xfrm_usersa_id *p = nlmsg_data(nlh);
+ struct xfrm_state *x;
+ struct sk_buff *resp_skb;
+ int err = -ESRCH;
+
+ x = xfrm_user_state_lookup(net, p, attrs, &err);
+ if (x == NULL)
+ goto out_noput;
+
+ resp_skb = xfrm_state_netlink(skb, x, nlh->nlmsg_seq);
+ if (IS_ERR(resp_skb)) {
+ err = PTR_ERR(resp_skb);
+ } else {
+ err = nlmsg_unicast(net->xfrm.nlsk, resp_skb, NETLINK_CB(skb).portid);
+ }
+ xfrm_state_put(x);
+out_noput:
+ return err;
+}
+
+static int xfrm_alloc_userspi(struct sk_buff *skb, struct nlmsghdr *nlh,
+ struct nlattr **attrs,
+ struct netlink_ext_ack *extack)
+{
+ struct net *net = sock_net(skb->sk);
+ struct xfrm_state *x;
+ struct xfrm_userspi_info *p;
+ struct xfrm_translator *xtr;
+ struct sk_buff *resp_skb;
+ xfrm_address_t *daddr;
+ int family;
+ int err;
+ u32 mark;
+ struct xfrm_mark m;
+ u32 if_id = 0;
+
+ p = nlmsg_data(nlh);
+ err = verify_spi_info(p->info.id.proto, p->min, p->max);
+ if (err)
+ goto out_noput;
+
+ family = p->info.family;
+ daddr = &p->info.id.daddr;
+
+ x = NULL;
+
+ mark = xfrm_mark_get(attrs, &m);
+
+ if (attrs[XFRMA_IF_ID])
+ if_id = nla_get_u32(attrs[XFRMA_IF_ID]);
+
+ if (p->info.seq) {
+ x = xfrm_find_acq_byseq(net, mark, p->info.seq);
+ if (x && !xfrm_addr_equal(&x->id.daddr, daddr, family)) {
+ xfrm_state_put(x);
+ x = NULL;
+ }
+ }
+
+ if (!x)
+ x = xfrm_find_acq(net, &m, p->info.mode, p->info.reqid,
+ if_id, p->info.id.proto, daddr,
+ &p->info.saddr, 1,
+ family);
+ err = -ENOENT;
+ if (x == NULL)
+ goto out_noput;
+
+ err = xfrm_alloc_spi(x, p->min, p->max);
+ if (err)
+ goto out;
+
+ resp_skb = xfrm_state_netlink(skb, x, nlh->nlmsg_seq);
+ if (IS_ERR(resp_skb)) {
+ err = PTR_ERR(resp_skb);
+ goto out;
+ }
+
+ xtr = xfrm_get_translator();
+ if (xtr) {
+ err = xtr->alloc_compat(skb, nlmsg_hdr(skb));
+
+ xfrm_put_translator(xtr);
+ if (err) {
+ kfree_skb(resp_skb);
+ goto out;
+ }
+ }
+
+ err = nlmsg_unicast(net->xfrm.nlsk, resp_skb, NETLINK_CB(skb).portid);
+
+out:
+ xfrm_state_put(x);
+out_noput:
+ return err;
+}
+
+static int verify_policy_dir(u8 dir, struct netlink_ext_ack *extack)
+{
+ switch (dir) {
+ case XFRM_POLICY_IN:
+ case XFRM_POLICY_OUT:
+ case XFRM_POLICY_FWD:
+ break;
+
+ default:
+ NL_SET_ERR_MSG(extack, "Invalid policy direction");
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
+static int verify_policy_type(u8 type, struct netlink_ext_ack *extack)
+{
+ switch (type) {
+ case XFRM_POLICY_TYPE_MAIN:
+#ifdef CONFIG_XFRM_SUB_POLICY
+ case XFRM_POLICY_TYPE_SUB:
+#endif
+ break;
+
+ default:
+ NL_SET_ERR_MSG(extack, "Invalid policy type");
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
+static int verify_newpolicy_info(struct xfrm_userpolicy_info *p,
+ struct netlink_ext_ack *extack)
+{
+ int ret;
+
+ switch (p->share) {
+ case XFRM_SHARE_ANY:
+ case XFRM_SHARE_SESSION:
+ case XFRM_SHARE_USER:
+ case XFRM_SHARE_UNIQUE:
+ break;
+
+ default:
+ NL_SET_ERR_MSG(extack, "Invalid policy share");
+ return -EINVAL;
+ }
+
+ switch (p->action) {
+ case XFRM_POLICY_ALLOW:
+ case XFRM_POLICY_BLOCK:
+ break;
+
+ default:
+ NL_SET_ERR_MSG(extack, "Invalid policy action");
+ return -EINVAL;
+ }
+
+ switch (p->sel.family) {
+ case AF_INET:
+ if (p->sel.prefixlen_d > 32 || p->sel.prefixlen_s > 32) {
+ NL_SET_ERR_MSG(extack, "Invalid prefix length in selector (must be <= 32 for IPv4)");
+ return -EINVAL;
+ }
+
+ break;
+
+ case AF_INET6:
+#if IS_ENABLED(CONFIG_IPV6)
+ if (p->sel.prefixlen_d > 128 || p->sel.prefixlen_s > 128) {
+ NL_SET_ERR_MSG(extack, "Invalid prefix length in selector (must be <= 128 for IPv6)");
+ return -EINVAL;
+ }
+
+ break;
+#else
+ NL_SET_ERR_MSG(extack, "IPv6 support disabled");
+ return -EAFNOSUPPORT;
+#endif
+
+ default:
+ NL_SET_ERR_MSG(extack, "Invalid selector family");
+ return -EINVAL;
+ }
+
+ ret = verify_policy_dir(p->dir, extack);
+ if (ret)
+ return ret;
+ if (p->index && (xfrm_policy_id2dir(p->index) != p->dir)) {
+ NL_SET_ERR_MSG(extack, "Policy index doesn't match direction");
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
+static int copy_from_user_sec_ctx(struct xfrm_policy *pol, struct nlattr **attrs)
+{
+ struct nlattr *rt = attrs[XFRMA_SEC_CTX];
+ struct xfrm_user_sec_ctx *uctx;
+
+ if (!rt)
+ return 0;
+
+ uctx = nla_data(rt);
+ return security_xfrm_policy_alloc(&pol->security, uctx, GFP_KERNEL);
+}
+
+static void copy_templates(struct xfrm_policy *xp, struct xfrm_user_tmpl *ut,
+ int nr)
+{
+ int i;
+
+ xp->xfrm_nr = nr;
+ for (i = 0; i < nr; i++, ut++) {
+ struct xfrm_tmpl *t = &xp->xfrm_vec[i];
+
+ memcpy(&t->id, &ut->id, sizeof(struct xfrm_id));
+ memcpy(&t->saddr, &ut->saddr,
+ sizeof(xfrm_address_t));
+ t->reqid = ut->reqid;
+ t->mode = ut->mode;
+ t->share = ut->share;
+ t->optional = ut->optional;
+ t->aalgos = ut->aalgos;
+ t->ealgos = ut->ealgos;
+ t->calgos = ut->calgos;
+ /* If all masks are ~0, then we allow all algorithms. */
+ t->allalgs = !~(t->aalgos & t->ealgos & t->calgos);
+ t->encap_family = ut->family;
+ }
+}
+
+static int validate_tmpl(int nr, struct xfrm_user_tmpl *ut, u16 family,
+ int dir, struct netlink_ext_ack *extack)
+{
+ u16 prev_family;
+ int i;
+
+ if (nr > XFRM_MAX_DEPTH) {
+ NL_SET_ERR_MSG(extack, "Template count must be <= XFRM_MAX_DEPTH (" __stringify(XFRM_MAX_DEPTH) ")");
+ return -EINVAL;
+ }
+
+ prev_family = family;
+
+ for (i = 0; i < nr; i++) {
+ /* We never validated the ut->family value, so many
+ * applications simply leave it at zero. The check was
+ * never made and ut->family was ignored because all
+ * templates could be assumed to have the same family as
+ * the policy itself. Now that we will have ipv4-in-ipv6
+ * and ipv6-in-ipv4 tunnels, this is no longer true.
+ */
+ if (!ut[i].family)
+ ut[i].family = family;
+
+ switch (ut[i].mode) {
+ case XFRM_MODE_TUNNEL:
+ case XFRM_MODE_BEET:
+ if (ut[i].optional && dir == XFRM_POLICY_OUT) {
+ NL_SET_ERR_MSG(extack, "Mode in optional template not allowed in outbound policy");
+ return -EINVAL;
+ }
+ break;
+ default:
+ if (ut[i].family != prev_family) {
+ NL_SET_ERR_MSG(extack, "Mode in template doesn't support a family change");
+ return -EINVAL;
+ }
+ break;
+ }
+ if (ut[i].mode >= XFRM_MODE_MAX) {
+ NL_SET_ERR_MSG(extack, "Mode in template must be < XFRM_MODE_MAX (" __stringify(XFRM_MODE_MAX) ")");
+ return -EINVAL;
+ }
+
+ prev_family = ut[i].family;
+
+ switch (ut[i].family) {
+ case AF_INET:
+ break;
+#if IS_ENABLED(CONFIG_IPV6)
+ case AF_INET6:
+ break;
+#endif
+ default:
+ NL_SET_ERR_MSG(extack, "Invalid family in template");
+ return -EINVAL;
+ }
+
+ if (!xfrm_id_proto_valid(ut[i].id.proto)) {
+ NL_SET_ERR_MSG(extack, "Invalid XFRM protocol in template");
+ return -EINVAL;
+ }
+ }
+
+ return 0;
+}
+
+static int copy_from_user_tmpl(struct xfrm_policy *pol, struct nlattr **attrs,
+ int dir, struct netlink_ext_ack *extack)
+{
+ struct nlattr *rt = attrs[XFRMA_TMPL];
+
+ if (!rt) {
+ pol->xfrm_nr = 0;
+ } else {
+ struct xfrm_user_tmpl *utmpl = nla_data(rt);
+ int nr = nla_len(rt) / sizeof(*utmpl);
+ int err;
+
+ err = validate_tmpl(nr, utmpl, pol->family, dir, extack);
+ if (err)
+ return err;
+
+ copy_templates(pol, utmpl, nr);
+ }
+ return 0;
+}
+
+static int copy_from_user_policy_type(u8 *tp, struct nlattr **attrs,
+ struct netlink_ext_ack *extack)
+{
+ struct nlattr *rt = attrs[XFRMA_POLICY_TYPE];
+ struct xfrm_userpolicy_type *upt;
+ u8 type = XFRM_POLICY_TYPE_MAIN;
+ int err;
+
+ if (rt) {
+ upt = nla_data(rt);
+ type = upt->type;
+ }
+
+ err = verify_policy_type(type, extack);
+ if (err)
+ return err;
+
+ *tp = type;
+ return 0;
+}
+
+static void copy_from_user_policy(struct xfrm_policy *xp, struct xfrm_userpolicy_info *p)
+{
+ xp->priority = p->priority;
+ xp->index = p->index;
+ memcpy(&xp->selector, &p->sel, sizeof(xp->selector));
+ memcpy(&xp->lft, &p->lft, sizeof(xp->lft));
+ xp->action = p->action;
+ xp->flags = p->flags;
+ xp->family = p->sel.family;
+ /* XXX xp->share = p->share; */
+}
+
+static void copy_to_user_policy(struct xfrm_policy *xp, struct xfrm_userpolicy_info *p, int dir)
+{
+ memset(p, 0, sizeof(*p));
+ memcpy(&p->sel, &xp->selector, sizeof(p->sel));
+ memcpy(&p->lft, &xp->lft, sizeof(p->lft));
+ memcpy(&p->curlft, &xp->curlft, sizeof(p->curlft));
+ p->priority = xp->priority;
+ p->index = xp->index;
+ p->sel.family = xp->family;
+ p->dir = dir;
+ p->action = xp->action;
+ p->flags = xp->flags;
+ p->share = XFRM_SHARE_ANY; /* XXX xp->share */
+}
+
+static struct xfrm_policy *xfrm_policy_construct(struct net *net,
+ struct xfrm_userpolicy_info *p,
+ struct nlattr **attrs,
+ int *errp,
+ struct netlink_ext_ack *extack)
+{
+ struct xfrm_policy *xp = xfrm_policy_alloc(net, GFP_KERNEL);
+ int err;
+
+ if (!xp) {
+ *errp = -ENOMEM;
+ return NULL;
+ }
+
+ copy_from_user_policy(xp, p);
+
+ err = copy_from_user_policy_type(&xp->type, attrs, extack);
+ if (err)
+ goto error;
+
+ if (!(err = copy_from_user_tmpl(xp, attrs, p->dir, extack)))
+ err = copy_from_user_sec_ctx(xp, attrs);
+ if (err)
+ goto error;
+
+ xfrm_mark_get(attrs, &xp->mark);
+
+ if (attrs[XFRMA_IF_ID])
+ xp->if_id = nla_get_u32(attrs[XFRMA_IF_ID]);
+
+ return xp;
+ error:
+ *errp = err;
+ xp->walk.dead = 1;
+ xfrm_policy_destroy(xp);
+ return NULL;
+}
+
+static int xfrm_add_policy(struct sk_buff *skb, struct nlmsghdr *nlh,
+ struct nlattr **attrs,
+ struct netlink_ext_ack *extack)
+{
+ struct net *net = sock_net(skb->sk);
+ struct xfrm_userpolicy_info *p = nlmsg_data(nlh);
+ struct xfrm_policy *xp;
+ struct km_event c;
+ int err;
+ int excl;
+
+ err = verify_newpolicy_info(p, extack);
+ if (err)
+ return err;
+ err = verify_sec_ctx_len(attrs, extack);
+ if (err)
+ return err;
+
+ xp = xfrm_policy_construct(net, p, attrs, &err, extack);
+ if (!xp)
+ return err;
+
+ /* shouldn't excl be based on nlh flags??
+ * Aha! this is anti-netlink really i.e more pfkey derived
+ * in netlink excl is a flag and you wouldn't need
+ * a type XFRM_MSG_UPDPOLICY - JHS */
+ excl = nlh->nlmsg_type == XFRM_MSG_NEWPOLICY;
+ err = xfrm_policy_insert(p->dir, xp, excl);
+ xfrm_audit_policy_add(xp, err ? 0 : 1, true);
+
+ if (err) {
+ security_xfrm_policy_free(xp->security);
+ kfree(xp);
+ return err;
+ }
+
+ c.event = nlh->nlmsg_type;
+ c.seq = nlh->nlmsg_seq;
+ c.portid = nlh->nlmsg_pid;
+ km_policy_notify(xp, p->dir, &c);
+
+ xfrm_pol_put(xp);
+
+ return 0;
+}
+
+static int copy_to_user_tmpl(struct xfrm_policy *xp, struct sk_buff *skb)
+{
+ struct xfrm_user_tmpl vec[XFRM_MAX_DEPTH];
+ int i;
+
+ if (xp->xfrm_nr == 0)
+ return 0;
+
+ for (i = 0; i < xp->xfrm_nr; i++) {
+ struct xfrm_user_tmpl *up = &vec[i];
+ struct xfrm_tmpl *kp = &xp->xfrm_vec[i];
+
+ memset(up, 0, sizeof(*up));
+ memcpy(&up->id, &kp->id, sizeof(up->id));
+ up->family = kp->encap_family;
+ memcpy(&up->saddr, &kp->saddr, sizeof(up->saddr));
+ up->reqid = kp->reqid;
+ up->mode = kp->mode;
+ up->share = kp->share;
+ up->optional = kp->optional;
+ up->aalgos = kp->aalgos;
+ up->ealgos = kp->ealgos;
+ up->calgos = kp->calgos;
+ }
+
+ return nla_put(skb, XFRMA_TMPL,
+ sizeof(struct xfrm_user_tmpl) * xp->xfrm_nr, vec);
+}
+
+static inline int copy_to_user_state_sec_ctx(struct xfrm_state *x, struct sk_buff *skb)
+{
+ if (x->security) {
+ return copy_sec_ctx(x->security, skb);
+ }
+ return 0;
+}
+
+static inline int copy_to_user_sec_ctx(struct xfrm_policy *xp, struct sk_buff *skb)
+{
+ if (xp->security)
+ return copy_sec_ctx(xp->security, skb);
+ return 0;
+}
+static inline unsigned int userpolicy_type_attrsize(void)
+{
+#ifdef CONFIG_XFRM_SUB_POLICY
+ return nla_total_size(sizeof(struct xfrm_userpolicy_type));
+#else
+ return 0;
+#endif
+}
+
+#ifdef CONFIG_XFRM_SUB_POLICY
+static int copy_to_user_policy_type(u8 type, struct sk_buff *skb)
+{
+ struct xfrm_userpolicy_type upt;
+
+ /* Sadly there are two holes in struct xfrm_userpolicy_type */
+ memset(&upt, 0, sizeof(upt));
+ upt.type = type;
+
+ return nla_put(skb, XFRMA_POLICY_TYPE, sizeof(upt), &upt);
+}
+
+#else
+static inline int copy_to_user_policy_type(u8 type, struct sk_buff *skb)
+{
+ return 0;
+}
+#endif
+
+static int dump_one_policy(struct xfrm_policy *xp, int dir, int count, void *ptr)
+{
+ struct xfrm_dump_info *sp = ptr;
+ struct xfrm_userpolicy_info *p;
+ struct sk_buff *in_skb = sp->in_skb;
+ struct sk_buff *skb = sp->out_skb;
+ struct xfrm_translator *xtr;
+ struct nlmsghdr *nlh;
+ int err;
+
+ nlh = nlmsg_put(skb, NETLINK_CB(in_skb).portid, sp->nlmsg_seq,
+ XFRM_MSG_NEWPOLICY, sizeof(*p), sp->nlmsg_flags);
+ if (nlh == NULL)
+ return -EMSGSIZE;
+
+ p = nlmsg_data(nlh);
+ copy_to_user_policy(xp, p, dir);
+ err = copy_to_user_tmpl(xp, skb);
+ if (!err)
+ err = copy_to_user_sec_ctx(xp, skb);
+ if (!err)
+ err = copy_to_user_policy_type(xp->type, skb);
+ if (!err)
+ err = xfrm_mark_put(skb, &xp->mark);
+ if (!err)
+ err = xfrm_if_id_put(skb, xp->if_id);
+ if (err) {
+ nlmsg_cancel(skb, nlh);
+ return err;
+ }
+ nlmsg_end(skb, nlh);
+
+ xtr = xfrm_get_translator();
+ if (xtr) {
+ err = xtr->alloc_compat(skb, nlh);
+
+ xfrm_put_translator(xtr);
+ if (err) {
+ nlmsg_cancel(skb, nlh);
+ return err;
+ }
+ }
+
+ return 0;
+}
+
+static int xfrm_dump_policy_done(struct netlink_callback *cb)
+{
+ struct xfrm_policy_walk *walk = (struct xfrm_policy_walk *)cb->args;
+ struct net *net = sock_net(cb->skb->sk);
+
+ xfrm_policy_walk_done(walk, net);
+ return 0;
+}
+
+static int xfrm_dump_policy_start(struct netlink_callback *cb)
+{
+ struct xfrm_policy_walk *walk = (struct xfrm_policy_walk *)cb->args;
+
+ BUILD_BUG_ON(sizeof(*walk) > sizeof(cb->args));
+
+ xfrm_policy_walk_init(walk, XFRM_POLICY_TYPE_ANY);
+ return 0;
+}
+
+static int xfrm_dump_policy(struct sk_buff *skb, struct netlink_callback *cb)
+{
+ struct net *net = sock_net(skb->sk);
+ struct xfrm_policy_walk *walk = (struct xfrm_policy_walk *)cb->args;
+ struct xfrm_dump_info info;
+
+ info.in_skb = cb->skb;
+ info.out_skb = skb;
+ info.nlmsg_seq = cb->nlh->nlmsg_seq;
+ info.nlmsg_flags = NLM_F_MULTI;
+
+ (void) xfrm_policy_walk(net, walk, dump_one_policy, &info);
+
+ return skb->len;
+}
+
+static struct sk_buff *xfrm_policy_netlink(struct sk_buff *in_skb,
+ struct xfrm_policy *xp,
+ int dir, u32 seq)
+{
+ struct xfrm_dump_info info;
+ struct sk_buff *skb;
+ int err;
+
+ skb = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
+ if (!skb)
+ return ERR_PTR(-ENOMEM);
+
+ info.in_skb = in_skb;
+ info.out_skb = skb;
+ info.nlmsg_seq = seq;
+ info.nlmsg_flags = 0;
+
+ err = dump_one_policy(xp, dir, 0, &info);
+ if (err) {
+ kfree_skb(skb);
+ return ERR_PTR(err);
+ }
+
+ return skb;
+}
+
+static int xfrm_notify_userpolicy(struct net *net)
+{
+ struct xfrm_userpolicy_default *up;
+ int len = NLMSG_ALIGN(sizeof(*up));
+ struct nlmsghdr *nlh;
+ struct sk_buff *skb;
+ int err;
+
+ skb = nlmsg_new(len, GFP_ATOMIC);
+ if (skb == NULL)
+ return -ENOMEM;
+
+ nlh = nlmsg_put(skb, 0, 0, XFRM_MSG_GETDEFAULT, sizeof(*up), 0);
+ if (nlh == NULL) {
+ kfree_skb(skb);
+ return -EMSGSIZE;
+ }
+
+ up = nlmsg_data(nlh);
+ up->in = net->xfrm.policy_default[XFRM_POLICY_IN];
+ up->fwd = net->xfrm.policy_default[XFRM_POLICY_FWD];
+ up->out = net->xfrm.policy_default[XFRM_POLICY_OUT];
+
+ nlmsg_end(skb, nlh);
+
+ rcu_read_lock();
+ err = xfrm_nlmsg_multicast(net, skb, 0, XFRMNLGRP_POLICY);
+ rcu_read_unlock();
+
+ return err;
+}
+
+static bool xfrm_userpolicy_is_valid(__u8 policy)
+{
+ return policy == XFRM_USERPOLICY_BLOCK ||
+ policy == XFRM_USERPOLICY_ACCEPT;
+}
+
+static int xfrm_set_default(struct sk_buff *skb, struct nlmsghdr *nlh,
+ struct nlattr **attrs, struct netlink_ext_ack *extack)
+{
+ struct net *net = sock_net(skb->sk);
+ struct xfrm_userpolicy_default *up = nlmsg_data(nlh);
+
+ if (xfrm_userpolicy_is_valid(up->in))
+ net->xfrm.policy_default[XFRM_POLICY_IN] = up->in;
+
+ if (xfrm_userpolicy_is_valid(up->fwd))
+ net->xfrm.policy_default[XFRM_POLICY_FWD] = up->fwd;
+
+ if (xfrm_userpolicy_is_valid(up->out))
+ net->xfrm.policy_default[XFRM_POLICY_OUT] = up->out;
+
+ rt_genid_bump_all(net);
+
+ xfrm_notify_userpolicy(net);
+ return 0;
+}
+
+static int xfrm_get_default(struct sk_buff *skb, struct nlmsghdr *nlh,
+ struct nlattr **attrs, struct netlink_ext_ack *extack)
+{
+ struct sk_buff *r_skb;
+ struct nlmsghdr *r_nlh;
+ struct net *net = sock_net(skb->sk);
+ struct xfrm_userpolicy_default *r_up;
+ int len = NLMSG_ALIGN(sizeof(struct xfrm_userpolicy_default));
+ u32 portid = NETLINK_CB(skb).portid;
+ u32 seq = nlh->nlmsg_seq;
+
+ r_skb = nlmsg_new(len, GFP_ATOMIC);
+ if (!r_skb)
+ return -ENOMEM;
+
+ r_nlh = nlmsg_put(r_skb, portid, seq, XFRM_MSG_GETDEFAULT, sizeof(*r_up), 0);
+ if (!r_nlh) {
+ kfree_skb(r_skb);
+ return -EMSGSIZE;
+ }
+
+ r_up = nlmsg_data(r_nlh);
+ r_up->in = net->xfrm.policy_default[XFRM_POLICY_IN];
+ r_up->fwd = net->xfrm.policy_default[XFRM_POLICY_FWD];
+ r_up->out = net->xfrm.policy_default[XFRM_POLICY_OUT];
+ nlmsg_end(r_skb, r_nlh);
+
+ return nlmsg_unicast(net->xfrm.nlsk, r_skb, portid);
+}
+
+static int xfrm_get_policy(struct sk_buff *skb, struct nlmsghdr *nlh,
+ struct nlattr **attrs,
+ struct netlink_ext_ack *extack)
+{
+ struct net *net = sock_net(skb->sk);
+ struct xfrm_policy *xp;
+ struct xfrm_userpolicy_id *p;
+ u8 type = XFRM_POLICY_TYPE_MAIN;
+ int err;
+ struct km_event c;
+ int delete;
+ struct xfrm_mark m;
+ u32 if_id = 0;
+
+ p = nlmsg_data(nlh);
+ delete = nlh->nlmsg_type == XFRM_MSG_DELPOLICY;
+
+ err = copy_from_user_policy_type(&type, attrs, extack);
+ if (err)
+ return err;
+
+ err = verify_policy_dir(p->dir, extack);
+ if (err)
+ return err;
+
+ if (attrs[XFRMA_IF_ID])
+ if_id = nla_get_u32(attrs[XFRMA_IF_ID]);
+
+ xfrm_mark_get(attrs, &m);
+
+ if (p->index)
+ xp = xfrm_policy_byid(net, &m, if_id, type, p->dir,
+ p->index, delete, &err);
+ else {
+ struct nlattr *rt = attrs[XFRMA_SEC_CTX];
+ struct xfrm_sec_ctx *ctx;
+
+ err = verify_sec_ctx_len(attrs, extack);
+ if (err)
+ return err;
+
+ ctx = NULL;
+ if (rt) {
+ struct xfrm_user_sec_ctx *uctx = nla_data(rt);
+
+ err = security_xfrm_policy_alloc(&ctx, uctx, GFP_KERNEL);
+ if (err)
+ return err;
+ }
+ xp = xfrm_policy_bysel_ctx(net, &m, if_id, type, p->dir,
+ &p->sel, ctx, delete, &err);
+ security_xfrm_policy_free(ctx);
+ }
+ if (xp == NULL)
+ return -ENOENT;
+
+ if (!delete) {
+ struct sk_buff *resp_skb;
+
+ resp_skb = xfrm_policy_netlink(skb, xp, p->dir, nlh->nlmsg_seq);
+ if (IS_ERR(resp_skb)) {
+ err = PTR_ERR(resp_skb);
+ } else {
+ err = nlmsg_unicast(net->xfrm.nlsk, resp_skb,
+ NETLINK_CB(skb).portid);
+ }
+ } else {
+ xfrm_audit_policy_delete(xp, err ? 0 : 1, true);
+
+ if (err != 0)
+ goto out;
+
+ c.data.byid = p->index;
+ c.event = nlh->nlmsg_type;
+ c.seq = nlh->nlmsg_seq;
+ c.portid = nlh->nlmsg_pid;
+ km_policy_notify(xp, p->dir, &c);
+ }
+
+out:
+ xfrm_pol_put(xp);
+ return err;
+}
+
+static int xfrm_flush_sa(struct sk_buff *skb, struct nlmsghdr *nlh,
+ struct nlattr **attrs,
+ struct netlink_ext_ack *extack)
+{
+ struct net *net = sock_net(skb->sk);
+ struct km_event c;
+ struct xfrm_usersa_flush *p = nlmsg_data(nlh);
+ int err;
+
+ err = xfrm_state_flush(net, p->proto, true, false);
+ if (err) {
+ if (err == -ESRCH) /* empty table */
+ return 0;
+ return err;
+ }
+ c.data.proto = p->proto;
+ c.event = nlh->nlmsg_type;
+ c.seq = nlh->nlmsg_seq;
+ c.portid = nlh->nlmsg_pid;
+ c.net = net;
+ km_state_notify(NULL, &c);
+
+ return 0;
+}
+
+static inline unsigned int xfrm_aevent_msgsize(struct xfrm_state *x)
+{
+ unsigned int replay_size = x->replay_esn ?
+ xfrm_replay_state_esn_len(x->replay_esn) :
+ sizeof(struct xfrm_replay_state);
+
+ return NLMSG_ALIGN(sizeof(struct xfrm_aevent_id))
+ + nla_total_size(replay_size)
+ + nla_total_size_64bit(sizeof(struct xfrm_lifetime_cur))
+ + nla_total_size(sizeof(struct xfrm_mark))
+ + nla_total_size(4) /* XFRM_AE_RTHR */
+ + nla_total_size(4); /* XFRM_AE_ETHR */
+}
+
+static int build_aevent(struct sk_buff *skb, struct xfrm_state *x, const struct km_event *c)
+{
+ struct xfrm_aevent_id *id;
+ struct nlmsghdr *nlh;
+ int err;
+
+ nlh = nlmsg_put(skb, c->portid, c->seq, XFRM_MSG_NEWAE, sizeof(*id), 0);
+ if (nlh == NULL)
+ return -EMSGSIZE;
+
+ id = nlmsg_data(nlh);
+ memset(&id->sa_id, 0, sizeof(id->sa_id));
+ memcpy(&id->sa_id.daddr, &x->id.daddr, sizeof(x->id.daddr));
+ id->sa_id.spi = x->id.spi;
+ id->sa_id.family = x->props.family;
+ id->sa_id.proto = x->id.proto;
+ memcpy(&id->saddr, &x->props.saddr, sizeof(x->props.saddr));
+ id->reqid = x->props.reqid;
+ id->flags = c->data.aevent;
+
+ if (x->replay_esn) {
+ err = nla_put(skb, XFRMA_REPLAY_ESN_VAL,
+ xfrm_replay_state_esn_len(x->replay_esn),
+ x->replay_esn);
+ } else {
+ err = nla_put(skb, XFRMA_REPLAY_VAL, sizeof(x->replay),
+ &x->replay);
+ }
+ if (err)
+ goto out_cancel;
+ err = nla_put_64bit(skb, XFRMA_LTIME_VAL, sizeof(x->curlft), &x->curlft,
+ XFRMA_PAD);
+ if (err)
+ goto out_cancel;
+
+ if (id->flags & XFRM_AE_RTHR) {
+ err = nla_put_u32(skb, XFRMA_REPLAY_THRESH, x->replay_maxdiff);
+ if (err)
+ goto out_cancel;
+ }
+ if (id->flags & XFRM_AE_ETHR) {
+ err = nla_put_u32(skb, XFRMA_ETIMER_THRESH,
+ x->replay_maxage * 10 / HZ);
+ if (err)
+ goto out_cancel;
+ }
+ err = xfrm_mark_put(skb, &x->mark);
+ if (err)
+ goto out_cancel;
+
+ err = xfrm_if_id_put(skb, x->if_id);
+ if (err)
+ goto out_cancel;
+
+ nlmsg_end(skb, nlh);
+ return 0;
+
+out_cancel:
+ nlmsg_cancel(skb, nlh);
+ return err;
+}
+
+static int xfrm_get_ae(struct sk_buff *skb, struct nlmsghdr *nlh,
+ struct nlattr **attrs, struct netlink_ext_ack *extack)
+{
+ struct net *net = sock_net(skb->sk);
+ struct xfrm_state *x;
+ struct sk_buff *r_skb;
+ int err;
+ struct km_event c;
+ u32 mark;
+ struct xfrm_mark m;
+ struct xfrm_aevent_id *p = nlmsg_data(nlh);
+ struct xfrm_usersa_id *id = &p->sa_id;
+
+ mark = xfrm_mark_get(attrs, &m);
+
+ x = xfrm_state_lookup(net, mark, &id->daddr, id->spi, id->proto, id->family);
+ if (x == NULL)
+ return -ESRCH;
+
+ r_skb = nlmsg_new(xfrm_aevent_msgsize(x), GFP_ATOMIC);
+ if (r_skb == NULL) {
+ xfrm_state_put(x);
+ return -ENOMEM;
+ }
+
+ /*
+ * XXX: is this lock really needed - none of the other
+ * gets lock (the concern is things getting updated
+ * while we are still reading) - jhs
+ */
+ spin_lock_bh(&x->lock);
+ c.data.aevent = p->flags;
+ c.seq = nlh->nlmsg_seq;
+ c.portid = nlh->nlmsg_pid;
+
+ err = build_aevent(r_skb, x, &c);
+ BUG_ON(err < 0);
+
+ err = nlmsg_unicast(net->xfrm.nlsk, r_skb, NETLINK_CB(skb).portid);
+ spin_unlock_bh(&x->lock);
+ xfrm_state_put(x);
+ return err;
+}
+
+static int xfrm_new_ae(struct sk_buff *skb, struct nlmsghdr *nlh,
+ struct nlattr **attrs, struct netlink_ext_ack *extack)
+{
+ struct net *net = sock_net(skb->sk);
+ struct xfrm_state *x;
+ struct km_event c;
+ int err = -EINVAL;
+ u32 mark = 0;
+ struct xfrm_mark m;
+ struct xfrm_aevent_id *p = nlmsg_data(nlh);
+ struct nlattr *rp = attrs[XFRMA_REPLAY_VAL];
+ struct nlattr *re = attrs[XFRMA_REPLAY_ESN_VAL];
+ struct nlattr *lt = attrs[XFRMA_LTIME_VAL];
+ struct nlattr *et = attrs[XFRMA_ETIMER_THRESH];
+ struct nlattr *rt = attrs[XFRMA_REPLAY_THRESH];
+
+ if (!lt && !rp && !re && !et && !rt)
+ return err;
+
+ /* pedantic mode - thou shalt sayeth replaceth */
+ if (!(nlh->nlmsg_flags&NLM_F_REPLACE))
+ return err;
+
+ mark = xfrm_mark_get(attrs, &m);
+
+ x = xfrm_state_lookup(net, mark, &p->sa_id.daddr, p->sa_id.spi, p->sa_id.proto, p->sa_id.family);
+ if (x == NULL)
+ return -ESRCH;
+
+ if (x->km.state != XFRM_STATE_VALID)
+ goto out;
+
+ err = xfrm_replay_verify_len(x->replay_esn, re);
+ if (err)
+ goto out;
+
+ spin_lock_bh(&x->lock);
+ xfrm_update_ae_params(x, attrs, 1);
+ spin_unlock_bh(&x->lock);
+
+ c.event = nlh->nlmsg_type;
+ c.seq = nlh->nlmsg_seq;
+ c.portid = nlh->nlmsg_pid;
+ c.data.aevent = XFRM_AE_CU;
+ km_state_notify(x, &c);
+ err = 0;
+out:
+ xfrm_state_put(x);
+ return err;
+}
+
+static int xfrm_flush_policy(struct sk_buff *skb, struct nlmsghdr *nlh,
+ struct nlattr **attrs,
+ struct netlink_ext_ack *extack)
+{
+ struct net *net = sock_net(skb->sk);
+ struct km_event c;
+ u8 type = XFRM_POLICY_TYPE_MAIN;
+ int err;
+
+ err = copy_from_user_policy_type(&type, attrs, extack);
+ if (err)
+ return err;
+
+ err = xfrm_policy_flush(net, type, true);
+ if (err) {
+ if (err == -ESRCH) /* empty table */
+ return 0;
+ return err;
+ }
+
+ c.data.type = type;
+ c.event = nlh->nlmsg_type;
+ c.seq = nlh->nlmsg_seq;
+ c.portid = nlh->nlmsg_pid;
+ c.net = net;
+ km_policy_notify(NULL, 0, &c);
+ return 0;
+}
+
+static int xfrm_add_pol_expire(struct sk_buff *skb, struct nlmsghdr *nlh,
+ struct nlattr **attrs,
+ struct netlink_ext_ack *extack)
+{
+ struct net *net = sock_net(skb->sk);
+ struct xfrm_policy *xp;
+ struct xfrm_user_polexpire *up = nlmsg_data(nlh);
+ struct xfrm_userpolicy_info *p = &up->pol;
+ u8 type = XFRM_POLICY_TYPE_MAIN;
+ int err = -ENOENT;
+ struct xfrm_mark m;
+ u32 if_id = 0;
+
+ err = copy_from_user_policy_type(&type, attrs, extack);
+ if (err)
+ return err;
+
+ err = verify_policy_dir(p->dir, extack);
+ if (err)
+ return err;
+
+ if (attrs[XFRMA_IF_ID])
+ if_id = nla_get_u32(attrs[XFRMA_IF_ID]);
+
+ xfrm_mark_get(attrs, &m);
+
+ if (p->index)
+ xp = xfrm_policy_byid(net, &m, if_id, type, p->dir, p->index,
+ 0, &err);
+ else {
+ struct nlattr *rt = attrs[XFRMA_SEC_CTX];
+ struct xfrm_sec_ctx *ctx;
+
+ err = verify_sec_ctx_len(attrs, extack);
+ if (err)
+ return err;
+
+ ctx = NULL;
+ if (rt) {
+ struct xfrm_user_sec_ctx *uctx = nla_data(rt);
+
+ err = security_xfrm_policy_alloc(&ctx, uctx, GFP_KERNEL);
+ if (err)
+ return err;
+ }
+ xp = xfrm_policy_bysel_ctx(net, &m, if_id, type, p->dir,
+ &p->sel, ctx, 0, &err);
+ security_xfrm_policy_free(ctx);
+ }
+ if (xp == NULL)
+ return -ENOENT;
+
+ if (unlikely(xp->walk.dead))
+ goto out;
+
+ err = 0;
+ if (up->hard) {
+ xfrm_policy_delete(xp, p->dir);
+ xfrm_audit_policy_delete(xp, 1, true);
+ }
+ km_policy_expired(xp, p->dir, up->hard, nlh->nlmsg_pid);
+
+out:
+ xfrm_pol_put(xp);
+ return err;
+}
+
+static int xfrm_add_sa_expire(struct sk_buff *skb, struct nlmsghdr *nlh,
+ struct nlattr **attrs,
+ struct netlink_ext_ack *extack)
+{
+ struct net *net = sock_net(skb->sk);
+ struct xfrm_state *x;
+ int err;
+ struct xfrm_user_expire *ue = nlmsg_data(nlh);
+ struct xfrm_usersa_info *p = &ue->state;
+ struct xfrm_mark m;
+ u32 mark = xfrm_mark_get(attrs, &m);
+
+ x = xfrm_state_lookup(net, mark, &p->id.daddr, p->id.spi, p->id.proto, p->family);
+
+ err = -ENOENT;
+ if (x == NULL)
+ return err;
+
+ spin_lock_bh(&x->lock);
+ err = -EINVAL;
+ if (x->km.state != XFRM_STATE_VALID)
+ goto out;
+ km_state_expired(x, ue->hard, nlh->nlmsg_pid);
+
+ if (ue->hard) {
+ __xfrm_state_delete(x);
+ xfrm_audit_state_delete(x, 1, true);
+ }
+ err = 0;
+out:
+ spin_unlock_bh(&x->lock);
+ xfrm_state_put(x);
+ return err;
+}
+
+static int xfrm_add_acquire(struct sk_buff *skb, struct nlmsghdr *nlh,
+ struct nlattr **attrs,
+ struct netlink_ext_ack *extack)
+{
+ struct net *net = sock_net(skb->sk);
+ struct xfrm_policy *xp;
+ struct xfrm_user_tmpl *ut;
+ int i;
+ struct nlattr *rt = attrs[XFRMA_TMPL];
+ struct xfrm_mark mark;
+
+ struct xfrm_user_acquire *ua = nlmsg_data(nlh);
+ struct xfrm_state *x = xfrm_state_alloc(net);
+ int err = -ENOMEM;
+
+ if (!x)
+ goto nomem;
+
+ xfrm_mark_get(attrs, &mark);
+
+ err = verify_newpolicy_info(&ua->policy, extack);
+ if (err)
+ goto free_state;
+ err = verify_sec_ctx_len(attrs, extack);
+ if (err)
+ goto free_state;
+
+ /* build an XP */
+ xp = xfrm_policy_construct(net, &ua->policy, attrs, &err, extack);
+ if (!xp)
+ goto free_state;
+
+ memcpy(&x->id, &ua->id, sizeof(ua->id));
+ memcpy(&x->props.saddr, &ua->saddr, sizeof(ua->saddr));
+ memcpy(&x->sel, &ua->sel, sizeof(ua->sel));
+ xp->mark.m = x->mark.m = mark.m;
+ xp->mark.v = x->mark.v = mark.v;
+ ut = nla_data(rt);
+ /* extract the templates and for each call km_key */
+ for (i = 0; i < xp->xfrm_nr; i++, ut++) {
+ struct xfrm_tmpl *t = &xp->xfrm_vec[i];
+ memcpy(&x->id, &t->id, sizeof(x->id));
+ x->props.mode = t->mode;
+ x->props.reqid = t->reqid;
+ x->props.family = ut->family;
+ t->aalgos = ua->aalgos;
+ t->ealgos = ua->ealgos;
+ t->calgos = ua->calgos;
+ err = km_query(x, t, xp);
+
+ }
+
+ xfrm_state_free(x);
+ kfree(xp);
+
+ return 0;
+
+free_state:
+ xfrm_state_free(x);
+nomem:
+ return err;
+}
+
+#ifdef CONFIG_XFRM_MIGRATE
+static int copy_from_user_migrate(struct xfrm_migrate *ma,
+ struct xfrm_kmaddress *k,
+ struct nlattr **attrs, int *num)
+{
+ struct nlattr *rt = attrs[XFRMA_MIGRATE];
+ struct xfrm_user_migrate *um;
+ int i, num_migrate;
+
+ if (k != NULL) {
+ struct xfrm_user_kmaddress *uk;
+
+ uk = nla_data(attrs[XFRMA_KMADDRESS]);
+ memcpy(&k->local, &uk->local, sizeof(k->local));
+ memcpy(&k->remote, &uk->remote, sizeof(k->remote));
+ k->family = uk->family;
+ k->reserved = uk->reserved;
+ }
+
+ um = nla_data(rt);
+ num_migrate = nla_len(rt) / sizeof(*um);
+
+ if (num_migrate <= 0 || num_migrate > XFRM_MAX_DEPTH)
+ return -EINVAL;
+
+ for (i = 0; i < num_migrate; i++, um++, ma++) {
+ memcpy(&ma->old_daddr, &um->old_daddr, sizeof(ma->old_daddr));
+ memcpy(&ma->old_saddr, &um->old_saddr, sizeof(ma->old_saddr));
+ memcpy(&ma->new_daddr, &um->new_daddr, sizeof(ma->new_daddr));
+ memcpy(&ma->new_saddr, &um->new_saddr, sizeof(ma->new_saddr));
+
+ ma->proto = um->proto;
+ ma->mode = um->mode;
+ ma->reqid = um->reqid;
+
+ ma->old_family = um->old_family;
+ ma->new_family = um->new_family;
+ }
+
+ *num = i;
+ return 0;
+}
+
+static int xfrm_do_migrate(struct sk_buff *skb, struct nlmsghdr *nlh,
+ struct nlattr **attrs, struct netlink_ext_ack *extack)
+{
+ struct xfrm_userpolicy_id *pi = nlmsg_data(nlh);
+ struct xfrm_migrate m[XFRM_MAX_DEPTH];
+ struct xfrm_kmaddress km, *kmp;
+ u8 type;
+ int err;
+ int n = 0;
+ struct net *net = sock_net(skb->sk);
+ struct xfrm_encap_tmpl *encap = NULL;
+ u32 if_id = 0;
+
+ if (attrs[XFRMA_MIGRATE] == NULL)
+ return -EINVAL;
+
+ kmp = attrs[XFRMA_KMADDRESS] ? &km : NULL;
+
+ err = copy_from_user_policy_type(&type, attrs, extack);
+ if (err)
+ return err;
+
+ err = copy_from_user_migrate((struct xfrm_migrate *)m, kmp, attrs, &n);
+ if (err)
+ return err;
+
+ if (!n)
+ return 0;
+
+ if (attrs[XFRMA_ENCAP]) {
+ encap = kmemdup(nla_data(attrs[XFRMA_ENCAP]),
+ sizeof(*encap), GFP_KERNEL);
+ if (!encap)
+ return -ENOMEM;
+ }
+
+ if (attrs[XFRMA_IF_ID])
+ if_id = nla_get_u32(attrs[XFRMA_IF_ID]);
+
+ err = xfrm_migrate(&pi->sel, pi->dir, type, m, n, kmp, net, encap, if_id);
+
+ kfree(encap);
+
+ return err;
+}
+#else
+static int xfrm_do_migrate(struct sk_buff *skb, struct nlmsghdr *nlh,
+ struct nlattr **attrs, struct netlink_ext_ack *extack)
+{
+ return -ENOPROTOOPT;
+}
+#endif
+
+#ifdef CONFIG_XFRM_MIGRATE
+static int copy_to_user_migrate(const struct xfrm_migrate *m, struct sk_buff *skb)
+{
+ struct xfrm_user_migrate um;
+
+ memset(&um, 0, sizeof(um));
+ um.proto = m->proto;
+ um.mode = m->mode;
+ um.reqid = m->reqid;
+ um.old_family = m->old_family;
+ memcpy(&um.old_daddr, &m->old_daddr, sizeof(um.old_daddr));
+ memcpy(&um.old_saddr, &m->old_saddr, sizeof(um.old_saddr));
+ um.new_family = m->new_family;
+ memcpy(&um.new_daddr, &m->new_daddr, sizeof(um.new_daddr));
+ memcpy(&um.new_saddr, &m->new_saddr, sizeof(um.new_saddr));
+
+ return nla_put(skb, XFRMA_MIGRATE, sizeof(um), &um);
+}
+
+static int copy_to_user_kmaddress(const struct xfrm_kmaddress *k, struct sk_buff *skb)
+{
+ struct xfrm_user_kmaddress uk;
+
+ memset(&uk, 0, sizeof(uk));
+ uk.family = k->family;
+ uk.reserved = k->reserved;
+ memcpy(&uk.local, &k->local, sizeof(uk.local));
+ memcpy(&uk.remote, &k->remote, sizeof(uk.remote));
+
+ return nla_put(skb, XFRMA_KMADDRESS, sizeof(uk), &uk);
+}
+
+static inline unsigned int xfrm_migrate_msgsize(int num_migrate, int with_kma,
+ int with_encp)
+{
+ return NLMSG_ALIGN(sizeof(struct xfrm_userpolicy_id))
+ + (with_kma ? nla_total_size(sizeof(struct xfrm_kmaddress)) : 0)
+ + (with_encp ? nla_total_size(sizeof(struct xfrm_encap_tmpl)) : 0)
+ + nla_total_size(sizeof(struct xfrm_user_migrate) * num_migrate)
+ + userpolicy_type_attrsize();
+}
+
+static int build_migrate(struct sk_buff *skb, const struct xfrm_migrate *m,
+ int num_migrate, const struct xfrm_kmaddress *k,
+ const struct xfrm_selector *sel,
+ const struct xfrm_encap_tmpl *encap, u8 dir, u8 type)
+{
+ const struct xfrm_migrate *mp;
+ struct xfrm_userpolicy_id *pol_id;
+ struct nlmsghdr *nlh;
+ int i, err;
+
+ nlh = nlmsg_put(skb, 0, 0, XFRM_MSG_MIGRATE, sizeof(*pol_id), 0);
+ if (nlh == NULL)
+ return -EMSGSIZE;
+
+ pol_id = nlmsg_data(nlh);
+ /* copy data from selector, dir, and type to the pol_id */
+ memset(pol_id, 0, sizeof(*pol_id));
+ memcpy(&pol_id->sel, sel, sizeof(pol_id->sel));
+ pol_id->dir = dir;
+
+ if (k != NULL) {
+ err = copy_to_user_kmaddress(k, skb);
+ if (err)
+ goto out_cancel;
+ }
+ if (encap) {
+ err = nla_put(skb, XFRMA_ENCAP, sizeof(*encap), encap);
+ if (err)
+ goto out_cancel;
+ }
+ err = copy_to_user_policy_type(type, skb);
+ if (err)
+ goto out_cancel;
+ for (i = 0, mp = m ; i < num_migrate; i++, mp++) {
+ err = copy_to_user_migrate(mp, skb);
+ if (err)
+ goto out_cancel;
+ }
+
+ nlmsg_end(skb, nlh);
+ return 0;
+
+out_cancel:
+ nlmsg_cancel(skb, nlh);
+ return err;
+}
+
+static int xfrm_send_migrate(const struct xfrm_selector *sel, u8 dir, u8 type,
+ const struct xfrm_migrate *m, int num_migrate,
+ const struct xfrm_kmaddress *k,
+ const struct xfrm_encap_tmpl *encap)
+{
+ struct net *net = &init_net;
+ struct sk_buff *skb;
+ int err;
+
+ skb = nlmsg_new(xfrm_migrate_msgsize(num_migrate, !!k, !!encap),
+ GFP_ATOMIC);
+ if (skb == NULL)
+ return -ENOMEM;
+
+ /* build migrate */
+ err = build_migrate(skb, m, num_migrate, k, sel, encap, dir, type);
+ BUG_ON(err < 0);
+
+ return xfrm_nlmsg_multicast(net, skb, 0, XFRMNLGRP_MIGRATE);
+}
+#else
+static int xfrm_send_migrate(const struct xfrm_selector *sel, u8 dir, u8 type,
+ const struct xfrm_migrate *m, int num_migrate,
+ const struct xfrm_kmaddress *k,
+ const struct xfrm_encap_tmpl *encap)
+{
+ return -ENOPROTOOPT;
+}
+#endif
+
+#define XMSGSIZE(type) sizeof(struct type)
+
+const int xfrm_msg_min[XFRM_NR_MSGTYPES] = {
+ [XFRM_MSG_NEWSA - XFRM_MSG_BASE] = XMSGSIZE(xfrm_usersa_info),
+ [XFRM_MSG_DELSA - XFRM_MSG_BASE] = XMSGSIZE(xfrm_usersa_id),
+ [XFRM_MSG_GETSA - XFRM_MSG_BASE] = XMSGSIZE(xfrm_usersa_id),
+ [XFRM_MSG_NEWPOLICY - XFRM_MSG_BASE] = XMSGSIZE(xfrm_userpolicy_info),
+ [XFRM_MSG_DELPOLICY - XFRM_MSG_BASE] = XMSGSIZE(xfrm_userpolicy_id),
+ [XFRM_MSG_GETPOLICY - XFRM_MSG_BASE] = XMSGSIZE(xfrm_userpolicy_id),
+ [XFRM_MSG_ALLOCSPI - XFRM_MSG_BASE] = XMSGSIZE(xfrm_userspi_info),
+ [XFRM_MSG_ACQUIRE - XFRM_MSG_BASE] = XMSGSIZE(xfrm_user_acquire),
+ [XFRM_MSG_EXPIRE - XFRM_MSG_BASE] = XMSGSIZE(xfrm_user_expire),
+ [XFRM_MSG_UPDPOLICY - XFRM_MSG_BASE] = XMSGSIZE(xfrm_userpolicy_info),
+ [XFRM_MSG_UPDSA - XFRM_MSG_BASE] = XMSGSIZE(xfrm_usersa_info),
+ [XFRM_MSG_POLEXPIRE - XFRM_MSG_BASE] = XMSGSIZE(xfrm_user_polexpire),
+ [XFRM_MSG_FLUSHSA - XFRM_MSG_BASE] = XMSGSIZE(xfrm_usersa_flush),
+ [XFRM_MSG_FLUSHPOLICY - XFRM_MSG_BASE] = 0,
+ [XFRM_MSG_NEWAE - XFRM_MSG_BASE] = XMSGSIZE(xfrm_aevent_id),
+ [XFRM_MSG_GETAE - XFRM_MSG_BASE] = XMSGSIZE(xfrm_aevent_id),
+ [XFRM_MSG_REPORT - XFRM_MSG_BASE] = XMSGSIZE(xfrm_user_report),
+ [XFRM_MSG_MIGRATE - XFRM_MSG_BASE] = XMSGSIZE(xfrm_userpolicy_id),
+ [XFRM_MSG_GETSADINFO - XFRM_MSG_BASE] = sizeof(u32),
+ [XFRM_MSG_NEWSPDINFO - XFRM_MSG_BASE] = sizeof(u32),
+ [XFRM_MSG_GETSPDINFO - XFRM_MSG_BASE] = sizeof(u32),
+ [XFRM_MSG_SETDEFAULT - XFRM_MSG_BASE] = XMSGSIZE(xfrm_userpolicy_default),
+ [XFRM_MSG_GETDEFAULT - XFRM_MSG_BASE] = XMSGSIZE(xfrm_userpolicy_default),
+};
+EXPORT_SYMBOL_GPL(xfrm_msg_min);
+
+#undef XMSGSIZE
+
+const struct nla_policy xfrma_policy[XFRMA_MAX+1] = {
+ [XFRMA_SA] = { .len = sizeof(struct xfrm_usersa_info)},
+ [XFRMA_POLICY] = { .len = sizeof(struct xfrm_userpolicy_info)},
+ [XFRMA_LASTUSED] = { .type = NLA_U64},
+ [XFRMA_ALG_AUTH_TRUNC] = { .len = sizeof(struct xfrm_algo_auth)},
+ [XFRMA_ALG_AEAD] = { .len = sizeof(struct xfrm_algo_aead) },
+ [XFRMA_ALG_AUTH] = { .len = sizeof(struct xfrm_algo) },
+ [XFRMA_ALG_CRYPT] = { .len = sizeof(struct xfrm_algo) },
+ [XFRMA_ALG_COMP] = { .len = sizeof(struct xfrm_algo) },
+ [XFRMA_ENCAP] = { .len = sizeof(struct xfrm_encap_tmpl) },
+ [XFRMA_TMPL] = { .len = sizeof(struct xfrm_user_tmpl) },
+ [XFRMA_SEC_CTX] = { .len = sizeof(struct xfrm_user_sec_ctx) },
+ [XFRMA_LTIME_VAL] = { .len = sizeof(struct xfrm_lifetime_cur) },
+ [XFRMA_REPLAY_VAL] = { .len = sizeof(struct xfrm_replay_state) },
+ [XFRMA_REPLAY_THRESH] = { .type = NLA_U32 },
+ [XFRMA_ETIMER_THRESH] = { .type = NLA_U32 },
+ [XFRMA_SRCADDR] = { .len = sizeof(xfrm_address_t) },
+ [XFRMA_COADDR] = { .len = sizeof(xfrm_address_t) },
+ [XFRMA_POLICY_TYPE] = { .len = sizeof(struct xfrm_userpolicy_type)},
+ [XFRMA_MIGRATE] = { .len = sizeof(struct xfrm_user_migrate) },
+ [XFRMA_KMADDRESS] = { .len = sizeof(struct xfrm_user_kmaddress) },
+ [XFRMA_MARK] = { .len = sizeof(struct xfrm_mark) },
+ [XFRMA_TFCPAD] = { .type = NLA_U32 },
+ [XFRMA_REPLAY_ESN_VAL] = { .len = sizeof(struct xfrm_replay_state_esn) },
+ [XFRMA_SA_EXTRA_FLAGS] = { .type = NLA_U32 },
+ [XFRMA_PROTO] = { .type = NLA_U8 },
+ [XFRMA_ADDRESS_FILTER] = { .len = sizeof(struct xfrm_address_filter) },
+ [XFRMA_OFFLOAD_DEV] = { .len = sizeof(struct xfrm_user_offload) },
+ [XFRMA_SET_MARK] = { .type = NLA_U32 },
+ [XFRMA_SET_MARK_MASK] = { .type = NLA_U32 },
+ [XFRMA_IF_ID] = { .type = NLA_U32 },
+ [XFRMA_MTIMER_THRESH] = { .type = NLA_U32 },
+};
+EXPORT_SYMBOL_GPL(xfrma_policy);
+
+static const struct nla_policy xfrma_spd_policy[XFRMA_SPD_MAX+1] = {
+ [XFRMA_SPD_IPV4_HTHRESH] = { .len = sizeof(struct xfrmu_spdhthresh) },
+ [XFRMA_SPD_IPV6_HTHRESH] = { .len = sizeof(struct xfrmu_spdhthresh) },
+};
+
+static const struct xfrm_link {
+ int (*doit)(struct sk_buff *, struct nlmsghdr *, struct nlattr **,
+ struct netlink_ext_ack *);
+ int (*start)(struct netlink_callback *);
+ int (*dump)(struct sk_buff *, struct netlink_callback *);
+ int (*done)(struct netlink_callback *);
+ const struct nla_policy *nla_pol;
+ int nla_max;
+} xfrm_dispatch[XFRM_NR_MSGTYPES] = {
+ [XFRM_MSG_NEWSA - XFRM_MSG_BASE] = { .doit = xfrm_add_sa },
+ [XFRM_MSG_DELSA - XFRM_MSG_BASE] = { .doit = xfrm_del_sa },
+ [XFRM_MSG_GETSA - XFRM_MSG_BASE] = { .doit = xfrm_get_sa,
+ .dump = xfrm_dump_sa,
+ .done = xfrm_dump_sa_done },
+ [XFRM_MSG_NEWPOLICY - XFRM_MSG_BASE] = { .doit = xfrm_add_policy },
+ [XFRM_MSG_DELPOLICY - XFRM_MSG_BASE] = { .doit = xfrm_get_policy },
+ [XFRM_MSG_GETPOLICY - XFRM_MSG_BASE] = { .doit = xfrm_get_policy,
+ .start = xfrm_dump_policy_start,
+ .dump = xfrm_dump_policy,
+ .done = xfrm_dump_policy_done },
+ [XFRM_MSG_ALLOCSPI - XFRM_MSG_BASE] = { .doit = xfrm_alloc_userspi },
+ [XFRM_MSG_ACQUIRE - XFRM_MSG_BASE] = { .doit = xfrm_add_acquire },
+ [XFRM_MSG_EXPIRE - XFRM_MSG_BASE] = { .doit = xfrm_add_sa_expire },
+ [XFRM_MSG_UPDPOLICY - XFRM_MSG_BASE] = { .doit = xfrm_add_policy },
+ [XFRM_MSG_UPDSA - XFRM_MSG_BASE] = { .doit = xfrm_add_sa },
+ [XFRM_MSG_POLEXPIRE - XFRM_MSG_BASE] = { .doit = xfrm_add_pol_expire},
+ [XFRM_MSG_FLUSHSA - XFRM_MSG_BASE] = { .doit = xfrm_flush_sa },
+ [XFRM_MSG_FLUSHPOLICY - XFRM_MSG_BASE] = { .doit = xfrm_flush_policy },
+ [XFRM_MSG_NEWAE - XFRM_MSG_BASE] = { .doit = xfrm_new_ae },
+ [XFRM_MSG_GETAE - XFRM_MSG_BASE] = { .doit = xfrm_get_ae },
+ [XFRM_MSG_MIGRATE - XFRM_MSG_BASE] = { .doit = xfrm_do_migrate },
+ [XFRM_MSG_GETSADINFO - XFRM_MSG_BASE] = { .doit = xfrm_get_sadinfo },
+ [XFRM_MSG_NEWSPDINFO - XFRM_MSG_BASE] = { .doit = xfrm_set_spdinfo,
+ .nla_pol = xfrma_spd_policy,
+ .nla_max = XFRMA_SPD_MAX },
+ [XFRM_MSG_GETSPDINFO - XFRM_MSG_BASE] = { .doit = xfrm_get_spdinfo },
+ [XFRM_MSG_SETDEFAULT - XFRM_MSG_BASE] = { .doit = xfrm_set_default },
+ [XFRM_MSG_GETDEFAULT - XFRM_MSG_BASE] = { .doit = xfrm_get_default },
+};
+
+static int xfrm_user_rcv_msg(struct sk_buff *skb, struct nlmsghdr *nlh,
+ struct netlink_ext_ack *extack)
+{
+ struct net *net = sock_net(skb->sk);
+ struct nlattr *attrs[XFRMA_MAX+1];
+ const struct xfrm_link *link;
+ struct nlmsghdr *nlh64 = NULL;
+ int type, err;
+
+ type = nlh->nlmsg_type;
+ if (type > XFRM_MSG_MAX)
+ return -EINVAL;
+
+ type -= XFRM_MSG_BASE;
+ link = &xfrm_dispatch[type];
+
+ /* All operations require privileges, even GET */
+ if (!netlink_net_capable(skb, CAP_NET_ADMIN))
+ return -EPERM;
+
+ if (in_compat_syscall()) {
+ struct xfrm_translator *xtr = xfrm_get_translator();
+
+ if (!xtr)
+ return -EOPNOTSUPP;
+
+ nlh64 = xtr->rcv_msg_compat(nlh, link->nla_max,
+ link->nla_pol, extack);
+ xfrm_put_translator(xtr);
+ if (IS_ERR(nlh64))
+ return PTR_ERR(nlh64);
+ if (nlh64)
+ nlh = nlh64;
+ }
+
+ if ((type == (XFRM_MSG_GETSA - XFRM_MSG_BASE) ||
+ type == (XFRM_MSG_GETPOLICY - XFRM_MSG_BASE)) &&
+ (nlh->nlmsg_flags & NLM_F_DUMP)) {
+ struct netlink_dump_control c = {
+ .start = link->start,
+ .dump = link->dump,
+ .done = link->done,
+ };
+
+ if (link->dump == NULL) {
+ err = -EINVAL;
+ goto err;
+ }
+
+ err = netlink_dump_start(net->xfrm.nlsk, skb, nlh, &c);
+ goto err;
+ }
+
+ err = nlmsg_parse_deprecated(nlh, xfrm_msg_min[type], attrs,
+ link->nla_max ? : XFRMA_MAX,
+ link->nla_pol ? : xfrma_policy, extack);
+ if (err < 0)
+ goto err;
+
+ if (link->doit == NULL) {
+ err = -EINVAL;
+ goto err;
+ }
+
+ err = link->doit(skb, nlh, attrs, extack);
+
+ /* We need to free skb allocated in xfrm_alloc_compat() before
+ * returning from this function, because consume_skb() won't take
+ * care of frag_list since netlink destructor sets
+ * sbk->head to NULL. (see netlink_skb_destructor())
+ */
+ if (skb_has_frag_list(skb)) {
+ kfree_skb(skb_shinfo(skb)->frag_list);
+ skb_shinfo(skb)->frag_list = NULL;
+ }
+
+err:
+ kvfree(nlh64);
+ return err;
+}
+
+static void xfrm_netlink_rcv(struct sk_buff *skb)
+{
+ struct net *net = sock_net(skb->sk);
+
+ mutex_lock(&net->xfrm.xfrm_cfg_mutex);
+ netlink_rcv_skb(skb, &xfrm_user_rcv_msg);
+ mutex_unlock(&net->xfrm.xfrm_cfg_mutex);
+}
+
+static inline unsigned int xfrm_expire_msgsize(void)
+{
+ return NLMSG_ALIGN(sizeof(struct xfrm_user_expire))
+ + nla_total_size(sizeof(struct xfrm_mark));
+}
+
+static int build_expire(struct sk_buff *skb, struct xfrm_state *x, const struct km_event *c)
+{
+ struct xfrm_user_expire *ue;
+ struct nlmsghdr *nlh;
+ int err;
+
+ nlh = nlmsg_put(skb, c->portid, 0, XFRM_MSG_EXPIRE, sizeof(*ue), 0);
+ if (nlh == NULL)
+ return -EMSGSIZE;
+
+ ue = nlmsg_data(nlh);
+ copy_to_user_state(x, &ue->state);
+ ue->hard = (c->data.hard != 0) ? 1 : 0;
+ /* clear the padding bytes */
+ memset_after(ue, 0, hard);
+
+ err = xfrm_mark_put(skb, &x->mark);
+ if (err)
+ return err;
+
+ err = xfrm_if_id_put(skb, x->if_id);
+ if (err)
+ return err;
+
+ nlmsg_end(skb, nlh);
+ return 0;
+}
+
+static int xfrm_exp_state_notify(struct xfrm_state *x, const struct km_event *c)
+{
+ struct net *net = xs_net(x);
+ struct sk_buff *skb;
+
+ skb = nlmsg_new(xfrm_expire_msgsize(), GFP_ATOMIC);
+ if (skb == NULL)
+ return -ENOMEM;
+
+ if (build_expire(skb, x, c) < 0) {
+ kfree_skb(skb);
+ return -EMSGSIZE;
+ }
+
+ return xfrm_nlmsg_multicast(net, skb, 0, XFRMNLGRP_EXPIRE);
+}
+
+static int xfrm_aevent_state_notify(struct xfrm_state *x, const struct km_event *c)
+{
+ struct net *net = xs_net(x);
+ struct sk_buff *skb;
+ int err;
+
+ skb = nlmsg_new(xfrm_aevent_msgsize(x), GFP_ATOMIC);
+ if (skb == NULL)
+ return -ENOMEM;
+
+ err = build_aevent(skb, x, c);
+ BUG_ON(err < 0);
+
+ return xfrm_nlmsg_multicast(net, skb, 0, XFRMNLGRP_AEVENTS);
+}
+
+static int xfrm_notify_sa_flush(const struct km_event *c)
+{
+ struct net *net = c->net;
+ struct xfrm_usersa_flush *p;
+ struct nlmsghdr *nlh;
+ struct sk_buff *skb;
+ int len = NLMSG_ALIGN(sizeof(struct xfrm_usersa_flush));
+
+ skb = nlmsg_new(len, GFP_ATOMIC);
+ if (skb == NULL)
+ return -ENOMEM;
+
+ nlh = nlmsg_put(skb, c->portid, c->seq, XFRM_MSG_FLUSHSA, sizeof(*p), 0);
+ if (nlh == NULL) {
+ kfree_skb(skb);
+ return -EMSGSIZE;
+ }
+
+ p = nlmsg_data(nlh);
+ p->proto = c->data.proto;
+
+ nlmsg_end(skb, nlh);
+
+ return xfrm_nlmsg_multicast(net, skb, 0, XFRMNLGRP_SA);
+}
+
+static inline unsigned int xfrm_sa_len(struct xfrm_state *x)
+{
+ unsigned int l = 0;
+ if (x->aead)
+ l += nla_total_size(aead_len(x->aead));
+ if (x->aalg) {
+ l += nla_total_size(sizeof(struct xfrm_algo) +
+ (x->aalg->alg_key_len + 7) / 8);
+ l += nla_total_size(xfrm_alg_auth_len(x->aalg));
+ }
+ if (x->ealg)
+ l += nla_total_size(xfrm_alg_len(x->ealg));
+ if (x->calg)
+ l += nla_total_size(sizeof(*x->calg));
+ if (x->encap)
+ l += nla_total_size(sizeof(*x->encap));
+ if (x->tfcpad)
+ l += nla_total_size(sizeof(x->tfcpad));
+ if (x->replay_esn)
+ l += nla_total_size(xfrm_replay_state_esn_len(x->replay_esn));
+ else
+ l += nla_total_size(sizeof(struct xfrm_replay_state));
+ if (x->security)
+ l += nla_total_size(sizeof(struct xfrm_user_sec_ctx) +
+ x->security->ctx_len);
+ if (x->coaddr)
+ l += nla_total_size(sizeof(*x->coaddr));
+ if (x->props.extra_flags)
+ l += nla_total_size(sizeof(x->props.extra_flags));
+ if (x->xso.dev)
+ l += nla_total_size(sizeof(struct xfrm_user_offload));
+ if (x->props.smark.v | x->props.smark.m) {
+ l += nla_total_size(sizeof(x->props.smark.v));
+ l += nla_total_size(sizeof(x->props.smark.m));
+ }
+ if (x->if_id)
+ l += nla_total_size(sizeof(x->if_id));
+
+ /* Must count x->lastused as it may become non-zero behind our back. */
+ l += nla_total_size_64bit(sizeof(u64));
+
+ if (x->mapping_maxage)
+ l += nla_total_size(sizeof(x->mapping_maxage));
+
+ return l;
+}
+
+static int xfrm_notify_sa(struct xfrm_state *x, const struct km_event *c)
+{
+ struct net *net = xs_net(x);
+ struct xfrm_usersa_info *p;
+ struct xfrm_usersa_id *id;
+ struct nlmsghdr *nlh;
+ struct sk_buff *skb;
+ unsigned int len = xfrm_sa_len(x);
+ unsigned int headlen;
+ int err;
+
+ headlen = sizeof(*p);
+ if (c->event == XFRM_MSG_DELSA) {
+ len += nla_total_size(headlen);
+ headlen = sizeof(*id);
+ len += nla_total_size(sizeof(struct xfrm_mark));
+ }
+ len += NLMSG_ALIGN(headlen);
+
+ skb = nlmsg_new(len, GFP_ATOMIC);
+ if (skb == NULL)
+ return -ENOMEM;
+
+ nlh = nlmsg_put(skb, c->portid, c->seq, c->event, headlen, 0);
+ err = -EMSGSIZE;
+ if (nlh == NULL)
+ goto out_free_skb;
+
+ p = nlmsg_data(nlh);
+ if (c->event == XFRM_MSG_DELSA) {
+ struct nlattr *attr;
+
+ id = nlmsg_data(nlh);
+ memset(id, 0, sizeof(*id));
+ memcpy(&id->daddr, &x->id.daddr, sizeof(id->daddr));
+ id->spi = x->id.spi;
+ id->family = x->props.family;
+ id->proto = x->id.proto;
+
+ attr = nla_reserve(skb, XFRMA_SA, sizeof(*p));
+ err = -EMSGSIZE;
+ if (attr == NULL)
+ goto out_free_skb;
+
+ p = nla_data(attr);
+ }
+ err = copy_to_user_state_extra(x, p, skb);
+ if (err)
+ goto out_free_skb;
+
+ nlmsg_end(skb, nlh);
+
+ return xfrm_nlmsg_multicast(net, skb, 0, XFRMNLGRP_SA);
+
+out_free_skb:
+ kfree_skb(skb);
+ return err;
+}
+
+static int xfrm_send_state_notify(struct xfrm_state *x, const struct km_event *c)
+{
+
+ switch (c->event) {
+ case XFRM_MSG_EXPIRE:
+ return xfrm_exp_state_notify(x, c);
+ case XFRM_MSG_NEWAE:
+ return xfrm_aevent_state_notify(x, c);
+ case XFRM_MSG_DELSA:
+ case XFRM_MSG_UPDSA:
+ case XFRM_MSG_NEWSA:
+ return xfrm_notify_sa(x, c);
+ case XFRM_MSG_FLUSHSA:
+ return xfrm_notify_sa_flush(c);
+ default:
+ printk(KERN_NOTICE "xfrm_user: Unknown SA event %d\n",
+ c->event);
+ break;
+ }
+
+ return 0;
+
+}
+
+static inline unsigned int xfrm_acquire_msgsize(struct xfrm_state *x,
+ struct xfrm_policy *xp)
+{
+ return NLMSG_ALIGN(sizeof(struct xfrm_user_acquire))
+ + nla_total_size(sizeof(struct xfrm_user_tmpl) * xp->xfrm_nr)
+ + nla_total_size(sizeof(struct xfrm_mark))
+ + nla_total_size(xfrm_user_sec_ctx_size(x->security))
+ + userpolicy_type_attrsize();
+}
+
+static int build_acquire(struct sk_buff *skb, struct xfrm_state *x,
+ struct xfrm_tmpl *xt, struct xfrm_policy *xp)
+{
+ __u32 seq = xfrm_get_acqseq();
+ struct xfrm_user_acquire *ua;
+ struct nlmsghdr *nlh;
+ int err;
+
+ nlh = nlmsg_put(skb, 0, 0, XFRM_MSG_ACQUIRE, sizeof(*ua), 0);
+ if (nlh == NULL)
+ return -EMSGSIZE;
+
+ ua = nlmsg_data(nlh);
+ memcpy(&ua->id, &x->id, sizeof(ua->id));
+ memcpy(&ua->saddr, &x->props.saddr, sizeof(ua->saddr));
+ memcpy(&ua->sel, &x->sel, sizeof(ua->sel));
+ copy_to_user_policy(xp, &ua->policy, XFRM_POLICY_OUT);
+ ua->aalgos = xt->aalgos;
+ ua->ealgos = xt->ealgos;
+ ua->calgos = xt->calgos;
+ ua->seq = x->km.seq = seq;
+
+ err = copy_to_user_tmpl(xp, skb);
+ if (!err)
+ err = copy_to_user_state_sec_ctx(x, skb);
+ if (!err)
+ err = copy_to_user_policy_type(xp->type, skb);
+ if (!err)
+ err = xfrm_mark_put(skb, &xp->mark);
+ if (!err)
+ err = xfrm_if_id_put(skb, xp->if_id);
+ if (err) {
+ nlmsg_cancel(skb, nlh);
+ return err;
+ }
+
+ nlmsg_end(skb, nlh);
+ return 0;
+}
+
+static int xfrm_send_acquire(struct xfrm_state *x, struct xfrm_tmpl *xt,
+ struct xfrm_policy *xp)
+{
+ struct net *net = xs_net(x);
+ struct sk_buff *skb;
+ int err;
+
+ skb = nlmsg_new(xfrm_acquire_msgsize(x, xp), GFP_ATOMIC);
+ if (skb == NULL)
+ return -ENOMEM;
+
+ err = build_acquire(skb, x, xt, xp);
+ BUG_ON(err < 0);
+
+ return xfrm_nlmsg_multicast(net, skb, 0, XFRMNLGRP_ACQUIRE);
+}
+
+/* User gives us xfrm_user_policy_info followed by an array of 0
+ * or more templates.
+ */
+static struct xfrm_policy *xfrm_compile_policy(struct sock *sk, int opt,
+ u8 *data, int len, int *dir)
+{
+ struct net *net = sock_net(sk);
+ struct xfrm_userpolicy_info *p = (struct xfrm_userpolicy_info *)data;
+ struct xfrm_user_tmpl *ut = (struct xfrm_user_tmpl *) (p + 1);
+ struct xfrm_policy *xp;
+ int nr;
+
+ switch (sk->sk_family) {
+ case AF_INET:
+ if (opt != IP_XFRM_POLICY) {
+ *dir = -EOPNOTSUPP;
+ return NULL;
+ }
+ break;
+#if IS_ENABLED(CONFIG_IPV6)
+ case AF_INET6:
+ if (opt != IPV6_XFRM_POLICY) {
+ *dir = -EOPNOTSUPP;
+ return NULL;
+ }
+ break;
+#endif
+ default:
+ *dir = -EINVAL;
+ return NULL;
+ }
+
+ *dir = -EINVAL;
+
+ if (len < sizeof(*p) ||
+ verify_newpolicy_info(p, NULL))
+ return NULL;
+
+ nr = ((len - sizeof(*p)) / sizeof(*ut));
+ if (validate_tmpl(nr, ut, p->sel.family, p->dir, NULL))
+ return NULL;
+
+ if (p->dir > XFRM_POLICY_OUT)
+ return NULL;
+
+ xp = xfrm_policy_alloc(net, GFP_ATOMIC);
+ if (xp == NULL) {
+ *dir = -ENOBUFS;
+ return NULL;
+ }
+
+ copy_from_user_policy(xp, p);
+ xp->type = XFRM_POLICY_TYPE_MAIN;
+ copy_templates(xp, ut, nr);
+
+ *dir = p->dir;
+
+ return xp;
+}
+
+static inline unsigned int xfrm_polexpire_msgsize(struct xfrm_policy *xp)
+{
+ return NLMSG_ALIGN(sizeof(struct xfrm_user_polexpire))
+ + nla_total_size(sizeof(struct xfrm_user_tmpl) * xp->xfrm_nr)
+ + nla_total_size(xfrm_user_sec_ctx_size(xp->security))
+ + nla_total_size(sizeof(struct xfrm_mark))
+ + userpolicy_type_attrsize();
+}
+
+static int build_polexpire(struct sk_buff *skb, struct xfrm_policy *xp,
+ int dir, const struct km_event *c)
+{
+ struct xfrm_user_polexpire *upe;
+ int hard = c->data.hard;
+ struct nlmsghdr *nlh;
+ int err;
+
+ nlh = nlmsg_put(skb, c->portid, 0, XFRM_MSG_POLEXPIRE, sizeof(*upe), 0);
+ if (nlh == NULL)
+ return -EMSGSIZE;
+
+ upe = nlmsg_data(nlh);
+ copy_to_user_policy(xp, &upe->pol, dir);
+ err = copy_to_user_tmpl(xp, skb);
+ if (!err)
+ err = copy_to_user_sec_ctx(xp, skb);
+ if (!err)
+ err = copy_to_user_policy_type(xp->type, skb);
+ if (!err)
+ err = xfrm_mark_put(skb, &xp->mark);
+ if (!err)
+ err = xfrm_if_id_put(skb, xp->if_id);
+ if (err) {
+ nlmsg_cancel(skb, nlh);
+ return err;
+ }
+ upe->hard = !!hard;
+
+ nlmsg_end(skb, nlh);
+ return 0;
+}
+
+static int xfrm_exp_policy_notify(struct xfrm_policy *xp, int dir, const struct km_event *c)
+{
+ struct net *net = xp_net(xp);
+ struct sk_buff *skb;
+ int err;
+
+ skb = nlmsg_new(xfrm_polexpire_msgsize(xp), GFP_ATOMIC);
+ if (skb == NULL)
+ return -ENOMEM;
+
+ err = build_polexpire(skb, xp, dir, c);
+ BUG_ON(err < 0);
+
+ return xfrm_nlmsg_multicast(net, skb, 0, XFRMNLGRP_EXPIRE);
+}
+
+static int xfrm_notify_policy(struct xfrm_policy *xp, int dir, const struct km_event *c)
+{
+ unsigned int len = nla_total_size(sizeof(struct xfrm_user_tmpl) * xp->xfrm_nr);
+ struct net *net = xp_net(xp);
+ struct xfrm_userpolicy_info *p;
+ struct xfrm_userpolicy_id *id;
+ struct nlmsghdr *nlh;
+ struct sk_buff *skb;
+ unsigned int headlen;
+ int err;
+
+ headlen = sizeof(*p);
+ if (c->event == XFRM_MSG_DELPOLICY) {
+ len += nla_total_size(headlen);
+ headlen = sizeof(*id);
+ }
+ len += userpolicy_type_attrsize();
+ len += nla_total_size(sizeof(struct xfrm_mark));
+ len += NLMSG_ALIGN(headlen);
+
+ skb = nlmsg_new(len, GFP_ATOMIC);
+ if (skb == NULL)
+ return -ENOMEM;
+
+ nlh = nlmsg_put(skb, c->portid, c->seq, c->event, headlen, 0);
+ err = -EMSGSIZE;
+ if (nlh == NULL)
+ goto out_free_skb;
+
+ p = nlmsg_data(nlh);
+ if (c->event == XFRM_MSG_DELPOLICY) {
+ struct nlattr *attr;
+
+ id = nlmsg_data(nlh);
+ memset(id, 0, sizeof(*id));
+ id->dir = dir;
+ if (c->data.byid)
+ id->index = xp->index;
+ else
+ memcpy(&id->sel, &xp->selector, sizeof(id->sel));
+
+ attr = nla_reserve(skb, XFRMA_POLICY, sizeof(*p));
+ err = -EMSGSIZE;
+ if (attr == NULL)
+ goto out_free_skb;
+
+ p = nla_data(attr);
+ }
+
+ copy_to_user_policy(xp, p, dir);
+ err = copy_to_user_tmpl(xp, skb);
+ if (!err)
+ err = copy_to_user_policy_type(xp->type, skb);
+ if (!err)
+ err = xfrm_mark_put(skb, &xp->mark);
+ if (!err)
+ err = xfrm_if_id_put(skb, xp->if_id);
+ if (err)
+ goto out_free_skb;
+
+ nlmsg_end(skb, nlh);
+
+ return xfrm_nlmsg_multicast(net, skb, 0, XFRMNLGRP_POLICY);
+
+out_free_skb:
+ kfree_skb(skb);
+ return err;
+}
+
+static int xfrm_notify_policy_flush(const struct km_event *c)
+{
+ struct net *net = c->net;
+ struct nlmsghdr *nlh;
+ struct sk_buff *skb;
+ int err;
+
+ skb = nlmsg_new(userpolicy_type_attrsize(), GFP_ATOMIC);
+ if (skb == NULL)
+ return -ENOMEM;
+
+ nlh = nlmsg_put(skb, c->portid, c->seq, XFRM_MSG_FLUSHPOLICY, 0, 0);
+ err = -EMSGSIZE;
+ if (nlh == NULL)
+ goto out_free_skb;
+ err = copy_to_user_policy_type(c->data.type, skb);
+ if (err)
+ goto out_free_skb;
+
+ nlmsg_end(skb, nlh);
+
+ return xfrm_nlmsg_multicast(net, skb, 0, XFRMNLGRP_POLICY);
+
+out_free_skb:
+ kfree_skb(skb);
+ return err;
+}
+
+static int xfrm_send_policy_notify(struct xfrm_policy *xp, int dir, const struct km_event *c)
+{
+
+ switch (c->event) {
+ case XFRM_MSG_NEWPOLICY:
+ case XFRM_MSG_UPDPOLICY:
+ case XFRM_MSG_DELPOLICY:
+ return xfrm_notify_policy(xp, dir, c);
+ case XFRM_MSG_FLUSHPOLICY:
+ return xfrm_notify_policy_flush(c);
+ case XFRM_MSG_POLEXPIRE:
+ return xfrm_exp_policy_notify(xp, dir, c);
+ default:
+ printk(KERN_NOTICE "xfrm_user: Unknown Policy event %d\n",
+ c->event);
+ }
+
+ return 0;
+
+}
+
+static inline unsigned int xfrm_report_msgsize(void)
+{
+ return NLMSG_ALIGN(sizeof(struct xfrm_user_report));
+}
+
+static int build_report(struct sk_buff *skb, u8 proto,
+ struct xfrm_selector *sel, xfrm_address_t *addr)
+{
+ struct xfrm_user_report *ur;
+ struct nlmsghdr *nlh;
+
+ nlh = nlmsg_put(skb, 0, 0, XFRM_MSG_REPORT, sizeof(*ur), 0);
+ if (nlh == NULL)
+ return -EMSGSIZE;
+
+ ur = nlmsg_data(nlh);
+ ur->proto = proto;
+ memcpy(&ur->sel, sel, sizeof(ur->sel));
+
+ if (addr) {
+ int err = nla_put(skb, XFRMA_COADDR, sizeof(*addr), addr);
+ if (err) {
+ nlmsg_cancel(skb, nlh);
+ return err;
+ }
+ }
+ nlmsg_end(skb, nlh);
+ return 0;
+}
+
+static int xfrm_send_report(struct net *net, u8 proto,
+ struct xfrm_selector *sel, xfrm_address_t *addr)
+{
+ struct sk_buff *skb;
+ int err;
+
+ skb = nlmsg_new(xfrm_report_msgsize(), GFP_ATOMIC);
+ if (skb == NULL)
+ return -ENOMEM;
+
+ err = build_report(skb, proto, sel, addr);
+ BUG_ON(err < 0);
+
+ return xfrm_nlmsg_multicast(net, skb, 0, XFRMNLGRP_REPORT);
+}
+
+static inline unsigned int xfrm_mapping_msgsize(void)
+{
+ return NLMSG_ALIGN(sizeof(struct xfrm_user_mapping));
+}
+
+static int build_mapping(struct sk_buff *skb, struct xfrm_state *x,
+ xfrm_address_t *new_saddr, __be16 new_sport)
+{
+ struct xfrm_user_mapping *um;
+ struct nlmsghdr *nlh;
+
+ nlh = nlmsg_put(skb, 0, 0, XFRM_MSG_MAPPING, sizeof(*um), 0);
+ if (nlh == NULL)
+ return -EMSGSIZE;
+
+ um = nlmsg_data(nlh);
+
+ memcpy(&um->id.daddr, &x->id.daddr, sizeof(um->id.daddr));
+ um->id.spi = x->id.spi;
+ um->id.family = x->props.family;
+ um->id.proto = x->id.proto;
+ memcpy(&um->new_saddr, new_saddr, sizeof(um->new_saddr));
+ memcpy(&um->old_saddr, &x->props.saddr, sizeof(um->old_saddr));
+ um->new_sport = new_sport;
+ um->old_sport = x->encap->encap_sport;
+ um->reqid = x->props.reqid;
+
+ nlmsg_end(skb, nlh);
+ return 0;
+}
+
+static int xfrm_send_mapping(struct xfrm_state *x, xfrm_address_t *ipaddr,
+ __be16 sport)
+{
+ struct net *net = xs_net(x);
+ struct sk_buff *skb;
+ int err;
+
+ if (x->id.proto != IPPROTO_ESP)
+ return -EINVAL;
+
+ if (!x->encap)
+ return -EINVAL;
+
+ skb = nlmsg_new(xfrm_mapping_msgsize(), GFP_ATOMIC);
+ if (skb == NULL)
+ return -ENOMEM;
+
+ err = build_mapping(skb, x, ipaddr, sport);
+ BUG_ON(err < 0);
+
+ return xfrm_nlmsg_multicast(net, skb, 0, XFRMNLGRP_MAPPING);
+}
+
+static bool xfrm_is_alive(const struct km_event *c)
+{
+ return (bool)xfrm_acquire_is_on(c->net);
+}
+
+static struct xfrm_mgr netlink_mgr = {
+ .notify = xfrm_send_state_notify,
+ .acquire = xfrm_send_acquire,
+ .compile_policy = xfrm_compile_policy,
+ .notify_policy = xfrm_send_policy_notify,
+ .report = xfrm_send_report,
+ .migrate = xfrm_send_migrate,
+ .new_mapping = xfrm_send_mapping,
+ .is_alive = xfrm_is_alive,
+};
+
+static int __net_init xfrm_user_net_init(struct net *net)
+{
+ struct sock *nlsk;
+ struct netlink_kernel_cfg cfg = {
+ .groups = XFRMNLGRP_MAX,
+ .input = xfrm_netlink_rcv,
+ };
+
+ nlsk = netlink_kernel_create(net, NETLINK_XFRM, &cfg);
+ if (nlsk == NULL)
+ return -ENOMEM;
+ net->xfrm.nlsk_stash = nlsk; /* Don't set to NULL */
+ rcu_assign_pointer(net->xfrm.nlsk, nlsk);
+ return 0;
+}
+
+static void __net_exit xfrm_user_net_pre_exit(struct net *net)
+{
+ RCU_INIT_POINTER(net->xfrm.nlsk, NULL);
+}
+
+static void __net_exit xfrm_user_net_exit(struct list_head *net_exit_list)
+{
+ struct net *net;
+
+ list_for_each_entry(net, net_exit_list, exit_list)
+ netlink_kernel_release(net->xfrm.nlsk_stash);
+}
+
+static struct pernet_operations xfrm_user_net_ops = {
+ .init = xfrm_user_net_init,
+ .pre_exit = xfrm_user_net_pre_exit,
+ .exit_batch = xfrm_user_net_exit,
+};
+
+static int __init xfrm_user_init(void)
+{
+ int rv;
+
+ printk(KERN_INFO "Initializing XFRM netlink socket\n");
+
+ rv = register_pernet_subsys(&xfrm_user_net_ops);
+ if (rv < 0)
+ return rv;
+ xfrm_register_km(&netlink_mgr);
+ return 0;
+}
+
+static void __exit xfrm_user_exit(void)
+{
+ xfrm_unregister_km(&netlink_mgr);
+ unregister_pernet_subsys(&xfrm_user_net_ops);
+}
+
+module_init(xfrm_user_init);
+module_exit(xfrm_user_exit);
+MODULE_LICENSE("GPL");
+MODULE_ALIAS_NET_PF_PROTO(PF_NETLINK, NETLINK_XFRM);
generated by cgit v1.2.3 (git 2.39.1) at 2024-11-20 03:20:06 +0000