diff options
Diffstat (limited to 'lib/isc/netmgr')
| -rw-r--r-- | lib/isc/netmgr/Makefile.in | 41 | ||||
| -rw-r--r-- | lib/isc/netmgr/netmgr-int.h | 1615 | ||||
| -rw-r--r-- | lib/isc/netmgr/netmgr.c | 3396 | ||||
| -rw-r--r-- | lib/isc/netmgr/tcp.c | 1456 | ||||
| -rw-r--r-- | lib/isc/netmgr/tcpdns.c | 1505 | ||||
| -rw-r--r-- | lib/isc/netmgr/udp.c | 1211 | ||||
| -rw-r--r-- | lib/isc/netmgr/uv-compat.c | 152 | ||||
| -rw-r--r-- | lib/isc/netmgr/uv-compat.h | 126 | ||||
| -rw-r--r-- | lib/isc/netmgr/uverr2result.c | 104 |
9 files changed, 9606 insertions, 0 deletions
diff --git a/lib/isc/netmgr/Makefile.in b/lib/isc/netmgr/Makefile.in new file mode 100644 index 0000000..1806cb4 --- /dev/null +++ b/lib/isc/netmgr/Makefile.in @@ -0,0 +1,41 @@ +# Copyright (C) Internet Systems Consortium, Inc. ("ISC") +# +# SPDX-License-Identifier: MPL-2.0 +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, you can obtain one at https://mozilla.org/MPL/2.0/. +# +# See the COPYRIGHT file distributed with this work for additional +# information regarding copyright ownership. + +srcdir = @srcdir@ +VPATH = @srcdir@ +top_srcdir = @top_srcdir@ + +CINCLUDES = -I${srcdir}/include \ + -I${srcdir}/../unix/include \ + -I${srcdir}/../pthreads/include \ + -I../include \ + -I${srcdir}/../include \ + -I${srcdir}/.. \ + ${OPENSSL_CFLAGS} \ + ${JSON_C_CFLAGS} \ + ${LIBUV_CFLAGS} \ + ${LIBXML2_CFLAGS} + +CDEFINES = +CWARNINGS = + +# Alphabetically +OBJS = netmgr.@O@ tcp.@O@ udp.@O@ \ + tcpdns.@O@ \ + uverr2result.@O@ uv-compat.@O@ + +# Alphabetically +SRCS = netmgr.c tcp.c udp.c tcpdns.c \ + uverr2result.c uv-compat.c + +TARGETS = ${OBJS} + +@BIND9_MAKE_RULES@ diff --git a/lib/isc/netmgr/netmgr-int.h b/lib/isc/netmgr/netmgr-int.h new file mode 100644 index 0000000..05fde1a --- /dev/null +++ b/lib/isc/netmgr/netmgr-int.h @@ -0,0 +1,1615 @@ +/* + * Copyright (C) Internet Systems Consortium, Inc. ("ISC") + * + * SPDX-License-Identifier: MPL-2.0 + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, you can obtain one at https://mozilla.org/MPL/2.0/. + * + * See the COPYRIGHT file distributed with this work for additional + * information regarding copyright ownership. + */ + +#pragma once + +#include <unistd.h> +#include <uv.h> + +#include <openssl/err.h> +#include <openssl/ssl.h> + +#include <isc/astack.h> +#include <isc/atomic.h> +#include <isc/barrier.h> +#include <isc/buffer.h> +#include <isc/condition.h> +#include <isc/magic.h> +#include <isc/mem.h> +#include <isc/netmgr.h> +#include <isc/quota.h> +#include <isc/random.h> +#include <isc/refcount.h> +#include <isc/region.h> +#include <isc/result.h> +#include <isc/rwlock.h> +#include <isc/sockaddr.h> +#include <isc/stats.h> +#include <isc/thread.h> +#include <isc/util.h> + +#include "uv-compat.h" + +#define ISC_NETMGR_TID_UNKNOWN -1 + +/* Must be different from ISC_NETMGR_TID_UNKNOWN */ +#define ISC_NETMGR_NON_INTERLOCKED -2 + +/* + * Receive buffers + */ +#if HAVE_DECL_UV_UDP_MMSG_CHUNK +/* + * The value 20 here is UV__MMSG_MAXWIDTH taken from the current libuv source, + * libuv will not receive more that 20 datagrams in a single recvmmsg call. + */ +#define ISC_NETMGR_UDP_RECVBUF_SIZE (20 * UINT16_MAX) +#else +/* + * A single DNS message size + */ +#define ISC_NETMGR_UDP_RECVBUF_SIZE UINT16_MAX +#endif + +/* + * The TCP receive buffer can fit one maximum sized DNS message plus its size, + * the receive buffer here affects TCP, DoT and DoH. + */ +#define ISC_NETMGR_TCP_RECVBUF_SIZE (sizeof(uint16_t) + UINT16_MAX) + +/* Pick the larger buffer */ +#define ISC_NETMGR_RECVBUF_SIZE \ + (ISC_NETMGR_UDP_RECVBUF_SIZE >= ISC_NETMGR_TCP_RECVBUF_SIZE \ + ? ISC_NETMGR_UDP_RECVBUF_SIZE \ + : ISC_NETMGR_TCP_RECVBUF_SIZE) + +/* + * Send buffer + */ +#define ISC_NETMGR_SENDBUF_SIZE (sizeof(uint16_t) + UINT16_MAX) + +/*% + * Regular TCP buffer size. + */ +#define NM_REG_BUF 4096 + +/*% + * Larger buffer for when the regular one isn't enough; this will + * hold two full DNS packets with lengths. netmgr receives 64k at + * most in TCPDNS connections, so there's no risk of overrun + * when using a buffer this size. + */ +#define NM_BIG_BUF ISC_NETMGR_TCP_RECVBUF_SIZE * 2 + +/* + * Define NETMGR_TRACE to activate tracing of handles and sockets. + * This will impair performance but enables us to quickly determine, + * if netmgr resources haven't been cleaned up on shutdown, which ones + * are still in use. + */ +#ifdef NETMGR_TRACE +#define TRACE_SIZE 8 + +void +isc__nm_dump_active(isc_nm_t *nm); + +#if defined(__linux__) +#include <syscall.h> +#define gettid() (uint32_t) syscall(SYS_gettid) +#elif defined(_WIN32) +#define gettid() (uint32_t) GetCurrentThreadId() +#else +#define gettid() (uint32_t) pthread_self() +#endif + +#ifdef NETMGR_TRACE_VERBOSE +#define NETMGR_TRACE_LOG(format, ...) \ + fprintf(stderr, "%" PRIu32 ":%d:%s:%u:%s:" format, gettid(), \ + isc_nm_tid(), file, line, func, __VA_ARGS__) +#else +#define NETMGR_TRACE_LOG(format, ...) \ + (void)file; \ + (void)line; \ + (void)func; +#endif + +#define FLARG_PASS , file, line, func +#define FLARG \ + , const char *file __attribute__((unused)), \ + unsigned int line __attribute__((unused)), \ + const char *func __attribute__((unused)) +#define FLARG_IEVENT(ievent) \ + const char *file = ievent->file; \ + unsigned int line = ievent->line; \ + const char *func = ievent->func; +#define FLARG_IEVENT_PASS(ievent) \ + ievent->file = file; \ + ievent->line = line; \ + ievent->func = func; +#define isc__nm_uvreq_get(req, sock) \ + isc___nm_uvreq_get(req, sock, __FILE__, __LINE__, __func__) +#define isc__nm_uvreq_put(req, sock) \ + isc___nm_uvreq_put(req, sock, __FILE__, __LINE__, __func__) +#define isc__nmsocket_init(sock, mgr, type, iface) \ + isc___nmsocket_init(sock, mgr, type, iface, __FILE__, __LINE__, \ + __func__) +#define isc__nmsocket_put(sockp) \ + isc___nmsocket_put(sockp, __FILE__, __LINE__, __func__) +#define isc__nmsocket_attach(sock, target) \ + isc___nmsocket_attach(sock, target, __FILE__, __LINE__, __func__) +#define isc__nmsocket_detach(socketp) \ + isc___nmsocket_detach(socketp, __FILE__, __LINE__, __func__) +#define isc__nmsocket_close(socketp) \ + isc___nmsocket_close(socketp, __FILE__, __LINE__, __func__) +#define isc__nmhandle_get(sock, peer, local) \ + isc___nmhandle_get(sock, peer, local, __FILE__, __LINE__, __func__) +#define isc__nmsocket_prep_destroy(sock) \ + isc___nmsocket_prep_destroy(sock, __FILE__, __LINE__, __func__) +#else +#define NETMGR_TRACE_LOG(format, ...) + +#define FLARG_PASS +#define FLARG +#define FLARG_IEVENT(ievent) +#define FLARG_IEVENT_PASS(ievent) +#define isc__nm_uvreq_get(req, sock) isc___nm_uvreq_get(req, sock) +#define isc__nm_uvreq_put(req, sock) isc___nm_uvreq_put(req, sock) +#define isc__nmsocket_init(sock, mgr, type, iface) \ + isc___nmsocket_init(sock, mgr, type, iface) +#define isc__nmsocket_put(sockp) isc___nmsocket_put(sockp) +#define isc__nmsocket_attach(sock, target) isc___nmsocket_attach(sock, target) +#define isc__nmsocket_detach(socketp) isc___nmsocket_detach(socketp) +#define isc__nmsocket_close(socketp) isc___nmsocket_close(socketp) +#define isc__nmhandle_get(sock, peer, local) \ + isc___nmhandle_get(sock, peer, local) +#define isc__nmsocket_prep_destroy(sock) isc___nmsocket_prep_destroy(sock) +#endif + +/* + * Queue types in the order of processing priority. + */ +typedef enum { + NETIEVENT_PRIORITY = 0, + NETIEVENT_PRIVILEGED = 1, + NETIEVENT_TASK = 2, + NETIEVENT_NORMAL = 3, + NETIEVENT_MAX = 4, +} netievent_type_t; + +typedef struct isc__nm_uvreq isc__nm_uvreq_t; +typedef struct isc__netievent isc__netievent_t; + +typedef ISC_LIST(isc__netievent_t) isc__netievent_list_t; + +typedef struct ievent { + isc_mutex_t lock; + isc_condition_t cond; + isc__netievent_list_t list; +} ievent_t; + +/* + * Single network event loop worker. + */ +typedef struct isc__networker { + isc_nm_t *mgr; + int id; /* thread id */ + uv_loop_t loop; /* libuv loop structure */ + uv_async_t async; /* async channel to send + * data to this networker */ + bool paused; + bool finished; + isc_thread_t thread; + ievent_t ievents[NETIEVENT_MAX]; + + isc_refcount_t references; + atomic_int_fast64_t pktcount; + char *recvbuf; + char *sendbuf; + bool recvbuf_inuse; +} isc__networker_t; + +/* + * A general handle for a connection bound to a networker. For UDP + * connections we have peer address here, so both TCP and UDP can be + * handled with a simple send-like function + */ +#define NMHANDLE_MAGIC ISC_MAGIC('N', 'M', 'H', 'D') +#define VALID_NMHANDLE(t) \ + (ISC_MAGIC_VALID(t, NMHANDLE_MAGIC) && \ + atomic_load(&(t)->references) > 0) + +typedef void (*isc__nm_closecb)(isc_nmhandle_t *); + +struct isc_nmhandle { + int magic; + isc_refcount_t references; + + /* + * The socket is not 'attached' in the traditional + * reference-counting sense. Instead, we keep all handles in an + * array in the socket object. This way, we don't have circular + * dependencies and we can close all handles when we're destroying + * the socket. + */ + isc_nmsocket_t *sock; + + isc_sockaddr_t peer; + isc_sockaddr_t local; + isc_nm_opaquecb_t doreset; /* reset extra callback, external */ + isc_nm_opaquecb_t dofree; /* free extra callback, external */ +#ifdef NETMGR_TRACE + void *backtrace[TRACE_SIZE]; + int backtrace_size; + LINK(isc_nmhandle_t) active_link; +#endif + void *opaque; + char extra[]; +}; + +typedef enum isc__netievent_type { + netievent_udpconnect, + netievent_udpclose, + netievent_udpsend, + netievent_udpread, + netievent_udpcancel, + + netievent_tcpconnect, + netievent_tcpclose, + netievent_tcpsend, + netievent_tcpstartread, + netievent_tcppauseread, + netievent_tcpaccept, + netievent_tcpcancel, + + netievent_tcpdnsaccept, + netievent_tcpdnsconnect, + netievent_tcpdnsclose, + netievent_tcpdnssend, + netievent_tcpdnsread, + netievent_tcpdnscancel, + + netievent_shutdown, + netievent_stop, + netievent_pause, + + netievent_connectcb, + netievent_readcb, + netievent_sendcb, + + netievent_detach, + netievent_close, + + netievent_task, + netievent_privilegedtask, + + /* + * event type values higher than this will be treated + * as high-priority events, which can be processed + * while the netmgr is pausing or paused. + */ + netievent_prio = 0xff, + + netievent_udplisten, + netievent_udpstop, + netievent_tcplisten, + netievent_tcpstop, + netievent_tcpdnslisten, + netievent_tcpdnsstop, + + netievent_resume, +} isc__netievent_type; + +typedef union { + isc_nm_recv_cb_t recv; + isc_nm_cb_t send; + isc_nm_cb_t connect; + isc_nm_accept_cb_t accept; +} isc__nm_cb_t; + +/* + * Wrapper around uv_req_t with 'our' fields in it. req->data should + * always point to its parent. Note that we always allocate more than + * sizeof(struct) because we make room for different req types; + */ +#define UVREQ_MAGIC ISC_MAGIC('N', 'M', 'U', 'R') +#define VALID_UVREQ(t) ISC_MAGIC_VALID(t, UVREQ_MAGIC) + +typedef struct isc__nm_uvreq isc__nm_uvreq_t; +struct isc__nm_uvreq { + int magic; + isc_nmsocket_t *sock; + isc_nmhandle_t *handle; + char tcplen[2]; /* The TCP DNS message length */ + uv_buf_t uvbuf; /* translated isc_region_t, to be + * sent or received */ + isc_sockaddr_t local; /* local address */ + isc_sockaddr_t peer; /* peer address */ + isc__nm_cb_t cb; /* callback */ + void *cbarg; /* callback argument */ + isc_nm_timer_t *timer; /* TCP write timer */ + + union { + uv_handle_t handle; + uv_req_t req; + uv_getaddrinfo_t getaddrinfo; + uv_getnameinfo_t getnameinfo; + uv_shutdown_t shutdown; + uv_write_t write; + uv_connect_t connect; + uv_udp_send_t udp_send; + uv_fs_t fs; + uv_work_t work; + } uv_req; + ISC_LINK(isc__nm_uvreq_t) link; +}; + +struct isc_nm_timer { + isc_refcount_t references; + uv_timer_t timer; + isc_nmhandle_t *handle; + isc_nm_timer_cb cb; + void *cbarg; +}; + +void * +isc__nm_get_netievent(isc_nm_t *mgr, isc__netievent_type type); +/*%< + * Allocate an ievent and set the type. + */ +void +isc__nm_put_netievent(isc_nm_t *mgr, void *ievent); + +/* + * The macros here are used to simulate the "inheritance" in C, there's the base + * netievent structure that contains just its own type and socket, and there are + * extended netievent types that also have handles or requests or other data. + * + * The macros here ensure that: + * + * 1. every netievent type has matching definition, declaration and + * implementation + * + * 2. we handle all the netievent types of same subclass the same, e.g. if the + * extended netievent contains handle, we always attach to the handle in + * the ctor and detach from the handle in dtor. + * + * There are three macros here for each netievent subclass: + * + * 1. NETIEVENT_*_TYPE(type) creates the typedef for each type; used below in + * this header + * + * 2. NETIEVENT_*_DECL(type) generates the declaration of the get and put + * functions (isc__nm_get_netievent_* and isc__nm_put_netievent_*); used + * below in this header + * + * 3. NETIEVENT_*_DEF(type) generates the definition of the functions; used + * either in netmgr.c or matching protocol file (e.g. udp.c, tcp.c, etc.) + */ + +#define NETIEVENT__SOCKET \ + isc__netievent_type type; \ + ISC_LINK(isc__netievent_t) link; \ + isc_nmsocket_t *sock; \ + const char *file; \ + unsigned int line; \ + const char *func + +typedef struct isc__netievent__socket { + NETIEVENT__SOCKET; +} isc__netievent__socket_t; + +#define NETIEVENT_SOCKET_TYPE(type) \ + typedef isc__netievent__socket_t isc__netievent_##type##_t; + +#define NETIEVENT_SOCKET_DECL(type) \ + isc__netievent_##type##_t *isc__nm_get_netievent_##type( \ + isc_nm_t *nm, isc_nmsocket_t *sock); \ + void isc__nm_put_netievent_##type(isc_nm_t *nm, \ + isc__netievent_##type##_t *ievent); + +#define NETIEVENT_SOCKET_DEF(type) \ + isc__netievent_##type##_t *isc__nm_get_netievent_##type( \ + isc_nm_t *nm, isc_nmsocket_t *sock) { \ + isc__netievent_##type##_t *ievent = \ + isc__nm_get_netievent(nm, netievent_##type); \ + isc__nmsocket_attach(sock, &ievent->sock); \ + \ + return (ievent); \ + } \ + \ + void isc__nm_put_netievent_##type(isc_nm_t *nm, \ + isc__netievent_##type##_t *ievent) { \ + isc__nmsocket_detach(&ievent->sock); \ + isc__nm_put_netievent(nm, ievent); \ + } + +typedef struct isc__netievent__socket_req { + NETIEVENT__SOCKET; + isc__nm_uvreq_t *req; +} isc__netievent__socket_req_t; + +#define NETIEVENT_SOCKET_REQ_TYPE(type) \ + typedef isc__netievent__socket_req_t isc__netievent_##type##_t; + +#define NETIEVENT_SOCKET_REQ_DECL(type) \ + isc__netievent_##type##_t *isc__nm_get_netievent_##type( \ + isc_nm_t *nm, isc_nmsocket_t *sock, isc__nm_uvreq_t *req); \ + void isc__nm_put_netievent_##type(isc_nm_t *nm, \ + isc__netievent_##type##_t *ievent); + +#define NETIEVENT_SOCKET_REQ_DEF(type) \ + isc__netievent_##type##_t *isc__nm_get_netievent_##type( \ + isc_nm_t *nm, isc_nmsocket_t *sock, isc__nm_uvreq_t *req) { \ + isc__netievent_##type##_t *ievent = \ + isc__nm_get_netievent(nm, netievent_##type); \ + isc__nmsocket_attach(sock, &ievent->sock); \ + ievent->req = req; \ + \ + return (ievent); \ + } \ + \ + void isc__nm_put_netievent_##type(isc_nm_t *nm, \ + isc__netievent_##type##_t *ievent) { \ + isc__nmsocket_detach(&ievent->sock); \ + isc__nm_put_netievent(nm, ievent); \ + } + +typedef struct isc__netievent__socket_req_result { + NETIEVENT__SOCKET; + isc__nm_uvreq_t *req; + isc_result_t result; +} isc__netievent__socket_req_result_t; + +#define NETIEVENT_SOCKET_REQ_RESULT_TYPE(type) \ + typedef isc__netievent__socket_req_result_t isc__netievent_##type##_t; + +#define NETIEVENT_SOCKET_REQ_RESULT_DECL(type) \ + isc__netievent_##type##_t *isc__nm_get_netievent_##type( \ + isc_nm_t *nm, isc_nmsocket_t *sock, isc__nm_uvreq_t *req, \ + isc_result_t result); \ + void isc__nm_put_netievent_##type(isc_nm_t *nm, \ + isc__netievent_##type##_t *ievent); + +#define NETIEVENT_SOCKET_REQ_RESULT_DEF(type) \ + isc__netievent_##type##_t *isc__nm_get_netievent_##type( \ + isc_nm_t *nm, isc_nmsocket_t *sock, isc__nm_uvreq_t *req, \ + isc_result_t result) { \ + isc__netievent_##type##_t *ievent = \ + isc__nm_get_netievent(nm, netievent_##type); \ + isc__nmsocket_attach(sock, &ievent->sock); \ + ievent->req = req; \ + ievent->result = result; \ + \ + return (ievent); \ + } \ + \ + void isc__nm_put_netievent_##type(isc_nm_t *nm, \ + isc__netievent_##type##_t *ievent) { \ + isc__nmsocket_detach(&ievent->sock); \ + isc__nm_put_netievent(nm, ievent); \ + } + +typedef struct isc__netievent__socket_handle { + NETIEVENT__SOCKET; + isc_nmhandle_t *handle; +} isc__netievent__socket_handle_t; + +#define NETIEVENT_SOCKET_HANDLE_TYPE(type) \ + typedef isc__netievent__socket_handle_t isc__netievent_##type##_t; + +#define NETIEVENT_SOCKET_HANDLE_DECL(type) \ + isc__netievent_##type##_t *isc__nm_get_netievent_##type( \ + isc_nm_t *nm, isc_nmsocket_t *sock, isc_nmhandle_t *handle); \ + void isc__nm_put_netievent_##type(isc_nm_t *nm, \ + isc__netievent_##type##_t *ievent); + +#define NETIEVENT_SOCKET_HANDLE_DEF(type) \ + isc__netievent_##type##_t *isc__nm_get_netievent_##type( \ + isc_nm_t *nm, isc_nmsocket_t *sock, isc_nmhandle_t *handle) { \ + isc__netievent_##type##_t *ievent = \ + isc__nm_get_netievent(nm, netievent_##type); \ + isc__nmsocket_attach(sock, &ievent->sock); \ + isc_nmhandle_attach(handle, &ievent->handle); \ + \ + return (ievent); \ + } \ + \ + void isc__nm_put_netievent_##type(isc_nm_t *nm, \ + isc__netievent_##type##_t *ievent) { \ + isc__nmsocket_detach(&ievent->sock); \ + isc_nmhandle_detach(&ievent->handle); \ + isc__nm_put_netievent(nm, ievent); \ + } + +typedef struct isc__netievent__socket_quota { + NETIEVENT__SOCKET; + isc_quota_t *quota; +} isc__netievent__socket_quota_t; + +#define NETIEVENT_SOCKET_QUOTA_TYPE(type) \ + typedef isc__netievent__socket_quota_t isc__netievent_##type##_t; + +#define NETIEVENT_SOCKET_QUOTA_DECL(type) \ + isc__netievent_##type##_t *isc__nm_get_netievent_##type( \ + isc_nm_t *nm, isc_nmsocket_t *sock, isc_quota_t *quota); \ + void isc__nm_put_netievent_##type(isc_nm_t *nm, \ + isc__netievent_##type##_t *ievent); + +#define NETIEVENT_SOCKET_QUOTA_DEF(type) \ + isc__netievent_##type##_t *isc__nm_get_netievent_##type( \ + isc_nm_t *nm, isc_nmsocket_t *sock, isc_quota_t *quota) { \ + isc__netievent_##type##_t *ievent = \ + isc__nm_get_netievent(nm, netievent_##type); \ + isc__nmsocket_attach(sock, &ievent->sock); \ + ievent->quota = quota; \ + \ + return (ievent); \ + } \ + \ + void isc__nm_put_netievent_##type(isc_nm_t *nm, \ + isc__netievent_##type##_t *ievent) { \ + isc__nmsocket_detach(&ievent->sock); \ + isc__nm_put_netievent(nm, ievent); \ + } + +typedef struct isc__netievent__task { + isc__netievent_type type; + ISC_LINK(isc__netievent_t) link; + isc_task_t *task; +} isc__netievent__task_t; + +#define NETIEVENT_TASK_TYPE(type) \ + typedef isc__netievent__task_t isc__netievent_##type##_t; + +#define NETIEVENT_TASK_DECL(type) \ + isc__netievent_##type##_t *isc__nm_get_netievent_##type( \ + isc_nm_t *nm, isc_task_t *task); \ + void isc__nm_put_netievent_##type(isc_nm_t *nm, \ + isc__netievent_##type##_t *ievent); + +#define NETIEVENT_TASK_DEF(type) \ + isc__netievent_##type##_t *isc__nm_get_netievent_##type( \ + isc_nm_t *nm, isc_task_t *task) { \ + isc__netievent_##type##_t *ievent = \ + isc__nm_get_netievent(nm, netievent_##type); \ + ievent->task = task; \ + \ + return (ievent); \ + } \ + \ + void isc__nm_put_netievent_##type(isc_nm_t *nm, \ + isc__netievent_##type##_t *ievent) { \ + ievent->task = NULL; \ + isc__nm_put_netievent(nm, ievent); \ + } + +typedef struct isc__netievent_udpsend { + NETIEVENT__SOCKET; + isc_sockaddr_t peer; + isc__nm_uvreq_t *req; +} isc__netievent_udpsend_t; + +typedef struct isc__netievent { + isc__netievent_type type; + ISC_LINK(isc__netievent_t) link; +} isc__netievent_t; + +#define NETIEVENT_TYPE(type) typedef isc__netievent_t isc__netievent_##type##_t; + +#define NETIEVENT_DECL(type) \ + isc__netievent_##type##_t *isc__nm_get_netievent_##type(isc_nm_t *nm); \ + void isc__nm_put_netievent_##type(isc_nm_t *nm, \ + isc__netievent_##type##_t *ievent); + +#define NETIEVENT_DEF(type) \ + isc__netievent_##type##_t *isc__nm_get_netievent_##type( \ + isc_nm_t *nm) { \ + isc__netievent_##type##_t *ievent = \ + isc__nm_get_netievent(nm, netievent_##type); \ + \ + return (ievent); \ + } \ + \ + void isc__nm_put_netievent_##type(isc_nm_t *nm, \ + isc__netievent_##type##_t *ievent) { \ + isc__nm_put_netievent(nm, ievent); \ + } + +typedef union { + isc__netievent_t ni; + isc__netievent__socket_t nis; + isc__netievent__socket_req_t nisr; + isc__netievent_udpsend_t nius; + isc__netievent__socket_quota_t nisq; +} isc__netievent_storage_t; + +/* + * Work item for a uv_work threadpool. + */ +typedef struct isc__nm_work { + isc_nm_t *netmgr; + uv_work_t req; + isc_nm_workcb_t cb; + isc_nm_after_workcb_t after_cb; + void *data; +} isc__nm_work_t; + +/* + * Network manager + */ +#define NM_MAGIC ISC_MAGIC('N', 'E', 'T', 'M') +#define VALID_NM(t) ISC_MAGIC_VALID(t, NM_MAGIC) + +struct isc_nm { + int magic; + isc_refcount_t references; + isc_mem_t *mctx; + int nworkers; + isc_mutex_t lock; + isc_condition_t wkstatecond; + isc_condition_t wkpausecond; + isc__networker_t *workers; + + isc_stats_t *stats; + + uint_fast32_t workers_running; + atomic_uint_fast32_t workers_paused; + atomic_uint_fast32_t maxudp; + + bool load_balance_sockets; + + atomic_bool paused; + + /* + * Active connections are being closed and new connections are + * no longer allowed. + */ + atomic_bool closing; + + /* + * A worker is actively waiting for other workers, for example to + * stop listening; that means no other thread can do the same thing + * or pause, or we'll deadlock. We have to either re-enqueue our + * event or wait for the other one to finish if we want to pause. + */ + atomic_int interlocked; + + /* + * Timeout values for TCP connections, corresponding to + * tcp-intiial-timeout, tcp-idle-timeout, tcp-keepalive-timeout, + * and tcp-advertised-timeout. Note that these are stored in + * milliseconds so they can be used directly with the libuv timer, + * but they are configured in tenths of seconds. + */ + atomic_uint_fast32_t init; + atomic_uint_fast32_t idle; + atomic_uint_fast32_t keepalive; + atomic_uint_fast32_t advertised; + + isc_barrier_t pausing; + isc_barrier_t resuming; + +#ifdef NETMGR_TRACE + ISC_LIST(isc_nmsocket_t) active_sockets; +#endif +}; + +typedef enum isc_nmsocket_type { + isc_nm_udpsocket, + isc_nm_udplistener, /* Aggregate of nm_udpsocks */ + isc_nm_tcpsocket, + isc_nm_tcplistener, + isc_nm_tcpdnslistener, + isc_nm_tcpdnssocket, +} isc_nmsocket_type; + +/*% + * A universal structure for either a single socket or a group of + * dup'd/SO_REUSE_PORT-using sockets listening on the same interface. + */ +#define NMSOCK_MAGIC ISC_MAGIC('N', 'M', 'S', 'K') +#define VALID_NMSOCK(t) ISC_MAGIC_VALID(t, NMSOCK_MAGIC) + +/*% + * Index into socket stat counter arrays. + */ +enum { + STATID_OPEN = 0, + STATID_OPENFAIL = 1, + STATID_CLOSE = 2, + STATID_BINDFAIL = 3, + STATID_CONNECTFAIL = 4, + STATID_CONNECT = 5, + STATID_ACCEPTFAIL = 6, + STATID_ACCEPT = 7, + STATID_SENDFAIL = 8, + STATID_RECVFAIL = 9, + STATID_ACTIVE = 10 +}; + +typedef void (*isc_nm_closehandlecb_t)(void *arg); +/*%< + * Opaque callback function, used for isc_nmhandle 'reset' and 'free' + * callbacks. + */ + +struct isc_nmsocket { + /*% Unlocked, RO */ + int magic; + int tid; + isc_nmsocket_type type; + isc_nm_t *mgr; + + /*% Parent socket for multithreaded listeners */ + isc_nmsocket_t *parent; + /*% Listener socket this connection was accepted on */ + isc_nmsocket_t *listener; + /*% Self socket */ + isc_nmsocket_t *self; + + isc_barrier_t startlistening; + isc_barrier_t stoplistening; + + /*% + * quota is the TCP client, attached when a TCP connection + * is established. pquota is a non-attached pointer to the + * TCP client quota, stored in listening sockets but only + * attached in connected sockets. + */ + isc_quota_t *quota; + isc_quota_t *pquota; + isc_quota_cb_t quotacb; + + /*% + * Socket statistics + */ + const isc_statscounter_t *statsindex; + + /*% + * TCP read/connect timeout timers. + */ + uv_timer_t read_timer; + uint64_t read_timeout; + uint64_t connect_timeout; + + /*% + * TCP write timeout timer. + */ + uint64_t write_timeout; + + /*% outer socket is for 'wrapped' sockets - e.g. tcpdns in tcp */ + isc_nmsocket_t *outer; + + /*% server socket for connections */ + isc_nmsocket_t *server; + + /*% Child sockets for multi-socket setups */ + isc_nmsocket_t *children; + uint_fast32_t nchildren; + isc_sockaddr_t iface; + isc_nmhandle_t *statichandle; + isc_nmhandle_t *outerhandle; + + /*% Extra data allocated at the end of each isc_nmhandle_t */ + size_t extrahandlesize; + + /*% TCP backlog */ + int backlog; + + /*% libuv data */ + uv_os_sock_t fd; + union uv_any_handle uv_handle; + + /*% Peer address */ + isc_sockaddr_t peer; + + /* Atomic */ + /*% Number of running (e.g. listening) child sockets */ + atomic_uint_fast32_t rchildren; + + /*% + * Socket is active if it's listening, working, etc. If it's + * closing, then it doesn't make a sense, for example, to + * push handles or reqs for reuse. + */ + atomic_bool active; + atomic_bool destroying; + + /*% + * Socket is closed if it's not active and all the possible + * callbacks were fired, there are no active handles, etc. + * If active==false but closed==false, that means the socket + * is closing. + */ + atomic_bool closing; + atomic_bool closed; + atomic_bool listening; + atomic_bool connecting; + atomic_bool connected; + bool accepting; + bool reading; + atomic_bool timedout; + isc_refcount_t references; + + /*% + * Established an outgoing connection, as client not server. + */ + atomic_bool client; + + /*% + * TCPDNS socket has been set not to pipeline. + */ + atomic_bool sequential; + + /*% + * The socket is processing read callback, this is guard to not read + * data before the readcb is back. + */ + bool processing; + + /*% + * A TCP socket has had isc_nm_pauseread() called. + */ + atomic_bool readpaused; + + /*% + * A TCP or TCPDNS socket has been set to use the keepalive + * timeout instead of the default idle timeout. + */ + atomic_bool keepalive; + + /*% + * 'spare' handles for that can be reused to avoid allocations, + * for UDP. + */ + isc_astack_t *inactivehandles; + isc_astack_t *inactivereqs; + + /*% + * Used to wait for TCP listening events to complete, and + * for the number of running children to reach zero during + * shutdown. + * + * We use two condition variables to prevent the race where the netmgr + * threads would be able to finish and destroy the socket before it's + * unlocked by the isc_nm_listen<proto>() function. So, the flow is as + * follows: + * + * 1. parent thread creates all children sockets and passes then to + * netthreads, looks at the signaling variable and WAIT(cond) until + * the childrens are done initializing + * + * 2. the events get picked by netthreads, calls the libuv API (and + * either succeeds or fails) and WAIT(scond) until all other + * children sockets in netthreads are initialized and the listening + * socket lock is unlocked + * + * 3. the control is given back to the parent thread which now either + * returns success or shutdowns the listener if an error has + * occured in the children netthread + * + * NOTE: The other approach would be doing an extra attach to the parent + * listening socket, and then detach it in the parent thread, but that + * breaks the promise that once the libuv socket is initialized on the + * nmsocket, the nmsocket needs to be handled only by matching + * netthread, so in fact that would add a complexity in a way that + * isc__nmsocket_detach would have to be converted to use an + * asynchrounous netievent. + */ + isc_mutex_t lock; + isc_condition_t cond; + isc_condition_t scond; + + /*% + * Used to pass a result back from listen or connect events. + */ + isc_result_t result; + + /*% + * Current number of active handles. + */ + atomic_int_fast32_t ah; + + /*% Buffer for TCPDNS processing */ + size_t buf_size; + size_t buf_len; + unsigned char *buf; + + /*% + * This function will be called with handle->sock + * as the argument whenever a handle's references drop + * to zero, after its reset callback has been called. + */ + isc_nm_closehandlecb_t closehandle_cb; + + isc_nmhandle_t *recv_handle; + isc_nm_recv_cb_t recv_cb; + void *recv_cbarg; + bool recv_read; + + isc_nm_cb_t connect_cb; + void *connect_cbarg; + + isc_nm_accept_cb_t accept_cb; + void *accept_cbarg; + + atomic_int_fast32_t active_child_connections; + +#ifdef NETMGR_TRACE + void *backtrace[TRACE_SIZE]; + int backtrace_size; + LINK(isc_nmsocket_t) active_link; + ISC_LIST(isc_nmhandle_t) active_handles; +#endif +}; + +bool +isc__nm_in_netthread(void); +/*% + * Returns 'true' if we're in the network thread. + */ + +void +isc__nm_maybe_enqueue_ievent(isc__networker_t *worker, isc__netievent_t *event); +/*%< + * If the caller is already in the matching nmthread, process the netievent + * directly, if not enqueue using isc__nm_enqueue_ievent(). + */ + +void +isc__nm_enqueue_ievent(isc__networker_t *worker, isc__netievent_t *event); +/*%< + * Enqueue an ievent onto a specific worker queue. (This the only safe + * way to use an isc__networker_t from another thread.) + */ + +void +isc__nm_free_uvbuf(isc_nmsocket_t *sock, const uv_buf_t *buf); +/*%< + * Free a buffer allocated for a receive operation. + * + * Note that as currently implemented, this doesn't actually + * free anything, marks the isc__networker's UDP receive buffer + * as "not in use". + */ + +isc_nmhandle_t * +isc___nmhandle_get(isc_nmsocket_t *sock, isc_sockaddr_t *peer, + isc_sockaddr_t *local FLARG); +/*%< + * Get a handle for the socket 'sock', allocating a new one + * if there isn't one available in 'sock->inactivehandles'. + * + * If 'peer' is not NULL, set the handle's peer address to 'peer', + * otherwise set it to 'sock->peer'. + * + * If 'local' is not NULL, set the handle's local address to 'local', + * otherwise set it to 'sock->iface->addr'. + * + * 'sock' will be attached to 'handle->sock'. The caller may need + * to detach the socket afterward. + */ + +isc__nm_uvreq_t * +isc___nm_uvreq_get(isc_nm_t *mgr, isc_nmsocket_t *sock FLARG); +/*%< + * Get a UV request structure for the socket 'sock', allocating a + * new one if there isn't one available in 'sock->inactivereqs'. + */ + +void +isc___nm_uvreq_put(isc__nm_uvreq_t **req, isc_nmsocket_t *sock FLARG); +/*%< + * Completes the use of a UV request structure, setting '*req' to NULL. + * + * The UV request is pushed onto the 'sock->inactivereqs' stack or, + * if that doesn't work, freed. + */ + +void +isc___nmsocket_init(isc_nmsocket_t *sock, isc_nm_t *mgr, isc_nmsocket_type type, + isc_sockaddr_t *iface FLARG); +/*%< + * Initialize socket 'sock', attach it to 'mgr', and set it to type 'type' + * and its interface to 'iface'. + */ + +void +isc___nmsocket_attach(isc_nmsocket_t *sock, isc_nmsocket_t **target FLARG); +/*%< + * Attach to a socket, increasing refcount + */ + +void +isc___nmsocket_detach(isc_nmsocket_t **socketp FLARG); +/*%< + * Detach from socket, decreasing refcount and possibly destroying the + * socket if it's no longer referenced. + */ + +void +isc___nmsocket_prep_destroy(isc_nmsocket_t *sock FLARG); +/*%< + * Market 'sock' as inactive, close it if necessary, and destroy it + * if there are no remaining references or active handles. + */ + +void +isc__nmsocket_shutdown(isc_nmsocket_t *sock); +/*%< + * Initiate the socket shutdown which actively calls the active + * callbacks. + */ + +bool +isc__nmsocket_active(isc_nmsocket_t *sock); +/*%< + * Determine whether 'sock' is active by checking 'sock->active' + * or, for child sockets, 'sock->parent->active'. + */ + +bool +isc__nmsocket_deactivate(isc_nmsocket_t *sock); +/*%< + * @brief Deactivate active socket + * + * Atomically deactive the socket by setting @p sock->active or, for child + * sockets, @p sock->parent->active to @c false + * + * @param[in] sock - valid nmsocket + * @return @c false if the socket was already inactive, @c true otherwise + */ + +void +isc__nmsocket_clearcb(isc_nmsocket_t *sock); +/*%< + * Clear the recv and accept callbacks in 'sock'. + */ + +void +isc__nmsocket_timer_stop(isc_nmsocket_t *sock); +void +isc__nmsocket_timer_start(isc_nmsocket_t *sock); +void +isc__nmsocket_timer_restart(isc_nmsocket_t *sock); +bool +isc__nmsocket_timer_running(isc_nmsocket_t *sock); +/*%< + * Start/stop/restart/check the timeout on the socket + */ + +void +isc__nm_connectcb(isc_nmsocket_t *sock, isc__nm_uvreq_t *uvreq, + isc_result_t eresult, bool async); + +void +isc__nm_async_connectcb(isc__networker_t *worker, isc__netievent_t *ev0); +/*%< + * Issue a connect callback on the socket, used to call the callback + */ + +void +isc__nm_readcb(isc_nmsocket_t *sock, isc__nm_uvreq_t *uvreq, + isc_result_t eresult); +void +isc__nm_async_readcb(isc__networker_t *worker, isc__netievent_t *ev0); + +/*%< + * Issue a read callback on the socket, used to call the callback + * on failed conditions when the event can't be scheduled on the uv loop. + * + */ + +void +isc__nm_sendcb(isc_nmsocket_t *sock, isc__nm_uvreq_t *uvreq, + isc_result_t eresult, bool async); +void +isc__nm_async_sendcb(isc__networker_t *worker, isc__netievent_t *ev0); +/*%< + * Issue a write callback on the socket, used to call the callback + * on failed conditions when the event can't be scheduled on the uv loop. + */ + +void +isc__nm_async_shutdown(isc__networker_t *worker, isc__netievent_t *ev0); +/*%< + * Walk through all uv handles, get the underlying sockets and issue + * close on them. + */ + +void +isc__nm_udp_send(isc_nmhandle_t *handle, const isc_region_t *region, + isc_nm_cb_t cb, void *cbarg); +/*%< + * Back-end implementation of isc_nm_send() for UDP handles. + */ + +void +isc__nm_udp_read(isc_nmhandle_t *handle, isc_nm_recv_cb_t cb, void *cbarg); +/* + * Back-end implementation of isc_nm_read() for UDP handles. + */ + +void +isc__nm_udp_close(isc_nmsocket_t *sock); +/*%< + * Close a UDP socket. + */ + +void +isc__nm_udp_cancelread(isc_nmhandle_t *handle); +/*%< + * Stop reading on a connected UDP handle. + */ + +void +isc__nm_udp_shutdown(isc_nmsocket_t *sock); +/*%< + * Called during the shutdown process to close and clean up connected + * sockets. + */ + +void +isc__nm_udp_stoplistening(isc_nmsocket_t *sock); +/*%< + * Stop listening on 'sock'. + */ + +void +isc__nm_udp_settimeout(isc_nmhandle_t *handle, uint32_t timeout); +/*%< + * Set or clear the recv timeout for the UDP socket associated with 'handle'. + */ + +void +isc__nm_async_udplisten(isc__networker_t *worker, isc__netievent_t *ev0); +void +isc__nm_async_udpconnect(isc__networker_t *worker, isc__netievent_t *ev0); +void +isc__nm_async_udpstop(isc__networker_t *worker, isc__netievent_t *ev0); +void +isc__nm_async_udpsend(isc__networker_t *worker, isc__netievent_t *ev0); +void +isc__nm_async_udpread(isc__networker_t *worker, isc__netievent_t *ev0); +void +isc__nm_async_udpcancel(isc__networker_t *worker, isc__netievent_t *ev0); +void +isc__nm_async_udpclose(isc__networker_t *worker, isc__netievent_t *ev0); +/*%< + * Callback handlers for asynchronous UDP events (listen, stoplisten, send). + */ + +void +isc__nm_tcp_send(isc_nmhandle_t *handle, const isc_region_t *region, + isc_nm_cb_t cb, void *cbarg); +/*%< + * Back-end implementation of isc_nm_send() for TCP handles. + */ + +void +isc__nm_tcp_read(isc_nmhandle_t *handle, isc_nm_recv_cb_t cb, void *cbarg); +/* + * Back-end implementation of isc_nm_read() for TCP handles. + */ + +void +isc__nm_tcp_close(isc_nmsocket_t *sock); +/*%< + * Close a TCP socket. + */ +void +isc__nm_tcp_pauseread(isc_nmhandle_t *handle); +/*%< + * Pause reading on this handle, while still remembering the callback. + */ + +void +isc__nm_tcp_resumeread(isc_nmhandle_t *handle); +/*%< + * Resume reading from socket. + * + */ + +void +isc__nm_tcp_shutdown(isc_nmsocket_t *sock); +/*%< + * Called during the shutdown process to close and clean up connected + * sockets. + */ + +void +isc__nm_tcp_cancelread(isc_nmhandle_t *handle); +/*%< + * Stop reading on a connected TCP handle. + */ + +void +isc__nm_tcp_stoplistening(isc_nmsocket_t *sock); +/*%< + * Stop listening on 'sock'. + */ + +int_fast32_t +isc__nm_tcp_listener_nactive(isc_nmsocket_t *sock); +/*%< + * Returns the number of active connections for the TCP listener socket. + */ + +void +isc__nm_tcp_settimeout(isc_nmhandle_t *handle, uint32_t timeout); +/*%< + * Set the read timeout for the TCP socket associated with 'handle'. + */ + +void +isc__nm_async_tcpconnect(isc__networker_t *worker, isc__netievent_t *ev0); +void +isc__nm_async_tcplisten(isc__networker_t *worker, isc__netievent_t *ev0); +void +isc__nm_async_tcpaccept(isc__networker_t *worker, isc__netievent_t *ev0); +void +isc__nm_async_tcpstop(isc__networker_t *worker, isc__netievent_t *ev0); +void +isc__nm_async_tcpsend(isc__networker_t *worker, isc__netievent_t *ev0); +void +isc__nm_async_startread(isc__networker_t *worker, isc__netievent_t *ev0); +void +isc__nm_async_pauseread(isc__networker_t *worker, isc__netievent_t *ev0); +void +isc__nm_async_tcpstartread(isc__networker_t *worker, isc__netievent_t *ev0); +void +isc__nm_async_tcppauseread(isc__networker_t *worker, isc__netievent_t *ev0); +void +isc__nm_async_tcpcancel(isc__networker_t *worker, isc__netievent_t *ev0); +void +isc__nm_async_tcpclose(isc__networker_t *worker, isc__netievent_t *ev0); +/*%< + * Callback handlers for asynchronous TCP events (connect, listen, + * stoplisten, send, read, pause, close). + */ + +void +isc__nm_async_tcpdnsaccept(isc__networker_t *worker, isc__netievent_t *ev0); +void +isc__nm_async_tcpdnsconnect(isc__networker_t *worker, isc__netievent_t *ev0); +void +isc__nm_async_tcpdnslisten(isc__networker_t *worker, isc__netievent_t *ev0); + +void +isc__nm_tcpdns_send(isc_nmhandle_t *handle, isc_region_t *region, + isc_nm_cb_t cb, void *cbarg); +/*%< + * Back-end implementation of isc_nm_send() for TCPDNS handles. + */ + +void +isc__nm_tcpdns_shutdown(isc_nmsocket_t *sock); + +void +isc__nm_tcpdns_close(isc_nmsocket_t *sock); +/*%< + * Close a TCPDNS socket. + */ + +void +isc__nm_tcpdns_stoplistening(isc_nmsocket_t *sock); +/*%< + * Stop listening on 'sock'. + */ + +void +isc__nm_tcpdns_settimeout(isc_nmhandle_t *handle, uint32_t timeout); +/*%< + * Set the read timeout and reset the timer for the TCPDNS socket + * associated with 'handle', and the TCP socket it wraps around. + */ + +void +isc__nm_async_tcpdnsaccept(isc__networker_t *worker, isc__netievent_t *ev0); +void +isc__nm_async_tcpdnsconnect(isc__networker_t *worker, isc__netievent_t *ev0); +void +isc__nm_async_tcpdnslisten(isc__networker_t *worker, isc__netievent_t *ev0); +void +isc__nm_async_tcpdnscancel(isc__networker_t *worker, isc__netievent_t *ev0); +void +isc__nm_async_tcpdnsclose(isc__networker_t *worker, isc__netievent_t *ev0); +void +isc__nm_async_tcpdnssend(isc__networker_t *worker, isc__netievent_t *ev0); +void +isc__nm_async_tcpdnsstop(isc__networker_t *worker, isc__netievent_t *ev0); +void +isc__nm_async_tcpdnsread(isc__networker_t *worker, isc__netievent_t *ev0); +/*%< + * Callback handlers for asynchronous TCPDNS events. + */ + +void +isc__nm_tcpdns_read(isc_nmhandle_t *handle, isc_nm_recv_cb_t cb, void *cbarg); +/* + * Back-end implementation of isc_nm_read() for TCPDNS handles. + */ + +void +isc__nm_tcpdns_cancelread(isc_nmhandle_t *handle); +/*%< + * Stop reading on a connected TCPDNS handle. + */ + +#define isc__nm_uverr2result(x) \ + isc___nm_uverr2result(x, true, __FILE__, __LINE__, __func__) +isc_result_t +isc___nm_uverr2result(int uverr, bool dolog, const char *file, + unsigned int line, const char *func); +/*%< + * Convert a libuv error value into an isc_result_t. The + * list of supported error values is not complete; new users + * of this function should add any expected errors that are + * not already there. + */ + +bool +isc__nm_acquire_interlocked(isc_nm_t *mgr); +/*%< + * Try to acquire interlocked state; return true if successful. + */ + +void +isc__nm_drop_interlocked(isc_nm_t *mgr); +/*%< + * Drop interlocked state; signal waiters. + */ + +void +isc__nm_acquire_interlocked_force(isc_nm_t *mgr); +/*%< + * Actively wait for interlocked state. + */ + +void +isc__nm_incstats(isc_nm_t *mgr, isc_statscounter_t counterid); +/*%< + * Increment socket-related statistics counters. + */ + +void +isc__nm_decstats(isc_nm_t *mgr, isc_statscounter_t counterid); +/*%< + * Decrement socket-related statistics counters. + */ + +isc_result_t +isc__nm_socket(int domain, int type, int protocol, uv_os_sock_t *sockp); +/*%< + * Platform independent socket() version + */ + +void +isc__nm_closesocket(uv_os_sock_t sock); +/*%< + * Platform independent closesocket() version + */ + +isc_result_t +isc__nm_socket_freebind(uv_os_sock_t fd, sa_family_t sa_family); +/*%< + * Set the IP_FREEBIND (or equivalent) socket option on the uv_handle + */ + +isc_result_t +isc__nm_socket_reuse(uv_os_sock_t fd); +/*%< + * Set the SO_REUSEADDR or SO_REUSEPORT (or equivalent) socket option on the fd + */ + +isc_result_t +isc__nm_socket_reuse_lb(uv_os_sock_t fd); +/*%< + * Set the SO_REUSEPORT_LB (or equivalent) socket option on the fd + */ + +isc_result_t +isc__nm_socket_incoming_cpu(uv_os_sock_t fd); +/*%< + * Set the SO_INCOMING_CPU socket option on the fd if available + */ + +isc_result_t +isc__nm_socket_disable_pmtud(uv_os_sock_t fd, sa_family_t sa_family); +/*%< + * Disable the Path MTU Discovery, either by disabling IP(V6)_DONTFRAG socket + * option, or setting the IP(V6)_MTU_DISCOVER socket option to IP_PMTUDISC_OMIT + */ + +isc_result_t +isc__nm_socket_connectiontimeout(uv_os_sock_t fd, int timeout_ms); +/*%< + * Set the connection timeout in milliseconds, on non-Linux platforms, + * the minimum value must be at least 1000 (1 second). + */ + +isc_result_t +isc__nm_socket_tcp_nodelay(uv_os_sock_t fd); +/*%< + * Disables Nagle's algorithm on a TCP socket (sets TCP_NODELAY). + */ + +/* + * typedef all the netievent types + */ + +NETIEVENT_SOCKET_TYPE(close); +NETIEVENT_SOCKET_TYPE(tcpclose); +NETIEVENT_SOCKET_TYPE(tcplisten); +NETIEVENT_SOCKET_TYPE(tcppauseread); +NETIEVENT_SOCKET_TYPE(tcpstop); +NETIEVENT_SOCKET_TYPE(udpclose); +NETIEVENT_SOCKET_TYPE(udplisten); +NETIEVENT_SOCKET_TYPE(udpread); +/* NETIEVENT_SOCKET_TYPE(udpsend); */ /* unique type, defined independently */ +NETIEVENT_SOCKET_TYPE(udpstop); + +NETIEVENT_SOCKET_TYPE(tcpdnsclose); +NETIEVENT_SOCKET_TYPE(tcpdnsread); +NETIEVENT_SOCKET_TYPE(tcpdnsstop); +NETIEVENT_SOCKET_TYPE(tcpdnslisten); +NETIEVENT_SOCKET_REQ_TYPE(tcpdnsconnect); +NETIEVENT_SOCKET_REQ_TYPE(tcpdnssend); +NETIEVENT_SOCKET_HANDLE_TYPE(tcpdnscancel); +NETIEVENT_SOCKET_QUOTA_TYPE(tcpdnsaccept); + +NETIEVENT_SOCKET_REQ_TYPE(tcpconnect); +NETIEVENT_SOCKET_REQ_TYPE(tcpsend); +NETIEVENT_SOCKET_TYPE(tcpstartread); +NETIEVENT_SOCKET_REQ_TYPE(udpconnect); + +NETIEVENT_SOCKET_REQ_RESULT_TYPE(connectcb); +NETIEVENT_SOCKET_REQ_RESULT_TYPE(readcb); +NETIEVENT_SOCKET_REQ_RESULT_TYPE(sendcb); + +NETIEVENT_SOCKET_HANDLE_TYPE(detach); +NETIEVENT_SOCKET_HANDLE_TYPE(tcpcancel); +NETIEVENT_SOCKET_HANDLE_TYPE(udpcancel); + +NETIEVENT_SOCKET_QUOTA_TYPE(tcpaccept); + +NETIEVENT_TYPE(pause); +NETIEVENT_TYPE(resume); +NETIEVENT_TYPE(shutdown); +NETIEVENT_TYPE(stop); + +NETIEVENT_TASK_TYPE(task); +NETIEVENT_TASK_TYPE(privilegedtask); + +/* Now declared the helper functions */ + +NETIEVENT_SOCKET_DECL(close); +NETIEVENT_SOCKET_DECL(tcpclose); +NETIEVENT_SOCKET_DECL(tcplisten); +NETIEVENT_SOCKET_DECL(tcppauseread); +NETIEVENT_SOCKET_DECL(tcpstartread); +NETIEVENT_SOCKET_DECL(tcpstop); +NETIEVENT_SOCKET_DECL(udpclose); +NETIEVENT_SOCKET_DECL(udplisten); +NETIEVENT_SOCKET_DECL(udpread); +NETIEVENT_SOCKET_DECL(udpsend); +NETIEVENT_SOCKET_DECL(udpstop); + +NETIEVENT_SOCKET_DECL(tcpdnsclose); +NETIEVENT_SOCKET_DECL(tcpdnsread); +NETIEVENT_SOCKET_DECL(tcpdnsstop); +NETIEVENT_SOCKET_DECL(tcpdnslisten); +NETIEVENT_SOCKET_REQ_DECL(tcpdnsconnect); +NETIEVENT_SOCKET_REQ_DECL(tcpdnssend); +NETIEVENT_SOCKET_HANDLE_DECL(tcpdnscancel); +NETIEVENT_SOCKET_QUOTA_DECL(tcpdnsaccept); + +NETIEVENT_SOCKET_REQ_DECL(tcpconnect); +NETIEVENT_SOCKET_REQ_DECL(tcpsend); +NETIEVENT_SOCKET_REQ_DECL(udpconnect); + +NETIEVENT_SOCKET_REQ_RESULT_DECL(connectcb); +NETIEVENT_SOCKET_REQ_RESULT_DECL(readcb); +NETIEVENT_SOCKET_REQ_RESULT_DECL(sendcb); + +NETIEVENT_SOCKET_HANDLE_DECL(udpcancel); +NETIEVENT_SOCKET_HANDLE_DECL(tcpcancel); +NETIEVENT_SOCKET_DECL(detach); + +NETIEVENT_SOCKET_QUOTA_DECL(tcpaccept); + +NETIEVENT_DECL(pause); +NETIEVENT_DECL(resume); +NETIEVENT_DECL(shutdown); +NETIEVENT_DECL(stop); + +NETIEVENT_TASK_DECL(task); +NETIEVENT_TASK_DECL(privilegedtask); + +void +isc__nm_udp_failed_read_cb(isc_nmsocket_t *sock, isc_result_t result); +void +isc__nm_tcp_failed_read_cb(isc_nmsocket_t *sock, isc_result_t result); +void +isc__nm_tcpdns_failed_read_cb(isc_nmsocket_t *sock, isc_result_t result); + +isc_result_t +isc__nm_tcpdns_processbuffer(isc_nmsocket_t *sock); + +isc__nm_uvreq_t * +isc__nm_get_read_req(isc_nmsocket_t *sock, isc_sockaddr_t *sockaddr); + +void +isc__nm_alloc_cb(uv_handle_t *handle, size_t size, uv_buf_t *buf); + +void +isc__nm_udp_read_cb(uv_udp_t *handle, ssize_t nrecv, const uv_buf_t *buf, + const struct sockaddr *addr, unsigned flags); +void +isc__nm_tcp_read_cb(uv_stream_t *stream, ssize_t nread, const uv_buf_t *buf); +void +isc__nm_tcpdns_read_cb(uv_stream_t *stream, ssize_t nread, const uv_buf_t *buf); + +isc_result_t +isc__nm_start_reading(isc_nmsocket_t *sock); +void +isc__nm_stop_reading(isc_nmsocket_t *sock); +isc_result_t +isc__nm_process_sock_buffer(isc_nmsocket_t *sock); +void +isc__nm_resume_processing(void *arg); +bool +isc__nmsocket_closing(isc_nmsocket_t *sock); +bool +isc__nm_closing(isc_nmsocket_t *sock); + +void +isc__nm_alloc_dnsbuf(isc_nmsocket_t *sock, size_t len); + +void +isc__nm_failed_send_cb(isc_nmsocket_t *sock, isc__nm_uvreq_t *req, + isc_result_t eresult); +void +isc__nm_failed_accept_cb(isc_nmsocket_t *sock, isc_result_t eresult); +void +isc__nm_failed_connect_cb(isc_nmsocket_t *sock, isc__nm_uvreq_t *req, + isc_result_t eresult, bool async); +void +isc__nm_failed_read_cb(isc_nmsocket_t *sock, isc_result_t result, bool async); + +void +isc__nm_accept_connection_log(isc_result_t result, bool can_log_quota); + +/* + * Timeout callbacks + */ +void +isc__nmsocket_connecttimeout_cb(uv_timer_t *timer); +void +isc__nmsocket_readtimeout_cb(uv_timer_t *timer); +void +isc__nmsocket_writetimeout_cb(void *data, isc_result_t eresult); + +/*%< + * + * Maximum number of simultaneous handles in flight supported for a single + * connected TCPDNS socket. This value was chosen arbitrarily, and may be + * changed in the future. + */ +#define STREAM_CLIENTS_PER_CONN 23 + +#define UV_RUNTIME_CHECK(func, ret) \ + if (ret != 0) { \ + isc_error_fatal(__FILE__, __LINE__, "%s failed: %s\n", #func, \ + uv_strerror(ret)); \ + } diff --git a/lib/isc/netmgr/netmgr.c b/lib/isc/netmgr/netmgr.c new file mode 100644 index 0000000..6f42ec9 --- /dev/null +++ b/lib/isc/netmgr/netmgr.c @@ -0,0 +1,3396 @@ +/* + * Copyright (C) Internet Systems Consortium, Inc. ("ISC") + * + * SPDX-License-Identifier: MPL-2.0 + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, you can obtain one at https://mozilla.org/MPL/2.0/. + * + * See the COPYRIGHT file distributed with this work for additional + * information regarding copyright ownership. + */ + +#include <inttypes.h> +#include <unistd.h> +#include <uv.h> +#ifdef HAVE_LIBCTRACE +#include <execinfo.h> +#endif /* ifdef HAVE_LIBCTRACE */ + +#include <isc/atomic.h> +#include <isc/barrier.h> +#include <isc/buffer.h> +#include <isc/condition.h> +#include <isc/errno.h> +#include <isc/list.h> +#include <isc/log.h> +#include <isc/magic.h> +#include <isc/mem.h> +#include <isc/netmgr.h> +#include <isc/print.h> +#include <isc/quota.h> +#include <isc/random.h> +#include <isc/refcount.h> +#include <isc/region.h> +#include <isc/result.h> +#include <isc/sockaddr.h> +#include <isc/stats.h> +#include <isc/strerr.h> +#include <isc/task.h> +#include <isc/thread.h> +#include <isc/util.h> + +#include "netmgr-int.h" +#include "netmgr_p.h" +#include "openssl_shim.h" +#include "trampoline_p.h" +#include "uv-compat.h" + +/*% + * How many isc_nmhandles and isc_nm_uvreqs will we be + * caching for reuse in a socket. + */ +#define ISC_NM_HANDLES_STACK_SIZE 600 +#define ISC_NM_REQS_STACK_SIZE 600 + +/*% + * Shortcut index arrays to get access to statistics counters. + */ + +static const isc_statscounter_t udp4statsindex[] = { + isc_sockstatscounter_udp4open, + isc_sockstatscounter_udp4openfail, + isc_sockstatscounter_udp4close, + isc_sockstatscounter_udp4bindfail, + isc_sockstatscounter_udp4connectfail, + isc_sockstatscounter_udp4connect, + -1, + -1, + isc_sockstatscounter_udp4sendfail, + isc_sockstatscounter_udp4recvfail, + isc_sockstatscounter_udp4active +}; + +static const isc_statscounter_t udp6statsindex[] = { + isc_sockstatscounter_udp6open, + isc_sockstatscounter_udp6openfail, + isc_sockstatscounter_udp6close, + isc_sockstatscounter_udp6bindfail, + isc_sockstatscounter_udp6connectfail, + isc_sockstatscounter_udp6connect, + -1, + -1, + isc_sockstatscounter_udp6sendfail, + isc_sockstatscounter_udp6recvfail, + isc_sockstatscounter_udp6active +}; + +static const isc_statscounter_t tcp4statsindex[] = { + isc_sockstatscounter_tcp4open, isc_sockstatscounter_tcp4openfail, + isc_sockstatscounter_tcp4close, isc_sockstatscounter_tcp4bindfail, + isc_sockstatscounter_tcp4connectfail, isc_sockstatscounter_tcp4connect, + isc_sockstatscounter_tcp4acceptfail, isc_sockstatscounter_tcp4accept, + isc_sockstatscounter_tcp4sendfail, isc_sockstatscounter_tcp4recvfail, + isc_sockstatscounter_tcp4active +}; + +static const isc_statscounter_t tcp6statsindex[] = { + isc_sockstatscounter_tcp6open, isc_sockstatscounter_tcp6openfail, + isc_sockstatscounter_tcp6close, isc_sockstatscounter_tcp6bindfail, + isc_sockstatscounter_tcp6connectfail, isc_sockstatscounter_tcp6connect, + isc_sockstatscounter_tcp6acceptfail, isc_sockstatscounter_tcp6accept, + isc_sockstatscounter_tcp6sendfail, isc_sockstatscounter_tcp6recvfail, + isc_sockstatscounter_tcp6active +}; + +#if 0 +/* XXX: not currently used */ +static const isc_statscounter_t unixstatsindex[] = { + isc_sockstatscounter_unixopen, + isc_sockstatscounter_unixopenfail, + isc_sockstatscounter_unixclose, + isc_sockstatscounter_unixbindfail, + isc_sockstatscounter_unixconnectfail, + isc_sockstatscounter_unixconnect, + isc_sockstatscounter_unixacceptfail, + isc_sockstatscounter_unixaccept, + isc_sockstatscounter_unixsendfail, + isc_sockstatscounter_unixrecvfail, + isc_sockstatscounter_unixactive +}; +#endif /* if 0 */ + +/* + * libuv is not thread safe, but has mechanisms to pass messages + * between threads. Each socket is owned by a thread. For UDP + * sockets we have a set of sockets for each interface and we can + * choose a sibling and send the message directly. For TCP, or if + * we're calling from a non-networking thread, we need to pass the + * request using async_cb. + */ + +#if defined(HAVE_THREAD_LOCAL) +#include <threads.h> +static thread_local int isc__nm_tid_v = ISC_NETMGR_TID_UNKNOWN; +#elif defined(HAVE___THREAD) +static __thread int isc__nm_tid_v = ISC_NETMGR_TID_UNKNOWN; +#elif HAVE___DECLSPEC_THREAD +__declspec(thread) int isc__nm_tid_v = ISC_NETMGR_TID_UNKNOWN; +#endif /* if defined(HAVE_THREAD_LOCAL) */ + +static void +nmsocket_maybe_destroy(isc_nmsocket_t *sock FLARG); +static void +nmhandle_free(isc_nmsocket_t *sock, isc_nmhandle_t *handle); +static isc_threadresult_t +nm_thread(isc_threadarg_t worker0); +static void +async_cb(uv_async_t *handle); + +static bool +process_netievent(isc__networker_t *worker, isc__netievent_t *ievent); +static isc_result_t +process_queue(isc__networker_t *worker, netievent_type_t type); +static void +wait_for_priority_queue(isc__networker_t *worker); +static void +drain_queue(isc__networker_t *worker, netievent_type_t type); + +static void +isc__nm_async_stop(isc__networker_t *worker, isc__netievent_t *ev0); +static void +isc__nm_async_pause(isc__networker_t *worker, isc__netievent_t *ev0); +static void +isc__nm_async_resume(isc__networker_t *worker, isc__netievent_t *ev0); +static void +isc__nm_async_detach(isc__networker_t *worker, isc__netievent_t *ev0); +static void +isc__nm_async_close(isc__networker_t *worker, isc__netievent_t *ev0); + +static void +isc__nm_threadpool_initialize(uint32_t workers); +static void +isc__nm_work_cb(uv_work_t *req); +static void +isc__nm_after_work_cb(uv_work_t *req, int status); + +void +isc__nmsocket_reset(isc_nmsocket_t *sock); + +/*%< + * Issue a 'handle closed' callback on the socket. + */ + +static void +nmhandle_detach_cb(isc_nmhandle_t **handlep FLARG); + +int +isc_nm_tid(void) { + return (isc__nm_tid_v); +} + +bool +isc__nm_in_netthread(void) { + return (isc__nm_tid_v >= 0); +} + +#ifdef WIN32 +static void +isc__nm_winsock_initialize(void) { + WORD wVersionRequested = MAKEWORD(2, 2); + WSADATA wsaData; + int result; + + result = WSAStartup(wVersionRequested, &wsaData); + if (result != 0) { + char strbuf[ISC_STRERRORSIZE]; + strerror_r(result, strbuf, sizeof(strbuf)); + UNEXPECTED_ERROR(__FILE__, __LINE__, + "WSAStartup() failed with error code %lu: %s", + result, strbuf); + } + + /* + * Confirm that the WinSock DLL supports version 2.2. Note that if the + * DLL supports versions greater than 2.2 in addition to 2.2, it will + * still return 2.2 in wVersion since that is the version we requested. + */ + if (LOBYTE(wsaData.wVersion) != 2 || HIBYTE(wsaData.wVersion) != 2) { + UNEXPECTED_ERROR(__FILE__, __LINE__, + "Unusable WinSock DLL version: %u.%u", + LOBYTE(wsaData.wVersion), + HIBYTE(wsaData.wVersion)); + } +} + +static void +isc__nm_winsock_destroy(void) { + WSACleanup(); +} +#endif /* WIN32 */ + +static void +isc__nm_threadpool_initialize(uint32_t workers) { + char buf[11]; + int r = uv_os_getenv("UV_THREADPOOL_SIZE", buf, + &(size_t){ sizeof(buf) }); + if (r == UV_ENOENT) { + snprintf(buf, sizeof(buf), "%" PRIu32, workers); + uv_os_setenv("UV_THREADPOOL_SIZE", buf); + } +} + +#if HAVE_DECL_UV_UDP_MMSG_FREE +#define MINIMAL_UV_VERSION UV_VERSION(1, 40, 0) +#elif HAVE_DECL_UV_UDP_RECVMMSG +#define MAXIMAL_UV_VERSION UV_VERSION(1, 39, 99) +#define MINIMAL_UV_VERSION UV_VERSION(1, 37, 0) +#elif _WIN32 +#define MINIMAL_UV_VERSION UV_VERSION(1, 0, 0) +#else +#define MAXIMAL_UV_VERSION UV_VERSION(1, 34, 99) +#define MINIMAL_UV_VERSION UV_VERSION(1, 0, 0) +#endif + +void +isc__netmgr_create(isc_mem_t *mctx, uint32_t workers, isc_nm_t **netmgrp) { + isc_nm_t *mgr = NULL; + char name[32]; + + REQUIRE(workers > 0); + +#ifdef MAXIMAL_UV_VERSION + if (uv_version() > MAXIMAL_UV_VERSION) { + isc_error_fatal(__FILE__, __LINE__, + "libuv version too new: running with libuv %s " + "when compiled with libuv %s will lead to " + "libuv failures", + uv_version_string(), UV_VERSION_STRING); + } +#endif /* MAXIMAL_UV_VERSION */ + + if (uv_version() < MINIMAL_UV_VERSION) { + isc_error_fatal(__FILE__, __LINE__, + "libuv version too old: running with libuv %s " + "when compiled with libuv %s will lead to " + "libuv failures", + uv_version_string(), UV_VERSION_STRING); + } + +#ifdef WIN32 + isc__nm_winsock_initialize(); +#endif /* WIN32 */ + + isc__nm_threadpool_initialize(workers); + + mgr = isc_mem_get(mctx, sizeof(*mgr)); + *mgr = (isc_nm_t){ .nworkers = workers }; + + isc_mem_attach(mctx, &mgr->mctx); + isc_mutex_init(&mgr->lock); + isc_condition_init(&mgr->wkstatecond); + isc_condition_init(&mgr->wkpausecond); + isc_refcount_init(&mgr->references, 1); + atomic_init(&mgr->maxudp, 0); + atomic_init(&mgr->interlocked, ISC_NETMGR_NON_INTERLOCKED); + atomic_init(&mgr->workers_paused, 0); + atomic_init(&mgr->paused, false); + atomic_init(&mgr->closing, false); +#if HAVE_SO_REUSEPORT_LB + mgr->load_balance_sockets = true; +#else + mgr->load_balance_sockets = false; +#endif + +#ifdef NETMGR_TRACE + ISC_LIST_INIT(mgr->active_sockets); +#endif + + /* + * Default TCP timeout values. + * May be updated by isc_nm_tcptimeouts(). + */ + atomic_init(&mgr->init, 30000); + atomic_init(&mgr->idle, 30000); + atomic_init(&mgr->keepalive, 30000); + atomic_init(&mgr->advertised, 30000); + + isc_barrier_init(&mgr->pausing, workers); + isc_barrier_init(&mgr->resuming, workers); + + mgr->workers = isc_mem_get(mctx, workers * sizeof(isc__networker_t)); + for (size_t i = 0; i < workers; i++) { + isc__networker_t *worker = &mgr->workers[i]; + int r; + + *worker = (isc__networker_t){ + .mgr = mgr, + .id = i, + }; + + r = uv_loop_init(&worker->loop); + UV_RUNTIME_CHECK(uv_loop_init, r); + + worker->loop.data = &mgr->workers[i]; + + r = uv_async_init(&worker->loop, &worker->async, async_cb); + UV_RUNTIME_CHECK(uv_async_init, r); + + for (size_t type = 0; type < NETIEVENT_MAX; type++) { + isc_mutex_init(&worker->ievents[type].lock); + isc_condition_init(&worker->ievents[type].cond); + ISC_LIST_INIT(worker->ievents[type].list); + } + + worker->recvbuf = isc_mem_get(mctx, ISC_NETMGR_RECVBUF_SIZE); + worker->sendbuf = isc_mem_get(mctx, ISC_NETMGR_SENDBUF_SIZE); + + /* + * We need to do this here and not in nm_thread to avoid a + * race - we could exit isc_nm_start, launch nm_destroy, + * and nm_thread would still not be up. + */ + mgr->workers_running++; + isc_thread_create(nm_thread, &mgr->workers[i], &worker->thread); + + snprintf(name, sizeof(name), "isc-net-%04zu", i); + isc_thread_setname(worker->thread, name); + } + + mgr->magic = NM_MAGIC; + *netmgrp = mgr; +} + +/* + * Free the resources of the network manager. + */ +static void +nm_destroy(isc_nm_t **mgr0) { + REQUIRE(VALID_NM(*mgr0)); + REQUIRE(!isc__nm_in_netthread()); + + isc_nm_t *mgr = *mgr0; + *mgr0 = NULL; + + isc_refcount_destroy(&mgr->references); + + mgr->magic = 0; + + for (int i = 0; i < mgr->nworkers; i++) { + isc__networker_t *worker = &mgr->workers[i]; + isc__netievent_t *event = isc__nm_get_netievent_stop(mgr); + isc__nm_enqueue_ievent(worker, event); + } + + LOCK(&mgr->lock); + while (mgr->workers_running > 0) { + WAIT(&mgr->wkstatecond, &mgr->lock); + } + UNLOCK(&mgr->lock); + + for (int i = 0; i < mgr->nworkers; i++) { + isc__networker_t *worker = &mgr->workers[i]; + int r; + + r = uv_loop_close(&worker->loop); + UV_RUNTIME_CHECK(uv_loop_close, r); + + for (size_t type = 0; type < NETIEVENT_MAX; type++) { + INSIST(ISC_LIST_EMPTY(worker->ievents[type].list)); + isc_condition_destroy(&worker->ievents[type].cond); + isc_mutex_destroy(&worker->ievents[type].lock); + } + + isc_mem_put(mgr->mctx, worker->sendbuf, + ISC_NETMGR_SENDBUF_SIZE); + isc_mem_put(mgr->mctx, worker->recvbuf, + ISC_NETMGR_RECVBUF_SIZE); + isc_thread_join(worker->thread, NULL); + } + + if (mgr->stats != NULL) { + isc_stats_detach(&mgr->stats); + } + + isc_barrier_destroy(&mgr->resuming); + isc_barrier_destroy(&mgr->pausing); + + isc_condition_destroy(&mgr->wkstatecond); + isc_condition_destroy(&mgr->wkpausecond); + isc_mutex_destroy(&mgr->lock); + + isc_mem_put(mgr->mctx, mgr->workers, + mgr->nworkers * sizeof(isc__networker_t)); + isc_mem_putanddetach(&mgr->mctx, mgr, sizeof(*mgr)); + +#ifdef WIN32 + isc__nm_winsock_destroy(); +#endif /* WIN32 */ +} + +static void +enqueue_pause(isc__networker_t *worker) { + isc__netievent_pause_t *event = + isc__nm_get_netievent_pause(worker->mgr); + isc__nm_enqueue_ievent(worker, (isc__netievent_t *)event); +} + +static void +isc__nm_async_pause(isc__networker_t *worker, isc__netievent_t *ev0) { + UNUSED(ev0); + REQUIRE(worker->paused == false); + + worker->paused = true; + uv_stop(&worker->loop); +} + +void +isc_nm_pause(isc_nm_t *mgr) { + REQUIRE(VALID_NM(mgr)); + REQUIRE(!atomic_load(&mgr->paused)); + + isc__nm_acquire_interlocked_force(mgr); + + if (isc__nm_in_netthread()) { + REQUIRE(isc_nm_tid() == 0); + } + + for (int i = 0; i < mgr->nworkers; i++) { + isc__networker_t *worker = &mgr->workers[i]; + if (i == isc_nm_tid()) { + isc__nm_async_pause(worker, NULL); + } else { + enqueue_pause(worker); + } + } + + if (isc__nm_in_netthread()) { + atomic_fetch_add(&mgr->workers_paused, 1); + isc_barrier_wait(&mgr->pausing); + } + + LOCK(&mgr->lock); + while (atomic_load(&mgr->workers_paused) != mgr->workers_running) { + WAIT(&mgr->wkstatecond, &mgr->lock); + } + UNLOCK(&mgr->lock); + + REQUIRE(atomic_compare_exchange_strong(&mgr->paused, &(bool){ false }, + true)); +} + +static void +enqueue_resume(isc__networker_t *worker) { + isc__netievent_resume_t *event = + isc__nm_get_netievent_resume(worker->mgr); + isc__nm_enqueue_ievent(worker, (isc__netievent_t *)event); +} + +static void +isc__nm_async_resume(isc__networker_t *worker, isc__netievent_t *ev0) { + UNUSED(ev0); + REQUIRE(worker->paused == true); + + worker->paused = false; +} + +void +isc_nm_resume(isc_nm_t *mgr) { + REQUIRE(VALID_NM(mgr)); + REQUIRE(atomic_load(&mgr->paused)); + + if (isc__nm_in_netthread()) { + REQUIRE(isc_nm_tid() == 0); + drain_queue(&mgr->workers[isc_nm_tid()], NETIEVENT_PRIORITY); + } + + for (int i = 0; i < mgr->nworkers; i++) { + isc__networker_t *worker = &mgr->workers[i]; + if (i == isc_nm_tid()) { + isc__nm_async_resume(worker, NULL); + } else { + enqueue_resume(worker); + } + } + + if (isc__nm_in_netthread()) { + drain_queue(&mgr->workers[isc_nm_tid()], NETIEVENT_PRIVILEGED); + + atomic_fetch_sub(&mgr->workers_paused, 1); + isc_barrier_wait(&mgr->resuming); + } + + LOCK(&mgr->lock); + while (atomic_load(&mgr->workers_paused) != 0) { + WAIT(&mgr->wkstatecond, &mgr->lock); + } + UNLOCK(&mgr->lock); + + REQUIRE(atomic_compare_exchange_strong(&mgr->paused, &(bool){ true }, + false)); + + isc__nm_drop_interlocked(mgr); +} + +void +isc_nm_attach(isc_nm_t *mgr, isc_nm_t **dst) { + REQUIRE(VALID_NM(mgr)); + REQUIRE(dst != NULL && *dst == NULL); + + isc_refcount_increment(&mgr->references); + + *dst = mgr; +} + +void +isc_nm_detach(isc_nm_t **mgr0) { + isc_nm_t *mgr = NULL; + + REQUIRE(mgr0 != NULL); + REQUIRE(VALID_NM(*mgr0)); + + mgr = *mgr0; + *mgr0 = NULL; + + if (isc_refcount_decrement(&mgr->references) == 1) { + nm_destroy(&mgr); + } +} + +void +isc__netmgr_shutdown(isc_nm_t *mgr) { + REQUIRE(VALID_NM(mgr)); + + atomic_store(&mgr->closing, true); + for (int i = 0; i < mgr->nworkers; i++) { + isc__netievent_t *event = NULL; + event = isc__nm_get_netievent_shutdown(mgr); + isc__nm_enqueue_ievent(&mgr->workers[i], event); + } +} + +void +isc__netmgr_destroy(isc_nm_t **netmgrp) { + isc_nm_t *mgr = NULL; + int counter = 0; + + REQUIRE(VALID_NM(*netmgrp)); + + mgr = *netmgrp; + + /* + * Close active connections. + */ + isc__netmgr_shutdown(mgr); + + /* + * Wait for the manager to be dereferenced elsewhere. + */ + while (isc_refcount_current(&mgr->references) > 1 && counter++ < 1000) { + uv_sleep(10); + } + +#ifdef NETMGR_TRACE + if (isc_refcount_current(&mgr->references) > 1) { + isc__nm_dump_active(mgr); + UNREACHABLE(); + } +#endif + + /* + * Now just patiently wait + */ + while (isc_refcount_current(&mgr->references) > 1) { + uv_sleep(10); + } + + /* + * Detach final reference. + */ + isc_nm_detach(netmgrp); +} + +void +isc_nm_maxudp(isc_nm_t *mgr, uint32_t maxudp) { + REQUIRE(VALID_NM(mgr)); + + atomic_store(&mgr->maxudp, maxudp); +} + +void +isc_nmhandle_setwritetimeout(isc_nmhandle_t *handle, uint64_t write_timeout) { + REQUIRE(VALID_NMHANDLE(handle)); + REQUIRE(VALID_NMSOCK(handle->sock)); + + handle->sock->write_timeout = write_timeout; +} + +void +isc_nm_settimeouts(isc_nm_t *mgr, uint32_t init, uint32_t idle, + uint32_t keepalive, uint32_t advertised) { + REQUIRE(VALID_NM(mgr)); + + atomic_store(&mgr->init, init); + atomic_store(&mgr->idle, idle); + atomic_store(&mgr->keepalive, keepalive); + atomic_store(&mgr->advertised, advertised); +} + +bool +isc_nm_getloadbalancesockets(isc_nm_t *mgr) { + REQUIRE(VALID_NM(mgr)); + + return (mgr->load_balance_sockets); +} + +void +isc_nm_setloadbalancesockets(isc_nm_t *mgr, bool enabled) { + REQUIRE(VALID_NM(mgr)); + +#if HAVE_SO_REUSEPORT_LB + mgr->load_balance_sockets = enabled; +#else + UNUSED(enabled); +#endif +} + +void +isc_nm_gettimeouts(isc_nm_t *mgr, uint32_t *initial, uint32_t *idle, + uint32_t *keepalive, uint32_t *advertised) { + REQUIRE(VALID_NM(mgr)); + + if (initial != NULL) { + *initial = atomic_load(&mgr->init); + } + + if (idle != NULL) { + *idle = atomic_load(&mgr->idle); + } + + if (keepalive != NULL) { + *keepalive = atomic_load(&mgr->keepalive); + } + + if (advertised != NULL) { + *advertised = atomic_load(&mgr->advertised); + } +} + +/* + * nm_thread is a single worker thread, that runs uv_run event loop + * until asked to stop. + * + * There are four queues for asynchronous events: + * + * 1. priority queue - netievents on the priority queue are run even when + * the taskmgr enters exclusive mode and the netmgr is paused. This + * is needed to properly start listening on the interfaces, free + * resources on shutdown, or resume from a pause. + * + * 2. privileged task queue - only privileged tasks are queued here and + * this is the first queue that gets processed when network manager + * is unpaused using isc_nm_resume(). All netmgr workers need to + * clean the privileged task queue before they all proceed to normal + * operation. Both task queues are processed when the workers are + * shutting down. + * + * 3. task queue - only (traditional) tasks are scheduled here, and this + * queue and the privileged task queue are both processed when the + * netmgr workers are finishing. This is needed to process the task + * shutdown events. + * + * 4. normal queue - this is the queue with netmgr events, e.g. reading, + * sending, callbacks, etc. + */ + +static isc_threadresult_t +nm_thread(isc_threadarg_t worker0) { + isc__networker_t *worker = (isc__networker_t *)worker0; + isc_nm_t *mgr = worker->mgr; + + isc__nm_tid_v = worker->id; + + while (true) { + /* + * uv_run() runs async_cb() in a loop, which processes + * all four event queues until a "pause" or "stop" event + * is encountered. On pause, we process only priority and + * privileged events until resuming. + */ + int r = uv_run(&worker->loop, UV_RUN_DEFAULT); + INSIST(r > 0 || worker->finished); + + if (worker->paused) { + INSIST(atomic_load(&mgr->interlocked) != isc_nm_tid()); + + atomic_fetch_add(&mgr->workers_paused, 1); + if (isc_barrier_wait(&mgr->pausing) != 0) { + LOCK(&mgr->lock); + SIGNAL(&mgr->wkstatecond); + UNLOCK(&mgr->lock); + } + + while (worker->paused) { + wait_for_priority_queue(worker); + } + + /* + * All workers must drain the privileged event + * queue before we resume from pause. + */ + drain_queue(worker, NETIEVENT_PRIVILEGED); + + atomic_fetch_sub(&mgr->workers_paused, 1); + if (isc_barrier_wait(&mgr->resuming) != 0) { + LOCK(&mgr->lock); + SIGNAL(&mgr->wkstatecond); + UNLOCK(&mgr->lock); + } + } + + if (r == 0) { + INSIST(worker->finished); + break; + } + + INSIST(!worker->finished); + } + + /* + * We are shutting down. Drain the queues. + */ + drain_queue(worker, NETIEVENT_PRIVILEGED); + drain_queue(worker, NETIEVENT_TASK); + + for (size_t type = 0; type < NETIEVENT_MAX; type++) { + LOCK(&worker->ievents[type].lock); + INSIST(ISC_LIST_EMPTY(worker->ievents[type].list)); + UNLOCK(&worker->ievents[type].lock); + } + + LOCK(&mgr->lock); + mgr->workers_running--; + SIGNAL(&mgr->wkstatecond); + UNLOCK(&mgr->lock); + + return ((isc_threadresult_t)0); +} + +static bool +process_all_queues(isc__networker_t *worker) { + bool reschedule = false; + /* + * The queue processing functions will return false when the + * system is pausing or stopping and we don't want to process + * the other queues in such case, but we need the async event + * to be rescheduled in the next uv_run(). + */ + for (size_t type = 0; type < NETIEVENT_MAX; type++) { + isc_result_t result = process_queue(worker, type); + switch (result) { + case ISC_R_SUSPEND: + reschedule = true; + break; + case ISC_R_EMPTY: + /* empty queue */ + break; + case ISC_R_SUCCESS: + reschedule = true; + break; + default: + UNREACHABLE(); + } + } + + return (reschedule); +} + +/* + * async_cb() is a universal callback for 'async' events sent to event loop. + * It's the only way to safely pass data to the libuv event loop. We use a + * single async event and a set of lockless queues of 'isc__netievent_t' + * structures passed from other threads. + */ +static void +async_cb(uv_async_t *handle) { + isc__networker_t *worker = (isc__networker_t *)handle->loop->data; + + if (process_all_queues(worker)) { + /* + * If we didn't process all the events, we need to enqueue + * async_cb to be run in the next iteration of the uv_loop + */ + uv_async_send(handle); + } +} + +static void +isc__nm_async_stop(isc__networker_t *worker, isc__netievent_t *ev0) { + UNUSED(ev0); + worker->finished = true; + /* Close the async handler */ + uv_close((uv_handle_t *)&worker->async, NULL); +} + +void +isc_nm_task_enqueue(isc_nm_t *nm, isc_task_t *task, int threadid) { + isc__netievent_t *event = NULL; + int tid; + isc__networker_t *worker = NULL; + + if (threadid == -1) { + tid = (int)isc_random_uniform(nm->nworkers); + } else { + tid = threadid % nm->nworkers; + } + + worker = &nm->workers[tid]; + + if (isc_task_privileged(task)) { + event = (isc__netievent_t *) + isc__nm_get_netievent_privilegedtask(nm, task); + } else { + event = (isc__netievent_t *)isc__nm_get_netievent_task(nm, + task); + } + + isc__nm_enqueue_ievent(worker, event); +} + +#define isc__nm_async_privilegedtask(worker, ev0) \ + isc__nm_async_task(worker, ev0) + +static void +isc__nm_async_task(isc__networker_t *worker, isc__netievent_t *ev0) { + isc__netievent_task_t *ievent = (isc__netievent_task_t *)ev0; + isc_result_t result; + + UNUSED(worker); + + result = isc_task_run(ievent->task); + + switch (result) { + case ISC_R_QUOTA: + isc_task_ready(ievent->task); + return; + case ISC_R_SUCCESS: + return; + default: + UNREACHABLE(); + } +} + +static void +wait_for_priority_queue(isc__networker_t *worker) { + isc_condition_t *cond = &worker->ievents[NETIEVENT_PRIORITY].cond; + isc_mutex_t *lock = &worker->ievents[NETIEVENT_PRIORITY].lock; + isc__netievent_list_t *list = + &(worker->ievents[NETIEVENT_PRIORITY].list); + + LOCK(lock); + while (ISC_LIST_EMPTY(*list)) { + WAIT(cond, lock); + } + UNLOCK(lock); + + drain_queue(worker, NETIEVENT_PRIORITY); +} + +static void +drain_queue(isc__networker_t *worker, netievent_type_t type) { + bool empty = false; + while (!empty) { + if (process_queue(worker, type) == ISC_R_EMPTY) { + LOCK(&worker->ievents[type].lock); + empty = ISC_LIST_EMPTY(worker->ievents[type].list); + UNLOCK(&worker->ievents[type].lock); + } + } +} + +/* + * The two macros here generate the individual cases for the process_netievent() + * function. The NETIEVENT_CASE(type) macro is the common case, and + * NETIEVENT_CASE_NOMORE(type) is a macro that causes the loop in the + * process_queue() to stop, e.g. it's only used for the netievent that + * stops/pauses processing the enqueued netievents. + */ +#define NETIEVENT_CASE(type) \ + case netievent_##type: { \ + isc__nm_async_##type(worker, ievent); \ + isc__nm_put_netievent_##type( \ + worker->mgr, (isc__netievent_##type##_t *)ievent); \ + return (true); \ + } + +#define NETIEVENT_CASE_NOMORE(type) \ + case netievent_##type: { \ + isc__nm_async_##type(worker, ievent); \ + isc__nm_put_netievent_##type(worker->mgr, ievent); \ + return (false); \ + } + +static bool +process_netievent(isc__networker_t *worker, isc__netievent_t *ievent) { + REQUIRE(worker->id == isc_nm_tid()); + + switch (ievent->type) { + /* Don't process more ievents when we are stopping */ + NETIEVENT_CASE_NOMORE(stop); + + NETIEVENT_CASE(privilegedtask); + NETIEVENT_CASE(task); + + NETIEVENT_CASE(udpconnect); + NETIEVENT_CASE(udplisten); + NETIEVENT_CASE(udpstop); + NETIEVENT_CASE(udpsend); + NETIEVENT_CASE(udpread); + NETIEVENT_CASE(udpcancel); + NETIEVENT_CASE(udpclose); + + NETIEVENT_CASE(tcpaccept); + NETIEVENT_CASE(tcpconnect); + NETIEVENT_CASE(tcplisten); + NETIEVENT_CASE(tcpstartread); + NETIEVENT_CASE(tcppauseread); + NETIEVENT_CASE(tcpsend); + NETIEVENT_CASE(tcpstop); + NETIEVENT_CASE(tcpcancel); + NETIEVENT_CASE(tcpclose); + + NETIEVENT_CASE(tcpdnsaccept); + NETIEVENT_CASE(tcpdnslisten); + NETIEVENT_CASE(tcpdnsconnect); + NETIEVENT_CASE(tcpdnssend); + NETIEVENT_CASE(tcpdnscancel); + NETIEVENT_CASE(tcpdnsclose); + NETIEVENT_CASE(tcpdnsread); + NETIEVENT_CASE(tcpdnsstop); + + NETIEVENT_CASE(connectcb); + NETIEVENT_CASE(readcb); + NETIEVENT_CASE(sendcb); + + NETIEVENT_CASE(close); + NETIEVENT_CASE(detach); + + NETIEVENT_CASE(shutdown); + NETIEVENT_CASE(resume); + NETIEVENT_CASE_NOMORE(pause); + default: + UNREACHABLE(); + } + return (true); +} + +static isc_result_t +process_queue(isc__networker_t *worker, netievent_type_t type) { + isc__netievent_t *ievent = NULL; + isc__netievent_list_t list; + + ISC_LIST_INIT(list); + + LOCK(&worker->ievents[type].lock); + ISC_LIST_MOVE(list, worker->ievents[type].list); + UNLOCK(&worker->ievents[type].lock); + + ievent = ISC_LIST_HEAD(list); + if (ievent == NULL) { + /* There's nothing scheduled */ + return (ISC_R_EMPTY); + } + + while (ievent != NULL) { + isc__netievent_t *next = ISC_LIST_NEXT(ievent, link); + ISC_LIST_DEQUEUE(list, ievent, link); + + if (!process_netievent(worker, ievent)) { + /* The netievent told us to stop */ + if (!ISC_LIST_EMPTY(list)) { + /* + * Reschedule the rest of the unprocessed + * events. + */ + LOCK(&worker->ievents[type].lock); + ISC_LIST_PREPENDLIST(worker->ievents[type].list, + list, link); + UNLOCK(&worker->ievents[type].lock); + } + return (ISC_R_SUSPEND); + } + + ievent = next; + } + + /* We processed at least one */ + return (ISC_R_SUCCESS); +} + +void * +isc__nm_get_netievent(isc_nm_t *mgr, isc__netievent_type type) { + isc__netievent_storage_t *event = isc_mem_get(mgr->mctx, + sizeof(*event)); + + *event = (isc__netievent_storage_t){ .ni.type = type }; + ISC_LINK_INIT(&(event->ni), link); + return (event); +} + +void +isc__nm_put_netievent(isc_nm_t *mgr, void *ievent) { + isc_mem_put(mgr->mctx, ievent, sizeof(isc__netievent_storage_t)); +} + +NETIEVENT_SOCKET_DEF(tcpclose); +NETIEVENT_SOCKET_DEF(tcplisten); +NETIEVENT_SOCKET_DEF(tcppauseread); +NETIEVENT_SOCKET_DEF(tcpstartread); +NETIEVENT_SOCKET_DEF(tcpstop); +NETIEVENT_SOCKET_DEF(udpclose); +NETIEVENT_SOCKET_DEF(udplisten); +NETIEVENT_SOCKET_DEF(udpread); +NETIEVENT_SOCKET_DEF(udpsend); +NETIEVENT_SOCKET_DEF(udpstop); + +NETIEVENT_SOCKET_DEF(tcpdnsclose); +NETIEVENT_SOCKET_DEF(tcpdnsread); +NETIEVENT_SOCKET_DEF(tcpdnsstop); +NETIEVENT_SOCKET_DEF(tcpdnslisten); +NETIEVENT_SOCKET_REQ_DEF(tcpdnsconnect); +NETIEVENT_SOCKET_REQ_DEF(tcpdnssend); +NETIEVENT_SOCKET_HANDLE_DEF(tcpdnscancel); +NETIEVENT_SOCKET_QUOTA_DEF(tcpdnsaccept); + +NETIEVENT_SOCKET_REQ_DEF(tcpconnect); +NETIEVENT_SOCKET_REQ_DEF(tcpsend); +NETIEVENT_SOCKET_REQ_DEF(udpconnect); +NETIEVENT_SOCKET_REQ_RESULT_DEF(connectcb); +NETIEVENT_SOCKET_REQ_RESULT_DEF(readcb); +NETIEVENT_SOCKET_REQ_RESULT_DEF(sendcb); + +NETIEVENT_SOCKET_DEF(detach); +NETIEVENT_SOCKET_HANDLE_DEF(tcpcancel); +NETIEVENT_SOCKET_HANDLE_DEF(udpcancel); + +NETIEVENT_SOCKET_QUOTA_DEF(tcpaccept); + +NETIEVENT_SOCKET_DEF(close); +NETIEVENT_DEF(pause); +NETIEVENT_DEF(resume); +NETIEVENT_DEF(shutdown); +NETIEVENT_DEF(stop); + +NETIEVENT_TASK_DEF(task); +NETIEVENT_TASK_DEF(privilegedtask); + +void +isc__nm_maybe_enqueue_ievent(isc__networker_t *worker, + isc__netievent_t *event) { + /* + * If we are already in the matching nmthread, process the ievent + * directly. + */ + if (worker->id == isc_nm_tid()) { + process_netievent(worker, event); + return; + } + + isc__nm_enqueue_ievent(worker, event); +} + +void +isc__nm_enqueue_ievent(isc__networker_t *worker, isc__netievent_t *event) { + netievent_type_t type; + + if (event->type > netievent_prio) { + type = NETIEVENT_PRIORITY; + } else { + switch (event->type) { + case netievent_prio: + UNREACHABLE(); + break; + case netievent_privilegedtask: + type = NETIEVENT_PRIVILEGED; + break; + case netievent_task: + type = NETIEVENT_TASK; + break; + default: + type = NETIEVENT_NORMAL; + break; + } + } + + /* + * We need to make sure this signal will be delivered and + * the queue will be processed. + */ + LOCK(&worker->ievents[type].lock); + ISC_LIST_ENQUEUE(worker->ievents[type].list, event, link); + if (type == NETIEVENT_PRIORITY) { + SIGNAL(&worker->ievents[type].cond); + } + UNLOCK(&worker->ievents[type].lock); + + uv_async_send(&worker->async); +} + +bool +isc__nmsocket_active(isc_nmsocket_t *sock) { + REQUIRE(VALID_NMSOCK(sock)); + if (sock->parent != NULL) { + return (atomic_load(&sock->parent->active)); + } + + return (atomic_load(&sock->active)); +} + +bool +isc__nmsocket_deactivate(isc_nmsocket_t *sock) { + REQUIRE(VALID_NMSOCK(sock)); + + if (sock->parent != NULL) { + return (atomic_compare_exchange_strong(&sock->parent->active, + &(bool){ true }, false)); + } + + return (atomic_compare_exchange_strong(&sock->active, &(bool){ true }, + false)); +} + +void +isc___nmsocket_attach(isc_nmsocket_t *sock, isc_nmsocket_t **target FLARG) { + REQUIRE(VALID_NMSOCK(sock)); + REQUIRE(target != NULL && *target == NULL); + + isc_nmsocket_t *rsock = NULL; + + if (sock->parent != NULL) { + rsock = sock->parent; + INSIST(rsock->parent == NULL); /* sanity check */ + } else { + rsock = sock; + } + + NETMGR_TRACE_LOG("isc__nmsocket_attach():%p->references = %" PRIuFAST32 + "\n", + rsock, isc_refcount_current(&rsock->references) + 1); + + isc_refcount_increment0(&rsock->references); + + *target = sock; +} + +/* + * Free all resources inside a socket (including its children if any). + */ +static void +nmsocket_cleanup(isc_nmsocket_t *sock, bool dofree FLARG) { + isc_nmhandle_t *handle = NULL; + isc__nm_uvreq_t *uvreq = NULL; + + REQUIRE(VALID_NMSOCK(sock)); + REQUIRE(!isc__nmsocket_active(sock)); + + NETMGR_TRACE_LOG("nmsocket_cleanup():%p->references = %" PRIuFAST32 + "\n", + sock, isc_refcount_current(&sock->references)); + + atomic_store(&sock->destroying, true); + + if (sock->parent == NULL && sock->children != NULL) { + /* + * We shouldn't be here unless there are no active handles, + * so we can clean up and free the children. + */ + for (size_t i = 0; i < sock->nchildren; i++) { + if (!atomic_load(&sock->children[i].destroying)) { + nmsocket_cleanup(&sock->children[i], + false FLARG_PASS); + } + } + + /* + * This was a parent socket: destroy the listening + * barriers that synchronized the children. + */ + isc_barrier_destroy(&sock->startlistening); + isc_barrier_destroy(&sock->stoplistening); + + /* + * Now free them. + */ + isc_mem_put(sock->mgr->mctx, sock->children, + sock->nchildren * sizeof(*sock)); + sock->children = NULL; + sock->nchildren = 0; + } + if (sock->statsindex != NULL) { + isc__nm_decstats(sock->mgr, sock->statsindex[STATID_ACTIVE]); + } + + sock->statichandle = NULL; + + if (sock->outerhandle != NULL) { + isc__nmhandle_detach(&sock->outerhandle FLARG_PASS); + } + + if (sock->outer != NULL) { + isc___nmsocket_detach(&sock->outer FLARG_PASS); + } + + while ((handle = isc_astack_pop(sock->inactivehandles)) != NULL) { + nmhandle_free(sock, handle); + } + + if (sock->buf != NULL) { + isc_mem_free(sock->mgr->mctx, sock->buf); + } + + if (sock->quota != NULL) { + isc_quota_detach(&sock->quota); + } + + sock->pquota = NULL; + + isc_astack_destroy(sock->inactivehandles); + + while ((uvreq = isc_astack_pop(sock->inactivereqs)) != NULL) { + isc_mem_put(sock->mgr->mctx, uvreq, sizeof(*uvreq)); + } + + isc_astack_destroy(sock->inactivereqs); + sock->magic = 0; + + isc_condition_destroy(&sock->scond); + isc_condition_destroy(&sock->cond); + isc_mutex_destroy(&sock->lock); +#ifdef NETMGR_TRACE + LOCK(&sock->mgr->lock); + ISC_LIST_UNLINK(sock->mgr->active_sockets, sock, active_link); + UNLOCK(&sock->mgr->lock); +#endif + if (dofree) { + isc_nm_t *mgr = sock->mgr; + isc_mem_put(mgr->mctx, sock, sizeof(*sock)); + isc_nm_detach(&mgr); + } else { + isc_nm_detach(&sock->mgr); + } +} + +static void +nmsocket_maybe_destroy(isc_nmsocket_t *sock FLARG) { + int active_handles; + bool destroy = false; + + NETMGR_TRACE_LOG("%s():%p->references = %" PRIuFAST32 "\n", __func__, + sock, isc_refcount_current(&sock->references)); + + if (sock->parent != NULL) { + /* + * This is a child socket and cannot be destroyed except + * as a side effect of destroying the parent, so let's go + * see if the parent is ready to be destroyed. + */ + nmsocket_maybe_destroy(sock->parent FLARG_PASS); + return; + } + + /* + * This is a parent socket (or a standalone). See whether the + * children have active handles before deciding whether to + * accept destruction. + */ + LOCK(&sock->lock); + if (atomic_load(&sock->active) || atomic_load(&sock->destroying) || + !atomic_load(&sock->closed) || atomic_load(&sock->references) != 0) + { + UNLOCK(&sock->lock); + return; + } + + active_handles = atomic_load(&sock->ah); + if (sock->children != NULL) { + for (size_t i = 0; i < sock->nchildren; i++) { + LOCK(&sock->children[i].lock); + active_handles += atomic_load(&sock->children[i].ah); + UNLOCK(&sock->children[i].lock); + } + } + + if (active_handles == 0 || sock->statichandle != NULL) { + destroy = true; + } + + NETMGR_TRACE_LOG("%s:%p->active_handles = %d, .statichandle = %p\n", + __func__, sock, active_handles, sock->statichandle); + + if (destroy) { + atomic_store(&sock->destroying, true); + UNLOCK(&sock->lock); + nmsocket_cleanup(sock, true FLARG_PASS); + } else { + UNLOCK(&sock->lock); + } +} + +void +isc___nmsocket_prep_destroy(isc_nmsocket_t *sock FLARG) { + REQUIRE(sock->parent == NULL); + + NETMGR_TRACE_LOG("isc___nmsocket_prep_destroy():%p->references = " + "%" PRIuFAST32 "\n", + sock, isc_refcount_current(&sock->references)); + + /* + * The final external reference to the socket is gone. We can try + * destroying the socket, but we have to wait for all the inflight + * handles to finish first. + */ + atomic_store(&sock->active, false); + + /* + * If the socket has children, they'll need to be marked inactive + * so they can be cleaned up too. + */ + if (sock->children != NULL) { + for (size_t i = 0; i < sock->nchildren; i++) { + atomic_store(&sock->children[i].active, false); + } + } + + /* + * If we're here then we already stopped listening; otherwise + * we'd have a hanging reference from the listening process. + * + * If it's a regular socket we may need to close it. + */ + if (!atomic_load(&sock->closed)) { + switch (sock->type) { + case isc_nm_udpsocket: + isc__nm_udp_close(sock); + return; + case isc_nm_tcpsocket: + isc__nm_tcp_close(sock); + return; + case isc_nm_tcpdnssocket: + isc__nm_tcpdns_close(sock); + return; + default: + break; + } + } + + nmsocket_maybe_destroy(sock FLARG_PASS); +} + +void +isc___nmsocket_detach(isc_nmsocket_t **sockp FLARG) { + REQUIRE(sockp != NULL && *sockp != NULL); + REQUIRE(VALID_NMSOCK(*sockp)); + + isc_nmsocket_t *sock = *sockp, *rsock = NULL; + *sockp = NULL; + + /* + * If the socket is a part of a set (a child socket) we are + * counting references for the whole set at the parent. + */ + if (sock->parent != NULL) { + rsock = sock->parent; + INSIST(rsock->parent == NULL); /* Sanity check */ + } else { + rsock = sock; + } + + NETMGR_TRACE_LOG("isc__nmsocket_detach():%p->references = %" PRIuFAST32 + "\n", + rsock, isc_refcount_current(&rsock->references) - 1); + + if (isc_refcount_decrement(&rsock->references) == 1) { + isc___nmsocket_prep_destroy(rsock FLARG_PASS); + } +} + +void +isc_nmsocket_close(isc_nmsocket_t **sockp) { + REQUIRE(sockp != NULL); + REQUIRE(VALID_NMSOCK(*sockp)); + REQUIRE((*sockp)->type == isc_nm_udplistener || + (*sockp)->type == isc_nm_tcplistener || + (*sockp)->type == isc_nm_tcpdnslistener); + + isc__nmsocket_detach(sockp); +} + +void +isc___nmsocket_init(isc_nmsocket_t *sock, isc_nm_t *mgr, isc_nmsocket_type type, + isc_sockaddr_t *iface FLARG) { + uint16_t family; + + REQUIRE(sock != NULL); + REQUIRE(mgr != NULL); + REQUIRE(iface != NULL); + + family = iface->type.sa.sa_family; + + *sock = (isc_nmsocket_t){ .type = type, + .iface = *iface, + .fd = -1, + .inactivehandles = isc_astack_new( + mgr->mctx, ISC_NM_HANDLES_STACK_SIZE), + .inactivereqs = isc_astack_new( + mgr->mctx, ISC_NM_REQS_STACK_SIZE) }; + +#if NETMGR_TRACE + sock->backtrace_size = backtrace(sock->backtrace, TRACE_SIZE); + ISC_LINK_INIT(sock, active_link); + ISC_LIST_INIT(sock->active_handles); + LOCK(&mgr->lock); + ISC_LIST_APPEND(mgr->active_sockets, sock, active_link); + UNLOCK(&mgr->lock); +#endif + + isc_nm_attach(mgr, &sock->mgr); + sock->uv_handle.handle.data = sock; + + ISC_LINK_INIT(&sock->quotacb, link); + + switch (type) { + case isc_nm_udpsocket: + case isc_nm_udplistener: + if (family == AF_INET) { + sock->statsindex = udp4statsindex; + } else { + sock->statsindex = udp6statsindex; + } + isc__nm_incstats(sock->mgr, sock->statsindex[STATID_ACTIVE]); + break; + case isc_nm_tcpsocket: + case isc_nm_tcplistener: + case isc_nm_tcpdnssocket: + case isc_nm_tcpdnslistener: + if (family == AF_INET) { + sock->statsindex = tcp4statsindex; + } else { + sock->statsindex = tcp6statsindex; + } + isc__nm_incstats(sock->mgr, sock->statsindex[STATID_ACTIVE]); + break; + default: + break; + } + + isc_mutex_init(&sock->lock); + isc_condition_init(&sock->cond); + isc_condition_init(&sock->scond); + isc_refcount_init(&sock->references, 1); + + NETMGR_TRACE_LOG("isc__nmsocket_init():%p->references = %" PRIuFAST32 + "\n", + sock, isc_refcount_current(&sock->references)); + + atomic_init(&sock->active, true); + atomic_init(&sock->sequential, false); + atomic_init(&sock->readpaused, false); + atomic_init(&sock->closing, false); + atomic_init(&sock->listening, 0); + atomic_init(&sock->closed, 0); + atomic_init(&sock->destroying, 0); + atomic_init(&sock->ah, 0); + atomic_init(&sock->client, 0); + atomic_init(&sock->connecting, false); + atomic_init(&sock->keepalive, false); + atomic_init(&sock->connected, false); + atomic_init(&sock->timedout, false); + + atomic_init(&sock->active_child_connections, 0); + + sock->magic = NMSOCK_MAGIC; +} + +void +isc__nmsocket_clearcb(isc_nmsocket_t *sock) { + REQUIRE(VALID_NMSOCK(sock)); + REQUIRE(!isc__nm_in_netthread() || sock->tid == isc_nm_tid()); + + sock->recv_cb = NULL; + sock->recv_cbarg = NULL; + sock->accept_cb = NULL; + sock->accept_cbarg = NULL; + sock->connect_cb = NULL; + sock->connect_cbarg = NULL; +} + +void +isc__nm_free_uvbuf(isc_nmsocket_t *sock, const uv_buf_t *buf) { + isc__networker_t *worker = NULL; + + REQUIRE(VALID_NMSOCK(sock)); + + worker = &sock->mgr->workers[sock->tid]; + REQUIRE(buf->base == worker->recvbuf); + + worker->recvbuf_inuse = false; +} + +static isc_nmhandle_t * +alloc_handle(isc_nmsocket_t *sock) { + isc_nmhandle_t *handle = + isc_mem_get(sock->mgr->mctx, + sizeof(isc_nmhandle_t) + sock->extrahandlesize); + + *handle = (isc_nmhandle_t){ .magic = NMHANDLE_MAGIC }; +#ifdef NETMGR_TRACE + ISC_LINK_INIT(handle, active_link); +#endif + isc_refcount_init(&handle->references, 1); + + return (handle); +} + +isc_nmhandle_t * +isc___nmhandle_get(isc_nmsocket_t *sock, isc_sockaddr_t *peer, + isc_sockaddr_t *local FLARG) { + isc_nmhandle_t *handle = NULL; + + REQUIRE(VALID_NMSOCK(sock)); + + handle = isc_astack_pop(sock->inactivehandles); + + if (handle == NULL) { + handle = alloc_handle(sock); + } else { + isc_refcount_init(&handle->references, 1); + INSIST(VALID_NMHANDLE(handle)); + } + + NETMGR_TRACE_LOG( + "isc__nmhandle_get():handle %p->references = %" PRIuFAST32 "\n", + handle, isc_refcount_current(&handle->references)); + + isc___nmsocket_attach(sock, &handle->sock FLARG_PASS); + +#if NETMGR_TRACE + handle->backtrace_size = backtrace(handle->backtrace, TRACE_SIZE); +#endif + + if (peer != NULL) { + handle->peer = *peer; + } else { + handle->peer = sock->peer; + } + + if (local != NULL) { + handle->local = *local; + } else { + handle->local = sock->iface; + } + + (void)atomic_fetch_add(&sock->ah, 1); + +#ifdef NETMGR_TRACE + LOCK(&sock->lock); + ISC_LIST_APPEND(sock->active_handles, handle, active_link); + UNLOCK(&sock->lock); +#endif + + switch (sock->type) { + case isc_nm_udpsocket: + case isc_nm_tcpdnssocket: + if (!atomic_load(&sock->client)) { + break; + } + FALLTHROUGH; + case isc_nm_tcpsocket: + INSIST(sock->statichandle == NULL); + + /* + * statichandle must be assigned, not attached; + * otherwise, if a handle was detached elsewhere + * it could never reach 0 references, and the + * handle and socket would never be freed. + */ + sock->statichandle = handle; + break; + default: + break; + } + + return (handle); +} + +void +isc__nmhandle_attach(isc_nmhandle_t *handle, isc_nmhandle_t **handlep FLARG) { + REQUIRE(VALID_NMHANDLE(handle)); + REQUIRE(handlep != NULL && *handlep == NULL); + + NETMGR_TRACE_LOG("isc__nmhandle_attach():handle %p->references = " + "%" PRIuFAST32 "\n", + handle, isc_refcount_current(&handle->references) + 1); + + isc_refcount_increment(&handle->references); + *handlep = handle; +} + +bool +isc_nmhandle_is_stream(isc_nmhandle_t *handle) { + REQUIRE(VALID_NMHANDLE(handle)); + + return (handle->sock->type == isc_nm_tcpsocket || + handle->sock->type == isc_nm_tcpdnssocket); +} + +static void +nmhandle_free(isc_nmsocket_t *sock, isc_nmhandle_t *handle) { + size_t extra = sock->extrahandlesize; + + isc_refcount_destroy(&handle->references); + + if (handle->dofree != NULL) { + handle->dofree(handle->opaque); + } + + *handle = (isc_nmhandle_t){ .magic = 0 }; + + isc_mem_put(sock->mgr->mctx, handle, sizeof(isc_nmhandle_t) + extra); +} + +static void +nmhandle_deactivate(isc_nmsocket_t *sock, isc_nmhandle_t *handle) { + bool reuse = false; + + /* + * We do all of this under lock to avoid races with socket + * destruction. We have to do this now, because at this point the + * socket is either unused or still attached to event->sock. + */ + LOCK(&sock->lock); + +#ifdef NETMGR_TRACE + ISC_LIST_UNLINK(sock->active_handles, handle, active_link); +#endif + + INSIST(atomic_fetch_sub(&sock->ah, 1) > 0); + +#if !__SANITIZE_ADDRESS__ && !__SANITIZE_THREAD__ + if (atomic_load(&sock->active)) { + reuse = isc_astack_trypush(sock->inactivehandles, handle); + } +#endif /* !__SANITIZE_ADDRESS__ && !__SANITIZE_THREAD__ */ + if (!reuse) { + nmhandle_free(sock, handle); + } + UNLOCK(&sock->lock); +} + +void +isc__nmhandle_detach(isc_nmhandle_t **handlep FLARG) { + isc_nmsocket_t *sock = NULL; + isc_nmhandle_t *handle = NULL; + + REQUIRE(handlep != NULL); + REQUIRE(VALID_NMHANDLE(*handlep)); + + handle = *handlep; + *handlep = NULL; + + /* + * If the closehandle_cb is set, it needs to run asynchronously to + * ensure correct ordering of the isc__nm_process_sock_buffer(). + */ + sock = handle->sock; + if (sock->tid == isc_nm_tid() && sock->closehandle_cb == NULL) { + nmhandle_detach_cb(&handle FLARG_PASS); + } else { + isc__netievent_detach_t *event = + isc__nm_get_netievent_detach(sock->mgr, sock); + /* + * we are using implicit "attach" as the last reference + * need to be destroyed explicitly in the async callback + */ + event->handle = handle; + FLARG_IEVENT_PASS(event); + isc__nm_enqueue_ievent(&sock->mgr->workers[sock->tid], + (isc__netievent_t *)event); + } +} + +void +isc__nmsocket_shutdown(isc_nmsocket_t *sock); + +static void +nmhandle_detach_cb(isc_nmhandle_t **handlep FLARG) { + isc_nmsocket_t *sock = NULL; + isc_nmhandle_t *handle = NULL; + + REQUIRE(handlep != NULL); + REQUIRE(VALID_NMHANDLE(*handlep)); + + handle = *handlep; + *handlep = NULL; + + NETMGR_TRACE_LOG("isc__nmhandle_detach():%p->references = %" PRIuFAST32 + "\n", + handle, isc_refcount_current(&handle->references) - 1); + + if (isc_refcount_decrement(&handle->references) > 1) { + return; + } + + /* We need an acquire memory barrier here */ + (void)isc_refcount_current(&handle->references); + + sock = handle->sock; + handle->sock = NULL; + + if (handle->doreset != NULL) { + handle->doreset(handle->opaque); + } + + nmhandle_deactivate(sock, handle); + + /* + * The handle is gone now. If the socket has a callback configured + * for that (e.g., to perform cleanup after request processing), + * call it now, or schedule it to run asynchronously. + */ + if (sock->closehandle_cb != NULL) { + if (sock->tid == isc_nm_tid()) { + sock->closehandle_cb(sock); + } else { + isc__netievent_close_t *event = + isc__nm_get_netievent_close(sock->mgr, sock); + isc__nm_enqueue_ievent(&sock->mgr->workers[sock->tid], + (isc__netievent_t *)event); + } + } + + if (handle == sock->statichandle) { + /* statichandle is assigned, not attached. */ + sock->statichandle = NULL; + } + + isc___nmsocket_detach(&sock FLARG_PASS); +} + +void * +isc_nmhandle_getdata(isc_nmhandle_t *handle) { + REQUIRE(VALID_NMHANDLE(handle)); + + return (handle->opaque); +} + +void +isc_nmhandle_setdata(isc_nmhandle_t *handle, void *arg, + isc_nm_opaquecb_t doreset, isc_nm_opaquecb_t dofree) { + REQUIRE(VALID_NMHANDLE(handle)); + + handle->opaque = arg; + handle->doreset = doreset; + handle->dofree = dofree; +} + +void +isc__nm_alloc_dnsbuf(isc_nmsocket_t *sock, size_t len) { + REQUIRE(len <= NM_BIG_BUF); + + if (sock->buf == NULL) { + /* We don't have the buffer at all */ + size_t alloc_len = len < NM_REG_BUF ? NM_REG_BUF : NM_BIG_BUF; + sock->buf = isc_mem_allocate(sock->mgr->mctx, alloc_len); + sock->buf_size = alloc_len; + } else { + /* We have the buffer but it's too small */ + sock->buf = isc_mem_reallocate(sock->mgr->mctx, sock->buf, + NM_BIG_BUF); + sock->buf_size = NM_BIG_BUF; + } +} + +void +isc__nm_failed_send_cb(isc_nmsocket_t *sock, isc__nm_uvreq_t *req, + isc_result_t eresult) { + REQUIRE(VALID_NMSOCK(sock)); + REQUIRE(VALID_UVREQ(req)); + + if (req->cb.send != NULL) { + isc__nm_sendcb(sock, req, eresult, true); + } else { + isc__nm_uvreq_put(&req, sock); + } +} + +void +isc__nm_failed_accept_cb(isc_nmsocket_t *sock, isc_result_t eresult) { + REQUIRE(sock->accepting); + REQUIRE(sock->server); + + /* + * Detach the quota early to make room for other connections; + * otherwise it'd be detached later asynchronously, and clog + * the quota unnecessarily. + */ + if (sock->quota != NULL) { + isc_quota_detach(&sock->quota); + } + + isc__nmsocket_detach(&sock->server); + + sock->accepting = false; + + switch (eresult) { + case ISC_R_NOTCONNECTED: + /* IGNORE: The client disconnected before we could accept */ + break; + default: + isc_log_write(isc_lctx, ISC_LOGCATEGORY_GENERAL, + ISC_LOGMODULE_NETMGR, ISC_LOG_ERROR, + "Accepting TCP connection failed: %s", + isc_result_totext(eresult)); + } +} + +void +isc__nm_failed_connect_cb(isc_nmsocket_t *sock, isc__nm_uvreq_t *req, + isc_result_t eresult, bool async) { + REQUIRE(VALID_NMSOCK(sock)); + REQUIRE(VALID_UVREQ(req)); + REQUIRE(sock->tid == isc_nm_tid()); + REQUIRE(req->cb.connect != NULL); + + isc__nmsocket_timer_stop(sock); + uv_handle_set_data((uv_handle_t *)&sock->read_timer, sock); + + INSIST(atomic_compare_exchange_strong(&sock->connecting, + &(bool){ true }, false)); + + isc__nmsocket_clearcb(sock); + isc__nm_connectcb(sock, req, eresult, async); + + isc__nmsocket_prep_destroy(sock); +} + +void +isc__nm_failed_read_cb(isc_nmsocket_t *sock, isc_result_t result, bool async) { + REQUIRE(VALID_NMSOCK(sock)); + UNUSED(async); + + switch (sock->type) { + case isc_nm_udpsocket: + isc__nm_udp_failed_read_cb(sock, result); + return; + case isc_nm_tcpsocket: + isc__nm_tcp_failed_read_cb(sock, result); + return; + case isc_nm_tcpdnssocket: + isc__nm_tcpdns_failed_read_cb(sock, result); + return; + default: + UNREACHABLE(); + } +} + +void +isc__nmsocket_connecttimeout_cb(uv_timer_t *timer) { + uv_connect_t *uvreq = uv_handle_get_data((uv_handle_t *)timer); + isc_nmsocket_t *sock = uv_handle_get_data((uv_handle_t *)uvreq->handle); + isc__nm_uvreq_t *req = uv_handle_get_data((uv_handle_t *)uvreq); + + REQUIRE(VALID_NMSOCK(sock)); + REQUIRE(sock->tid == isc_nm_tid()); + REQUIRE(atomic_load(&sock->connecting)); + REQUIRE(VALID_UVREQ(req)); + REQUIRE(VALID_NMHANDLE(req->handle)); + + isc__nmsocket_timer_stop(sock); + + /* + * Mark the connection as timed out and shutdown the socket. + */ + + INSIST(atomic_compare_exchange_strong(&sock->timedout, &(bool){ false }, + true)); + isc__nmsocket_clearcb(sock); + isc__nmsocket_shutdown(sock); +} + +void +isc__nm_accept_connection_log(isc_result_t result, bool can_log_quota) { + int level; + + switch (result) { + case ISC_R_SUCCESS: + case ISC_R_NOCONN: + return; + case ISC_R_QUOTA: + case ISC_R_SOFTQUOTA: + if (!can_log_quota) { + return; + } + level = ISC_LOG_INFO; + break; + case ISC_R_NOTCONNECTED: + level = ISC_LOG_INFO; + break; + default: + level = ISC_LOG_ERROR; + } + + isc_log_write(isc_lctx, ISC_LOGCATEGORY_GENERAL, ISC_LOGMODULE_NETMGR, + level, "Accepting TCP connection failed: %s", + isc_result_totext(result)); +} + +void +isc__nmsocket_writetimeout_cb(void *data, isc_result_t eresult) { + isc__nm_uvreq_t *req = data; + isc_nmsocket_t *sock = NULL; + + REQUIRE(eresult == ISC_R_TIMEDOUT); + REQUIRE(VALID_UVREQ(req)); + REQUIRE(VALID_NMSOCK(req->sock)); + + sock = req->sock; + + isc__nmsocket_reset(sock); +} + +void +isc__nmsocket_readtimeout_cb(uv_timer_t *timer) { + isc_nmsocket_t *sock = uv_handle_get_data((uv_handle_t *)timer); + + REQUIRE(VALID_NMSOCK(sock)); + REQUIRE(sock->tid == isc_nm_tid()); + REQUIRE(sock->reading); + + if (atomic_load(&sock->client)) { + uv_timer_stop(timer); + + if (sock->recv_cb != NULL) { + isc__nm_uvreq_t *req = isc__nm_get_read_req(sock, NULL); + isc__nm_readcb(sock, req, ISC_R_TIMEDOUT); + } + + if (!isc__nmsocket_timer_running(sock)) { + isc__nmsocket_clearcb(sock); + isc__nm_failed_read_cb(sock, ISC_R_CANCELED, false); + } + } else { + isc__nm_failed_read_cb(sock, ISC_R_TIMEDOUT, false); + } +} + +void +isc__nmsocket_timer_restart(isc_nmsocket_t *sock) { + REQUIRE(VALID_NMSOCK(sock)); + + if (atomic_load(&sock->connecting)) { + int r; + + if (sock->connect_timeout == 0) { + return; + } + + r = uv_timer_start(&sock->read_timer, + isc__nmsocket_connecttimeout_cb, + sock->connect_timeout + 10, 0); + UV_RUNTIME_CHECK(uv_timer_start, r); + + } else { + int r; + + if (sock->read_timeout == 0) { + return; + } + + r = uv_timer_start(&sock->read_timer, + isc__nmsocket_readtimeout_cb, + sock->read_timeout, 0); + UV_RUNTIME_CHECK(uv_timer_start, r); + } +} + +bool +isc__nmsocket_timer_running(isc_nmsocket_t *sock) { + REQUIRE(VALID_NMSOCK(sock)); + + return (uv_is_active((uv_handle_t *)&sock->read_timer)); +} + +void +isc__nmsocket_timer_start(isc_nmsocket_t *sock) { + REQUIRE(VALID_NMSOCK(sock)); + + if (isc__nmsocket_timer_running(sock)) { + return; + } + + isc__nmsocket_timer_restart(sock); +} + +void +isc__nmsocket_timer_stop(isc_nmsocket_t *sock) { + int r; + + REQUIRE(VALID_NMSOCK(sock)); + + /* uv_timer_stop() is idempotent, no need to check if running */ + + r = uv_timer_stop(&sock->read_timer); + UV_RUNTIME_CHECK(uv_timer_stop, r); +} + +isc__nm_uvreq_t * +isc__nm_get_read_req(isc_nmsocket_t *sock, isc_sockaddr_t *sockaddr) { + isc__nm_uvreq_t *req = NULL; + + req = isc__nm_uvreq_get(sock->mgr, sock); + req->cb.recv = sock->recv_cb; + req->cbarg = sock->recv_cbarg; + + switch (sock->type) { + case isc_nm_tcpsocket: + isc_nmhandle_attach(sock->statichandle, &req->handle); + break; + default: + if (atomic_load(&sock->client)) { + isc_nmhandle_attach(sock->statichandle, &req->handle); + } else { + req->handle = isc__nmhandle_get(sock, sockaddr, NULL); + } + break; + } + + return (req); +} + +/*%< + * Allocator callback for read operations. + * + * Note this doesn't actually allocate anything, it just assigns the + * worker's receive buffer to a socket, and marks it as "in use". + */ +void +isc__nm_alloc_cb(uv_handle_t *handle, size_t size, uv_buf_t *buf) { + isc_nmsocket_t *sock = uv_handle_get_data(handle); + isc__networker_t *worker = NULL; + + REQUIRE(VALID_NMSOCK(sock)); + REQUIRE(isc__nm_in_netthread()); + /* + * The size provided by libuv is only suggested size, and it always + * defaults to 64 * 1024 in the current versions of libuv (see + * src/unix/udp.c and src/unix/stream.c). + */ + UNUSED(size); + + worker = &sock->mgr->workers[sock->tid]; + INSIST(!worker->recvbuf_inuse); + INSIST(worker->recvbuf != NULL); + + switch (sock->type) { + case isc_nm_udpsocket: + buf->len = ISC_NETMGR_UDP_RECVBUF_SIZE; + break; + case isc_nm_tcpsocket: + case isc_nm_tcpdnssocket: + buf->len = ISC_NETMGR_TCP_RECVBUF_SIZE; + break; + default: + UNREACHABLE(); + } + + REQUIRE(buf->len <= ISC_NETMGR_RECVBUF_SIZE); + buf->base = worker->recvbuf; + + worker->recvbuf_inuse = true; +} + +isc_result_t +isc__nm_start_reading(isc_nmsocket_t *sock) { + isc_result_t result = ISC_R_SUCCESS; + int r; + + if (sock->reading) { + return (ISC_R_SUCCESS); + } + + switch (sock->type) { + case isc_nm_udpsocket: + r = uv_udp_recv_start(&sock->uv_handle.udp, isc__nm_alloc_cb, + isc__nm_udp_read_cb); + break; + case isc_nm_tcpsocket: + r = uv_read_start(&sock->uv_handle.stream, isc__nm_alloc_cb, + isc__nm_tcp_read_cb); + break; + case isc_nm_tcpdnssocket: + r = uv_read_start(&sock->uv_handle.stream, isc__nm_alloc_cb, + isc__nm_tcpdns_read_cb); + break; + default: + UNREACHABLE(); + } + + if (r != 0) { + result = isc__nm_uverr2result(r); + } else { + sock->reading = true; + } + + return (result); +} + +void +isc__nm_stop_reading(isc_nmsocket_t *sock) { + int r; + + if (!sock->reading) { + return; + } + + switch (sock->type) { + case isc_nm_udpsocket: + r = uv_udp_recv_stop(&sock->uv_handle.udp); + UV_RUNTIME_CHECK(uv_udp_recv_stop, r); + break; + case isc_nm_tcpsocket: + case isc_nm_tcpdnssocket: + r = uv_read_stop(&sock->uv_handle.stream); + UV_RUNTIME_CHECK(uv_read_stop, r); + break; + default: + UNREACHABLE(); + } + sock->reading = false; +} + +bool +isc__nm_closing(isc_nmsocket_t *sock) { + return (atomic_load(&sock->mgr->closing)); +} + +bool +isc__nmsocket_closing(isc_nmsocket_t *sock) { + return (!isc__nmsocket_active(sock) || atomic_load(&sock->closing) || + atomic_load(&sock->mgr->closing) || + (sock->server != NULL && !isc__nmsocket_active(sock->server))); +} + +static isc_result_t +processbuffer(isc_nmsocket_t *sock) { + switch (sock->type) { + case isc_nm_tcpdnssocket: + return (isc__nm_tcpdns_processbuffer(sock)); + default: + UNREACHABLE(); + } +} + +/* + * Process a DNS message. + * + * If we only have an incomplete DNS message, we don't touch any + * timers. If we do have a full message, reset the timer. + * + * Stop reading if this is a client socket, or if the server socket + * has been set to sequential mode, or the number of queries we are + * processing simultaneously has reached the clients-per-connection + * limit. In this case we'll be called again by resume_processing() + * later. + */ +isc_result_t +isc__nm_process_sock_buffer(isc_nmsocket_t *sock) { + for (;;) { + int_fast32_t ah = atomic_load(&sock->ah); + isc_result_t result = processbuffer(sock); + switch (result) { + case ISC_R_NOMORE: + /* + * Don't reset the timer until we have a + * full DNS message. + */ + result = isc__nm_start_reading(sock); + if (result != ISC_R_SUCCESS) { + return (result); + } + /* + * Start the timer only if there are no externally used + * active handles, there's always one active handle + * attached internally to sock->recv_handle in + * accept_connection() + */ + if (ah == 1) { + isc__nmsocket_timer_start(sock); + } + goto done; + case ISC_R_CANCELED: + isc__nmsocket_timer_stop(sock); + isc__nm_stop_reading(sock); + goto done; + case ISC_R_SUCCESS: + /* + * Stop the timer on the successful message read, this + * also allows to restart the timer when we have no more + * data. + */ + isc__nmsocket_timer_stop(sock); + + if (atomic_load(&sock->client) || + atomic_load(&sock->sequential) || + ah >= STREAM_CLIENTS_PER_CONN) + { + isc__nm_stop_reading(sock); + goto done; + } + break; + default: + UNREACHABLE(); + } + } +done: + return (ISC_R_SUCCESS); +} + +void +isc__nm_resume_processing(void *arg) { + isc_nmsocket_t *sock = (isc_nmsocket_t *)arg; + + REQUIRE(VALID_NMSOCK(sock)); + REQUIRE(sock->tid == isc_nm_tid()); + REQUIRE(!atomic_load(&sock->client)); + + if (isc__nmsocket_closing(sock)) { + return; + } + + isc__nm_process_sock_buffer(sock); +} + +void +isc_nmhandle_cleartimeout(isc_nmhandle_t *handle) { + REQUIRE(VALID_NMHANDLE(handle)); + REQUIRE(VALID_NMSOCK(handle->sock)); + + switch (handle->sock->type) { + default: + handle->sock->read_timeout = 0; + + if (uv_is_active((uv_handle_t *)&handle->sock->read_timer)) { + isc__nmsocket_timer_stop(handle->sock); + } + } +} + +void +isc_nmhandle_settimeout(isc_nmhandle_t *handle, uint32_t timeout) { + REQUIRE(VALID_NMHANDLE(handle)); + REQUIRE(VALID_NMSOCK(handle->sock)); + + switch (handle->sock->type) { + default: + handle->sock->read_timeout = timeout; + isc__nmsocket_timer_restart(handle->sock); + } +} + +void +isc_nmhandle_keepalive(isc_nmhandle_t *handle, bool value) { + isc_nmsocket_t *sock = NULL; + + REQUIRE(VALID_NMHANDLE(handle)); + REQUIRE(VALID_NMSOCK(handle->sock)); + + sock = handle->sock; + + switch (sock->type) { + case isc_nm_tcpsocket: + case isc_nm_tcpdnssocket: + atomic_store(&sock->keepalive, value); + sock->read_timeout = value ? atomic_load(&sock->mgr->keepalive) + : atomic_load(&sock->mgr->idle); + sock->write_timeout = value ? atomic_load(&sock->mgr->keepalive) + : atomic_load(&sock->mgr->idle); + break; + default: + /* + * For any other protocol, this is a no-op. + */ + return; + } +} + +void * +isc_nmhandle_getextra(isc_nmhandle_t *handle) { + REQUIRE(VALID_NMHANDLE(handle)); + + return (handle->extra); +} + +isc_sockaddr_t +isc_nmhandle_peeraddr(isc_nmhandle_t *handle) { + REQUIRE(VALID_NMHANDLE(handle)); + + return (handle->peer); +} + +isc_sockaddr_t +isc_nmhandle_localaddr(isc_nmhandle_t *handle) { + REQUIRE(VALID_NMHANDLE(handle)); + + return (handle->local); +} + +isc_nm_t * +isc_nmhandle_netmgr(isc_nmhandle_t *handle) { + REQUIRE(VALID_NMHANDLE(handle)); + REQUIRE(VALID_NMSOCK(handle->sock)); + + return (handle->sock->mgr); +} + +isc__nm_uvreq_t * +isc___nm_uvreq_get(isc_nm_t *mgr, isc_nmsocket_t *sock FLARG) { + isc__nm_uvreq_t *req = NULL; + + REQUIRE(VALID_NM(mgr)); + REQUIRE(VALID_NMSOCK(sock)); + + if (sock != NULL && isc__nmsocket_active(sock)) { + /* Try to reuse one */ + req = isc_astack_pop(sock->inactivereqs); + } + + if (req == NULL) { + req = isc_mem_get(mgr->mctx, sizeof(*req)); + } + + *req = (isc__nm_uvreq_t){ .magic = 0 }; + ISC_LINK_INIT(req, link); + req->uv_req.req.data = req; + isc___nmsocket_attach(sock, &req->sock FLARG_PASS); + req->magic = UVREQ_MAGIC; + + return (req); +} + +void +isc___nm_uvreq_put(isc__nm_uvreq_t **req0, isc_nmsocket_t *sock FLARG) { + isc__nm_uvreq_t *req = NULL; + isc_nmhandle_t *handle = NULL; + + REQUIRE(req0 != NULL); + REQUIRE(VALID_UVREQ(*req0)); + + req = *req0; + *req0 = NULL; + + INSIST(sock == req->sock); + + req->magic = 0; + + /* + * We need to save this first to make sure that handle, + * sock, and the netmgr won't all disappear. + */ + handle = req->handle; + req->handle = NULL; + +#if !__SANITIZE_ADDRESS__ && !__SANITIZE_THREAD__ + if (!isc__nmsocket_active(sock) || + !isc_astack_trypush(sock->inactivereqs, req)) + { + isc_mem_put(sock->mgr->mctx, req, sizeof(*req)); + } +#else /* !__SANITIZE_ADDRESS__ && !__SANITIZE_THREAD__ */ + isc_mem_put(sock->mgr->mctx, req, sizeof(*req)); +#endif /* !__SANITIZE_ADDRESS__ && !__SANITIZE_THREAD__ */ + + if (handle != NULL) { + isc__nmhandle_detach(&handle FLARG_PASS); + } + + isc___nmsocket_detach(&sock FLARG_PASS); +} + +void +isc_nm_send(isc_nmhandle_t *handle, isc_region_t *region, isc_nm_cb_t cb, + void *cbarg) { + REQUIRE(VALID_NMHANDLE(handle)); + + switch (handle->sock->type) { + case isc_nm_udpsocket: + case isc_nm_udplistener: + isc__nm_udp_send(handle, region, cb, cbarg); + break; + case isc_nm_tcpsocket: + isc__nm_tcp_send(handle, region, cb, cbarg); + break; + case isc_nm_tcpdnssocket: + isc__nm_tcpdns_send(handle, region, cb, cbarg); + break; + default: + UNREACHABLE(); + } +} + +void +isc_nm_read(isc_nmhandle_t *handle, isc_nm_recv_cb_t cb, void *cbarg) { + REQUIRE(VALID_NMHANDLE(handle)); + + /* + * This is always called via callback (from accept or connect), and + * caller must attach to the handle, so the references always need to be + * at least 2. + */ + REQUIRE(isc_refcount_current(&handle->references) >= 2); + + switch (handle->sock->type) { + case isc_nm_udpsocket: + isc__nm_udp_read(handle, cb, cbarg); + break; + case isc_nm_tcpsocket: + isc__nm_tcp_read(handle, cb, cbarg); + break; + case isc_nm_tcpdnssocket: + isc__nm_tcpdns_read(handle, cb, cbarg); + break; + default: + UNREACHABLE(); + } +} + +void +isc_nm_cancelread(isc_nmhandle_t *handle) { + REQUIRE(VALID_NMHANDLE(handle)); + + switch (handle->sock->type) { + case isc_nm_udpsocket: + isc__nm_udp_cancelread(handle); + break; + case isc_nm_tcpsocket: + isc__nm_tcp_cancelread(handle); + break; + case isc_nm_tcpdnssocket: + isc__nm_tcpdns_cancelread(handle); + break; + default: + UNREACHABLE(); + } +} + +void +isc_nm_pauseread(isc_nmhandle_t *handle) { + REQUIRE(VALID_NMHANDLE(handle)); + + isc_nmsocket_t *sock = handle->sock; + + switch (sock->type) { + case isc_nm_tcpsocket: + isc__nm_tcp_pauseread(handle); + break; + default: + UNREACHABLE(); + } +} + +void +isc_nm_resumeread(isc_nmhandle_t *handle) { + REQUIRE(VALID_NMHANDLE(handle)); + + isc_nmsocket_t *sock = handle->sock; + + switch (sock->type) { + case isc_nm_tcpsocket: + isc__nm_tcp_resumeread(handle); + break; + default: + UNREACHABLE(); + } +} + +void +isc_nm_stoplistening(isc_nmsocket_t *sock) { + REQUIRE(VALID_NMSOCK(sock)); + + switch (sock->type) { + case isc_nm_udplistener: + isc__nm_udp_stoplistening(sock); + break; + case isc_nm_tcpdnslistener: + isc__nm_tcpdns_stoplistening(sock); + break; + case isc_nm_tcplistener: + isc__nm_tcp_stoplistening(sock); + break; + default: + UNREACHABLE(); + } +} + +void +isc__nm_connectcb(isc_nmsocket_t *sock, isc__nm_uvreq_t *uvreq, + isc_result_t eresult, bool async) { + REQUIRE(VALID_NMSOCK(sock)); + REQUIRE(VALID_UVREQ(uvreq)); + REQUIRE(VALID_NMHANDLE(uvreq->handle)); + + if (!async) { + isc__netievent_connectcb_t ievent = { .sock = sock, + .req = uvreq, + .result = eresult }; + isc__nm_async_connectcb(NULL, (isc__netievent_t *)&ievent); + } else { + isc__netievent_connectcb_t *ievent = + isc__nm_get_netievent_connectcb(sock->mgr, sock, uvreq, + eresult); + isc__nm_enqueue_ievent(&sock->mgr->workers[sock->tid], + (isc__netievent_t *)ievent); + } +} + +void +isc__nm_async_connectcb(isc__networker_t *worker, isc__netievent_t *ev0) { + isc__netievent_connectcb_t *ievent = (isc__netievent_connectcb_t *)ev0; + isc_nmsocket_t *sock = ievent->sock; + isc__nm_uvreq_t *uvreq = ievent->req; + isc_result_t eresult = ievent->result; + + UNUSED(worker); + + REQUIRE(VALID_NMSOCK(sock)); + REQUIRE(VALID_UVREQ(uvreq)); + REQUIRE(VALID_NMHANDLE(uvreq->handle)); + REQUIRE(ievent->sock->tid == isc_nm_tid()); + REQUIRE(uvreq->cb.connect != NULL); + + uvreq->cb.connect(uvreq->handle, eresult, uvreq->cbarg); + + isc__nm_uvreq_put(&uvreq, sock); +} + +void +isc__nm_readcb(isc_nmsocket_t *sock, isc__nm_uvreq_t *uvreq, + isc_result_t eresult) { + REQUIRE(VALID_NMSOCK(sock)); + REQUIRE(VALID_UVREQ(uvreq)); + REQUIRE(VALID_NMHANDLE(uvreq->handle)); + + if (eresult == ISC_R_SUCCESS || eresult == ISC_R_TIMEDOUT) { + isc__netievent_readcb_t ievent = { .sock = sock, + .req = uvreq, + .result = eresult }; + + isc__nm_async_readcb(NULL, (isc__netievent_t *)&ievent); + } else { + isc__netievent_readcb_t *ievent = isc__nm_get_netievent_readcb( + sock->mgr, sock, uvreq, eresult); + isc__nm_enqueue_ievent(&sock->mgr->workers[sock->tid], + (isc__netievent_t *)ievent); + } +} + +void +isc__nm_async_readcb(isc__networker_t *worker, isc__netievent_t *ev0) { + isc__netievent_readcb_t *ievent = (isc__netievent_readcb_t *)ev0; + isc_nmsocket_t *sock = ievent->sock; + isc__nm_uvreq_t *uvreq = ievent->req; + isc_result_t eresult = ievent->result; + isc_region_t region; + + UNUSED(worker); + + REQUIRE(VALID_NMSOCK(sock)); + REQUIRE(VALID_UVREQ(uvreq)); + REQUIRE(VALID_NMHANDLE(uvreq->handle)); + REQUIRE(sock->tid == isc_nm_tid()); + + region.base = (unsigned char *)uvreq->uvbuf.base; + region.length = uvreq->uvbuf.len; + + uvreq->cb.recv(uvreq->handle, eresult, ®ion, uvreq->cbarg); + + isc__nm_uvreq_put(&uvreq, sock); +} + +void +isc__nm_sendcb(isc_nmsocket_t *sock, isc__nm_uvreq_t *uvreq, + isc_result_t eresult, bool async) { + REQUIRE(VALID_NMSOCK(sock)); + REQUIRE(VALID_UVREQ(uvreq)); + REQUIRE(VALID_NMHANDLE(uvreq->handle)); + + if (!async) { + isc__netievent_sendcb_t ievent = { .sock = sock, + .req = uvreq, + .result = eresult }; + isc__nm_async_sendcb(NULL, (isc__netievent_t *)&ievent); + return; + } + + isc__netievent_sendcb_t *ievent = + isc__nm_get_netievent_sendcb(sock->mgr, sock, uvreq, eresult); + isc__nm_enqueue_ievent(&sock->mgr->workers[sock->tid], + (isc__netievent_t *)ievent); +} + +void +isc__nm_async_sendcb(isc__networker_t *worker, isc__netievent_t *ev0) { + isc__netievent_sendcb_t *ievent = (isc__netievent_sendcb_t *)ev0; + isc_nmsocket_t *sock = ievent->sock; + isc__nm_uvreq_t *uvreq = ievent->req; + isc_result_t eresult = ievent->result; + + UNUSED(worker); + + REQUIRE(VALID_NMSOCK(sock)); + REQUIRE(VALID_UVREQ(uvreq)); + REQUIRE(VALID_NMHANDLE(uvreq->handle)); + REQUIRE(sock->tid == isc_nm_tid()); + + uvreq->cb.send(uvreq->handle, eresult, uvreq->cbarg); + + isc__nm_uvreq_put(&uvreq, sock); +} + +static void +isc__nm_async_close(isc__networker_t *worker, isc__netievent_t *ev0) { + isc__netievent_close_t *ievent = (isc__netievent_close_t *)ev0; + isc_nmsocket_t *sock = ievent->sock; + + REQUIRE(VALID_NMSOCK(ievent->sock)); + REQUIRE(sock->tid == isc_nm_tid()); + REQUIRE(sock->closehandle_cb != NULL); + + UNUSED(worker); + + ievent->sock->closehandle_cb(sock); +} + +void +isc__nm_async_detach(isc__networker_t *worker, isc__netievent_t *ev0) { + isc__netievent_detach_t *ievent = (isc__netievent_detach_t *)ev0; + FLARG_IEVENT(ievent); + + REQUIRE(VALID_NMSOCK(ievent->sock)); + REQUIRE(VALID_NMHANDLE(ievent->handle)); + REQUIRE(ievent->sock->tid == isc_nm_tid()); + + UNUSED(worker); + + nmhandle_detach_cb(&ievent->handle FLARG_PASS); +} + +static void +reset_shutdown(uv_handle_t *handle) { + isc_nmsocket_t *sock = uv_handle_get_data(handle); + + isc__nmsocket_shutdown(sock); + isc__nmsocket_detach(&sock); +} + +void +isc__nmsocket_reset(isc_nmsocket_t *sock) { + REQUIRE(VALID_NMSOCK(sock)); + + switch (sock->type) { + case isc_nm_tcpsocket: + case isc_nm_tcpdnssocket: + /* + * This can be called from the TCP write timeout. + */ + REQUIRE(sock->parent == NULL); + break; + default: + UNREACHABLE(); + break; + } + + if (!uv_is_closing(&sock->uv_handle.handle) && + uv_is_active(&sock->uv_handle.handle)) + { + /* + * The real shutdown will be handled in the respective + * close functions. + */ + isc__nmsocket_attach(sock, &(isc_nmsocket_t *){ NULL }); + int r = uv_tcp_close_reset(&sock->uv_handle.tcp, + reset_shutdown); + UV_RUNTIME_CHECK(uv_tcp_close_reset, r); + } else { + isc__nmsocket_shutdown(sock); + } +} + +void +isc__nmsocket_shutdown(isc_nmsocket_t *sock) { + REQUIRE(VALID_NMSOCK(sock)); + switch (sock->type) { + case isc_nm_udpsocket: + isc__nm_udp_shutdown(sock); + break; + case isc_nm_tcpsocket: + isc__nm_tcp_shutdown(sock); + break; + case isc_nm_tcpdnssocket: + isc__nm_tcpdns_shutdown(sock); + break; + case isc_nm_udplistener: + case isc_nm_tcplistener: + case isc_nm_tcpdnslistener: + return; + default: + UNREACHABLE(); + } +} + +static void +shutdown_walk_cb(uv_handle_t *handle, void *arg) { + isc_nmsocket_t *sock = uv_handle_get_data(handle); + UNUSED(arg); + + if (uv_is_closing(handle)) { + return; + } + + switch (handle->type) { + case UV_UDP: + isc__nmsocket_shutdown(sock); + return; + case UV_TCP: + switch (sock->type) { + case isc_nm_tcpsocket: + case isc_nm_tcpdnssocket: + if (sock->parent == NULL) { + /* Reset the TCP connections on shutdown */ + isc__nmsocket_reset(sock); + return; + } + FALLTHROUGH; + default: + isc__nmsocket_shutdown(sock); + } + + return; + default: + return; + } +} + +void +isc__nm_async_shutdown(isc__networker_t *worker, isc__netievent_t *ev0) { + UNUSED(ev0); + + uv_walk(&worker->loop, shutdown_walk_cb, NULL); +} + +bool +isc__nm_acquire_interlocked(isc_nm_t *mgr) { + if (!isc__nm_in_netthread()) { + return (false); + } + + LOCK(&mgr->lock); + bool success = atomic_compare_exchange_strong( + &mgr->interlocked, &(int){ ISC_NETMGR_NON_INTERLOCKED }, + isc_nm_tid()); + + UNLOCK(&mgr->lock); + return (success); +} + +void +isc__nm_drop_interlocked(isc_nm_t *mgr) { + if (!isc__nm_in_netthread()) { + return; + } + + LOCK(&mgr->lock); + int tid = atomic_exchange(&mgr->interlocked, + ISC_NETMGR_NON_INTERLOCKED); + INSIST(tid != ISC_NETMGR_NON_INTERLOCKED); + BROADCAST(&mgr->wkstatecond); + UNLOCK(&mgr->lock); +} + +void +isc__nm_acquire_interlocked_force(isc_nm_t *mgr) { + if (!isc__nm_in_netthread()) { + return; + } + + LOCK(&mgr->lock); + while (!atomic_compare_exchange_strong( + &mgr->interlocked, &(int){ ISC_NETMGR_NON_INTERLOCKED }, + isc_nm_tid())) + { + WAIT(&mgr->wkstatecond, &mgr->lock); + } + UNLOCK(&mgr->lock); +} + +void +isc_nm_setstats(isc_nm_t *mgr, isc_stats_t *stats) { + REQUIRE(VALID_NM(mgr)); + REQUIRE(mgr->stats == NULL); + REQUIRE(isc_stats_ncounters(stats) == isc_sockstatscounter_max); + + isc_stats_attach(stats, &mgr->stats); +} + +void +isc__nm_incstats(isc_nm_t *mgr, isc_statscounter_t counterid) { + REQUIRE(VALID_NM(mgr)); + REQUIRE(counterid != -1); + + if (mgr->stats != NULL) { + isc_stats_increment(mgr->stats, counterid); + } +} + +void +isc__nm_decstats(isc_nm_t *mgr, isc_statscounter_t counterid) { + REQUIRE(VALID_NM(mgr)); + REQUIRE(counterid != -1); + + if (mgr->stats != NULL) { + isc_stats_decrement(mgr->stats, counterid); + } +} + +isc_result_t +isc__nm_socket(int domain, int type, int protocol, uv_os_sock_t *sockp) { +#ifdef WIN32 + SOCKET sock; + sock = socket(domain, type, protocol); + if (sock == INVALID_SOCKET) { + char strbuf[ISC_STRERRORSIZE]; + DWORD socket_errno = WSAGetLastError(); + switch (socket_errno) { + case WSAEMFILE: + case WSAENOBUFS: + return (ISC_R_NORESOURCES); + + case WSAEPROTONOSUPPORT: + case WSAEPFNOSUPPORT: + case WSAEAFNOSUPPORT: + return (ISC_R_FAMILYNOSUPPORT); + default: + strerror_r(socket_errno, strbuf, sizeof(strbuf)); + UNEXPECTED_ERROR( + __FILE__, __LINE__, + "socket() failed with error code %lu: %s", + socket_errno, strbuf); + return (ISC_R_UNEXPECTED); + } + } +#else + int sock = socket(domain, type, protocol); + if (sock < 0) { + return (isc_errno_toresult(errno)); + } +#endif + *sockp = (uv_os_sock_t)sock; + return (ISC_R_SUCCESS); +} + +void +isc__nm_closesocket(uv_os_sock_t sock) { +#ifdef WIN32 + closesocket(sock); +#else + close(sock); +#endif +} + +#define setsockopt_on(socket, level, name) \ + setsockopt(socket, level, name, &(int){ 1 }, sizeof(int)) + +#define setsockopt_off(socket, level, name) \ + setsockopt(socket, level, name, &(int){ 0 }, sizeof(int)) + +isc_result_t +isc__nm_socket_freebind(uv_os_sock_t fd, sa_family_t sa_family) { + /* + * Set the IP_FREEBIND (or equivalent option) on the uv_handle. + */ +#ifdef IP_FREEBIND + UNUSED(sa_family); + if (setsockopt_on(fd, IPPROTO_IP, IP_FREEBIND) == -1) { + return (ISC_R_FAILURE); + } + return (ISC_R_SUCCESS); +#elif defined(IP_BINDANY) || defined(IPV6_BINDANY) + if (sa_family == AF_INET) { +#if defined(IP_BINDANY) + if (setsockopt_on(fd, IPPROTO_IP, IP_BINDANY) == -1) { + return (ISC_R_FAILURE); + } + return (ISC_R_SUCCESS); +#endif + } else if (sa_family == AF_INET6) { +#if defined(IPV6_BINDANY) + if (setsockopt_on(fd, IPPROTO_IPV6, IPV6_BINDANY) == -1) { + return (ISC_R_FAILURE); + } + return (ISC_R_SUCCESS); +#endif + } + return (ISC_R_NOTIMPLEMENTED); +#elif defined(SO_BINDANY) + UNUSED(sa_family); + if (setsockopt_on(fd, SOL_SOCKET, SO_BINDANY) == -1) { + return (ISC_R_FAILURE); + } + return (ISC_R_SUCCESS); +#else + UNUSED(fd); + UNUSED(sa_family); + return (ISC_R_NOTIMPLEMENTED); +#endif +} + +isc_result_t +isc__nm_socket_reuse(uv_os_sock_t fd) { + /* + * Generally, the SO_REUSEADDR socket option allows reuse of + * local addresses. + * + * On the BSDs, SO_REUSEPORT implies SO_REUSEADDR but with some + * additional refinements for programs that use multicast. + * + * On Linux, SO_REUSEPORT has different semantics: it _shares_ the port + * rather than steal it from the current listener, so we don't use it + * here, but rather in isc__nm_socket_reuse_lb(). + * + * On Windows, it also allows a socket to forcibly bind to a port in use + * by another socket. + */ + +#if defined(SO_REUSEPORT) && !defined(__linux__) + if (setsockopt_on(fd, SOL_SOCKET, SO_REUSEPORT) == -1) { + return (ISC_R_FAILURE); + } + return (ISC_R_SUCCESS); +#elif defined(SO_REUSEADDR) + if (setsockopt_on(fd, SOL_SOCKET, SO_REUSEADDR) == -1) { + return (ISC_R_FAILURE); + } + return (ISC_R_SUCCESS); +#else + UNUSED(fd); + return (ISC_R_NOTIMPLEMENTED); +#endif +} + +isc_result_t +isc__nm_socket_reuse_lb(uv_os_sock_t fd) { + /* + * On FreeBSD 12+, SO_REUSEPORT_LB socket option allows sockets to be + * bound to an identical socket address. For UDP sockets, the use of + * this option can provide better distribution of incoming datagrams to + * multiple processes (or threads) as compared to the traditional + * technique of having multiple processes compete to receive datagrams + * on the same socket. + * + * On Linux, the same thing is achieved simply with SO_REUSEPORT. + */ +#if defined(SO_REUSEPORT_LB) + if (setsockopt_on(fd, SOL_SOCKET, SO_REUSEPORT_LB) == -1) { + return (ISC_R_FAILURE); + } else { + return (ISC_R_SUCCESS); + } +#elif defined(SO_REUSEPORT) && defined(__linux__) + if (setsockopt_on(fd, SOL_SOCKET, SO_REUSEPORT) == -1) { + return (ISC_R_FAILURE); + } else { + return (ISC_R_SUCCESS); + } +#else + UNUSED(fd); + return (ISC_R_NOTIMPLEMENTED); +#endif +} + +isc_result_t +isc__nm_socket_incoming_cpu(uv_os_sock_t fd) { +#ifdef SO_INCOMING_CPU + if (setsockopt_on(fd, SOL_SOCKET, SO_INCOMING_CPU) == -1) { + return (ISC_R_FAILURE); + } else { + return (ISC_R_SUCCESS); + } +#else + UNUSED(fd); +#endif + return (ISC_R_NOTIMPLEMENTED); +} + +isc_result_t +isc__nm_socket_disable_pmtud(uv_os_sock_t fd, sa_family_t sa_family) { + /* + * Disable the Path MTU Discovery on IP packets + */ + if (sa_family == AF_INET6) { +#if defined(IPV6_DONTFRAG) + if (setsockopt_off(fd, IPPROTO_IPV6, IPV6_DONTFRAG) == -1) { + return (ISC_R_FAILURE); + } else { + return (ISC_R_SUCCESS); + } +#elif defined(IPV6_MTU_DISCOVER) && defined(IP_PMTUDISC_OMIT) + if (setsockopt(fd, IPPROTO_IPV6, IPV6_MTU_DISCOVER, + &(int){ IP_PMTUDISC_OMIT }, sizeof(int)) == -1) + { + return (ISC_R_FAILURE); + } else { + return (ISC_R_SUCCESS); + } +#else + UNUSED(fd); +#endif + } else if (sa_family == AF_INET) { +#if defined(IP_DONTFRAG) + if (setsockopt_off(fd, IPPROTO_IP, IP_DONTFRAG) == -1) { + return (ISC_R_FAILURE); + } else { + return (ISC_R_SUCCESS); + } +#elif defined(IP_MTU_DISCOVER) && defined(IP_PMTUDISC_OMIT) + if (setsockopt(fd, IPPROTO_IP, IP_MTU_DISCOVER, + &(int){ IP_PMTUDISC_OMIT }, sizeof(int)) == -1) + { + return (ISC_R_FAILURE); + } else { + return (ISC_R_SUCCESS); + } +#else + UNUSED(fd); +#endif + } else { + return (ISC_R_FAMILYNOSUPPORT); + } + + return (ISC_R_NOTIMPLEMENTED); +} + +#if defined(_WIN32) +#define TIMEOUT_TYPE DWORD +#define TIMEOUT_DIV 1000 +#define TIMEOUT_OPTNAME TCP_MAXRT +#elif defined(TCP_CONNECTIONTIMEOUT) +#define TIMEOUT_TYPE int +#define TIMEOUT_DIV 1000 +#define TIMEOUT_OPTNAME TCP_CONNECTIONTIMEOUT +#elif defined(TCP_RXT_CONNDROPTIME) +#define TIMEOUT_TYPE int +#define TIMEOUT_DIV 1000 +#define TIMEOUT_OPTNAME TCP_RXT_CONNDROPTIME +#elif defined(TCP_USER_TIMEOUT) +#define TIMEOUT_TYPE unsigned int +#define TIMEOUT_DIV 1 +#define TIMEOUT_OPTNAME TCP_USER_TIMEOUT +#elif defined(TCP_KEEPINIT) +#define TIMEOUT_TYPE int +#define TIMEOUT_DIV 1000 +#define TIMEOUT_OPTNAME TCP_KEEPINIT +#endif + +isc_result_t +isc__nm_socket_connectiontimeout(uv_os_sock_t fd, int timeout_ms) { +#if defined(TIMEOUT_OPTNAME) + TIMEOUT_TYPE timeout = timeout_ms / TIMEOUT_DIV; + + if (timeout == 0) { + timeout = 1; + } + + if (setsockopt(fd, IPPROTO_TCP, TIMEOUT_OPTNAME, &timeout, + sizeof(timeout)) == -1) + { + return (ISC_R_FAILURE); + } + + return (ISC_R_SUCCESS); +#else + UNUSED(fd); + UNUSED(timeout_ms); + + return (ISC_R_SUCCESS); +#endif +} + +isc_result_t +isc__nm_socket_tcp_nodelay(uv_os_sock_t fd) { +#ifdef TCP_NODELAY + if (setsockopt_on(fd, IPPROTO_TCP, TCP_NODELAY) == -1) { + return (ISC_R_FAILURE); + } else { + return (ISC_R_SUCCESS); + } +#else + UNUSED(fd); + return (ISC_R_SUCCESS); +#endif +} + +static isc_threadresult_t +isc__nm_work_run(isc_threadarg_t arg) { + isc__nm_work_t *work = (isc__nm_work_t *)arg; + + work->cb(work->data); + + return ((isc_threadresult_t)0); +} + +static void +isc__nm_work_cb(uv_work_t *req) { + isc__nm_work_t *work = uv_req_get_data((uv_req_t *)req); + + if (isc_tid_v == SIZE_MAX) { + isc__trampoline_t *trampoline_arg = + isc__trampoline_get(isc__nm_work_run, work); + (void)isc__trampoline_run(trampoline_arg); + } else { + (void)isc__nm_work_run((isc_threadarg_t)work); + } +} + +static void +isc__nm_after_work_cb(uv_work_t *req, int status) { + isc_result_t result = ISC_R_SUCCESS; + isc__nm_work_t *work = uv_req_get_data((uv_req_t *)req); + isc_nm_t *netmgr = work->netmgr; + + if (status != 0) { + result = isc__nm_uverr2result(status); + } + + work->after_cb(work->data, result); + + isc_mem_put(netmgr->mctx, work, sizeof(*work)); + + isc_nm_detach(&netmgr); +} + +void +isc_nm_work_offload(isc_nm_t *netmgr, isc_nm_workcb_t work_cb, + isc_nm_after_workcb_t after_work_cb, void *data) { + isc__networker_t *worker = NULL; + isc__nm_work_t *work = NULL; + int r; + + REQUIRE(isc__nm_in_netthread()); + REQUIRE(VALID_NM(netmgr)); + + worker = &netmgr->workers[isc_nm_tid()]; + + work = isc_mem_get(netmgr->mctx, sizeof(*work)); + *work = (isc__nm_work_t){ + .cb = work_cb, + .after_cb = after_work_cb, + .data = data, + }; + + isc_nm_attach(netmgr, &work->netmgr); + + uv_req_set_data((uv_req_t *)&work->req, work); + + r = uv_queue_work(&worker->loop, &work->req, isc__nm_work_cb, + isc__nm_after_work_cb); + UV_RUNTIME_CHECK(uv_queue_work, r); +} + +void +isc_nm_timer_create(isc_nmhandle_t *handle, isc_nm_timer_cb cb, void *cbarg, + isc_nm_timer_t **timerp) { + isc__networker_t *worker = NULL; + isc_nmsocket_t *sock = NULL; + isc_nm_timer_t *timer = NULL; + int r; + + REQUIRE(isc__nm_in_netthread()); + REQUIRE(VALID_NMHANDLE(handle)); + REQUIRE(VALID_NMSOCK(handle->sock)); + + sock = handle->sock; + worker = &sock->mgr->workers[isc_nm_tid()]; + + timer = isc_mem_get(sock->mgr->mctx, sizeof(*timer)); + *timer = (isc_nm_timer_t){ .cb = cb, .cbarg = cbarg }; + isc_refcount_init(&timer->references, 1); + isc_nmhandle_attach(handle, &timer->handle); + + r = uv_timer_init(&worker->loop, &timer->timer); + UV_RUNTIME_CHECK(uv_timer_init, r); + + uv_handle_set_data((uv_handle_t *)&timer->timer, timer); + + *timerp = timer; +} + +void +isc_nm_timer_attach(isc_nm_timer_t *timer, isc_nm_timer_t **timerp) { + REQUIRE(timer != NULL); + REQUIRE(timerp != NULL && *timerp == NULL); + + isc_refcount_increment(&timer->references); + *timerp = timer; +} + +static void +timer_destroy(uv_handle_t *uvhandle) { + isc_nm_timer_t *timer = uv_handle_get_data(uvhandle); + isc_nmhandle_t *handle = timer->handle; + isc_mem_t *mctx = timer->handle->sock->mgr->mctx; + + isc_mem_put(mctx, timer, sizeof(*timer)); + + isc_nmhandle_detach(&handle); +} + +void +isc_nm_timer_detach(isc_nm_timer_t **timerp) { + isc_nm_timer_t *timer = NULL; + isc_nmhandle_t *handle = NULL; + + REQUIRE(timerp != NULL && *timerp != NULL); + + timer = *timerp; + *timerp = NULL; + + handle = timer->handle; + + REQUIRE(isc__nm_in_netthread()); + REQUIRE(VALID_NMHANDLE(handle)); + REQUIRE(VALID_NMSOCK(handle->sock)); + + if (isc_refcount_decrement(&timer->references) == 1) { + int r = uv_timer_stop(&timer->timer); + UV_RUNTIME_CHECK(uv_timer_stop, r); + uv_close((uv_handle_t *)&timer->timer, timer_destroy); + } +} + +static void +timer_cb(uv_timer_t *uvtimer) { + isc_nm_timer_t *timer = uv_handle_get_data((uv_handle_t *)uvtimer); + + REQUIRE(timer->cb != NULL); + + timer->cb(timer->cbarg, ISC_R_TIMEDOUT); +} + +void +isc_nm_timer_start(isc_nm_timer_t *timer, uint64_t timeout) { + int r = uv_timer_start(&timer->timer, timer_cb, timeout, 0); + UV_RUNTIME_CHECK(uv_timer_start, r); +} + +void +isc_nm_timer_stop(isc_nm_timer_t *timer) { + int r = uv_timer_stop(&timer->timer); + UV_RUNTIME_CHECK(uv_timer_stop, r); +} + +#ifdef NETMGR_TRACE +/* + * Dump all active sockets in netmgr. We output to stderr + * as the logger might be already shut down. + */ + +static const char * +nmsocket_type_totext(isc_nmsocket_type type) { + switch (type) { + case isc_nm_udpsocket: + return ("isc_nm_udpsocket"); + case isc_nm_udplistener: + return ("isc_nm_udplistener"); + case isc_nm_tcpsocket: + return ("isc_nm_tcpsocket"); + case isc_nm_tcplistener: + return ("isc_nm_tcplistener"); + case isc_nm_tcpdnslistener: + return ("isc_nm_tcpdnslistener"); + case isc_nm_tcpdnssocket: + return ("isc_nm_tcpdnssocket"); + default: + UNREACHABLE(); + } +} + +static void +nmhandle_dump(isc_nmhandle_t *handle) { + fprintf(stderr, "Active handle %p, refs %" PRIuFAST32 "\n", handle, + isc_refcount_current(&handle->references)); + fprintf(stderr, "Created by:\n"); + backtrace_symbols_fd(handle->backtrace, handle->backtrace_size, + STDERR_FILENO); + fprintf(stderr, "\n\n"); +} + +static void +nmsocket_dump(isc_nmsocket_t *sock) { + isc_nmhandle_t *handle = NULL; + + LOCK(&sock->lock); + fprintf(stderr, "\n=================\n"); + fprintf(stderr, "Active %s socket %p, type %s, refs %" PRIuFAST32 "\n", + atomic_load(&sock->client) ? "client" : "server", sock, + nmsocket_type_totext(sock->type), + isc_refcount_current(&sock->references)); + fprintf(stderr, + "Parent %p, listener %p, server %p, statichandle = " + "%p\n", + sock->parent, sock->listener, sock->server, sock->statichandle); + fprintf(stderr, "Flags:%s%s%s%s%s\n", + atomic_load(&sock->active) ? " active" : "", + atomic_load(&sock->closing) ? " closing" : "", + atomic_load(&sock->destroying) ? " destroying" : "", + atomic_load(&sock->connecting) ? " connecting" : "", + sock->accepting ? " accepting" : ""); + fprintf(stderr, "Created by:\n"); + backtrace_symbols_fd(sock->backtrace, sock->backtrace_size, + STDERR_FILENO); + fprintf(stderr, "\n"); + + for (handle = ISC_LIST_HEAD(sock->active_handles); handle != NULL; + handle = ISC_LIST_NEXT(handle, active_link)) + { + static bool first = true; + if (first) { + fprintf(stderr, "Active handles:\n"); + first = false; + } + nmhandle_dump(handle); + } + + fprintf(stderr, "\n"); + UNLOCK(&sock->lock); +} + +void +isc__nm_dump_active(isc_nm_t *nm) { + isc_nmsocket_t *sock = NULL; + + REQUIRE(VALID_NM(nm)); + + LOCK(&nm->lock); + for (sock = ISC_LIST_HEAD(nm->active_sockets); sock != NULL; + sock = ISC_LIST_NEXT(sock, active_link)) + { + static bool first = true; + if (first) { + fprintf(stderr, "Outstanding sockets\n"); + first = false; + } + nmsocket_dump(sock); + } + UNLOCK(&nm->lock); +} +#endif diff --git a/lib/isc/netmgr/tcp.c b/lib/isc/netmgr/tcp.c new file mode 100644 index 0000000..821d6c4 --- /dev/null +++ b/lib/isc/netmgr/tcp.c @@ -0,0 +1,1456 @@ +/* + * Copyright (C) Internet Systems Consortium, Inc. ("ISC") + * + * SPDX-License-Identifier: MPL-2.0 + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, you can obtain one at https://mozilla.org/MPL/2.0/. + * + * See the COPYRIGHT file distributed with this work for additional + * information regarding copyright ownership. + */ + +#include <libgen.h> +#include <unistd.h> +#include <uv.h> + +#include <isc/atomic.h> +#include <isc/barrier.h> +#include <isc/buffer.h> +#include <isc/condition.h> +#include <isc/errno.h> +#include <isc/log.h> +#include <isc/magic.h> +#include <isc/mem.h> +#include <isc/netmgr.h> +#include <isc/quota.h> +#include <isc/random.h> +#include <isc/refcount.h> +#include <isc/region.h> +#include <isc/result.h> +#include <isc/sockaddr.h> +#include <isc/stdtime.h> +#include <isc/thread.h> +#include <isc/util.h> + +#include "netmgr-int.h" +#include "uv-compat.h" + +static atomic_uint_fast32_t last_tcpquota_log = 0; + +static bool +can_log_tcp_quota(void) { + isc_stdtime_t now, last; + + isc_stdtime_get(&now); + last = atomic_exchange_relaxed(&last_tcpquota_log, now); + if (now != last) { + return (true); + } + + return (false); +} + +static isc_result_t +tcp_connect_direct(isc_nmsocket_t *sock, isc__nm_uvreq_t *req); + +static void +tcp_close_direct(isc_nmsocket_t *sock); + +static isc_result_t +tcp_send_direct(isc_nmsocket_t *sock, isc__nm_uvreq_t *req); +static void +tcp_connect_cb(uv_connect_t *uvreq, int status); + +static void +tcp_connection_cb(uv_stream_t *server, int status); + +static void +tcp_close_cb(uv_handle_t *uvhandle); + +static isc_result_t +accept_connection(isc_nmsocket_t *ssock, isc_quota_t *quota); + +static void +quota_accept_cb(isc_quota_t *quota, void *sock0); + +static void +failed_accept_cb(isc_nmsocket_t *sock, isc_result_t eresult); + +static void +stop_tcp_parent(isc_nmsocket_t *sock); +static void +stop_tcp_child(isc_nmsocket_t *sock); + +static void +failed_accept_cb(isc_nmsocket_t *sock, isc_result_t eresult) { + REQUIRE(sock->accepting); + REQUIRE(sock->server); + + /* + * Detach the quota early to make room for other connections; + * otherwise it'd be detached later asynchronously, and clog + * the quota unnecessarily. + */ + if (sock->quota != NULL) { + isc_quota_detach(&sock->quota); + } + + isc__nmsocket_detach(&sock->server); + + sock->accepting = false; + + switch (eresult) { + case ISC_R_NOTCONNECTED: + /* IGNORE: The client disconnected before we could accept */ + break; + default: + isc_log_write(isc_lctx, ISC_LOGCATEGORY_GENERAL, + ISC_LOGMODULE_NETMGR, ISC_LOG_ERROR, + "Accepting TCP connection failed: %s", + isc_result_totext(eresult)); + } +} + +static isc_result_t +tcp_connect_direct(isc_nmsocket_t *sock, isc__nm_uvreq_t *req) { + isc__networker_t *worker = NULL; + isc_result_t result = ISC_R_UNSET; + int r; + + REQUIRE(VALID_NMSOCK(sock)); + REQUIRE(VALID_UVREQ(req)); + + REQUIRE(isc__nm_in_netthread()); + REQUIRE(sock->tid == isc_nm_tid()); + + worker = &sock->mgr->workers[sock->tid]; + + atomic_store(&sock->connecting, true); + + /* 2 minute timeout */ + result = isc__nm_socket_connectiontimeout(sock->fd, 120 * 1000); + RUNTIME_CHECK(result == ISC_R_SUCCESS); + + r = uv_tcp_init(&worker->loop, &sock->uv_handle.tcp); + UV_RUNTIME_CHECK(uv_tcp_init, r); + uv_handle_set_data(&sock->uv_handle.handle, sock); + + r = uv_timer_init(&worker->loop, &sock->read_timer); + UV_RUNTIME_CHECK(uv_timer_init, r); + uv_handle_set_data((uv_handle_t *)&sock->read_timer, sock); + + r = uv_tcp_open(&sock->uv_handle.tcp, sock->fd); + if (r != 0) { + isc__nm_closesocket(sock->fd); + isc__nm_incstats(sock->mgr, sock->statsindex[STATID_OPENFAIL]); + goto done; + } + isc__nm_incstats(sock->mgr, sock->statsindex[STATID_OPEN]); + + if (req->local.length != 0) { + r = uv_tcp_bind(&sock->uv_handle.tcp, &req->local.type.sa, 0); + if (r != 0) { + isc__nm_incstats(sock->mgr, + sock->statsindex[STATID_BINDFAIL]); + goto done; + } + } + + uv_handle_set_data(&req->uv_req.handle, req); + r = uv_tcp_connect(&req->uv_req.connect, &sock->uv_handle.tcp, + &req->peer.type.sa, tcp_connect_cb); + if (r != 0) { + isc__nm_incstats(sock->mgr, + sock->statsindex[STATID_CONNECTFAIL]); + goto done; + } + isc__nm_incstats(sock->mgr, sock->statsindex[STATID_CONNECT]); + + uv_handle_set_data((uv_handle_t *)&sock->read_timer, + &req->uv_req.connect); + isc__nmsocket_timer_start(sock); + + atomic_store(&sock->connected, true); + +done: + result = isc__nm_uverr2result(r); + LOCK(&sock->lock); + sock->result = result; + SIGNAL(&sock->cond); + if (!atomic_load(&sock->active)) { + WAIT(&sock->scond, &sock->lock); + } + INSIST(atomic_load(&sock->active)); + UNLOCK(&sock->lock); + + return (result); +} + +void +isc__nm_async_tcpconnect(isc__networker_t *worker, isc__netievent_t *ev0) { + isc__netievent_tcpconnect_t *ievent = + (isc__netievent_tcpconnect_t *)ev0; + isc_nmsocket_t *sock = ievent->sock; + isc__nm_uvreq_t *req = ievent->req; + isc_result_t result = ISC_R_SUCCESS; + + UNUSED(worker); + + REQUIRE(VALID_NMSOCK(sock)); + REQUIRE(sock->type == isc_nm_tcpsocket); + REQUIRE(sock->parent == NULL); + REQUIRE(sock->tid == isc_nm_tid()); + + result = tcp_connect_direct(sock, req); + if (result != ISC_R_SUCCESS) { + atomic_store(&sock->active, false); + if (sock->fd != (uv_os_sock_t)(-1)) { + isc__nm_tcp_close(sock); + } + isc__nm_connectcb(sock, req, result, true); + } + + /* + * The sock is now attached to the handle. + */ + isc__nmsocket_detach(&sock); +} + +static void +tcp_connect_cb(uv_connect_t *uvreq, int status) { + isc_result_t result; + isc__nm_uvreq_t *req = NULL; + isc_nmsocket_t *sock = uv_handle_get_data((uv_handle_t *)uvreq->handle); + struct sockaddr_storage ss; + int r; + + REQUIRE(VALID_NMSOCK(sock)); + REQUIRE(sock->tid == isc_nm_tid()); + + isc__nmsocket_timer_stop(sock); + uv_handle_set_data((uv_handle_t *)&sock->read_timer, sock); + + req = uv_handle_get_data((uv_handle_t *)uvreq); + + REQUIRE(VALID_UVREQ(req)); + REQUIRE(VALID_NMHANDLE(req->handle)); + + if (atomic_load(&sock->timedout)) { + result = ISC_R_TIMEDOUT; + goto error; + } + + if (!atomic_load(&sock->connecting)) { + /* + * The connect was cancelled from timeout; just clean up + * the req. + */ + isc__nm_uvreq_put(&req, sock); + return; + } else if (isc__nmsocket_closing(sock)) { + /* Socket was closed midflight by isc__nm_tcp_shutdown() */ + result = ISC_R_CANCELED; + goto error; + } else if (status == UV_ETIMEDOUT) { + /* Timeout status code here indicates hard error */ + result = ISC_R_TIMEDOUT; + goto error; + } else if (status != 0) { + result = isc__nm_uverr2result(status); + goto error; + } + + isc__nm_incstats(sock->mgr, sock->statsindex[STATID_CONNECT]); + r = uv_tcp_getpeername(&sock->uv_handle.tcp, (struct sockaddr *)&ss, + &(int){ sizeof(ss) }); + if (r != 0) { + result = isc__nm_uverr2result(r); + goto error; + } + + atomic_store(&sock->connecting, false); + + result = isc_sockaddr_fromsockaddr(&sock->peer, (struct sockaddr *)&ss); + RUNTIME_CHECK(result == ISC_R_SUCCESS); + + isc__nm_connectcb(sock, req, ISC_R_SUCCESS, false); + + return; + +error: + isc__nm_failed_connect_cb(sock, req, result, false); +} + +void +isc_nm_tcpconnect(isc_nm_t *mgr, isc_sockaddr_t *local, isc_sockaddr_t *peer, + isc_nm_cb_t cb, void *cbarg, unsigned int timeout, + size_t extrahandlesize) { + isc_result_t result = ISC_R_SUCCESS; + isc_nmsocket_t *sock = NULL; + isc__netievent_tcpconnect_t *ievent = NULL; + isc__nm_uvreq_t *req = NULL; + sa_family_t sa_family; + + REQUIRE(VALID_NM(mgr)); + REQUIRE(local != NULL); + REQUIRE(peer != NULL); + + sa_family = peer->type.sa.sa_family; + + sock = isc_mem_get(mgr->mctx, sizeof(*sock)); + isc__nmsocket_init(sock, mgr, isc_nm_tcpsocket, local); + + sock->extrahandlesize = extrahandlesize; + sock->connect_timeout = timeout; + sock->result = ISC_R_UNSET; + sock->fd = (uv_os_sock_t)-1; + atomic_init(&sock->client, true); + + req = isc__nm_uvreq_get(mgr, sock); + req->cb.connect = cb; + req->cbarg = cbarg; + req->peer = *peer; + req->local = *local; + req->handle = isc__nmhandle_get(sock, &req->peer, &sock->iface); + + result = isc__nm_socket(sa_family, SOCK_STREAM, 0, &sock->fd); + if (result != ISC_R_SUCCESS) { + if (isc__nm_in_netthread()) { + sock->tid = isc_nm_tid(); + isc__nmsocket_clearcb(sock); + isc__nm_connectcb(sock, req, result, false); + } else { + isc__nmsocket_clearcb(sock); + sock->tid = isc_random_uniform(mgr->nworkers); + isc__nm_connectcb(sock, req, result, true); + } + atomic_store(&sock->closed, true); + isc__nmsocket_detach(&sock); + return; + } + + ievent = isc__nm_get_netievent_tcpconnect(mgr, sock, req); + + if (isc__nm_in_netthread()) { + atomic_store(&sock->active, true); + sock->tid = isc_nm_tid(); + isc__nm_async_tcpconnect(&mgr->workers[sock->tid], + (isc__netievent_t *)ievent); + isc__nm_put_netievent_tcpconnect(mgr, ievent); + } else { + atomic_init(&sock->active, false); + sock->tid = isc_random_uniform(mgr->nworkers); + isc__nm_enqueue_ievent(&mgr->workers[sock->tid], + (isc__netievent_t *)ievent); + } + LOCK(&sock->lock); + while (sock->result == ISC_R_UNSET) { + WAIT(&sock->cond, &sock->lock); + } + atomic_store(&sock->active, true); + BROADCAST(&sock->scond); + UNLOCK(&sock->lock); +} + +static uv_os_sock_t +isc__nm_tcp_lb_socket(isc_nm_t *mgr, sa_family_t sa_family) { + isc_result_t result; + uv_os_sock_t sock; + + result = isc__nm_socket(sa_family, SOCK_STREAM, 0, &sock); + RUNTIME_CHECK(result == ISC_R_SUCCESS); + + (void)isc__nm_socket_incoming_cpu(sock); + + /* FIXME: set mss */ + + result = isc__nm_socket_reuse(sock); + RUNTIME_CHECK(result == ISC_R_SUCCESS); + +#ifndef _WIN32 + if (mgr->load_balance_sockets) { + result = isc__nm_socket_reuse_lb(sock); + RUNTIME_CHECK(result == ISC_R_SUCCESS); + } +#endif + + return (sock); +} + +static void +start_tcp_child(isc_nm_t *mgr, isc_sockaddr_t *iface, isc_nmsocket_t *sock, + uv_os_sock_t fd, int tid) { + isc__netievent_tcplisten_t *ievent = NULL; + isc_nmsocket_t *csock = &sock->children[tid]; + + isc__nmsocket_init(csock, mgr, isc_nm_tcpsocket, iface); + csock->parent = sock; + csock->accept_cb = sock->accept_cb; + csock->accept_cbarg = sock->accept_cbarg; + csock->extrahandlesize = sock->extrahandlesize; + csock->backlog = sock->backlog; + csock->tid = tid; + /* + * We don't attach to quota, just assign - to avoid + * increasing quota unnecessarily. + */ + csock->pquota = sock->pquota; + isc_quota_cb_init(&csock->quotacb, quota_accept_cb, csock); + +#ifdef _WIN32 + UNUSED(fd); + csock->fd = isc__nm_tcp_lb_socket(mgr, iface->type.sa.sa_family); +#else + if (mgr->load_balance_sockets) { + UNUSED(fd); + csock->fd = isc__nm_tcp_lb_socket(mgr, + iface->type.sa.sa_family); + } else { + csock->fd = dup(fd); + } +#endif + REQUIRE(csock->fd >= 0); + + ievent = isc__nm_get_netievent_tcplisten(mgr, csock); + isc__nm_maybe_enqueue_ievent(&mgr->workers[tid], + (isc__netievent_t *)ievent); +} + +static void +enqueue_stoplistening(isc_nmsocket_t *sock) { + isc__netievent_tcpstop_t *ievent = + isc__nm_get_netievent_tcpstop(sock->mgr, sock); + isc__nm_enqueue_ievent(&sock->mgr->workers[sock->tid], + (isc__netievent_t *)ievent); +} + +isc_result_t +isc_nm_listentcp(isc_nm_t *mgr, isc_sockaddr_t *iface, + isc_nm_accept_cb_t accept_cb, void *accept_cbarg, + size_t extrahandlesize, int backlog, isc_quota_t *quota, + isc_nmsocket_t **sockp) { + isc_result_t result = ISC_R_SUCCESS; + isc_nmsocket_t *sock = NULL; + size_t children_size = 0; + uv_os_sock_t fd = -1; + + REQUIRE(VALID_NM(mgr)); + + sock = isc_mem_get(mgr->mctx, sizeof(*sock)); + isc__nmsocket_init(sock, mgr, isc_nm_tcplistener, iface); + + atomic_init(&sock->rchildren, 0); +#if defined(WIN32) + sock->nchildren = 1; +#else + sock->nchildren = mgr->nworkers; +#endif + children_size = sock->nchildren * sizeof(sock->children[0]); + sock->children = isc_mem_get(mgr->mctx, children_size); + memset(sock->children, 0, children_size); + + sock->result = ISC_R_UNSET; + + sock->accept_cb = accept_cb; + sock->accept_cbarg = accept_cbarg; + sock->extrahandlesize = extrahandlesize; + sock->backlog = backlog; + sock->pquota = quota; + + sock->tid = 0; + sock->fd = -1; + +#ifndef _WIN32 + if (!mgr->load_balance_sockets) { + fd = isc__nm_tcp_lb_socket(mgr, iface->type.sa.sa_family); + } +#endif + + isc_barrier_init(&sock->startlistening, sock->nchildren); + + for (size_t i = 0; i < sock->nchildren; i++) { + if ((int)i == isc_nm_tid()) { + continue; + } + start_tcp_child(mgr, iface, sock, fd, i); + } + + if (isc__nm_in_netthread()) { + start_tcp_child(mgr, iface, sock, fd, isc_nm_tid()); + } + +#ifndef _WIN32 + if (!mgr->load_balance_sockets) { + isc__nm_closesocket(fd); + } +#endif + + LOCK(&sock->lock); + while (atomic_load(&sock->rchildren) != sock->nchildren) { + WAIT(&sock->cond, &sock->lock); + } + result = sock->result; + atomic_store(&sock->active, true); + UNLOCK(&sock->lock); + + INSIST(result != ISC_R_UNSET); + + if (result == ISC_R_SUCCESS) { + REQUIRE(atomic_load(&sock->rchildren) == sock->nchildren); + *sockp = sock; + } else { + atomic_store(&sock->active, false); + enqueue_stoplistening(sock); + isc_nmsocket_close(&sock); + } + + return (result); +} + +void +isc__nm_async_tcplisten(isc__networker_t *worker, isc__netievent_t *ev0) { + isc__netievent_tcplisten_t *ievent = (isc__netievent_tcplisten_t *)ev0; + sa_family_t sa_family; + int r; + int flags = 0; + isc_nmsocket_t *sock = NULL; + isc_result_t result; + isc_nm_t *mgr; + + REQUIRE(VALID_NMSOCK(ievent->sock)); + REQUIRE(ievent->sock->tid == isc_nm_tid()); + REQUIRE(VALID_NMSOCK(ievent->sock->parent)); + + sock = ievent->sock; + sa_family = sock->iface.type.sa.sa_family; + mgr = sock->mgr; + + REQUIRE(sock->type == isc_nm_tcpsocket); + REQUIRE(sock->parent != NULL); + REQUIRE(sock->tid == isc_nm_tid()); + + /* TODO: set min mss */ + + r = uv_tcp_init(&worker->loop, &sock->uv_handle.tcp); + UV_RUNTIME_CHECK(uv_tcp_init, r); + + uv_handle_set_data(&sock->uv_handle.handle, sock); + /* This keeps the socket alive after everything else is gone */ + isc__nmsocket_attach(sock, &(isc_nmsocket_t *){ NULL }); + + r = uv_timer_init(&worker->loop, &sock->read_timer); + UV_RUNTIME_CHECK(uv_timer_init, r); + uv_handle_set_data((uv_handle_t *)&sock->read_timer, sock); + + LOCK(&sock->parent->lock); + + r = uv_tcp_open(&sock->uv_handle.tcp, sock->fd); + if (r < 0) { + isc__nm_closesocket(sock->fd); + isc__nm_incstats(sock->mgr, sock->statsindex[STATID_OPENFAIL]); + goto done; + } + isc__nm_incstats(sock->mgr, sock->statsindex[STATID_OPEN]); + + if (sa_family == AF_INET6) { + flags = UV_TCP_IPV6ONLY; + } + +#ifdef _WIN32 + r = isc_uv_tcp_freebind(&sock->uv_handle.tcp, &sock->iface.type.sa, + flags); + if (r < 0) { + isc__nm_incstats(sock->mgr, sock->statsindex[STATID_BINDFAIL]); + goto done; + } +#else + if (mgr->load_balance_sockets) { + r = isc_uv_tcp_freebind(&sock->uv_handle.tcp, + &sock->iface.type.sa, flags); + if (r < 0) { + isc__nm_incstats(sock->mgr, + sock->statsindex[STATID_BINDFAIL]); + goto done; + } + } else { + if (sock->parent->fd == -1) { + r = isc_uv_tcp_freebind(&sock->uv_handle.tcp, + &sock->iface.type.sa, flags); + if (r < 0) { + isc__nm_incstats(sock->mgr, STATID_BINDFAIL); + goto done; + } + sock->parent->uv_handle.tcp.flags = + sock->uv_handle.tcp.flags; + sock->parent->fd = sock->fd; + } else { + /* The socket is already bound, just copy the flags */ + sock->uv_handle.tcp.flags = + sock->parent->uv_handle.tcp.flags; + } + } +#endif + + /* + * The callback will run in the same thread uv_listen() was called + * from, so a race with tcp_connection_cb() isn't possible. + */ + r = uv_listen((uv_stream_t *)&sock->uv_handle.tcp, sock->backlog, + tcp_connection_cb); + if (r != 0) { + isc_log_write(isc_lctx, ISC_LOGCATEGORY_GENERAL, + ISC_LOGMODULE_NETMGR, ISC_LOG_ERROR, + "uv_listen failed: %s", + isc_result_totext(isc__nm_uverr2result(r))); + isc__nm_incstats(sock->mgr, sock->statsindex[STATID_BINDFAIL]); + goto done; + } + + atomic_store(&sock->listening, true); + +done: + result = isc__nm_uverr2result(r); + if (result != ISC_R_SUCCESS) { + sock->pquota = NULL; + } + + atomic_fetch_add(&sock->parent->rchildren, 1); + if (sock->parent->result == ISC_R_UNSET) { + sock->parent->result = result; + } + SIGNAL(&sock->parent->cond); + UNLOCK(&sock->parent->lock); + + isc_barrier_wait(&sock->parent->startlistening); +} + +static void +tcp_connection_cb(uv_stream_t *server, int status) { + isc_nmsocket_t *ssock = uv_handle_get_data((uv_handle_t *)server); + isc_result_t result; + isc_quota_t *quota = NULL; + + if (status != 0) { + result = isc__nm_uverr2result(status); + goto done; + } + + REQUIRE(VALID_NMSOCK(ssock)); + REQUIRE(ssock->tid == isc_nm_tid()); + + if (isc__nmsocket_closing(ssock)) { + result = ISC_R_CANCELED; + goto done; + } + + if (ssock->pquota != NULL) { + result = isc_quota_attach_cb(ssock->pquota, "a, + &ssock->quotacb); + if (result == ISC_R_QUOTA) { + isc__nm_incstats(ssock->mgr, + ssock->statsindex[STATID_ACCEPTFAIL]); + goto done; + } + } + + result = accept_connection(ssock, quota); +done: + isc__nm_accept_connection_log(result, can_log_tcp_quota()); +} + +void +isc__nm_tcp_stoplistening(isc_nmsocket_t *sock) { + REQUIRE(VALID_NMSOCK(sock)); + REQUIRE(sock->type == isc_nm_tcplistener); + + if (!atomic_compare_exchange_strong(&sock->closing, &(bool){ false }, + true)) + { + UNREACHABLE(); + } + + if (!isc__nm_in_netthread()) { + enqueue_stoplistening(sock); + } else { + stop_tcp_parent(sock); + } +} + +void +isc__nm_async_tcpstop(isc__networker_t *worker, isc__netievent_t *ev0) { + isc__netievent_tcpstop_t *ievent = (isc__netievent_tcpstop_t *)ev0; + isc_nmsocket_t *sock = ievent->sock; + + UNUSED(worker); + + REQUIRE(VALID_NMSOCK(sock)); + REQUIRE(sock->tid == isc_nm_tid()); + + if (sock->parent != NULL) { + stop_tcp_child(sock); + return; + } + + stop_tcp_parent(sock); +} + +void +isc__nm_tcp_failed_read_cb(isc_nmsocket_t *sock, isc_result_t result) { + REQUIRE(VALID_NMSOCK(sock)); + REQUIRE(result != ISC_R_SUCCESS); + + isc__nmsocket_timer_stop(sock); + isc__nm_stop_reading(sock); + + if (!sock->recv_read) { + goto destroy; + } + sock->recv_read = false; + + if (sock->recv_cb != NULL) { + isc__nm_uvreq_t *req = isc__nm_get_read_req(sock, NULL); + isc__nmsocket_clearcb(sock); + isc__nm_readcb(sock, req, result); + } + +destroy: + isc__nmsocket_prep_destroy(sock); + + /* + * We need to detach from quota after the read callback function had a + * chance to be executed. + */ + if (sock->quota != NULL) { + isc_quota_detach(&sock->quota); + } +} + +void +isc__nm_tcp_read(isc_nmhandle_t *handle, isc_nm_recv_cb_t cb, void *cbarg) { + REQUIRE(VALID_NMHANDLE(handle)); + REQUIRE(VALID_NMSOCK(handle->sock)); + + isc_nmsocket_t *sock = handle->sock; + isc__netievent_tcpstartread_t *ievent = NULL; + + REQUIRE(sock->type == isc_nm_tcpsocket); + REQUIRE(sock->statichandle == handle); + REQUIRE(sock->tid == isc_nm_tid()); + REQUIRE(!sock->recv_read); + + sock->recv_cb = cb; + sock->recv_cbarg = cbarg; + sock->recv_read = true; + if (sock->read_timeout == 0) { + sock->read_timeout = + (atomic_load(&sock->keepalive) + ? atomic_load(&sock->mgr->keepalive) + : atomic_load(&sock->mgr->idle)); + } + + ievent = isc__nm_get_netievent_tcpstartread(sock->mgr, sock); + + /* + * This MUST be done asynchronously, no matter which thread we're + * in. The callback function for isc_nm_read() often calls + * isc_nm_read() again; if we tried to do that synchronously + * we'd clash in processbuffer() and grow the stack indefinitely. + */ + isc__nm_enqueue_ievent(&sock->mgr->workers[sock->tid], + (isc__netievent_t *)ievent); + + return; +} + +void +isc__nm_async_tcpstartread(isc__networker_t *worker, isc__netievent_t *ev0) { + isc__netievent_tcpstartread_t *ievent = + (isc__netievent_tcpstartread_t *)ev0; + isc_nmsocket_t *sock = ievent->sock; + isc_result_t result; + + REQUIRE(VALID_NMSOCK(sock)); + REQUIRE(sock->tid == isc_nm_tid()); + UNUSED(worker); + + if (isc__nmsocket_closing(sock)) { + result = ISC_R_CANCELED; + } else { + result = isc__nm_start_reading(sock); + } + + if (result != ISC_R_SUCCESS) { + sock->reading = true; + isc__nm_tcp_failed_read_cb(sock, result); + return; + } + + isc__nmsocket_timer_start(sock); +} + +void +isc__nm_tcp_pauseread(isc_nmhandle_t *handle) { + isc__netievent_tcppauseread_t *ievent = NULL; + isc_nmsocket_t *sock = NULL; + + REQUIRE(VALID_NMHANDLE(handle)); + + sock = handle->sock; + + REQUIRE(VALID_NMSOCK(sock)); + + if (!atomic_compare_exchange_strong(&sock->readpaused, &(bool){ false }, + true)) + { + return; + } + + ievent = isc__nm_get_netievent_tcppauseread(sock->mgr, sock); + + isc__nm_maybe_enqueue_ievent(&sock->mgr->workers[sock->tid], + (isc__netievent_t *)ievent); + + return; +} + +void +isc__nm_async_tcppauseread(isc__networker_t *worker, isc__netievent_t *ev0) { + isc__netievent_tcppauseread_t *ievent = + (isc__netievent_tcppauseread_t *)ev0; + isc_nmsocket_t *sock = ievent->sock; + + REQUIRE(VALID_NMSOCK(sock)); + REQUIRE(sock->tid == isc_nm_tid()); + UNUSED(worker); + + isc__nmsocket_timer_stop(sock); + isc__nm_stop_reading(sock); +} + +void +isc__nm_tcp_resumeread(isc_nmhandle_t *handle) { + REQUIRE(VALID_NMHANDLE(handle)); + REQUIRE(VALID_NMSOCK(handle->sock)); + + isc__netievent_tcpstartread_t *ievent = NULL; + isc_nmsocket_t *sock = handle->sock; + + REQUIRE(sock->tid == isc_nm_tid()); + + if (sock->recv_cb == NULL) { + /* We are no longer reading */ + return; + } + + if (!isc__nmsocket_active(sock)) { + sock->reading = true; + isc__nm_tcp_failed_read_cb(sock, ISC_R_CANCELED); + return; + } + + if (!atomic_compare_exchange_strong(&sock->readpaused, &(bool){ true }, + false)) + { + return; + } + + ievent = isc__nm_get_netievent_tcpstartread(sock->mgr, sock); + + isc__nm_maybe_enqueue_ievent(&sock->mgr->workers[sock->tid], + (isc__netievent_t *)ievent); +} + +void +isc__nm_tcp_read_cb(uv_stream_t *stream, ssize_t nread, const uv_buf_t *buf) { + isc_nmsocket_t *sock = uv_handle_get_data((uv_handle_t *)stream); + isc__nm_uvreq_t *req = NULL; + + REQUIRE(VALID_NMSOCK(sock)); + REQUIRE(sock->tid == isc_nm_tid()); + REQUIRE(sock->reading); + REQUIRE(buf != NULL); + + if (isc__nmsocket_closing(sock)) { + isc__nm_tcp_failed_read_cb(sock, ISC_R_CANCELED); + goto free; + } + + if (nread < 0) { + if (nread != UV_EOF) { + isc__nm_incstats(sock->mgr, + sock->statsindex[STATID_RECVFAIL]); + } + + isc__nm_tcp_failed_read_cb(sock, isc__nm_uverr2result(nread)); + + goto free; + } + + req = isc__nm_get_read_req(sock, NULL); + + /* + * The callback will be called synchronously because the + * result is ISC_R_SUCCESS, so we don't need to retain + * the buffer + */ + req->uvbuf.base = buf->base; + req->uvbuf.len = nread; + + if (!atomic_load(&sock->client)) { + sock->read_timeout = + (atomic_load(&sock->keepalive) + ? atomic_load(&sock->mgr->keepalive) + : atomic_load(&sock->mgr->idle)); + } + + isc__nm_readcb(sock, req, ISC_R_SUCCESS); + + /* The readcb could have paused the reading */ + if (sock->reading) { + /* The timer will be updated */ + isc__nmsocket_timer_restart(sock); + } + +free: + if (nread < 0) { + /* + * The buffer may be a null buffer on error. + */ + if (buf->base == NULL && buf->len == 0) { + return; + } + } + + isc__nm_free_uvbuf(sock, buf); +} + +static void +quota_accept_cb(isc_quota_t *quota, void *sock0) { + isc_nmsocket_t *sock = (isc_nmsocket_t *)sock0; + isc__netievent_tcpaccept_t *ievent = NULL; + + REQUIRE(VALID_NMSOCK(sock)); + + /* + * Create a tcpaccept event and pass it using the async channel. + */ + ievent = isc__nm_get_netievent_tcpaccept(sock->mgr, sock, quota); + isc__nm_maybe_enqueue_ievent(&sock->mgr->workers[sock->tid], + (isc__netievent_t *)ievent); +} + +/* + * This is called after we get a quota_accept_cb() callback. + */ +void +isc__nm_async_tcpaccept(isc__networker_t *worker, isc__netievent_t *ev0) { + isc__netievent_tcpaccept_t *ievent = (isc__netievent_tcpaccept_t *)ev0; + isc_nmsocket_t *sock = ievent->sock; + isc_result_t result; + + UNUSED(worker); + + REQUIRE(VALID_NMSOCK(sock)); + REQUIRE(sock->tid == isc_nm_tid()); + + result = accept_connection(sock, ievent->quota); + isc__nm_accept_connection_log(result, can_log_tcp_quota()); +} + +static isc_result_t +accept_connection(isc_nmsocket_t *ssock, isc_quota_t *quota) { + isc_nmsocket_t *csock = NULL; + isc__networker_t *worker = NULL; + int r; + isc_result_t result; + struct sockaddr_storage ss; + isc_sockaddr_t local; + isc_nmhandle_t *handle = NULL; + + REQUIRE(VALID_NMSOCK(ssock)); + REQUIRE(ssock->tid == isc_nm_tid()); + + if (isc__nmsocket_closing(ssock)) { + if (quota != NULL) { + isc_quota_detach("a); + } + return (ISC_R_CANCELED); + } + + csock = isc_mem_get(ssock->mgr->mctx, sizeof(isc_nmsocket_t)); + isc__nmsocket_init(csock, ssock->mgr, isc_nm_tcpsocket, &ssock->iface); + csock->tid = ssock->tid; + csock->extrahandlesize = ssock->extrahandlesize; + isc__nmsocket_attach(ssock, &csock->server); + csock->recv_cb = ssock->recv_cb; + csock->recv_cbarg = ssock->recv_cbarg; + csock->quota = quota; + csock->accepting = true; + + worker = &csock->mgr->workers[isc_nm_tid()]; + + r = uv_tcp_init(&worker->loop, &csock->uv_handle.tcp); + UV_RUNTIME_CHECK(uv_tcp_init, r); + uv_handle_set_data(&csock->uv_handle.handle, csock); + + r = uv_timer_init(&worker->loop, &csock->read_timer); + UV_RUNTIME_CHECK(uv_timer_init, r); + uv_handle_set_data((uv_handle_t *)&csock->read_timer, csock); + + r = uv_accept(&ssock->uv_handle.stream, &csock->uv_handle.stream); + if (r != 0) { + result = isc__nm_uverr2result(r); + goto failure; + } + + r = uv_tcp_getpeername(&csock->uv_handle.tcp, (struct sockaddr *)&ss, + &(int){ sizeof(ss) }); + if (r != 0) { + result = isc__nm_uverr2result(r); + goto failure; + } + + result = isc_sockaddr_fromsockaddr(&csock->peer, + (struct sockaddr *)&ss); + if (result != ISC_R_SUCCESS) { + goto failure; + } + + r = uv_tcp_getsockname(&csock->uv_handle.tcp, (struct sockaddr *)&ss, + &(int){ sizeof(ss) }); + if (r != 0) { + result = isc__nm_uverr2result(r); + goto failure; + } + + result = isc_sockaddr_fromsockaddr(&local, (struct sockaddr *)&ss); + if (result != ISC_R_SUCCESS) { + goto failure; + } + + handle = isc__nmhandle_get(csock, NULL, &local); + + result = ssock->accept_cb(handle, ISC_R_SUCCESS, ssock->accept_cbarg); + if (result != ISC_R_SUCCESS) { + isc_nmhandle_detach(&handle); + goto failure; + } + + csock->accepting = false; + + isc__nm_incstats(csock->mgr, csock->statsindex[STATID_ACCEPT]); + + csock->read_timeout = atomic_load(&csock->mgr->init); + + atomic_fetch_add(&ssock->parent->active_child_connections, 1); + + /* + * The acceptcb needs to attach to the handle if it wants to keep the + * connection alive + */ + isc_nmhandle_detach(&handle); + + /* + * sock is now attached to the handle. + */ + isc__nmsocket_detach(&csock); + + return (ISC_R_SUCCESS); + +failure: + atomic_store(&csock->active, false); + + failed_accept_cb(csock, result); + + isc__nmsocket_prep_destroy(csock); + + isc__nmsocket_detach(&csock); + + return (result); +} + +void +isc__nm_tcp_send(isc_nmhandle_t *handle, const isc_region_t *region, + isc_nm_cb_t cb, void *cbarg) { + REQUIRE(VALID_NMHANDLE(handle)); + REQUIRE(VALID_NMSOCK(handle->sock)); + + isc_nmsocket_t *sock = handle->sock; + isc__netievent_tcpsend_t *ievent = NULL; + isc__nm_uvreq_t *uvreq = NULL; + + REQUIRE(sock->type == isc_nm_tcpsocket); + + uvreq = isc__nm_uvreq_get(sock->mgr, sock); + uvreq->uvbuf.base = (char *)region->base; + uvreq->uvbuf.len = region->length; + + isc_nmhandle_attach(handle, &uvreq->handle); + + uvreq->cb.send = cb; + uvreq->cbarg = cbarg; + + ievent = isc__nm_get_netievent_tcpsend(sock->mgr, sock, uvreq); + isc__nm_maybe_enqueue_ievent(&sock->mgr->workers[sock->tid], + (isc__netievent_t *)ievent); + + return; +} + +static void +tcp_send_cb(uv_write_t *req, int status) { + isc__nm_uvreq_t *uvreq = (isc__nm_uvreq_t *)req->data; + isc_nmsocket_t *sock = NULL; + + REQUIRE(VALID_UVREQ(uvreq)); + REQUIRE(VALID_NMSOCK(uvreq->sock)); + + sock = uvreq->sock; + + isc_nm_timer_stop(uvreq->timer); + isc_nm_timer_detach(&uvreq->timer); + + if (status < 0) { + isc__nm_incstats(sock->mgr, sock->statsindex[STATID_SENDFAIL]); + isc__nm_failed_send_cb(sock, uvreq, + isc__nm_uverr2result(status)); + return; + } + + isc__nm_sendcb(sock, uvreq, ISC_R_SUCCESS, false); +} + +/* + * Handle 'tcpsend' async event - send a packet on the socket + */ +void +isc__nm_async_tcpsend(isc__networker_t *worker, isc__netievent_t *ev0) { + isc_result_t result; + isc__netievent_tcpsend_t *ievent = (isc__netievent_tcpsend_t *)ev0; + isc_nmsocket_t *sock = ievent->sock; + isc__nm_uvreq_t *uvreq = ievent->req; + + REQUIRE(sock->type == isc_nm_tcpsocket); + REQUIRE(sock->tid == isc_nm_tid()); + UNUSED(worker); + + if (sock->write_timeout == 0) { + sock->write_timeout = + (atomic_load(&sock->keepalive) + ? atomic_load(&sock->mgr->keepalive) + : atomic_load(&sock->mgr->idle)); + } + + result = tcp_send_direct(sock, uvreq); + if (result != ISC_R_SUCCESS) { + isc__nm_incstats(sock->mgr, sock->statsindex[STATID_SENDFAIL]); + isc__nm_failed_send_cb(sock, uvreq, result); + } +} + +static isc_result_t +tcp_send_direct(isc_nmsocket_t *sock, isc__nm_uvreq_t *req) { + REQUIRE(VALID_NMSOCK(sock)); + REQUIRE(VALID_UVREQ(req)); + REQUIRE(sock->tid == isc_nm_tid()); + REQUIRE(sock->type == isc_nm_tcpsocket); + + int r; + + if (isc__nmsocket_closing(sock)) { + return (ISC_R_CANCELED); + } + + r = uv_write(&req->uv_req.write, &sock->uv_handle.stream, &req->uvbuf, + 1, tcp_send_cb); + if (r < 0) { + return (isc__nm_uverr2result(r)); + } + + isc_nm_timer_create(req->handle, isc__nmsocket_writetimeout_cb, req, + &req->timer); + if (sock->write_timeout > 0) { + isc_nm_timer_start(req->timer, sock->write_timeout); + } + + return (ISC_R_SUCCESS); +} + +static void +tcp_stop_cb(uv_handle_t *handle) { + isc_nmsocket_t *sock = uv_handle_get_data(handle); + uv_handle_set_data(handle, NULL); + + REQUIRE(VALID_NMSOCK(sock)); + REQUIRE(sock->tid == isc_nm_tid()); + REQUIRE(atomic_load(&sock->closing)); + + if (!atomic_compare_exchange_strong(&sock->closed, &(bool){ false }, + true)) + { + UNREACHABLE(); + } + + isc__nm_incstats(sock->mgr, sock->statsindex[STATID_CLOSE]); + + atomic_store(&sock->listening, false); + + isc__nmsocket_detach(&sock); +} + +static void +tcp_close_sock(isc_nmsocket_t *sock) { + REQUIRE(VALID_NMSOCK(sock)); + REQUIRE(sock->tid == isc_nm_tid()); + REQUIRE(atomic_load(&sock->closing)); + + if (!atomic_compare_exchange_strong(&sock->closed, &(bool){ false }, + true)) + { + UNREACHABLE(); + } + + isc__nm_incstats(sock->mgr, sock->statsindex[STATID_CLOSE]); + + if (sock->server != NULL) { + isc__nmsocket_detach(&sock->server); + } + + atomic_store(&sock->connected, false); + + isc__nmsocket_prep_destroy(sock); +} + +static void +tcp_close_cb(uv_handle_t *handle) { + isc_nmsocket_t *sock = uv_handle_get_data(handle); + uv_handle_set_data(handle, NULL); + + tcp_close_sock(sock); +} + +static void +read_timer_close_cb(uv_handle_t *handle) { + isc_nmsocket_t *sock = uv_handle_get_data(handle); + uv_handle_set_data(handle, NULL); + + if (sock->parent) { + uv_close(&sock->uv_handle.handle, tcp_stop_cb); + } else if (uv_is_closing(&sock->uv_handle.handle)) { + tcp_close_sock(sock); + } else { + uv_close(&sock->uv_handle.handle, tcp_close_cb); + } +} + +static void +stop_tcp_child(isc_nmsocket_t *sock) { + REQUIRE(sock->type == isc_nm_tcpsocket); + REQUIRE(sock->tid == isc_nm_tid()); + + if (!atomic_compare_exchange_strong(&sock->closing, &(bool){ false }, + true)) + { + return; + } + + tcp_close_direct(sock); + + atomic_fetch_sub(&sock->parent->rchildren, 1); + + isc_barrier_wait(&sock->parent->stoplistening); +} + +static void +stop_tcp_parent(isc_nmsocket_t *sock) { + isc_nmsocket_t *csock = NULL; + + REQUIRE(VALID_NMSOCK(sock)); + REQUIRE(sock->tid == isc_nm_tid()); + REQUIRE(sock->type == isc_nm_tcplistener); + + isc_barrier_init(&sock->stoplistening, sock->nchildren); + + for (size_t i = 0; i < sock->nchildren; i++) { + csock = &sock->children[i]; + REQUIRE(VALID_NMSOCK(csock)); + + if ((int)i == isc_nm_tid()) { + /* + * We need to schedule closing the other sockets first + */ + continue; + } + + atomic_store(&csock->active, false); + enqueue_stoplistening(csock); + } + + csock = &sock->children[isc_nm_tid()]; + atomic_store(&csock->active, false); + stop_tcp_child(csock); + + atomic_store(&sock->closed, true); + isc__nmsocket_prep_destroy(sock); +} + +static void +tcp_close_direct(isc_nmsocket_t *sock) { + REQUIRE(VALID_NMSOCK(sock)); + REQUIRE(sock->tid == isc_nm_tid()); + REQUIRE(atomic_load(&sock->closing)); + + if (sock->server != NULL) { + REQUIRE(VALID_NMSOCK(sock->server)); + REQUIRE(VALID_NMSOCK(sock->server->parent)); + if (sock->server->parent != NULL) { + atomic_fetch_sub( + &sock->server->parent->active_child_connections, + 1); + } + } + + if (sock->quota != NULL) { + isc_quota_detach(&sock->quota); + } + + isc__nmsocket_timer_stop(sock); + isc__nm_stop_reading(sock); + + uv_handle_set_data((uv_handle_t *)&sock->read_timer, sock); + uv_close((uv_handle_t *)&sock->read_timer, read_timer_close_cb); +} + +void +isc__nm_tcp_close(isc_nmsocket_t *sock) { + REQUIRE(VALID_NMSOCK(sock)); + REQUIRE(sock->type == isc_nm_tcpsocket); + REQUIRE(!isc__nmsocket_active(sock)); + + if (!atomic_compare_exchange_strong(&sock->closing, &(bool){ false }, + true)) + { + return; + } + + if (sock->tid == isc_nm_tid()) { + tcp_close_direct(sock); + } else { + /* + * We need to create an event and pass it using async channel + */ + isc__netievent_tcpclose_t *ievent = + isc__nm_get_netievent_tcpclose(sock->mgr, sock); + + isc__nm_enqueue_ievent(&sock->mgr->workers[sock->tid], + (isc__netievent_t *)ievent); + } +} + +void +isc__nm_async_tcpclose(isc__networker_t *worker, isc__netievent_t *ev0) { + isc__netievent_tcpclose_t *ievent = (isc__netievent_tcpclose_t *)ev0; + isc_nmsocket_t *sock = ievent->sock; + + REQUIRE(VALID_NMSOCK(sock)); + REQUIRE(sock->tid == isc_nm_tid()); + + UNUSED(worker); + + tcp_close_direct(sock); +} + +static void +tcp_close_connect_cb(uv_handle_t *handle) { + isc_nmsocket_t *sock = uv_handle_get_data(handle); + + REQUIRE(VALID_NMSOCK(sock)); + + REQUIRE(isc__nm_in_netthread()); + REQUIRE(sock->tid == isc_nm_tid()); + + isc__nmsocket_prep_destroy(sock); + isc__nmsocket_detach(&sock); +} + +void +isc__nm_tcp_shutdown(isc_nmsocket_t *sock) { + REQUIRE(VALID_NMSOCK(sock)); + REQUIRE(sock->tid == isc_nm_tid()); + REQUIRE(sock->type == isc_nm_tcpsocket); + + /* + * If the socket is active, mark it inactive and + * continue. If it isn't active, stop now. + */ + if (!isc__nmsocket_deactivate(sock)) { + return; + } + + if (sock->accepting) { + return; + } + + if (atomic_load(&sock->connecting)) { + isc_nmsocket_t *tsock = NULL; + isc__nmsocket_attach(sock, &tsock); + uv_close(&sock->uv_handle.handle, tcp_close_connect_cb); + return; + } + + if (sock->statichandle != NULL) { + isc__nm_tcp_failed_read_cb(sock, ISC_R_CANCELED); + return; + } + + /* + * Otherwise, we just send the socket to abyss... + */ + if (sock->parent == NULL) { + isc__nmsocket_prep_destroy(sock); + } +} + +void +isc__nm_tcp_cancelread(isc_nmhandle_t *handle) { + isc_nmsocket_t *sock = NULL; + isc__netievent_tcpcancel_t *ievent = NULL; + + REQUIRE(VALID_NMHANDLE(handle)); + + sock = handle->sock; + + REQUIRE(VALID_NMSOCK(sock)); + REQUIRE(sock->type == isc_nm_tcpsocket); + + ievent = isc__nm_get_netievent_tcpcancel(sock->mgr, sock, handle); + isc__nm_enqueue_ievent(&sock->mgr->workers[sock->tid], + (isc__netievent_t *)ievent); +} + +void +isc__nm_async_tcpcancel(isc__networker_t *worker, isc__netievent_t *ev0) { + isc__netievent_tcpcancel_t *ievent = (isc__netievent_tcpcancel_t *)ev0; + isc_nmsocket_t *sock = ievent->sock; + + REQUIRE(VALID_NMSOCK(sock)); + REQUIRE(sock->tid == isc_nm_tid()); + UNUSED(worker); + + uv_timer_stop(&sock->read_timer); + + isc__nm_tcp_failed_read_cb(sock, ISC_R_EOF); +} + +int_fast32_t +isc__nm_tcp_listener_nactive(isc_nmsocket_t *listener) { + int_fast32_t nactive; + + REQUIRE(VALID_NMSOCK(listener)); + + nactive = atomic_load(&listener->active_child_connections); + INSIST(nactive >= 0); + return (nactive); +} diff --git a/lib/isc/netmgr/tcpdns.c b/lib/isc/netmgr/tcpdns.c new file mode 100644 index 0000000..bd593eb --- /dev/null +++ b/lib/isc/netmgr/tcpdns.c @@ -0,0 +1,1505 @@ +/* + * Copyright (C) Internet Systems Consortium, Inc. ("ISC") + * + * SPDX-License-Identifier: MPL-2.0 + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, you can obtain one at https://mozilla.org/MPL/2.0/. + * + * See the COPYRIGHT file distributed with this work for additional + * information regarding copyright ownership. + */ + +#include <libgen.h> +#include <unistd.h> +#include <uv.h> + +#include <isc/atomic.h> +#include <isc/barrier.h> +#include <isc/buffer.h> +#include <isc/condition.h> +#include <isc/errno.h> +#include <isc/log.h> +#include <isc/magic.h> +#include <isc/mem.h> +#include <isc/netmgr.h> +#include <isc/quota.h> +#include <isc/random.h> +#include <isc/refcount.h> +#include <isc/region.h> +#include <isc/result.h> +#include <isc/sockaddr.h> +#include <isc/stdtime.h> +#include <isc/thread.h> +#include <isc/util.h> + +#include "netmgr-int.h" +#include "uv-compat.h" + +static atomic_uint_fast32_t last_tcpdnsquota_log = 0; + +static bool +can_log_tcpdns_quota(void) { + isc_stdtime_t now, last; + + isc_stdtime_get(&now); + last = atomic_exchange_relaxed(&last_tcpdnsquota_log, now); + if (now != last) { + return (true); + } + + return (false); +} + +static isc_result_t +tcpdns_connect_direct(isc_nmsocket_t *sock, isc__nm_uvreq_t *req); + +static void +tcpdns_close_direct(isc_nmsocket_t *sock); + +static void +tcpdns_connect_cb(uv_connect_t *uvreq, int status); + +static void +tcpdns_connection_cb(uv_stream_t *server, int status); + +static void +tcpdns_close_cb(uv_handle_t *uvhandle); + +static isc_result_t +accept_connection(isc_nmsocket_t *ssock, isc_quota_t *quota); + +static void +quota_accept_cb(isc_quota_t *quota, void *sock0); + +static void +stop_tcpdns_parent(isc_nmsocket_t *sock); +static void +stop_tcpdns_child(isc_nmsocket_t *sock); + +static isc_result_t +tcpdns_connect_direct(isc_nmsocket_t *sock, isc__nm_uvreq_t *req) { + isc__networker_t *worker = NULL; + isc_result_t result = ISC_R_UNSET; + int r; + + REQUIRE(VALID_NMSOCK(sock)); + REQUIRE(VALID_UVREQ(req)); + + REQUIRE(isc__nm_in_netthread()); + REQUIRE(sock->tid == isc_nm_tid()); + + worker = &sock->mgr->workers[sock->tid]; + + atomic_store(&sock->connecting, true); + + r = uv_tcp_init(&worker->loop, &sock->uv_handle.tcp); + UV_RUNTIME_CHECK(uv_tcp_init, r); + uv_handle_set_data(&sock->uv_handle.handle, sock); + + r = uv_timer_init(&worker->loop, &sock->read_timer); + UV_RUNTIME_CHECK(uv_timer_init, r); + uv_handle_set_data((uv_handle_t *)&sock->read_timer, sock); + + if (isc__nm_closing(sock)) { + result = ISC_R_CANCELED; + goto error; + } + + r = uv_tcp_open(&sock->uv_handle.tcp, sock->fd); + if (r != 0) { + isc__nm_closesocket(sock->fd); + isc__nm_incstats(sock->mgr, sock->statsindex[STATID_OPENFAIL]); + goto done; + } + isc__nm_incstats(sock->mgr, sock->statsindex[STATID_OPEN]); + + if (req->local.length != 0) { + r = uv_tcp_bind(&sock->uv_handle.tcp, &req->local.type.sa, 0); + /* + * In case of shared socket UV_EINVAL will be returned and needs + * to be ignored + */ + if (r != 0 && r != UV_EINVAL) { + isc__nm_incstats(sock->mgr, + sock->statsindex[STATID_BINDFAIL]); + goto done; + } + } + + uv_handle_set_data(&req->uv_req.handle, req); + r = uv_tcp_connect(&req->uv_req.connect, &sock->uv_handle.tcp, + &req->peer.type.sa, tcpdns_connect_cb); + if (r != 0) { + isc__nm_incstats(sock->mgr, + sock->statsindex[STATID_CONNECTFAIL]); + goto done; + } + isc__nm_incstats(sock->mgr, sock->statsindex[STATID_CONNECT]); + + uv_handle_set_data((uv_handle_t *)&sock->read_timer, + &req->uv_req.connect); + isc__nmsocket_timer_start(sock); + + atomic_store(&sock->connected, true); + +done: + result = isc__nm_uverr2result(r); +error: + LOCK(&sock->lock); + sock->result = result; + SIGNAL(&sock->cond); + if (!atomic_load(&sock->active)) { + WAIT(&sock->scond, &sock->lock); + } + INSIST(atomic_load(&sock->active)); + UNLOCK(&sock->lock); + + return (result); +} + +void +isc__nm_async_tcpdnsconnect(isc__networker_t *worker, isc__netievent_t *ev0) { + isc__netievent_tcpdnsconnect_t *ievent = + (isc__netievent_tcpdnsconnect_t *)ev0; + isc_nmsocket_t *sock = ievent->sock; + isc__nm_uvreq_t *req = ievent->req; + isc_result_t result = ISC_R_SUCCESS; + + UNUSED(worker); + + REQUIRE(VALID_NMSOCK(sock)); + REQUIRE(sock->type == isc_nm_tcpdnssocket); + REQUIRE(sock->parent == NULL); + REQUIRE(sock->tid == isc_nm_tid()); + + result = tcpdns_connect_direct(sock, req); + if (result != ISC_R_SUCCESS) { + isc__nmsocket_clearcb(sock); + isc__nm_connectcb(sock, req, result, true); + atomic_store(&sock->active, false); + isc__nm_tcpdns_close(sock); + } + + /* + * The sock is now attached to the handle. + */ + isc__nmsocket_detach(&sock); +} + +static void +tcpdns_connect_cb(uv_connect_t *uvreq, int status) { + isc_result_t result; + isc__nm_uvreq_t *req = NULL; + isc_nmsocket_t *sock = uv_handle_get_data((uv_handle_t *)uvreq->handle); + struct sockaddr_storage ss; + int r; + + REQUIRE(VALID_NMSOCK(sock)); + REQUIRE(sock->tid == isc_nm_tid()); + + isc__nmsocket_timer_stop(sock); + uv_handle_set_data((uv_handle_t *)&sock->read_timer, sock); + + req = uv_handle_get_data((uv_handle_t *)uvreq); + + REQUIRE(VALID_UVREQ(req)); + REQUIRE(VALID_NMHANDLE(req->handle)); + + if (atomic_load(&sock->timedout)) { + result = ISC_R_TIMEDOUT; + goto error; + } + + if (isc__nmsocket_closing(sock)) { + /* Socket was closed midflight by isc__nm_tcpdns_shutdown() */ + result = ISC_R_CANCELED; + goto error; + } else if (status == UV_ETIMEDOUT) { + /* Timeout status code here indicates hard error */ + result = ISC_R_TIMEDOUT; + goto error; + } else if (status != 0) { + result = isc__nm_uverr2result(status); + goto error; + } + + isc__nm_incstats(sock->mgr, sock->statsindex[STATID_CONNECT]); + r = uv_tcp_getpeername(&sock->uv_handle.tcp, (struct sockaddr *)&ss, + &(int){ sizeof(ss) }); + if (r != 0) { + result = isc__nm_uverr2result(r); + goto error; + } + + atomic_store(&sock->connecting, false); + + result = isc_sockaddr_fromsockaddr(&sock->peer, (struct sockaddr *)&ss); + RUNTIME_CHECK(result == ISC_R_SUCCESS); + + isc__nm_connectcb(sock, req, ISC_R_SUCCESS, false); + + return; + +error: + isc__nm_failed_connect_cb(sock, req, result, false); +} + +void +isc_nm_tcpdnsconnect(isc_nm_t *mgr, isc_sockaddr_t *local, isc_sockaddr_t *peer, + isc_nm_cb_t cb, void *cbarg, unsigned int timeout, + size_t extrahandlesize) { + isc_result_t result = ISC_R_SUCCESS; + isc_nmsocket_t *sock = NULL; + isc__netievent_tcpdnsconnect_t *ievent = NULL; + isc__nm_uvreq_t *req = NULL; + sa_family_t sa_family; + + REQUIRE(VALID_NM(mgr)); + REQUIRE(local != NULL); + REQUIRE(peer != NULL); + + sa_family = peer->type.sa.sa_family; + + sock = isc_mem_get(mgr->mctx, sizeof(*sock)); + isc__nmsocket_init(sock, mgr, isc_nm_tcpdnssocket, local); + + sock->extrahandlesize = extrahandlesize; + sock->connect_timeout = timeout; + sock->result = ISC_R_UNSET; + atomic_init(&sock->client, true); + + req = isc__nm_uvreq_get(mgr, sock); + req->cb.connect = cb; + req->cbarg = cbarg; + req->peer = *peer; + req->local = *local; + req->handle = isc__nmhandle_get(sock, &req->peer, &sock->iface); + + result = isc__nm_socket(sa_family, SOCK_STREAM, 0, &sock->fd); + if (result != ISC_R_SUCCESS) { + if (isc__nm_in_netthread()) { + sock->tid = isc_nm_tid(); + } + isc__nmsocket_clearcb(sock); + isc__nm_connectcb(sock, req, result, true); + atomic_store(&sock->closed, true); + isc__nmsocket_detach(&sock); + return; + } + + /* 2 minute timeout */ + result = isc__nm_socket_connectiontimeout(sock->fd, 120 * 1000); + RUNTIME_CHECK(result == ISC_R_SUCCESS); + + ievent = isc__nm_get_netievent_tcpdnsconnect(mgr, sock, req); + + if (isc__nm_in_netthread()) { + atomic_store(&sock->active, true); + sock->tid = isc_nm_tid(); + isc__nm_async_tcpdnsconnect(&mgr->workers[sock->tid], + (isc__netievent_t *)ievent); + isc__nm_put_netievent_tcpdnsconnect(mgr, ievent); + } else { + atomic_init(&sock->active, false); + sock->tid = isc_random_uniform(mgr->nworkers); + isc__nm_enqueue_ievent(&mgr->workers[sock->tid], + (isc__netievent_t *)ievent); + } + + LOCK(&sock->lock); + while (sock->result == ISC_R_UNSET) { + WAIT(&sock->cond, &sock->lock); + } + atomic_store(&sock->active, true); + BROADCAST(&sock->scond); + UNLOCK(&sock->lock); +} + +static uv_os_sock_t +isc__nm_tcpdns_lb_socket(isc_nm_t *mgr, sa_family_t sa_family) { + isc_result_t result; + uv_os_sock_t sock; + + result = isc__nm_socket(sa_family, SOCK_STREAM, 0, &sock); + RUNTIME_CHECK(result == ISC_R_SUCCESS); + + (void)isc__nm_socket_incoming_cpu(sock); + + /* FIXME: set mss */ + + result = isc__nm_socket_reuse(sock); + RUNTIME_CHECK(result == ISC_R_SUCCESS); + +#ifndef _WIN32 + if (mgr->load_balance_sockets) { + result = isc__nm_socket_reuse_lb(sock); + RUNTIME_CHECK(result == ISC_R_SUCCESS); + } +#endif + + return (sock); +} + +static void +enqueue_stoplistening(isc_nmsocket_t *sock) { + isc__netievent_tcpdnsstop_t *ievent = + isc__nm_get_netievent_tcpdnsstop(sock->mgr, sock); + isc__nm_enqueue_ievent(&sock->mgr->workers[sock->tid], + (isc__netievent_t *)ievent); +} + +static void +start_tcpdns_child(isc_nm_t *mgr, isc_sockaddr_t *iface, isc_nmsocket_t *sock, + uv_os_sock_t fd, int tid) { + isc__netievent_tcpdnslisten_t *ievent = NULL; + isc_nmsocket_t *csock = &sock->children[tid]; + + isc__nmsocket_init(csock, mgr, isc_nm_tcpdnssocket, iface); + csock->parent = sock; + csock->accept_cb = sock->accept_cb; + csock->accept_cbarg = sock->accept_cbarg; + csock->recv_cb = sock->recv_cb; + csock->recv_cbarg = sock->recv_cbarg; + csock->extrahandlesize = sock->extrahandlesize; + csock->backlog = sock->backlog; + csock->tid = tid; + /* + * We don't attach to quota, just assign - to avoid + * increasing quota unnecessarily. + */ + csock->pquota = sock->pquota; + isc_quota_cb_init(&csock->quotacb, quota_accept_cb, csock); + +#ifdef _WIN32 + UNUSED(fd); + csock->fd = isc__nm_tcpdns_lb_socket(mgr, iface->type.sa.sa_family); +#else + if (mgr->load_balance_sockets) { + UNUSED(fd); + csock->fd = isc__nm_tcpdns_lb_socket(mgr, + iface->type.sa.sa_family); + } else { + csock->fd = dup(fd); + } +#endif + REQUIRE(csock->fd >= 0); + + ievent = isc__nm_get_netievent_tcpdnslisten(mgr, csock); + isc__nm_maybe_enqueue_ievent(&mgr->workers[tid], + (isc__netievent_t *)ievent); +} +isc_result_t +isc_nm_listentcpdns(isc_nm_t *mgr, isc_sockaddr_t *iface, + isc_nm_recv_cb_t recv_cb, void *recv_cbarg, + isc_nm_accept_cb_t accept_cb, void *accept_cbarg, + size_t extrahandlesize, int backlog, isc_quota_t *quota, + isc_nmsocket_t **sockp) { + isc_result_t result = ISC_R_SUCCESS; + isc_nmsocket_t *sock = NULL; + size_t children_size = 0; + uv_os_sock_t fd = -1; + + REQUIRE(VALID_NM(mgr)); + + sock = isc_mem_get(mgr->mctx, sizeof(*sock)); + isc__nmsocket_init(sock, mgr, isc_nm_tcpdnslistener, iface); + + atomic_init(&sock->rchildren, 0); +#if defined(WIN32) + sock->nchildren = 1; +#else + sock->nchildren = mgr->nworkers; +#endif + children_size = sock->nchildren * sizeof(sock->children[0]); + sock->children = isc_mem_get(mgr->mctx, children_size); + memset(sock->children, 0, children_size); + + sock->result = ISC_R_UNSET; + sock->accept_cb = accept_cb; + sock->accept_cbarg = accept_cbarg; + sock->recv_cb = recv_cb; + sock->recv_cbarg = recv_cbarg; + sock->extrahandlesize = extrahandlesize; + sock->backlog = backlog; + sock->pquota = quota; + + sock->tid = 0; + sock->fd = -1; + +#ifndef _WIN32 + if (!mgr->load_balance_sockets) { + fd = isc__nm_tcpdns_lb_socket(mgr, iface->type.sa.sa_family); + } +#endif + + isc_barrier_init(&sock->startlistening, sock->nchildren); + + for (size_t i = 0; i < sock->nchildren; i++) { + if ((int)i == isc_nm_tid()) { + continue; + } + start_tcpdns_child(mgr, iface, sock, fd, i); + } + + if (isc__nm_in_netthread()) { + start_tcpdns_child(mgr, iface, sock, fd, isc_nm_tid()); + } + +#ifndef _WIN32 + if (!mgr->load_balance_sockets) { + isc__nm_closesocket(fd); + } +#endif + + LOCK(&sock->lock); + while (atomic_load(&sock->rchildren) != sock->nchildren) { + WAIT(&sock->cond, &sock->lock); + } + result = sock->result; + atomic_store(&sock->active, true); + UNLOCK(&sock->lock); + + INSIST(result != ISC_R_UNSET); + + if (result == ISC_R_SUCCESS) { + REQUIRE(atomic_load(&sock->rchildren) == sock->nchildren); + *sockp = sock; + } else { + atomic_store(&sock->active, false); + enqueue_stoplistening(sock); + isc_nmsocket_close(&sock); + } + + return (result); +} + +void +isc__nm_async_tcpdnslisten(isc__networker_t *worker, isc__netievent_t *ev0) { + isc__netievent_tcpdnslisten_t *ievent = + (isc__netievent_tcpdnslisten_t *)ev0; + sa_family_t sa_family; + int r; + int flags = 0; + isc_nmsocket_t *sock = NULL; + isc_result_t result = ISC_R_UNSET; + isc_nm_t *mgr = NULL; + + REQUIRE(VALID_NMSOCK(ievent->sock)); + REQUIRE(ievent->sock->tid == isc_nm_tid()); + REQUIRE(VALID_NMSOCK(ievent->sock->parent)); + + sock = ievent->sock; + sa_family = sock->iface.type.sa.sa_family; + mgr = sock->mgr; + + REQUIRE(sock->type == isc_nm_tcpdnssocket); + REQUIRE(sock->parent != NULL); + REQUIRE(sock->tid == isc_nm_tid()); + + /* TODO: set min mss */ + + r = uv_tcp_init(&worker->loop, &sock->uv_handle.tcp); + UV_RUNTIME_CHECK(uv_tcp_init, r); + uv_handle_set_data(&sock->uv_handle.handle, sock); + /* This keeps the socket alive after everything else is gone */ + isc__nmsocket_attach(sock, &(isc_nmsocket_t *){ NULL }); + + r = uv_timer_init(&worker->loop, &sock->read_timer); + UV_RUNTIME_CHECK(uv_timer_init, r); + uv_handle_set_data((uv_handle_t *)&sock->read_timer, sock); + + LOCK(&sock->parent->lock); + + r = uv_tcp_open(&sock->uv_handle.tcp, sock->fd); + if (r < 0) { + isc__nm_closesocket(sock->fd); + isc__nm_incstats(sock->mgr, sock->statsindex[STATID_OPENFAIL]); + goto done; + } + isc__nm_incstats(sock->mgr, sock->statsindex[STATID_OPEN]); + + if (sa_family == AF_INET6) { + flags = UV_TCP_IPV6ONLY; + } + +#ifdef _WIN32 + r = isc_uv_tcp_freebind(&sock->uv_handle.tcp, &sock->iface.type.sa, + flags); + if (r < 0) { + isc__nm_incstats(sock->mgr, sock->statsindex[STATID_BINDFAIL]); + goto done; + } +#else + if (mgr->load_balance_sockets) { + r = isc_uv_tcp_freebind(&sock->uv_handle.tcp, + &sock->iface.type.sa, flags); + if (r < 0) { + isc__nm_incstats(sock->mgr, + sock->statsindex[STATID_BINDFAIL]); + goto done; + } + } else { + if (sock->parent->fd == -1) { + r = isc_uv_tcp_freebind(&sock->uv_handle.tcp, + &sock->iface.type.sa, flags); + if (r < 0) { + isc__nm_incstats(sock->mgr, STATID_BINDFAIL); + goto done; + } + sock->parent->uv_handle.tcp.flags = + sock->uv_handle.tcp.flags; + sock->parent->fd = sock->fd; + } else { + /* The socket is already bound, just copy the flags */ + sock->uv_handle.tcp.flags = + sock->parent->uv_handle.tcp.flags; + } + } +#endif + + /* + * The callback will run in the same thread uv_listen() was called + * from, so a race with tcpdns_connection_cb() isn't possible. + */ + r = uv_listen((uv_stream_t *)&sock->uv_handle.tcp, sock->backlog, + tcpdns_connection_cb); + if (r != 0) { + isc_log_write(isc_lctx, ISC_LOGCATEGORY_GENERAL, + ISC_LOGMODULE_NETMGR, ISC_LOG_ERROR, + "uv_listen failed: %s", + isc_result_totext(isc__nm_uverr2result(r))); + isc__nm_incstats(sock->mgr, sock->statsindex[STATID_BINDFAIL]); + goto done; + } + + atomic_store(&sock->listening, true); + +done: + result = isc__nm_uverr2result(r); + if (result != ISC_R_SUCCESS) { + sock->pquota = NULL; + } + + atomic_fetch_add(&sock->parent->rchildren, 1); + if (sock->parent->result == ISC_R_UNSET) { + sock->parent->result = result; + } + SIGNAL(&sock->parent->cond); + UNLOCK(&sock->parent->lock); + + isc_barrier_wait(&sock->parent->startlistening); +} + +static void +tcpdns_connection_cb(uv_stream_t *server, int status) { + isc_nmsocket_t *ssock = uv_handle_get_data((uv_handle_t *)server); + isc_result_t result; + isc_quota_t *quota = NULL; + + if (status != 0) { + result = isc__nm_uverr2result(status); + goto done; + } + + REQUIRE(VALID_NMSOCK(ssock)); + REQUIRE(ssock->tid == isc_nm_tid()); + + if (isc__nmsocket_closing(ssock)) { + result = ISC_R_CANCELED; + goto done; + } + + if (ssock->pquota != NULL) { + result = isc_quota_attach_cb(ssock->pquota, "a, + &ssock->quotacb); + if (result == ISC_R_QUOTA) { + isc__nm_incstats(ssock->mgr, + ssock->statsindex[STATID_ACCEPTFAIL]); + goto done; + } + } + + result = accept_connection(ssock, quota); +done: + isc__nm_accept_connection_log(result, can_log_tcpdns_quota()); +} + +void +isc__nm_tcpdns_stoplistening(isc_nmsocket_t *sock) { + REQUIRE(VALID_NMSOCK(sock)); + REQUIRE(sock->type == isc_nm_tcpdnslistener); + + if (!atomic_compare_exchange_strong(&sock->closing, &(bool){ false }, + true)) + { + UNREACHABLE(); + } + + if (!isc__nm_in_netthread()) { + enqueue_stoplistening(sock); + } else { + stop_tcpdns_parent(sock); + } +} + +void +isc__nm_async_tcpdnsstop(isc__networker_t *worker, isc__netievent_t *ev0) { + isc__netievent_tcpdnsstop_t *ievent = + (isc__netievent_tcpdnsstop_t *)ev0; + isc_nmsocket_t *sock = ievent->sock; + + UNUSED(worker); + + REQUIRE(VALID_NMSOCK(sock)); + REQUIRE(sock->tid == isc_nm_tid()); + + if (sock->parent != NULL) { + stop_tcpdns_child(sock); + return; + } + + stop_tcpdns_parent(sock); +} + +void +isc__nm_tcpdns_failed_read_cb(isc_nmsocket_t *sock, isc_result_t result) { + REQUIRE(VALID_NMSOCK(sock)); + REQUIRE(result != ISC_R_SUCCESS); + + isc__nmsocket_timer_stop(sock); + isc__nm_stop_reading(sock); + + if (!sock->recv_read) { + goto destroy; + } + sock->recv_read = false; + + if (sock->recv_cb != NULL) { + isc__nm_uvreq_t *req = isc__nm_get_read_req(sock, NULL); + isc__nmsocket_clearcb(sock); + isc__nm_readcb(sock, req, result); + } + +destroy: + isc__nmsocket_prep_destroy(sock); + + /* + * We need to detach from quota after the read callback function had a + * chance to be executed. + */ + if (sock->quota != NULL) { + isc_quota_detach(&sock->quota); + } +} + +void +isc__nm_tcpdns_read(isc_nmhandle_t *handle, isc_nm_recv_cb_t cb, void *cbarg) { + REQUIRE(VALID_NMHANDLE(handle)); + REQUIRE(VALID_NMSOCK(handle->sock)); + + isc_nmsocket_t *sock = handle->sock; + isc__netievent_tcpdnsread_t *ievent = NULL; + + REQUIRE(sock->type == isc_nm_tcpdnssocket); + REQUIRE(sock->statichandle == handle); + REQUIRE(sock->tid == isc_nm_tid()); + REQUIRE(!sock->recv_read); + + sock->recv_cb = cb; + sock->recv_cbarg = cbarg; + sock->recv_read = true; + if (sock->read_timeout == 0) { + sock->read_timeout = + (atomic_load(&sock->keepalive) + ? atomic_load(&sock->mgr->keepalive) + : atomic_load(&sock->mgr->idle)); + } + + ievent = isc__nm_get_netievent_tcpdnsread(sock->mgr, sock); + + /* + * This MUST be done asynchronously, no matter which thread we're + * in. The callback function for isc_nm_read() often calls + * isc_nm_read() again; if we tried to do that synchronously + * we'd clash in processbuffer() and grow the stack indefinitely. + */ + isc__nm_enqueue_ievent(&sock->mgr->workers[sock->tid], + (isc__netievent_t *)ievent); + + return; +} + +void +isc__nm_async_tcpdnsread(isc__networker_t *worker, isc__netievent_t *ev0) { + isc__netievent_tcpdnsread_t *ievent = + (isc__netievent_tcpdnsread_t *)ev0; + isc_nmsocket_t *sock = ievent->sock; + isc_result_t result; + + UNUSED(worker); + + REQUIRE(VALID_NMSOCK(sock)); + REQUIRE(sock->tid == isc_nm_tid()); + + if (isc__nmsocket_closing(sock)) { + result = ISC_R_CANCELED; + } else { + result = isc__nm_process_sock_buffer(sock); + } + + if (result != ISC_R_SUCCESS) { + sock->reading = true; + isc__nm_failed_read_cb(sock, result, false); + } +} + +/* + * Process a single packet from the incoming buffer. + * + * Return ISC_R_SUCCESS and attach 'handlep' to a handle if something + * was processed; return ISC_R_NOMORE if there isn't a full message + * to be processed. + * + * The caller will need to unreference the handle. + */ +isc_result_t +isc__nm_tcpdns_processbuffer(isc_nmsocket_t *sock) { + size_t len; + isc__nm_uvreq_t *req = NULL; + isc_nmhandle_t *handle = NULL; + + REQUIRE(VALID_NMSOCK(sock)); + REQUIRE(sock->tid == isc_nm_tid()); + + if (isc__nmsocket_closing(sock)) { + return (ISC_R_CANCELED); + } + + /* + * If we don't even have the length yet, we can't do + * anything. + */ + if (sock->buf_len < 2) { + return (ISC_R_NOMORE); + } + + /* + * Process the first packet from the buffer, leaving + * the rest (if any) for later. + */ + len = ntohs(*(uint16_t *)sock->buf); + if (len > sock->buf_len - 2) { + return (ISC_R_NOMORE); + } + + req = isc__nm_get_read_req(sock, NULL); + REQUIRE(VALID_UVREQ(req)); + + /* + * We need to launch the resume_processing after the buffer has + * been consumed, thus we need to delay the detaching the handle. + */ + isc_nmhandle_attach(req->handle, &handle); + + /* + * The callback will be called synchronously because the + * result is ISC_R_SUCCESS, so we don't need to have + * the buffer on the heap + */ + req->uvbuf.base = (char *)sock->buf + 2; + req->uvbuf.len = len; + + /* + * If isc__nm_tcpdns_read() was called, it will be satisfied by single + * DNS message in the next call. + */ + sock->recv_read = false; + + /* + * The assertion failure here means that there's a errnoneous extra + * nmhandle detach happening in the callback and resume_processing gets + * called while we are still processing the buffer. + */ + REQUIRE(sock->processing == false); + sock->processing = true; + isc__nm_readcb(sock, req, ISC_R_SUCCESS); + sock->processing = false; + + len += 2; + sock->buf_len -= len; + if (sock->buf_len > 0) { + memmove(sock->buf, sock->buf + len, sock->buf_len); + } + + isc_nmhandle_detach(&handle); + + return (ISC_R_SUCCESS); +} + +void +isc__nm_tcpdns_read_cb(uv_stream_t *stream, ssize_t nread, + const uv_buf_t *buf) { + isc_nmsocket_t *sock = uv_handle_get_data((uv_handle_t *)stream); + uint8_t *base = NULL; + size_t len; + isc_result_t result; + + REQUIRE(VALID_NMSOCK(sock)); + REQUIRE(sock->tid == isc_nm_tid()); + REQUIRE(sock->reading); + REQUIRE(buf != NULL); + + if (isc__nmsocket_closing(sock)) { + isc__nm_failed_read_cb(sock, ISC_R_CANCELED, true); + goto free; + } + + if (nread < 0) { + if (nread != UV_EOF) { + isc__nm_incstats(sock->mgr, + sock->statsindex[STATID_RECVFAIL]); + } + + isc__nm_failed_read_cb(sock, isc__nm_uverr2result(nread), true); + goto free; + } + + base = (uint8_t *)buf->base; + len = nread; + + /* + * FIXME: We can avoid the memmove here if we know we have received full + * packet; e.g. we should be smarter, a.s. there are just few situations + * + * The tcp_alloc_buf should be smarter and point the uv_read_start to + * the position where previous read has ended in the sock->buf, that way + * the data could be read directly into sock->buf. + */ + + if (sock->buf_len + len > sock->buf_size) { + isc__nm_alloc_dnsbuf(sock, sock->buf_len + len); + } + memmove(sock->buf + sock->buf_len, base, len); + sock->buf_len += len; + + if (!atomic_load(&sock->client)) { + sock->read_timeout = atomic_load(&sock->mgr->idle); + } + + result = isc__nm_process_sock_buffer(sock); + if (result != ISC_R_SUCCESS) { + isc__nm_failed_read_cb(sock, result, true); + } +free: + if (nread < 0) { + /* + * The buffer may be a null buffer on error. + */ + if (buf->base == NULL && buf->len == 0) { + return; + } + } + + isc__nm_free_uvbuf(sock, buf); +} + +static void +quota_accept_cb(isc_quota_t *quota, void *sock0) { + isc_nmsocket_t *sock = (isc_nmsocket_t *)sock0; + + REQUIRE(VALID_NMSOCK(sock)); + + /* + * Create a tcpdnsaccept event and pass it using the async channel. + */ + + isc__netievent_tcpdnsaccept_t *ievent = + isc__nm_get_netievent_tcpdnsaccept(sock->mgr, sock, quota); + isc__nm_maybe_enqueue_ievent(&sock->mgr->workers[sock->tid], + (isc__netievent_t *)ievent); +} + +/* + * This is called after we get a quota_accept_cb() callback. + */ +void +isc__nm_async_tcpdnsaccept(isc__networker_t *worker, isc__netievent_t *ev0) { + isc__netievent_tcpdnsaccept_t *ievent = + (isc__netievent_tcpdnsaccept_t *)ev0; + isc_result_t result; + + UNUSED(worker); + + REQUIRE(VALID_NMSOCK(ievent->sock)); + REQUIRE(ievent->sock->tid == isc_nm_tid()); + + result = accept_connection(ievent->sock, ievent->quota); + isc__nm_accept_connection_log(result, can_log_tcpdns_quota()); +} + +static isc_result_t +accept_connection(isc_nmsocket_t *ssock, isc_quota_t *quota) { + isc_nmsocket_t *csock = NULL; + isc__networker_t *worker = NULL; + int r; + isc_result_t result; + struct sockaddr_storage peer_ss; + struct sockaddr_storage local_ss; + isc_sockaddr_t local; + isc_nmhandle_t *handle = NULL; + + REQUIRE(VALID_NMSOCK(ssock)); + REQUIRE(ssock->tid == isc_nm_tid()); + + if (isc__nmsocket_closing(ssock)) { + if (quota != NULL) { + isc_quota_detach("a); + } + return (ISC_R_CANCELED); + } + + REQUIRE(ssock->accept_cb != NULL); + + csock = isc_mem_get(ssock->mgr->mctx, sizeof(isc_nmsocket_t)); + isc__nmsocket_init(csock, ssock->mgr, isc_nm_tcpdnssocket, + &ssock->iface); + csock->tid = ssock->tid; + csock->extrahandlesize = ssock->extrahandlesize; + isc__nmsocket_attach(ssock, &csock->server); + csock->recv_cb = ssock->recv_cb; + csock->recv_cbarg = ssock->recv_cbarg; + csock->quota = quota; + csock->accepting = true; + + worker = &csock->mgr->workers[csock->tid]; + + r = uv_tcp_init(&worker->loop, &csock->uv_handle.tcp); + UV_RUNTIME_CHECK(uv_tcp_init, r); + uv_handle_set_data(&csock->uv_handle.handle, csock); + + r = uv_timer_init(&worker->loop, &csock->read_timer); + UV_RUNTIME_CHECK(uv_timer_init, r); + uv_handle_set_data((uv_handle_t *)&csock->read_timer, csock); + + r = uv_accept(&ssock->uv_handle.stream, &csock->uv_handle.stream); + if (r != 0) { + result = isc__nm_uverr2result(r); + goto failure; + } + + r = uv_tcp_getpeername(&csock->uv_handle.tcp, + (struct sockaddr *)&peer_ss, + &(int){ sizeof(peer_ss) }); + if (r != 0) { + result = isc__nm_uverr2result(r); + goto failure; + } + + result = isc_sockaddr_fromsockaddr(&csock->peer, + (struct sockaddr *)&peer_ss); + if (result != ISC_R_SUCCESS) { + goto failure; + } + + r = uv_tcp_getsockname(&csock->uv_handle.tcp, + (struct sockaddr *)&local_ss, + &(int){ sizeof(local_ss) }); + if (r != 0) { + result = isc__nm_uverr2result(r); + goto failure; + } + + result = isc_sockaddr_fromsockaddr(&local, + (struct sockaddr *)&local_ss); + if (result != ISC_R_SUCCESS) { + goto failure; + } + + /* + * The handle will be either detached on acceptcb failure or in the + * readcb. + */ + handle = isc__nmhandle_get(csock, NULL, &local); + + result = ssock->accept_cb(handle, ISC_R_SUCCESS, ssock->accept_cbarg); + if (result != ISC_R_SUCCESS) { + isc_nmhandle_detach(&handle); + goto failure; + } + + csock->accepting = false; + + isc__nm_incstats(csock->mgr, csock->statsindex[STATID_ACCEPT]); + + csock->read_timeout = atomic_load(&csock->mgr->init); + + csock->closehandle_cb = isc__nm_resume_processing; + + /* + * We need to keep the handle alive until we fail to read or connection + * is closed by the other side, it will be detached via + * prep_destroy()->tcpdns_close_direct(). + */ + isc_nmhandle_attach(handle, &csock->recv_handle); + result = isc__nm_process_sock_buffer(csock); + if (result != ISC_R_SUCCESS) { + isc_nmhandle_detach(&csock->recv_handle); + isc_nmhandle_detach(&handle); + goto failure; + } + + /* + * The initial timer has been set, update the read timeout for the next + * reads. + */ + csock->read_timeout = (atomic_load(&csock->keepalive) + ? atomic_load(&csock->mgr->keepalive) + : atomic_load(&csock->mgr->idle)); + + isc_nmhandle_detach(&handle); + + /* + * sock is now attached to the handle. + */ + isc__nmsocket_detach(&csock); + + return (ISC_R_SUCCESS); + +failure: + + atomic_store(&csock->active, false); + + isc__nm_failed_accept_cb(csock, result); + + isc__nmsocket_prep_destroy(csock); + + isc__nmsocket_detach(&csock); + + return (result); +} + +void +isc__nm_tcpdns_send(isc_nmhandle_t *handle, isc_region_t *region, + isc_nm_cb_t cb, void *cbarg) { + REQUIRE(VALID_NMHANDLE(handle)); + REQUIRE(VALID_NMSOCK(handle->sock)); + + isc_nmsocket_t *sock = handle->sock; + isc__netievent_tcpdnssend_t *ievent = NULL; + isc__nm_uvreq_t *uvreq = NULL; + + REQUIRE(sock->type == isc_nm_tcpdnssocket); + + uvreq = isc__nm_uvreq_get(sock->mgr, sock); + *(uint16_t *)uvreq->tcplen = htons(region->length); + uvreq->uvbuf.base = (char *)region->base; + uvreq->uvbuf.len = region->length; + + isc_nmhandle_attach(handle, &uvreq->handle); + + uvreq->cb.send = cb; + uvreq->cbarg = cbarg; + + ievent = isc__nm_get_netievent_tcpdnssend(sock->mgr, sock, uvreq); + isc__nm_maybe_enqueue_ievent(&sock->mgr->workers[sock->tid], + (isc__netievent_t *)ievent); + + return; +} + +static void +tcpdns_send_cb(uv_write_t *req, int status) { + isc__nm_uvreq_t *uvreq = (isc__nm_uvreq_t *)req->data; + isc_nmsocket_t *sock = NULL; + + REQUIRE(VALID_UVREQ(uvreq)); + REQUIRE(VALID_NMSOCK(uvreq->sock)); + + sock = uvreq->sock; + + isc_nm_timer_stop(uvreq->timer); + isc_nm_timer_detach(&uvreq->timer); + + if (status < 0) { + isc__nm_incstats(sock->mgr, sock->statsindex[STATID_SENDFAIL]); + isc__nm_failed_send_cb(sock, uvreq, + isc__nm_uverr2result(status)); + return; + } + + isc__nm_sendcb(sock, uvreq, ISC_R_SUCCESS, false); +} + +/* + * Handle 'tcpsend' async event - send a packet on the socket + */ +void +isc__nm_async_tcpdnssend(isc__networker_t *worker, isc__netievent_t *ev0) { + isc__netievent_tcpdnssend_t *ievent = + (isc__netievent_tcpdnssend_t *)ev0; + + REQUIRE(VALID_UVREQ(ievent->req)); + REQUIRE(VALID_NMSOCK(ievent->sock)); + REQUIRE(ievent->sock->type == isc_nm_tcpdnssocket); + REQUIRE(ievent->sock->tid == isc_nm_tid()); + + isc_result_t result; + isc_nmsocket_t *sock = ievent->sock; + isc__nm_uvreq_t *uvreq = ievent->req; + + if (sock->write_timeout == 0) { + sock->write_timeout = + (atomic_load(&sock->keepalive) + ? atomic_load(&sock->mgr->keepalive) + : atomic_load(&sock->mgr->idle)); + } + + uv_buf_t bufs[2] = { { .base = uvreq->tcplen, .len = 2 }, + { .base = uvreq->uvbuf.base, + .len = uvreq->uvbuf.len } }; + int nbufs = 2; + int r; + + UNUSED(worker); + + if (isc__nmsocket_closing(sock)) { + result = ISC_R_CANCELED; + goto fail; + } + + r = uv_try_write(&sock->uv_handle.stream, bufs, nbufs); + + if (r == (int)(bufs[0].len + bufs[1].len)) { + /* Wrote everything */ + isc__nm_sendcb(sock, uvreq, ISC_R_SUCCESS, true); + return; + } + + if (r == 1) { + /* Partial write of DNSMSG length */ + bufs[0].base = uvreq->tcplen + 1; + bufs[0].len = 1; + } else if (r > 0) { + /* Partial write of DNSMSG */ + nbufs = 1; + bufs[0].base = uvreq->uvbuf.base + (r - 2); + bufs[0].len = uvreq->uvbuf.len - (r - 2); + } else if (r == UV_ENOSYS || r == UV_EAGAIN) { + /* uv_try_write not supported, send asynchronously */ + } else { + /* error sending data */ + result = isc__nm_uverr2result(r); + goto fail; + } + + r = uv_write(&uvreq->uv_req.write, &sock->uv_handle.stream, bufs, nbufs, + tcpdns_send_cb); + if (r < 0) { + result = isc__nm_uverr2result(r); + goto fail; + } + + isc_nm_timer_create(uvreq->handle, isc__nmsocket_writetimeout_cb, uvreq, + &uvreq->timer); + if (sock->write_timeout > 0) { + isc_nm_timer_start(uvreq->timer, sock->write_timeout); + } + + return; +fail: + if (result != ISC_R_SUCCESS) { + isc__nm_incstats(sock->mgr, sock->statsindex[STATID_SENDFAIL]); + isc__nm_failed_send_cb(sock, uvreq, result); + } +} + +static void +tcpdns_stop_cb(uv_handle_t *handle) { + isc_nmsocket_t *sock = uv_handle_get_data(handle); + + REQUIRE(VALID_NMSOCK(sock)); + REQUIRE(sock->tid == isc_nm_tid()); + REQUIRE(atomic_load(&sock->closing)); + + uv_handle_set_data(handle, NULL); + + if (!atomic_compare_exchange_strong(&sock->closed, &(bool){ false }, + true)) + { + UNREACHABLE(); + } + + isc__nm_incstats(sock->mgr, sock->statsindex[STATID_CLOSE]); + + atomic_store(&sock->listening, false); + + isc__nmsocket_detach(&sock); +} + +static void +tcpdns_close_sock(isc_nmsocket_t *sock) { + REQUIRE(VALID_NMSOCK(sock)); + REQUIRE(sock->tid == isc_nm_tid()); + REQUIRE(atomic_load(&sock->closing)); + + if (!atomic_compare_exchange_strong(&sock->closed, &(bool){ false }, + true)) + { + UNREACHABLE(); + } + + isc__nm_incstats(sock->mgr, sock->statsindex[STATID_CLOSE]); + + if (sock->server != NULL) { + isc__nmsocket_detach(&sock->server); + } + + atomic_store(&sock->connected, false); + + isc__nmsocket_prep_destroy(sock); +} + +static void +tcpdns_close_cb(uv_handle_t *handle) { + isc_nmsocket_t *sock = uv_handle_get_data(handle); + + uv_handle_set_data(handle, NULL); + + tcpdns_close_sock(sock); +} + +static void +read_timer_close_cb(uv_handle_t *timer) { + isc_nmsocket_t *sock = uv_handle_get_data(timer); + uv_handle_set_data(timer, NULL); + + REQUIRE(VALID_NMSOCK(sock)); + + if (sock->parent) { + uv_close(&sock->uv_handle.handle, tcpdns_stop_cb); + } else if (uv_is_closing(&sock->uv_handle.handle)) { + tcpdns_close_sock(sock); + } else { + uv_close(&sock->uv_handle.handle, tcpdns_close_cb); + } +} + +static void +stop_tcpdns_child(isc_nmsocket_t *sock) { + REQUIRE(sock->type == isc_nm_tcpdnssocket); + REQUIRE(sock->tid == isc_nm_tid()); + + if (!atomic_compare_exchange_strong(&sock->closing, &(bool){ false }, + true)) + { + return; + } + + tcpdns_close_direct(sock); + + atomic_fetch_sub(&sock->parent->rchildren, 1); + + isc_barrier_wait(&sock->parent->stoplistening); +} + +static void +stop_tcpdns_parent(isc_nmsocket_t *sock) { + isc_nmsocket_t *csock = NULL; + + REQUIRE(VALID_NMSOCK(sock)); + REQUIRE(sock->tid == isc_nm_tid()); + REQUIRE(sock->type == isc_nm_tcpdnslistener); + + isc_barrier_init(&sock->stoplistening, sock->nchildren); + + for (size_t i = 0; i < sock->nchildren; i++) { + csock = &sock->children[i]; + REQUIRE(VALID_NMSOCK(csock)); + + if ((int)i == isc_nm_tid()) { + /* + * We need to schedule closing the other sockets first + */ + continue; + } + + atomic_store(&csock->active, false); + enqueue_stoplistening(csock); + } + + csock = &sock->children[isc_nm_tid()]; + atomic_store(&csock->active, false); + stop_tcpdns_child(csock); + + atomic_store(&sock->closed, true); + isc__nmsocket_prep_destroy(sock); +} + +static void +tcpdns_close_direct(isc_nmsocket_t *sock) { + REQUIRE(VALID_NMSOCK(sock)); + REQUIRE(sock->tid == isc_nm_tid()); + REQUIRE(atomic_load(&sock->closing)); + + if (sock->quota != NULL) { + isc_quota_detach(&sock->quota); + } + + if (sock->recv_handle != NULL) { + isc_nmhandle_detach(&sock->recv_handle); + } + + isc__nmsocket_timer_stop(sock); + isc__nm_stop_reading(sock); + + uv_handle_set_data((uv_handle_t *)&sock->read_timer, sock); + uv_close((uv_handle_t *)&sock->read_timer, read_timer_close_cb); +} + +void +isc__nm_tcpdns_close(isc_nmsocket_t *sock) { + REQUIRE(VALID_NMSOCK(sock)); + REQUIRE(sock->type == isc_nm_tcpdnssocket); + REQUIRE(!isc__nmsocket_active(sock)); + + if (!atomic_compare_exchange_strong(&sock->closing, &(bool){ false }, + true)) + { + return; + } + + if (sock->tid == isc_nm_tid()) { + tcpdns_close_direct(sock); + } else { + /* + * We need to create an event and pass it using async channel + */ + isc__netievent_tcpdnsclose_t *ievent = + isc__nm_get_netievent_tcpdnsclose(sock->mgr, sock); + + isc__nm_enqueue_ievent(&sock->mgr->workers[sock->tid], + (isc__netievent_t *)ievent); + } +} + +void +isc__nm_async_tcpdnsclose(isc__networker_t *worker, isc__netievent_t *ev0) { + isc__netievent_tcpdnsclose_t *ievent = + (isc__netievent_tcpdnsclose_t *)ev0; + isc_nmsocket_t *sock = ievent->sock; + + UNUSED(worker); + + REQUIRE(VALID_NMSOCK(sock)); + REQUIRE(sock->tid == isc_nm_tid()); + + tcpdns_close_direct(sock); +} + +static void +tcpdns_close_connect_cb(uv_handle_t *handle) { + isc_nmsocket_t *sock = uv_handle_get_data(handle); + + REQUIRE(VALID_NMSOCK(sock)); + + REQUIRE(isc__nm_in_netthread()); + REQUIRE(sock->tid == isc_nm_tid()); + + isc__nmsocket_prep_destroy(sock); + isc__nmsocket_detach(&sock); +} + +void +isc__nm_tcpdns_shutdown(isc_nmsocket_t *sock) { + REQUIRE(VALID_NMSOCK(sock)); + REQUIRE(sock->tid == isc_nm_tid()); + REQUIRE(sock->type == isc_nm_tcpdnssocket); + + /* + * If the socket is active, mark it inactive and + * continue. If it isn't active, stop now. + */ + if (!isc__nmsocket_deactivate(sock)) { + return; + } + + if (sock->accepting) { + return; + } + + if (atomic_load(&sock->connecting)) { + isc_nmsocket_t *tsock = NULL; + isc__nmsocket_attach(sock, &tsock); + uv_close(&sock->uv_handle.handle, tcpdns_close_connect_cb); + return; + } + + if (sock->statichandle != NULL) { + isc__nm_failed_read_cb(sock, ISC_R_CANCELED, false); + return; + } + + /* + * Otherwise, we just send the socket to abyss... + */ + if (sock->parent == NULL) { + isc__nmsocket_prep_destroy(sock); + } +} + +void +isc__nm_tcpdns_cancelread(isc_nmhandle_t *handle) { + isc_nmsocket_t *sock = NULL; + isc__netievent_tcpdnscancel_t *ievent = NULL; + + REQUIRE(VALID_NMHANDLE(handle)); + + sock = handle->sock; + + REQUIRE(VALID_NMSOCK(sock)); + REQUIRE(sock->type == isc_nm_tcpdnssocket); + + ievent = isc__nm_get_netievent_tcpdnscancel(sock->mgr, sock, handle); + isc__nm_enqueue_ievent(&sock->mgr->workers[sock->tid], + (isc__netievent_t *)ievent); +} + +void +isc__nm_async_tcpdnscancel(isc__networker_t *worker, isc__netievent_t *ev0) { + isc__netievent_tcpdnscancel_t *ievent = + (isc__netievent_tcpdnscancel_t *)ev0; + isc_nmsocket_t *sock = ievent->sock; + + UNUSED(worker); + + REQUIRE(VALID_NMSOCK(sock)); + REQUIRE(sock->tid == isc_nm_tid()); + + isc__nm_failed_read_cb(sock, ISC_R_EOF, false); +} + +void +isc_nm_tcpdns_sequential(isc_nmhandle_t *handle) { + isc_nmsocket_t *sock = NULL; + + REQUIRE(VALID_NMHANDLE(handle)); + REQUIRE(VALID_NMSOCK(handle->sock)); + REQUIRE(handle->sock->type == isc_nm_tcpdnssocket); + + sock = handle->sock; + + /* + * We don't want pipelining on this connection. That means + * that we need to pause after reading each request, and + * resume only after the request has been processed. This + * is done in resume_processing(), which is the socket's + * closehandle_cb callback, called whenever a handle + * is released. + */ + + isc__nmsocket_timer_stop(sock); + isc__nm_stop_reading(sock); + atomic_store(&sock->sequential, true); +} diff --git a/lib/isc/netmgr/udp.c b/lib/isc/netmgr/udp.c new file mode 100644 index 0000000..00f9d40 --- /dev/null +++ b/lib/isc/netmgr/udp.c @@ -0,0 +1,1211 @@ +/* + * Copyright (C) Internet Systems Consortium, Inc. ("ISC") + * + * SPDX-License-Identifier: MPL-2.0 + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, you can obtain one at https://mozilla.org/MPL/2.0/. + * + * See the COPYRIGHT file distributed with this work for additional + * information regarding copyright ownership. + */ + +#include <unistd.h> +#include <uv.h> + +#include <isc/atomic.h> +#include <isc/barrier.h> +#include <isc/buffer.h> +#include <isc/condition.h> +#include <isc/errno.h> +#include <isc/magic.h> +#include <isc/mem.h> +#include <isc/netmgr.h> +#include <isc/random.h> +#include <isc/refcount.h> +#include <isc/region.h> +#include <isc/result.h> +#include <isc/sockaddr.h> +#include <isc/thread.h> +#include <isc/util.h> + +#include "netmgr-int.h" +#include "uv-compat.h" + +static isc_result_t +udp_send_direct(isc_nmsocket_t *sock, isc__nm_uvreq_t *req, + isc_sockaddr_t *peer); + +static void +udp_recv_cb(uv_udp_t *handle, ssize_t nrecv, const uv_buf_t *buf, + const struct sockaddr *addr, unsigned flags); + +static void +udp_send_cb(uv_udp_send_t *req, int status); + +static void +udp_close_cb(uv_handle_t *handle); + +static void +read_timer_close_cb(uv_handle_t *handle); + +static void +udp_close_direct(isc_nmsocket_t *sock); + +static void +stop_udp_parent(isc_nmsocket_t *sock); +static void +stop_udp_child(isc_nmsocket_t *sock); + +static uv_os_sock_t +isc__nm_udp_lb_socket(isc_nm_t *mgr, sa_family_t sa_family) { + isc_result_t result; + uv_os_sock_t sock; + + result = isc__nm_socket(sa_family, SOCK_DGRAM, 0, &sock); + RUNTIME_CHECK(result == ISC_R_SUCCESS); + + (void)isc__nm_socket_incoming_cpu(sock); + (void)isc__nm_socket_disable_pmtud(sock, sa_family); + + result = isc__nm_socket_reuse(sock); + RUNTIME_CHECK(result == ISC_R_SUCCESS); + +#ifndef _WIN32 + if (mgr->load_balance_sockets) { + result = isc__nm_socket_reuse_lb(sock); + RUNTIME_CHECK(result == ISC_R_SUCCESS); + } +#endif + + return (sock); +} + +static void +start_udp_child(isc_nm_t *mgr, isc_sockaddr_t *iface, isc_nmsocket_t *sock, + uv_os_sock_t fd, int tid) { + isc_nmsocket_t *csock; + isc__netievent_udplisten_t *ievent = NULL; + + csock = &sock->children[tid]; + + isc__nmsocket_init(csock, mgr, isc_nm_udpsocket, iface); + csock->parent = sock; + csock->iface = sock->iface; + csock->reading = true; + csock->recv_cb = sock->recv_cb; + csock->recv_cbarg = sock->recv_cbarg; + csock->extrahandlesize = sock->extrahandlesize; + csock->tid = tid; + +#ifdef _WIN32 + UNUSED(fd); + csock->fd = isc__nm_udp_lb_socket(mgr, iface->type.sa.sa_family); +#else + if (mgr->load_balance_sockets) { + UNUSED(fd); + csock->fd = isc__nm_udp_lb_socket(mgr, + iface->type.sa.sa_family); + } else { + csock->fd = dup(fd); + } +#endif + REQUIRE(csock->fd >= 0); + + ievent = isc__nm_get_netievent_udplisten(mgr, csock); + isc__nm_maybe_enqueue_ievent(&mgr->workers[tid], + (isc__netievent_t *)ievent); +} + +static void +enqueue_stoplistening(isc_nmsocket_t *sock) { + isc__netievent_udpstop_t *ievent = + isc__nm_get_netievent_udpstop(sock->mgr, sock); + isc__nm_enqueue_ievent(&sock->mgr->workers[sock->tid], + (isc__netievent_t *)ievent); +} + +isc_result_t +isc_nm_listenudp(isc_nm_t *mgr, isc_sockaddr_t *iface, isc_nm_recv_cb_t cb, + void *cbarg, size_t extrahandlesize, isc_nmsocket_t **sockp) { + isc_result_t result = ISC_R_SUCCESS; + isc_nmsocket_t *sock = NULL; + size_t children_size = 0; + REQUIRE(VALID_NM(mgr)); + uv_os_sock_t fd = -1; + + /* + * We are creating mgr->nworkers duplicated sockets, one + * socket for each worker thread. + */ + sock = isc_mem_get(mgr->mctx, sizeof(isc_nmsocket_t)); + isc__nmsocket_init(sock, mgr, isc_nm_udplistener, iface); + + atomic_init(&sock->rchildren, 0); +#if defined(WIN32) + sock->nchildren = 1; +#else + sock->nchildren = mgr->nworkers; +#endif + + children_size = sock->nchildren * sizeof(sock->children[0]); + sock->children = isc_mem_get(mgr->mctx, children_size); + memset(sock->children, 0, children_size); + + sock->recv_cb = cb; + sock->recv_cbarg = cbarg; + sock->extrahandlesize = extrahandlesize; + sock->result = ISC_R_UNSET; + + sock->tid = 0; + sock->fd = -1; + +#ifndef _WIN32 + if (!mgr->load_balance_sockets) { + fd = isc__nm_udp_lb_socket(mgr, iface->type.sa.sa_family); + } +#endif + + isc_barrier_init(&sock->startlistening, sock->nchildren); + + for (size_t i = 0; i < sock->nchildren; i++) { + if ((int)i == isc_nm_tid()) { + continue; + } + start_udp_child(mgr, iface, sock, fd, i); + } + + if (isc__nm_in_netthread()) { + start_udp_child(mgr, iface, sock, fd, isc_nm_tid()); + } + +#ifndef _WIN32 + if (!mgr->load_balance_sockets) { + isc__nm_closesocket(fd); + } +#endif + + LOCK(&sock->lock); + while (atomic_load(&sock->rchildren) != sock->nchildren) { + WAIT(&sock->cond, &sock->lock); + } + result = sock->result; + atomic_store(&sock->active, true); + UNLOCK(&sock->lock); + + INSIST(result != ISC_R_UNSET); + + if (result == ISC_R_SUCCESS) { + REQUIRE(atomic_load(&sock->rchildren) == sock->nchildren); + *sockp = sock; + } else { + atomic_store(&sock->active, false); + enqueue_stoplistening(sock); + isc_nmsocket_close(&sock); + } + + return (result); +} + +/* + * Asynchronous 'udplisten' call handler: start listening on a UDP socket. + */ +void +isc__nm_async_udplisten(isc__networker_t *worker, isc__netievent_t *ev0) { + isc__netievent_udplisten_t *ievent = (isc__netievent_udplisten_t *)ev0; + isc_nmsocket_t *sock = NULL; + int r, uv_bind_flags = 0; + int uv_init_flags = 0; + sa_family_t sa_family; + isc_result_t result = ISC_R_UNSET; + isc_nm_t *mgr = NULL; + + REQUIRE(VALID_NMSOCK(ievent->sock)); + REQUIRE(ievent->sock->tid == isc_nm_tid()); + REQUIRE(VALID_NMSOCK(ievent->sock->parent)); + + sock = ievent->sock; + sa_family = sock->iface.type.sa.sa_family; + mgr = sock->mgr; + + REQUIRE(sock->type == isc_nm_udpsocket); + REQUIRE(sock->parent != NULL); + REQUIRE(sock->tid == isc_nm_tid()); + +#if HAVE_DECL_UV_UDP_RECVMMSG + uv_init_flags |= UV_UDP_RECVMMSG; +#endif + r = uv_udp_init_ex(&worker->loop, &sock->uv_handle.udp, uv_init_flags); + UV_RUNTIME_CHECK(uv_udp_init_ex, r); + uv_handle_set_data(&sock->uv_handle.handle, sock); + /* This keeps the socket alive after everything else is gone */ + isc__nmsocket_attach(sock, &(isc_nmsocket_t *){ NULL }); + + r = uv_timer_init(&worker->loop, &sock->read_timer); + UV_RUNTIME_CHECK(uv_timer_init, r); + uv_handle_set_data((uv_handle_t *)&sock->read_timer, sock); + + LOCK(&sock->parent->lock); + + r = uv_udp_open(&sock->uv_handle.udp, sock->fd); + if (r < 0) { + isc__nm_closesocket(sock->fd); + isc__nm_incstats(sock->mgr, sock->statsindex[STATID_OPENFAIL]); + goto done; + } + isc__nm_incstats(sock->mgr, sock->statsindex[STATID_OPEN]); + + if (sa_family == AF_INET6) { + uv_bind_flags |= UV_UDP_IPV6ONLY; + } + +#ifdef _WIN32 + r = isc_uv_udp_freebind(&sock->uv_handle.udp, + &sock->parent->iface.type.sa, uv_bind_flags); + if (r < 0) { + isc__nm_incstats(sock->mgr, sock->statsindex[STATID_BINDFAIL]); + goto done; + } +#else + if (mgr->load_balance_sockets) { + r = isc_uv_udp_freebind(&sock->uv_handle.udp, + &sock->parent->iface.type.sa, + uv_bind_flags); + if (r < 0) { + isc__nm_incstats(sock->mgr, + sock->statsindex[STATID_BINDFAIL]); + goto done; + } + } else { + if (sock->parent->fd == -1) { + /* This thread is first, bind the socket */ + r = isc_uv_udp_freebind(&sock->uv_handle.udp, + &sock->parent->iface.type.sa, + uv_bind_flags); + if (r < 0) { + isc__nm_incstats(sock->mgr, STATID_BINDFAIL); + goto done; + } + sock->parent->uv_handle.udp.flags = + sock->uv_handle.udp.flags; + sock->parent->fd = sock->fd; + } else { + /* The socket is already bound, just copy the flags */ + sock->uv_handle.udp.flags = + sock->parent->uv_handle.udp.flags; + } + } +#endif + +#ifdef ISC_RECV_BUFFER_SIZE + uv_recv_buffer_size(&sock->uv_handle.handle, + &(int){ ISC_RECV_BUFFER_SIZE }); +#endif +#ifdef ISC_SEND_BUFFER_SIZE + uv_send_buffer_size(&sock->uv_handle.handle, + &(int){ ISC_SEND_BUFFER_SIZE }); +#endif + r = uv_udp_recv_start(&sock->uv_handle.udp, isc__nm_alloc_cb, + udp_recv_cb); + if (r != 0) { + isc__nm_incstats(sock->mgr, sock->statsindex[STATID_BINDFAIL]); + goto done; + } + + atomic_store(&sock->listening, true); + +done: + result = isc__nm_uverr2result(r); + atomic_fetch_add(&sock->parent->rchildren, 1); + if (sock->parent->result == ISC_R_UNSET) { + sock->parent->result = result; + } + SIGNAL(&sock->parent->cond); + UNLOCK(&sock->parent->lock); + + isc_barrier_wait(&sock->parent->startlistening); +} + +void +isc__nm_udp_stoplistening(isc_nmsocket_t *sock) { + REQUIRE(VALID_NMSOCK(sock)); + REQUIRE(sock->type == isc_nm_udplistener); + + if (!atomic_compare_exchange_strong(&sock->closing, &(bool){ false }, + true)) + { + UNREACHABLE(); + } + + if (!isc__nm_in_netthread()) { + enqueue_stoplistening(sock); + } else { + stop_udp_parent(sock); + } +} + +/* + * Asynchronous 'udpstop' call handler: stop listening on a UDP socket. + */ +void +isc__nm_async_udpstop(isc__networker_t *worker, isc__netievent_t *ev0) { + isc__netievent_udpstop_t *ievent = (isc__netievent_udpstop_t *)ev0; + isc_nmsocket_t *sock = ievent->sock; + + UNUSED(worker); + + REQUIRE(VALID_NMSOCK(sock)); + REQUIRE(sock->tid == isc_nm_tid()); + + if (sock->parent != NULL) { + stop_udp_child(sock); + return; + } + + stop_udp_parent(sock); +} + +/* + * udp_recv_cb handles incoming UDP packet from uv. The buffer here is + * reused for a series of packets, so we need to allocate a new one. + * This new one can be reused to send the response then. + */ +static void +udp_recv_cb(uv_udp_t *handle, ssize_t nrecv, const uv_buf_t *buf, + const struct sockaddr *addr, unsigned flags) { + isc_nmsocket_t *sock = uv_handle_get_data((uv_handle_t *)handle); + isc__nm_uvreq_t *req = NULL; + uint32_t maxudp; + isc_sockaddr_t sockaddr; + isc_result_t result; + + REQUIRE(VALID_NMSOCK(sock)); + REQUIRE(sock->tid == isc_nm_tid()); + REQUIRE(sock->reading); + + /* + * When using recvmmsg(2), if no errors occur, there will be a final + * callback with nrecv set to 0, addr set to NULL and the buffer + * pointing at the initially allocated data with the UV_UDP_MMSG_CHUNK + * flag cleared and the UV_UDP_MMSG_FREE flag set. + */ +#if HAVE_DECL_UV_UDP_MMSG_FREE + if ((flags & UV_UDP_MMSG_FREE) == UV_UDP_MMSG_FREE) { + INSIST(nrecv == 0); + INSIST(addr == NULL); + goto free; + } +#else + UNUSED(flags); +#endif + + /* + * - If we're simulating a firewall blocking UDP packets + * bigger than 'maxudp' bytes for testing purposes. + */ + maxudp = atomic_load(&sock->mgr->maxudp); + if ((maxudp != 0 && (uint32_t)nrecv > maxudp)) { + /* + * We need to keep the read_cb intact in case, so the + * readtimeout_cb can trigger and not crash because of + * missing read_req. + */ + goto free; + } + + /* + * - If addr == NULL, in which case it's the end of stream; + * we can free the buffer and bail. + */ + if (addr == NULL) { + isc__nm_failed_read_cb(sock, ISC_R_EOF, false); + goto free; + } + + /* + * - If the socket is no longer active. + */ + if (!isc__nmsocket_active(sock)) { + isc__nm_failed_read_cb(sock, ISC_R_CANCELED, false); + goto free; + } + + if (nrecv < 0) { + isc__nm_failed_read_cb(sock, isc__nm_uverr2result(nrecv), + false); + goto free; + } + + result = isc_sockaddr_fromsockaddr(&sockaddr, addr); + RUNTIME_CHECK(result == ISC_R_SUCCESS); + + req = isc__nm_get_read_req(sock, &sockaddr); + + /* + * The callback will be called synchronously, because result is + * ISC_R_SUCCESS, so we are ok of passing the buf directly. + */ + req->uvbuf.base = buf->base; + req->uvbuf.len = nrecv; + + sock->recv_read = false; + + REQUIRE(!sock->processing); + sock->processing = true; + isc__nm_readcb(sock, req, ISC_R_SUCCESS); + sock->processing = false; + +free: +#if HAVE_DECL_UV_UDP_MMSG_CHUNK + /* + * When using recvmmsg(2), chunks will have the UV_UDP_MMSG_CHUNK flag + * set, those must not be freed. + */ + if ((flags & UV_UDP_MMSG_CHUNK) == UV_UDP_MMSG_CHUNK) { + return; + } +#endif + + /* + * When using recvmmsg(2), if a UDP socket error occurs, nrecv will be < + * 0. In either scenario, the callee can now safely free the provided + * buffer. + */ + if (nrecv < 0) { + /* + * The buffer may be a null buffer on error. + */ + if (buf->base == NULL && buf->len == 0) { + return; + } + } + + isc__nm_free_uvbuf(sock, buf); +} + +/* + * Send the data in 'region' to a peer via a UDP socket. We try to find + * a proper sibling/child socket so that we won't have to jump to + * another thread. + */ +void +isc__nm_udp_send(isc_nmhandle_t *handle, const isc_region_t *region, + isc_nm_cb_t cb, void *cbarg) { + isc_nmsocket_t *sock = handle->sock; + isc_nmsocket_t *rsock = NULL; + isc_sockaddr_t *peer = &handle->peer; + isc__nm_uvreq_t *uvreq = NULL; + uint32_t maxudp = atomic_load(&sock->mgr->maxudp); + int ntid; + + INSIST(sock->type == isc_nm_udpsocket); + + /* + * We're simulating a firewall blocking UDP packets bigger than + * 'maxudp' bytes, for testing purposes. + * + * The client would ordinarily have unreferenced the handle + * in the callback, but that won't happen in this case, so + * we need to do so here. + */ + if (maxudp != 0 && region->length > maxudp) { + isc_nmhandle_detach(&handle); + return; + } + + if (atomic_load(&sock->client)) { + /* + * When we are sending from the client socket, we directly use + * the socket provided. + */ + rsock = sock; + goto send; + } else { + /* + * When we are sending from the server socket, we either use the + * socket associated with the network thread we are in, or we + * use the thread from the socket associated with the handle. + */ + INSIST(sock->parent != NULL); + +#if defined(WIN32) + /* On Windows, we have only a single listening listener */ + rsock = sock; +#else + if (isc__nm_in_netthread()) { + ntid = isc_nm_tid(); + } else { + ntid = sock->tid; + } + rsock = &sock->parent->children[ntid]; +#endif + } + +send: + uvreq = isc__nm_uvreq_get(rsock->mgr, rsock); + uvreq->uvbuf.base = (char *)region->base; + uvreq->uvbuf.len = region->length; + + isc_nmhandle_attach(handle, &uvreq->handle); + + uvreq->cb.send = cb; + uvreq->cbarg = cbarg; + + if (isc_nm_tid() == rsock->tid) { + REQUIRE(rsock->tid == isc_nm_tid()); + isc__netievent_udpsend_t ievent = { .sock = rsock, + .req = uvreq, + .peer = *peer }; + + isc__nm_async_udpsend(NULL, (isc__netievent_t *)&ievent); + } else { + isc__netievent_udpsend_t *ievent = + isc__nm_get_netievent_udpsend(sock->mgr, rsock); + ievent->peer = *peer; + ievent->req = uvreq; + + isc__nm_enqueue_ievent(&sock->mgr->workers[rsock->tid], + (isc__netievent_t *)ievent); + } +} + +/* + * Asynchronous 'udpsend' event handler: send a packet on a UDP socket. + */ +void +isc__nm_async_udpsend(isc__networker_t *worker, isc__netievent_t *ev0) { + isc_result_t result; + isc__netievent_udpsend_t *ievent = (isc__netievent_udpsend_t *)ev0; + isc_nmsocket_t *sock = ievent->sock; + isc__nm_uvreq_t *uvreq = ievent->req; + + REQUIRE(sock->type == isc_nm_udpsocket); + REQUIRE(sock->tid == isc_nm_tid()); + UNUSED(worker); + + if (isc__nmsocket_closing(sock)) { + isc__nm_failed_send_cb(sock, uvreq, ISC_R_CANCELED); + return; + } + + result = udp_send_direct(sock, uvreq, &ievent->peer); + if (result != ISC_R_SUCCESS) { + isc__nm_incstats(sock->mgr, sock->statsindex[STATID_SENDFAIL]); + isc__nm_failed_send_cb(sock, uvreq, result); + } +} + +static void +udp_send_cb(uv_udp_send_t *req, int status) { + isc_result_t result = ISC_R_SUCCESS; + isc__nm_uvreq_t *uvreq = uv_handle_get_data((uv_handle_t *)req); + isc_nmsocket_t *sock = NULL; + + REQUIRE(VALID_UVREQ(uvreq)); + REQUIRE(VALID_NMHANDLE(uvreq->handle)); + + sock = uvreq->sock; + + REQUIRE(sock->tid == isc_nm_tid()); + + if (status < 0) { + result = isc__nm_uverr2result(status); + isc__nm_incstats(sock->mgr, sock->statsindex[STATID_SENDFAIL]); + } + + isc__nm_sendcb(sock, uvreq, result, false); +} + +/* + * udp_send_direct sends buf to a peer on a socket. Sock has to be in + * the same thread as the callee. + */ +static isc_result_t +udp_send_direct(isc_nmsocket_t *sock, isc__nm_uvreq_t *req, + isc_sockaddr_t *peer) { + const struct sockaddr *sa = &peer->type.sa; + int r; + + REQUIRE(VALID_NMSOCK(sock)); + REQUIRE(VALID_UVREQ(req)); + REQUIRE(sock->tid == isc_nm_tid()); + REQUIRE(sock->type == isc_nm_udpsocket); + + if (isc__nmsocket_closing(sock)) { + return (ISC_R_CANCELED); + } + +#if UV_VERSION_HEX >= UV_VERSION(1, 27, 0) + /* + * If we used uv_udp_connect() (and not the shim version for + * older versions of libuv), then the peer address has to be + * set to NULL or else uv_udp_send() could fail or assert, + * depending on the libuv version. + */ + if (atomic_load(&sock->connected)) { + sa = NULL; + } +#endif + + r = uv_udp_send(&req->uv_req.udp_send, &sock->uv_handle.udp, + &req->uvbuf, 1, sa, udp_send_cb); + if (r < 0) { + return (isc__nm_uverr2result(r)); + } + + return (ISC_R_SUCCESS); +} + +static isc_result_t +udp_connect_direct(isc_nmsocket_t *sock, isc__nm_uvreq_t *req) { + isc__networker_t *worker = NULL; + int uv_bind_flags = UV_UDP_REUSEADDR; + isc_result_t result = ISC_R_UNSET; + int tries = 3; + int r; + + REQUIRE(isc__nm_in_netthread()); + REQUIRE(sock->tid == isc_nm_tid()); + + worker = &sock->mgr->workers[isc_nm_tid()]; + + atomic_store(&sock->connecting, true); + + r = uv_udp_init(&worker->loop, &sock->uv_handle.udp); + UV_RUNTIME_CHECK(uv_udp_init, r); + uv_handle_set_data(&sock->uv_handle.handle, sock); + + r = uv_timer_init(&worker->loop, &sock->read_timer); + UV_RUNTIME_CHECK(uv_timer_init, r); + uv_handle_set_data((uv_handle_t *)&sock->read_timer, sock); + + r = uv_udp_open(&sock->uv_handle.udp, sock->fd); + if (r != 0) { + isc__nm_incstats(sock->mgr, sock->statsindex[STATID_OPENFAIL]); + goto done; + } + isc__nm_incstats(sock->mgr, sock->statsindex[STATID_OPEN]); + + if (sock->iface.type.sa.sa_family == AF_INET6) { + uv_bind_flags |= UV_UDP_IPV6ONLY; + } + + r = uv_udp_bind(&sock->uv_handle.udp, &sock->iface.type.sa, + uv_bind_flags); + if (r != 0) { + isc__nm_incstats(sock->mgr, sock->statsindex[STATID_BINDFAIL]); + goto done; + } + +#ifdef ISC_RECV_BUFFER_SIZE + uv_recv_buffer_size(&sock->uv_handle.handle, + &(int){ ISC_RECV_BUFFER_SIZE }); +#endif +#ifdef ISC_SEND_BUFFER_SIZE + uv_send_buffer_size(&sock->uv_handle.handle, + &(int){ ISC_SEND_BUFFER_SIZE }); +#endif + + /* + * On FreeBSD the UDP connect() call sometimes results in a + * spurious transient EADDRINUSE. Try a few more times before + * giving up. + */ + do { + r = isc_uv_udp_connect(&sock->uv_handle.udp, + &req->peer.type.sa); + } while (r == UV_EADDRINUSE && --tries > 0); + if (r != 0) { + isc__nm_incstats(sock->mgr, + sock->statsindex[STATID_CONNECTFAIL]); + goto done; + } + isc__nm_incstats(sock->mgr, sock->statsindex[STATID_CONNECT]); + + atomic_store(&sock->connecting, false); + atomic_store(&sock->connected, true); + +done: + result = isc__nm_uverr2result(r); + + LOCK(&sock->lock); + sock->result = result; + SIGNAL(&sock->cond); + if (!atomic_load(&sock->active)) { + WAIT(&sock->scond, &sock->lock); + } + INSIST(atomic_load(&sock->active)); + UNLOCK(&sock->lock); + + return (result); +} + +/* + * Asynchronous 'udpconnect' call handler: open a new UDP socket and + * call the 'open' callback with a handle. + */ +void +isc__nm_async_udpconnect(isc__networker_t *worker, isc__netievent_t *ev0) { + isc__netievent_udpconnect_t *ievent = + (isc__netievent_udpconnect_t *)ev0; + isc_nmsocket_t *sock = ievent->sock; + isc__nm_uvreq_t *req = ievent->req; + isc_result_t result; + + UNUSED(worker); + + REQUIRE(VALID_NMSOCK(sock)); + REQUIRE(sock->type == isc_nm_udpsocket); + REQUIRE(sock->parent == NULL); + REQUIRE(sock->tid == isc_nm_tid()); + + result = udp_connect_direct(sock, req); + if (result != ISC_R_SUCCESS) { + atomic_store(&sock->active, false); + isc__nm_udp_close(sock); + isc__nm_connectcb(sock, req, result, true); + } else { + /* + * The callback has to be called after the socket has been + * initialized + */ + isc__nm_connectcb(sock, req, ISC_R_SUCCESS, true); + } + + /* + * The sock is now attached to the handle. + */ + isc__nmsocket_detach(&sock); +} + +void +isc_nm_udpconnect(isc_nm_t *mgr, isc_sockaddr_t *local, isc_sockaddr_t *peer, + isc_nm_cb_t cb, void *cbarg, unsigned int timeout, + size_t extrahandlesize) { + isc_result_t result = ISC_R_SUCCESS; + isc_nmsocket_t *sock = NULL; + isc__netievent_udpconnect_t *event = NULL; + isc__nm_uvreq_t *req = NULL; + sa_family_t sa_family; + + REQUIRE(VALID_NM(mgr)); + REQUIRE(local != NULL); + REQUIRE(peer != NULL); + + sa_family = peer->type.sa.sa_family; + + sock = isc_mem_get(mgr->mctx, sizeof(isc_nmsocket_t)); + isc__nmsocket_init(sock, mgr, isc_nm_udpsocket, local); + + sock->connect_cb = cb; + sock->connect_cbarg = cbarg; + sock->read_timeout = timeout; + sock->extrahandlesize = extrahandlesize; + sock->peer = *peer; + sock->result = ISC_R_UNSET; + atomic_init(&sock->client, true); + + req = isc__nm_uvreq_get(mgr, sock); + req->cb.connect = cb; + req->cbarg = cbarg; + req->peer = *peer; + req->local = *local; + req->handle = isc__nmhandle_get(sock, &req->peer, &sock->iface); + + result = isc__nm_socket(sa_family, SOCK_DGRAM, 0, &sock->fd); + if (result != ISC_R_SUCCESS) { + if (isc__nm_in_netthread()) { + sock->tid = isc_nm_tid(); + } + isc__nmsocket_clearcb(sock); + isc__nm_connectcb(sock, req, result, true); + atomic_store(&sock->closed, true); + isc__nmsocket_detach(&sock); + return; + } + + result = isc__nm_socket_reuse(sock->fd); + RUNTIME_CHECK(result == ISC_R_SUCCESS || + result == ISC_R_NOTIMPLEMENTED); + + result = isc__nm_socket_reuse_lb(sock->fd); + RUNTIME_CHECK(result == ISC_R_SUCCESS || + result == ISC_R_NOTIMPLEMENTED); + + (void)isc__nm_socket_incoming_cpu(sock->fd); + + (void)isc__nm_socket_disable_pmtud(sock->fd, sa_family); + + event = isc__nm_get_netievent_udpconnect(mgr, sock, req); + + if (isc__nm_in_netthread()) { + atomic_store(&sock->active, true); + sock->tid = isc_nm_tid(); + isc__nm_async_udpconnect(&mgr->workers[sock->tid], + (isc__netievent_t *)event); + isc__nm_put_netievent_udpconnect(mgr, event); + } else { + atomic_init(&sock->active, false); + sock->tid = isc_random_uniform(mgr->nworkers); + isc__nm_enqueue_ievent(&mgr->workers[sock->tid], + (isc__netievent_t *)event); + } + LOCK(&sock->lock); + while (sock->result == ISC_R_UNSET) { + WAIT(&sock->cond, &sock->lock); + } + atomic_store(&sock->active, true); + BROADCAST(&sock->scond); + UNLOCK(&sock->lock); +} + +void +isc__nm_udp_read_cb(uv_udp_t *handle, ssize_t nrecv, const uv_buf_t *buf, + const struct sockaddr *addr, unsigned flags) { + isc_nmsocket_t *sock = uv_handle_get_data((uv_handle_t *)handle); + REQUIRE(VALID_NMSOCK(sock)); + + udp_recv_cb(handle, nrecv, buf, addr, flags); + /* + * If a caller calls isc_nm_read() on a listening socket, we can + * get here, but we MUST NOT stop reading from the listener + * socket. The only difference between listener and connected + * sockets is that the former has sock->parent set and later + * does not. + */ + if (!sock->parent) { + isc__nm_stop_reading(sock); + } +} + +void +isc__nm_udp_failed_read_cb(isc_nmsocket_t *sock, isc_result_t result) { + REQUIRE(VALID_NMSOCK(sock)); + REQUIRE(result != ISC_R_SUCCESS); + + if (atomic_load(&sock->client)) { + isc__nmsocket_timer_stop(sock); + isc__nm_stop_reading(sock); + + if (!sock->recv_read) { + goto destroy; + } + sock->recv_read = false; + + if (sock->recv_cb != NULL) { + isc__nm_uvreq_t *req = isc__nm_get_read_req(sock, NULL); + isc__nmsocket_clearcb(sock); + isc__nm_readcb(sock, req, result); + } + + destroy: + isc__nmsocket_prep_destroy(sock); + return; + } + + /* + * For UDP server socket, we don't have child socket via + * "accept", so we: + * - we continue to read + * - we don't clear the callbacks + * - we don't destroy it (only stoplistening could do that) + */ + if (!sock->recv_read) { + return; + } + sock->recv_read = false; + + if (sock->recv_cb != NULL) { + isc__nm_uvreq_t *req = isc__nm_get_read_req(sock, NULL); + isc__nm_readcb(sock, req, result); + } +} + +/* + * Asynchronous 'udpread' call handler: start or resume reading on a + * socket; pause reading and call the 'recv' callback after each + * datagram. + */ +void +isc__nm_async_udpread(isc__networker_t *worker, isc__netievent_t *ev0) { + isc__netievent_udpread_t *ievent = (isc__netievent_udpread_t *)ev0; + isc_nmsocket_t *sock = ievent->sock; + isc_result_t result; + + UNUSED(worker); + + REQUIRE(VALID_NMSOCK(sock)); + REQUIRE(sock->tid == isc_nm_tid()); + + if (isc__nmsocket_closing(sock)) { + result = ISC_R_CANCELED; + } else { + result = isc__nm_start_reading(sock); + } + + if (result != ISC_R_SUCCESS) { + sock->reading = true; + isc__nm_failed_read_cb(sock, result, false); + return; + } + + isc__nmsocket_timer_start(sock); +} + +void +isc__nm_udp_read(isc_nmhandle_t *handle, isc_nm_recv_cb_t cb, void *cbarg) { + REQUIRE(VALID_NMHANDLE(handle)); + REQUIRE(VALID_NMSOCK(handle->sock)); + + isc_nmsocket_t *sock = handle->sock; + + REQUIRE(sock->type == isc_nm_udpsocket); + REQUIRE(sock->statichandle == handle); + REQUIRE(sock->tid == isc_nm_tid()); + REQUIRE(!sock->recv_read); + + sock->recv_cb = cb; + sock->recv_cbarg = cbarg; + sock->recv_read = true; + + if (!sock->reading && sock->tid == isc_nm_tid()) { + isc__netievent_udpread_t ievent = { .sock = sock }; + isc__nm_async_udpread(NULL, (isc__netievent_t *)&ievent); + } else { + isc__netievent_udpread_t *ievent = + isc__nm_get_netievent_udpread(sock->mgr, sock); + isc__nm_enqueue_ievent(&sock->mgr->workers[sock->tid], + (isc__netievent_t *)ievent); + } +} + +static void +udp_stop_cb(uv_handle_t *handle) { + isc_nmsocket_t *sock = uv_handle_get_data(handle); + uv_handle_set_data(handle, NULL); + + REQUIRE(VALID_NMSOCK(sock)); + REQUIRE(sock->tid == isc_nm_tid()); + REQUIRE(atomic_load(&sock->closing)); + + if (!atomic_compare_exchange_strong(&sock->closed, &(bool){ false }, + true)) + { + UNREACHABLE(); + } + + isc__nm_incstats(sock->mgr, sock->statsindex[STATID_CLOSE]); + + atomic_store(&sock->listening, false); + + isc__nmsocket_detach(&sock); +} + +static void +udp_close_cb(uv_handle_t *handle) { + isc_nmsocket_t *sock = uv_handle_get_data(handle); + uv_handle_set_data(handle, NULL); + + REQUIRE(VALID_NMSOCK(sock)); + REQUIRE(sock->tid == isc_nm_tid()); + REQUIRE(atomic_load(&sock->closing)); + + if (!atomic_compare_exchange_strong(&sock->closed, &(bool){ false }, + true)) + { + UNREACHABLE(); + } + + isc__nm_incstats(sock->mgr, sock->statsindex[STATID_CLOSE]); + + if (sock->server != NULL) { + isc__nmsocket_detach(&sock->server); + } + + atomic_store(&sock->connected, false); + atomic_store(&sock->listening, false); + + isc__nmsocket_prep_destroy(sock); +} + +static void +read_timer_close_cb(uv_handle_t *handle) { + isc_nmsocket_t *sock = uv_handle_get_data(handle); + uv_handle_set_data(handle, NULL); + + if (sock->parent) { + uv_close(&sock->uv_handle.handle, udp_stop_cb); + } else { + uv_close(&sock->uv_handle.handle, udp_close_cb); + } +} + +static void +stop_udp_child(isc_nmsocket_t *sock) { + REQUIRE(sock->type == isc_nm_udpsocket); + REQUIRE(sock->tid == isc_nm_tid()); + + if (!atomic_compare_exchange_strong(&sock->closing, &(bool){ false }, + true)) + { + return; + } + + udp_close_direct(sock); + + atomic_fetch_sub(&sock->parent->rchildren, 1); + + isc_barrier_wait(&sock->parent->stoplistening); +} + +static void +stop_udp_parent(isc_nmsocket_t *sock) { + isc_nmsocket_t *csock = NULL; + + REQUIRE(VALID_NMSOCK(sock)); + REQUIRE(sock->tid == isc_nm_tid()); + REQUIRE(sock->type == isc_nm_udplistener); + + isc_barrier_init(&sock->stoplistening, sock->nchildren); + + for (size_t i = 0; i < sock->nchildren; i++) { + csock = &sock->children[i]; + REQUIRE(VALID_NMSOCK(csock)); + + if ((int)i == isc_nm_tid()) { + /* + * We need to schedule closing the other sockets first + */ + continue; + } + + atomic_store(&csock->active, false); + enqueue_stoplistening(csock); + } + + csock = &sock->children[isc_nm_tid()]; + atomic_store(&csock->active, false); + stop_udp_child(csock); + + atomic_store(&sock->closed, true); + isc__nmsocket_prep_destroy(sock); +} + +static void +udp_close_direct(isc_nmsocket_t *sock) { + REQUIRE(VALID_NMSOCK(sock)); + REQUIRE(sock->tid == isc_nm_tid()); + + uv_handle_set_data((uv_handle_t *)&sock->read_timer, sock); + uv_close((uv_handle_t *)&sock->read_timer, read_timer_close_cb); +} + +void +isc__nm_async_udpclose(isc__networker_t *worker, isc__netievent_t *ev0) { + isc__netievent_udpclose_t *ievent = (isc__netievent_udpclose_t *)ev0; + isc_nmsocket_t *sock = ievent->sock; + + REQUIRE(VALID_NMSOCK(sock)); + REQUIRE(sock->tid == isc_nm_tid()); + UNUSED(worker); + + udp_close_direct(sock); +} + +void +isc__nm_udp_close(isc_nmsocket_t *sock) { + REQUIRE(VALID_NMSOCK(sock)); + REQUIRE(sock->type == isc_nm_udpsocket); + REQUIRE(!isc__nmsocket_active(sock)); + + if (!atomic_compare_exchange_strong(&sock->closing, &(bool){ false }, + true)) + { + return; + } + + if (sock->tid == isc_nm_tid()) { + udp_close_direct(sock); + } else { + isc__netievent_udpclose_t *ievent = + isc__nm_get_netievent_udpclose(sock->mgr, sock); + isc__nm_enqueue_ievent(&sock->mgr->workers[sock->tid], + (isc__netievent_t *)ievent); + } +} + +void +isc__nm_udp_shutdown(isc_nmsocket_t *sock) { + REQUIRE(VALID_NMSOCK(sock)); + REQUIRE(sock->tid == isc_nm_tid()); + REQUIRE(sock->type == isc_nm_udpsocket); + + /* + * If the socket is active, mark it inactive and + * continue. If it isn't active, stop now. + */ + if (!isc__nmsocket_deactivate(sock)) { + return; + } + + /* + * If the socket is connecting, the cancel will happen in the + * async_udpconnect() due socket being inactive now. + */ + if (atomic_load(&sock->connecting)) { + return; + } + + /* + * When the client detaches the last handle, the + * sock->statichandle would be NULL, in that case, nobody is + * interested in the callback. + */ + if (sock->statichandle != NULL) { + isc__nm_failed_read_cb(sock, ISC_R_CANCELED, false); + return; + } + + /* + * Otherwise, we just send the socket to abyss... + */ + if (sock->parent == NULL) { + isc__nmsocket_prep_destroy(sock); + } +} + +void +isc__nm_udp_cancelread(isc_nmhandle_t *handle) { + isc_nmsocket_t *sock = NULL; + isc__netievent_udpcancel_t *ievent = NULL; + + REQUIRE(VALID_NMHANDLE(handle)); + + sock = handle->sock; + + REQUIRE(VALID_NMSOCK(sock)); + REQUIRE(sock->type == isc_nm_udpsocket); + + ievent = isc__nm_get_netievent_udpcancel(sock->mgr, sock, handle); + + isc__nm_enqueue_ievent(&sock->mgr->workers[sock->tid], + (isc__netievent_t *)ievent); +} + +void +isc__nm_async_udpcancel(isc__networker_t *worker, isc__netievent_t *ev0) { + isc__netievent_udpcancel_t *ievent = (isc__netievent_udpcancel_t *)ev0; + isc_nmsocket_t *sock = NULL; + + UNUSED(worker); + + REQUIRE(VALID_NMSOCK(ievent->sock)); + + sock = ievent->sock; + + REQUIRE(sock->tid == isc_nm_tid()); + REQUIRE(atomic_load(&sock->client)); + + isc__nm_failed_read_cb(sock, ISC_R_EOF, false); +} diff --git a/lib/isc/netmgr/uv-compat.c b/lib/isc/netmgr/uv-compat.c new file mode 100644 index 0000000..a1fc309 --- /dev/null +++ b/lib/isc/netmgr/uv-compat.c @@ -0,0 +1,152 @@ +/* + * Copyright (C) Internet Systems Consortium, Inc. ("ISC") + * + * SPDX-License-Identifier: MPL-2.0 + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, you can obtain one at https://mozilla.org/MPL/2.0/. + * + * See the COPYRIGHT file distributed with this work for additional + * information regarding copyright ownership. + */ + +#include "uv-compat.h" +#include <unistd.h> + +#include <isc/util.h> + +#include "netmgr-int.h" + +#if UV_VERSION_HEX < UV_VERSION(1, 27, 0) +int +isc_uv_udp_connect(uv_udp_t *handle, const struct sockaddr *addr) { + int err = 0; + + do { + int addrlen = (addr->sa_family == AF_INET) + ? sizeof(struct sockaddr_in) + : sizeof(struct sockaddr_in6); +#ifdef WIN32 + err = connect(handle->socket, addr, addrlen); +#else /* WIN32 */ + err = connect(handle->io_watcher.fd, addr, addrlen); +#endif /* WIN32 */ + } while (err == -1 && errno == EINTR); + + if (err) { +#ifdef WIN32 + return (uv_translate_sys_error(err)); +#else /* WIN32 */ +#if UV_VERSION_HEX >= UV_VERSION(1, 10, 0) + return (uv_translate_sys_error(errno)); +#else + return (-errno); +#endif /* UV_VERSION_HEX >= UV_VERSION(1, 10, 0) */ +#endif /* WIN32 */ + } + + return (0); +} +#endif /* UV_VERSION_HEX < UV_VERSION(1, 27, 0) */ + +#if UV_VERSION_HEX < UV_VERSION(1, 32, 0) +int +uv_tcp_close_reset(uv_tcp_t *handle, uv_close_cb close_cb) { + if (setsockopt(handle->io_watcher.fd, SOL_SOCKET, SO_LINGER, + &(struct linger){ 1, 0 }, sizeof(struct linger)) == -1) + { +#if UV_VERSION_HEX >= UV_VERSION(1, 10, 0) + return (uv_translate_sys_error(errno)); +#else + return (-errno); +#endif /* UV_VERSION_HEX >= UV_VERSION(1, 10, 0) */ + } + + uv_close((uv_handle_t *)handle, close_cb); + return (0); +} +#endif /* UV_VERSION_HEX < UV_VERSION(1, 32, 0) */ + +int +isc_uv_udp_freebind(uv_udp_t *handle, const struct sockaddr *addr, + unsigned int flags) { + int r; + uv_os_sock_t fd; + + r = uv_fileno((const uv_handle_t *)handle, (uv_os_fd_t *)&fd); + if (r < 0) { + return (r); + } + +#if defined(WIN32) + REQUIRE(fd != INVALID_SOCKET); +#endif + + r = uv_udp_bind(handle, addr, flags); + if (r == UV_EADDRNOTAVAIL && + isc__nm_socket_freebind(fd, addr->sa_family) == ISC_R_SUCCESS) + { + /* + * Retry binding with IP_FREEBIND (or equivalent option) if the + * address is not available. This helps with IPv6 tentative + * addresses which are reported by the route socket, although + * named is not yet able to properly bind to them. + */ + r = uv_udp_bind(handle, addr, flags); + } + + return (r); +} + +static int +isc__uv_tcp_bind_now(uv_tcp_t *handle, const struct sockaddr *addr, + unsigned int flags) { + int r; + struct sockaddr_storage sname; + int snamelen = sizeof(sname); + + r = uv_tcp_bind(handle, addr, flags); + if (r < 0) { + return (r); + } + + /* + * uv_tcp_bind() uses a delayed error, initially returning + * success even if bind() fails. By calling uv_tcp_getsockname() + * here we can find out whether the bind() call was successful. + */ + r = uv_tcp_getsockname(handle, (struct sockaddr *)&sname, &snamelen); + if (r < 0) { + return (r); + } + + return (0); +} + +int +isc_uv_tcp_freebind(uv_tcp_t *handle, const struct sockaddr *addr, + unsigned int flags) { + int r; + uv_os_sock_t fd; + + r = uv_fileno((const uv_handle_t *)handle, (uv_os_fd_t *)&fd); + if (r < 0) { + return (r); + } + + r = isc__uv_tcp_bind_now(handle, addr, flags); + if (r == UV_EADDRNOTAVAIL && + isc__nm_socket_freebind(fd, addr->sa_family) == ISC_R_SUCCESS) + { + /* + * Retry binding with IP_FREEBIND (or equivalent option) if the + * address is not available. This helps with IPv6 tentative + * addresses which are reported by the route socket, although + * named is not yet able to properly bind to them. + */ + r = isc__uv_tcp_bind_now(handle, addr, flags); + } + + return (r); +} diff --git a/lib/isc/netmgr/uv-compat.h b/lib/isc/netmgr/uv-compat.h new file mode 100644 index 0000000..3a10387 --- /dev/null +++ b/lib/isc/netmgr/uv-compat.h @@ -0,0 +1,126 @@ +/* + * Copyright (C) Internet Systems Consortium, Inc. ("ISC") + * + * SPDX-License-Identifier: MPL-2.0 + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, you can obtain one at https://mozilla.org/MPL/2.0/. + * + * See the COPYRIGHT file distributed with this work for additional + * information regarding copyright ownership. + */ + +#pragma once +#include <uv.h> + +/* + * These functions were introduced in newer libuv, but we still + * want BIND9 compile on older ones so we emulate them. + * They're inline to avoid conflicts when running with a newer + * library version. + */ + +#define UV_VERSION(major, minor, patch) ((major << 16) | (minor << 8) | (patch)) + +/* + * Copied verbatim from libuv/src/version.c + */ + +#define UV_STRINGIFY(v) UV_STRINGIFY_HELPER(v) +#define UV_STRINGIFY_HELPER(v) #v + +#define UV_VERSION_STRING_BASE \ + UV_STRINGIFY(UV_VERSION_MAJOR) \ + "." UV_STRINGIFY(UV_VERSION_MINOR) "." UV_STRINGIFY(UV_VERSION_PATCH) + +#if UV_VERSION_IS_RELEASE +#define UV_VERSION_STRING UV_VERSION_STRING_BASE +#else +#define UV_VERSION_STRING UV_VERSION_STRING_BASE "-" UV_VERSION_SUFFIX +#endif + +#if !defined(UV__ERR) +#define UV__ERR(x) (-(x)) +#endif + +#if UV_VERSION_HEX < UV_VERSION(1, 19, 0) +static inline void * +uv_handle_get_data(const uv_handle_t *handle) { + return (handle->data); +} + +static inline void +uv_handle_set_data(uv_handle_t *handle, void *data) { + handle->data = data; +} + +static inline void * +uv_req_get_data(const uv_req_t *req) { + return (req->data); +} + +static inline void +uv_req_set_data(uv_req_t *req, void *data) { + req->data = data; +} +#endif /* UV_VERSION_HEX < UV_VERSION(1, 19, 0) */ + +#if UV_VERSION_HEX < UV_VERSION(1, 32, 0) +int +uv_tcp_close_reset(uv_tcp_t *handle, uv_close_cb close_cb); +#endif + +#if UV_VERSION_HEX < UV_VERSION(1, 34, 0) +#define uv_sleep(msec) usleep(msec * 1000) +#endif /* UV_VERSION_HEX < UV_VERSION(1, 34, 0) */ + +#if UV_VERSION_HEX < UV_VERSION(1, 27, 0) +int +isc_uv_udp_connect(uv_udp_t *handle, const struct sockaddr *addr); +/*%< + * Associate the UDP handle to a remote address and port, so every message sent + * by this handle is automatically sent to that destination. + * + * NOTE: This is just a limited shim for uv_udp_connect() as it requires the + * handle to be bound. + */ +#else /* UV_VERSION_HEX < UV_VERSION(1, 27, 0) */ +#define isc_uv_udp_connect uv_udp_connect +#endif /* UV_VERSION_HEX < UV_VERSION(1, 27, 0) */ + +#if UV_VERSION_HEX < UV_VERSION(1, 12, 0) +#include <stdlib.h> +#include <string.h> + +static inline int +uv_os_getenv(const char *name, char *buffer, size_t *size) { + size_t len; + char *buf = getenv(name); + + if (buf == NULL) { + return (UV_ENOENT); + } + + len = strlen(buf) + 1; + if (len > *size) { + *size = len; + return (UV_ENOBUFS); + } + + *size = len; + memmove(buffer, buf, len); + + return (0); +} + +#define uv_os_setenv(name, value) setenv(name, value, 0) +#endif /* UV_VERSION_HEX < UV_VERSION(1, 12, 0) */ + +int +isc_uv_udp_freebind(uv_udp_t *handle, const struct sockaddr *addr, + unsigned int flags); + +int +isc_uv_tcp_freebind(uv_tcp_t *handle, const struct sockaddr *addr, + unsigned int flags); diff --git a/lib/isc/netmgr/uverr2result.c b/lib/isc/netmgr/uverr2result.c new file mode 100644 index 0000000..5ce953d --- /dev/null +++ b/lib/isc/netmgr/uverr2result.c @@ -0,0 +1,104 @@ +/* + * Copyright (C) Internet Systems Consortium, Inc. ("ISC") + * + * SPDX-License-Identifier: MPL-2.0 + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, you can obtain one at https://mozilla.org/MPL/2.0/. + * + * See the COPYRIGHT file distributed with this work for additional + * information regarding copyright ownership. + */ + +#include <stdbool.h> +#include <uv.h> + +#include <isc/result.h> +#include <isc/strerr.h> +#include <isc/string.h> +#include <isc/util.h> + +#include "netmgr-int.h" + +/*% + * Convert a libuv error value into an isc_result_t. The + * list of supported error values is not complete; new users + * of this function should add any expected errors that are + * not already there. + */ +isc_result_t +isc___nm_uverr2result(int uverr, bool dolog, const char *file, + unsigned int line, const char *func) { + switch (uverr) { + case 0: + return (ISC_R_SUCCESS); + case UV_ENOTDIR: + case UV_ELOOP: + case UV_EINVAL: /* XXX sometimes this is not for files */ + case UV_ENAMETOOLONG: + case UV_EBADF: + return (ISC_R_INVALIDFILE); + case UV_ENOENT: + return (ISC_R_FILENOTFOUND); + case UV_EAGAIN: + return (ISC_R_NOCONN); + case UV_EACCES: + case UV_EPERM: + return (ISC_R_NOPERM); + case UV_EEXIST: + return (ISC_R_FILEEXISTS); + case UV_EIO: + return (ISC_R_IOERROR); + case UV_ENOMEM: + return (ISC_R_NOMEMORY); + case UV_ENFILE: + case UV_EMFILE: + return (ISC_R_TOOMANYOPENFILES); + case UV_ENOSPC: + return (ISC_R_DISCFULL); + case UV_EPIPE: + case UV_ECONNRESET: + case UV_ECONNABORTED: + return (ISC_R_CONNECTIONRESET); + case UV_ENOTCONN: + return (ISC_R_NOTCONNECTED); + case UV_ETIMEDOUT: + return (ISC_R_TIMEDOUT); + case UV_ENOBUFS: + return (ISC_R_NORESOURCES); + case UV_EAFNOSUPPORT: + return (ISC_R_FAMILYNOSUPPORT); + case UV_ENETDOWN: + return (ISC_R_NETDOWN); + case UV_EHOSTDOWN: + return (ISC_R_HOSTDOWN); + case UV_ENETUNREACH: + return (ISC_R_NETUNREACH); + case UV_EHOSTUNREACH: + return (ISC_R_HOSTUNREACH); + case UV_EADDRINUSE: + return (ISC_R_ADDRINUSE); + case UV_EADDRNOTAVAIL: + return (ISC_R_ADDRNOTAVAIL); + case UV_ECONNREFUSED: + return (ISC_R_CONNREFUSED); + case UV_ECANCELED: + return (ISC_R_CANCELED); + case UV_EOF: + return (ISC_R_EOF); + case UV_EMSGSIZE: + return (ISC_R_MAXSIZE); + case UV_ENOTSUP: + return (ISC_R_FAMILYNOSUPPORT); + default: + if (dolog) { + UNEXPECTED_ERROR( + file, line, + "unable to convert libuv " + "error code in %s to isc_result: %d: %s", + func, uverr, uv_strerror(uverr)); + } + return (ISC_R_UNEXPECTED); + } +} |