diff options
Diffstat (limited to 'src')
30 files changed, 818 insertions, 451 deletions
diff --git a/src/HttpServer.cc b/src/HttpServer.cc index b59cecd..6b28d1b 100644 --- a/src/HttpServer.cc +++ b/src/HttpServer.cc @@ -750,34 +750,40 @@ int Http2Handler::read_tls() { ERR_clear_error(); - auto rv = SSL_read(ssl_, buf.data(), buf.size()); - - if (rv <= 0) { - auto err = SSL_get_error(ssl_, rv); - switch (err) { - case SSL_ERROR_WANT_READ: - return write_(*this); - case SSL_ERROR_WANT_WRITE: - // renegotiation started - return -1; - default: - return -1; + for (;;) { + auto rv = SSL_read(ssl_, buf.data(), buf.size()); + + if (rv <= 0) { + auto err = SSL_get_error(ssl_, rv); + switch (err) { + case SSL_ERROR_WANT_READ: + return write_(*this); + case SSL_ERROR_WANT_WRITE: + // renegotiation started + return -1; + default: + return -1; + } } - } - auto nread = rv; + auto nread = rv; - if (get_config()->hexdump) { - util::hexdump(stdout, buf.data(), nread); - } + if (get_config()->hexdump) { + util::hexdump(stdout, buf.data(), nread); + } - rv = nghttp2_session_mem_recv2(session_, buf.data(), nread); - if (rv < 0) { - if (rv != NGHTTP2_ERR_BAD_CLIENT_MAGIC) { - std::cerr << "nghttp2_session_mem_recv2() returned error: " - << nghttp2_strerror(rv) << std::endl; + rv = nghttp2_session_mem_recv2(session_, buf.data(), nread); + if (rv < 0) { + if (rv != NGHTTP2_ERR_BAD_CLIENT_MAGIC) { + std::cerr << "nghttp2_session_mem_recv2() returned error: " + << nghttp2_strerror(rv) << std::endl; + } + return -1; + } + + if (SSL_pending(ssl_) == 0) { + break; } - return -1; } return write_(*this); diff --git a/src/h2load.cc b/src/h2load.cc index 8136a9f..4f9f00e 100644 --- a/src/h2load.cc +++ b/src/h2load.cc @@ -87,15 +87,6 @@ bool recorded(const std::chrono::steady_clock::time_point &t) { } } // namespace -namespace { -std::ofstream keylog_file; -void keylog_callback(const SSL *ssl, const char *line) { - keylog_file.write(line, strlen(line)); - keylog_file.put('\n'); - keylog_file.flush(); -} -} // namespace - Config::Config() : ciphers(tls::DEFAULT_CIPHER_LIST), tls13_ciphers("TLS_AES_128_GCM_SHA256:TLS_AES_256_GCM_SHA384:TLS_" @@ -2977,12 +2968,10 @@ int main(int argc, char **argv) { SSL_CTX_set_alpn_protos(ssl_ctx, proto_list.data(), proto_list.size()); - auto keylog_filename = getenv("SSLKEYLOGFILE"); - if (keylog_filename) { - keylog_file.open(keylog_filename, std::ios_base::app); - if (keylog_file) { - SSL_CTX_set_keylog_callback(ssl_ctx, keylog_callback); - } + if (tls::setup_keylog_callback(ssl_ctx) != 0) { + std::cerr << "Failed to setup keylog" << std::endl; + + exit(EXIT_FAILURE); } #if defined(NGHTTP2_OPENSSL_IS_BORINGSSL) && defined(HAVE_LIBBROTLI) @@ -3002,7 +2991,7 @@ int main(int argc, char **argv) { shared_nva.emplace_back("user-agent", user_agent); // list header fields that can be overridden. - auto override_hdrs = make_array<std::string>(":authority", ":host", ":method", + auto override_hdrs = make_array<std::string>(":authority", "host", ":method", ":scheme", "user-agent"); for (auto &kv : config.custom_headers) { @@ -3010,7 +2999,7 @@ int main(int argc, char **argv) { kv.name) != std::end(override_hdrs)) { // override header for (auto &nv : shared_nva) { - if ((nv.name == ":authority" && kv.name == ":host") || + if ((nv.name == ":authority" && kv.name == "host") || (nv.name == kv.name)) { nv.value = kv.value; } diff --git a/src/nghttp.cc b/src/nghttp.cc index f670320..6684374 100644 --- a/src/nghttp.cc +++ b/src/nghttp.cc @@ -2322,6 +2322,14 @@ int communicate( goto fin; } #endif // NGHTTP2_OPENSSL_IS_BORINGSSL && HAVE_LIBBROTLI + + if (tls::setup_keylog_callback(ssl_ctx) != 0) { + std::cerr << "[ERROR] Failed to setup keylog" << std::endl; + + result = -1; + + goto fin; + } } { HttpClient client{callbacks, loop, ssl_ctx}; diff --git a/src/shrpx.cc b/src/shrpx.cc index b42054c..89a7787 100644 --- a/src/shrpx.cc +++ b/src/shrpx.cc @@ -141,11 +141,13 @@ constexpr auto ENV_ACCEPT_PREFIX = StringRef::from_lit("NGHTTPX_ACCEPT_"); constexpr auto ENV_ORIG_PID = StringRef::from_lit("NGHTTPX_ORIG_PID"); // Prefix of environment variables to tell new binary the QUIC IPC -// file descriptor and CID prefix of the lingering worker process. -// The value must be comma separated parameters: -// <FD>,<CID_PREFIX_0>,<CID_PREFIX_1>,... <FD> is the file -// descriptor. <CID_PREFIX_I> is the I-th CID prefix in hex encoded -// string. +// file descriptor and Worker ID of the lingering worker process. The +// value must be comma separated parameters: +// +// <FD>,<WORKER_ID_0>,<WORKER_ID_1>,...,<WORKER_ID_I> +// +// <FD> is the file descriptor. <WORKER_ID_I> is the I-th Worker ID +// in hex encoded string. constexpr auto ENV_QUIC_WORKER_PROCESS_PREFIX = StringRef::from_lit("NGHTTPX_QUIC_WORKER_PROCESS_"); @@ -203,9 +205,7 @@ struct WorkerProcess { WorkerProcess(struct ev_loop *loop, pid_t worker_pid, int ipc_fd #ifdef ENABLE_HTTP3 , - int quic_ipc_fd, - const std::vector<std::array<uint8_t, SHRPX_QUIC_CID_PREFIXLEN>> - &cid_prefixes + int quic_ipc_fd, std::vector<WorkerID> worker_ids, uint16_t seq #endif // ENABLE_HTTP3 ) : loop(loop), @@ -214,7 +214,8 @@ struct WorkerProcess { #ifdef ENABLE_HTTP3 , quic_ipc_fd(quic_ipc_fd), - cid_prefixes(cid_prefixes) + worker_ids(std::move(worker_ids)), + seq(seq) #endif // ENABLE_HTTP3 { ev_child_init(&worker_process_childev, worker_process_child_cb, worker_pid, @@ -245,7 +246,8 @@ struct WorkerProcess { std::chrono::steady_clock::time_point termination_deadline; #ifdef ENABLE_HTTP3 int quic_ipc_fd; - std::vector<std::array<uint8_t, SHRPX_QUIC_CID_PREFIXLEN>> cid_prefixes; + std::vector<WorkerID> worker_ids; + uint16_t seq; #endif // ENABLE_HTTP3 }; @@ -255,6 +257,10 @@ void reload_config(); namespace { std::deque<std::unique_ptr<WorkerProcess>> worker_processes; + +#ifdef ENABLE_HTTP3 +uint16_t worker_process_seq; +#endif // ENABLE_HTTP3 } // namespace namespace { @@ -582,9 +588,10 @@ void exec_binary() { s += util::utos(i + 1); s += '='; s += util::utos(wp->quic_ipc_fd); - for (auto &cid_prefix : wp->cid_prefixes) { + for (auto &wid : wp->worker_ids) { s += ','; - s += util::format_hex(cid_prefix); + s += util::format_hex(reinterpret_cast<const unsigned char *>(&wid), + sizeof(wid)); } quic_lwps.emplace_back(s); @@ -1223,7 +1230,7 @@ std::vector<QUICLingeringWorkerProcess> namespace { std::vector<QUICLingeringWorkerProcess> get_inherited_quic_lingering_worker_process_from_env() { - std::vector<QUICLingeringWorkerProcess> iwps; + std::vector<QUICLingeringWorkerProcess> lwps; for (size_t i = 1;; ++i) { auto name = ENV_QUIC_WORKER_PROCESS_PREFIX.str(); @@ -1258,26 +1265,27 @@ get_inherited_quic_lingering_worker_process_from_env() { util::make_socket_closeonexec(fd); - std::vector<std::array<uint8_t, SHRPX_QUIC_CID_PREFIXLEN>> cid_prefixes; + std::vector<WorkerID> worker_ids; auto p = end_fd + 1; for (;;) { auto end = std::find(p, envend, ','); - auto hex_cid_prefix = StringRef{p, end}; - if (hex_cid_prefix.size() != SHRPX_QUIC_CID_PREFIXLEN * 2 || - !util::is_hex_string(hex_cid_prefix)) { - LOG(WARN) << "Found invalid CID prefix=" << hex_cid_prefix; + auto hex_wid = StringRef{p, end}; + if (hex_wid.size() != SHRPX_QUIC_WORKER_IDLEN * 2 || + !util::is_hex_string(hex_wid)) { + LOG(WARN) << "Found invalid WorkerID=" << hex_wid; break; } if (LOG_ENABLED(INFO)) { - LOG(INFO) << "Inherit worker process CID prefix=" << hex_cid_prefix; + LOG(INFO) << "Inherit worker process WorkerID=" << hex_wid; } - cid_prefixes.emplace_back(); + worker_ids.emplace_back(); - util::decode_hex(std::begin(cid_prefixes.back()), hex_cid_prefix); + util::decode_hex(reinterpret_cast<uint8_t *>(&worker_ids.back()), + hex_wid); if (end == envend) { break; @@ -1286,10 +1294,20 @@ get_inherited_quic_lingering_worker_process_from_env() { p = end + 1; } - iwps.emplace_back(std::move(cid_prefixes), fd); + lwps.emplace_back(std::move(worker_ids), fd); } - return iwps; + if (!lwps.empty()) { + const auto &lwp = lwps.back(); + + if (!lwp.worker_ids.empty() && + worker_process_seq <= lwp.worker_ids[0].worker_process) { + worker_process_seq = lwp.worker_ids[0].worker_process; + ++worker_process_seq; + } + } + + return lwps; } } // namespace #endif // ENABLE_HTTP3 @@ -1418,32 +1436,33 @@ int create_quic_ipc_socket(std::array<int, 2> &quic_ipc_fd) { } // namespace namespace { -int generate_cid_prefix( - std::vector<std::array<uint8_t, SHRPX_QUIC_CID_PREFIXLEN>> &cid_prefixes, - const Config *config) { +int generate_worker_id(std::vector<WorkerID> &worker_ids, uint16_t wp_seq, + const Config *config) { auto &apiconf = config->api; auto &quicconf = config->quic; - size_t num_cid_prefix; + size_t num_wid; if (config->single_thread) { - num_cid_prefix = 1; + num_wid = 1; } else { - num_cid_prefix = config->num_worker; + num_wid = config->num_worker; // API endpoint occupies the one dedicated worker thread. - // Although such worker never gets QUIC traffic, we create CID - // prefix for it to make code a bit simpler. + // Although such worker never gets QUIC traffic, we create Worker + // ID for it to make code a bit simpler. if (apiconf.enabled) { - ++num_cid_prefix; + ++num_wid; } } - cid_prefixes.resize(num_cid_prefix); + worker_ids.resize(num_wid); - for (auto &cid_prefix : cid_prefixes) { - if (create_cid_prefix(cid_prefix.data(), quicconf.server_id.data()) != 0) { - return -1; - } + uint16_t idx = 0; + + for (auto &wid : worker_ids) { + wid.server = quicconf.server_id; + wid.worker_process = wp_seq; + wid.thread = idx++; } return 0; @@ -1458,7 +1477,7 @@ collect_quic_lingering_worker_processes() { std::end(inherited_quic_lingering_worker_processes)}; for (auto &wp : worker_processes) { - quic_lwps.emplace_back(wp->cid_prefixes, wp->quic_ipc_fd); + quic_lwps.emplace_back(wp->worker_ids, wp->quic_ipc_fd); } return quic_lwps; @@ -1596,19 +1615,17 @@ namespace { // |main_ipc_fd|. In child process, we will close file descriptors // which are inherited from previous configuration/process, but not // used in the current configuration. -pid_t fork_worker_process( - int &main_ipc_fd +pid_t fork_worker_process(int &main_ipc_fd #ifdef ENABLE_HTTP3 - , - int &wp_quic_ipc_fd + , + int &wp_quic_ipc_fd #endif // ENABLE_HTTP3 - , - const std::vector<InheritedAddr> &iaddrs + , + const std::vector<InheritedAddr> &iaddrs #ifdef ENABLE_HTTP3 - , - const std::vector<std::array<uint8_t, SHRPX_QUIC_CID_PREFIXLEN>> - &cid_prefixes, - const std::vector<QUICLingeringWorkerProcess> &quic_lwps + , + std::vector<WorkerID> worker_ids, + std::vector<QUICLingeringWorkerProcess> quic_lwps #endif // ENABLE_HTTP3 ) { std::array<char, STRERROR_BUFSIZE> errbuf; @@ -1714,9 +1731,9 @@ pid_t fork_worker_process( .ipc_fd = ipc_fd[0], .ready_ipc_fd = worker_process_ready_ipc_fd[1], #ifdef ENABLE_HTTP3 - .cid_prefixes = cid_prefixes, + .worker_ids = std::move(worker_ids), .quic_ipc_fd = quic_ipc_fd[0], - .quic_lingering_worker_processes = quic_lwps, + .quic_lingering_worker_processes = std::move(quic_lwps), #endif // ENABLE_HTTP3 }; rv = worker_process_event_loop(&wpconf); @@ -1835,9 +1852,9 @@ int event_loop() { auto quic_lwps = collect_quic_lingering_worker_processes(); - std::vector<std::array<uint8_t, SHRPX_QUIC_CID_PREFIXLEN>> cid_prefixes; + std::vector<WorkerID> worker_ids; - if (generate_cid_prefix(cid_prefixes, config) != 0) { + if (generate_worker_id(worker_ids, worker_process_seq, config) != 0) { return -1; } #endif // ENABLE_HTTP3 @@ -1858,7 +1875,7 @@ int event_loop() { {} #ifdef ENABLE_HTTP3 , - cid_prefixes, quic_lwps + worker_ids, std::move(quic_lwps) #endif // ENABLE_HTTP3 ); @@ -1869,12 +1886,13 @@ int event_loop() { ev_timer_init(&worker_process_grace_period_timer, worker_process_grace_period_timercb, 0., 0.); - worker_process_add(std::make_unique<WorkerProcess>(loop, pid, ipc_fd + worker_process_add(std::make_unique<WorkerProcess>( + loop, pid, ipc_fd #ifdef ENABLE_HTTP3 - , - quic_ipc_fd, cid_prefixes + , + quic_ipc_fd, std::move(worker_ids), worker_process_seq++ #endif // ENABLE_HTTP3 - )); + )); // Write PID file when we are ready to accept connection from peer. // This makes easier to write restart script for nghttpx. Because @@ -2006,6 +2024,7 @@ void fill_default_config(Config *config) { httpconf.xfp.add = true; httpconf.xfp.strip_incoming = true; httpconf.early_data.strip_incoming = true; + httpconf.timeout.header = 1_min; auto &http2conf = config->http2; { @@ -2088,7 +2107,8 @@ void fill_default_config(Config *config) { static_cast<ev_tstamp>(NGTCP2_DEFAULT_INITIAL_RTT) / NGTCP2_SECONDS; } - if (RAND_bytes(quicconf.server_id.data(), quicconf.server_id.size()) != 1) { + if (RAND_bytes(reinterpret_cast<unsigned char *>(&quicconf.server_id), + sizeof(quicconf.server_id)) != 1) { assert(0); abort(); } @@ -2132,20 +2152,17 @@ void fill_default_config(Config *config) { auto &upstreamconf = connconf.upstream; { auto &timeoutconf = upstreamconf.timeout; - // Read timeout for HTTP2 upstream connection - timeoutconf.http2_read = 3_min; + // Idle timeout for HTTP2 upstream connection + timeoutconf.http2_idle = 3_min; - // Read timeout for HTTP3 upstream connection - timeoutconf.http3_read = 3_min; - - // Read timeout for non-HTTP2 upstream connection - timeoutconf.read = 1_min; + // Idle timeout for HTTP3 upstream connection + timeoutconf.http3_idle = 3_min; // Write timeout for HTTP2/non-HTTP2 upstream connection timeoutconf.write = 30_s; - // Keep alive timeout for HTTP/1 upstream connection - timeoutconf.idle_read = 1_min; + // Keep alive (idle) timeout for HTTP/1 upstream connection + timeoutconf.idle = 1_min; } } @@ -2644,18 +2661,18 @@ Performance: this option will be simply ignored. Timeout: - --frontend-http2-read-timeout=<DURATION> - Specify read timeout for HTTP/2 frontend connection. - Default: )" - << util::duration_str(config->conn.upstream.timeout.http2_read) << R"( - --frontend-http3-read-timeout=<DURATION> - Specify read timeout for HTTP/3 frontend connection. + --frontend-http2-idle-timeout=<DURATION> + Specify idle timeout for HTTP/2 frontend connection. If + no active streams exist for this duration, connection is + closed. Default: )" - << util::duration_str(config->conn.upstream.timeout.http3_read) << R"( - --frontend-read-timeout=<DURATION> - Specify read timeout for HTTP/1.1 frontend connection. + << util::duration_str(config->conn.upstream.timeout.http2_idle) << R"( + --frontend-http3-idle-timeout=<DURATION> + Specify idle timeout for HTTP/3 frontend connection. If + no active streams exist for this duration, connection is + closed. Default: )" - << util::duration_str(config->conn.upstream.timeout.read) << R"( + << util::duration_str(config->conn.upstream.timeout.http3_idle) << R"( --frontend-write-timeout=<DURATION> Specify write timeout for all frontend connections. Default: )" @@ -2664,7 +2681,15 @@ Timeout: Specify keep-alive timeout for frontend HTTP/1 connection. Default: )" - << util::duration_str(config->conn.upstream.timeout.idle_read) << R"( + << util::duration_str(config->conn.upstream.timeout.idle) << R"( + --frontend-header-timeout=<DURATION> + Specify duration that the server waits for an HTTP + request header fields to be received completely. On + timeout, HTTP/1 and HTTP/2 connections are closed. For + HTTP/3, the stream is shutdown, and the connection + itself is left intact. + Default: )" + << util::duration_str(config->http.timeout.header) << R"( --stream-read-timeout=<DURATION> Specify read timeout for HTTP/2 streams. 0 means no timeout. @@ -3530,12 +3555,12 @@ HTTP/3 and QUIC: encrypting tokens and Connection IDs. It is not used to encrypt QUIC packets. Each line of this file must contain exactly 136 bytes hex-encoded string (when - decoded the byte string is 68 bytes long). The first 2 + decoded the byte string is 68 bytes long). The first 3 bits of decoded byte string are used to identify the keying material. An empty line or a line which starts '#' is ignored. The file can contain more than one - keying materials. Because the identifier is 2 bits, at - most 4 keying materials are read and the remaining data + keying materials. Because the identifier is 3 bits, at + most 8 keying materials are read and the remaining data is discarded. The first keying material in the file is primarily used for encryption and decryption for new connection. The other ones are used to decrypt data for @@ -3997,9 +4022,10 @@ void reload_config() { auto quic_lwps = collect_quic_lingering_worker_processes(); - std::vector<std::array<uint8_t, SHRPX_QUIC_CID_PREFIXLEN>> cid_prefixes; + std::vector<WorkerID> worker_ids; - if (generate_cid_prefix(cid_prefixes, new_config.get()) != 0) { + if (generate_worker_id(worker_ids, worker_process_seq, new_config.get()) != + 0) { close_not_inherited_fd(new_config.get(), iaddrs); return; } @@ -4020,7 +4046,7 @@ void reload_config() { iaddrs #ifdef ENABLE_HTTP3 , - cid_prefixes, quic_lwps + worker_ids, std::move(quic_lwps) #endif // ENABLE_HTTP3 ); @@ -4035,12 +4061,13 @@ void reload_config() { close_unused_inherited_addr(iaddrs); - worker_process_add(std::make_unique<WorkerProcess>(loop, pid, ipc_fd + worker_process_add(std::make_unique<WorkerProcess>( + loop, pid, ipc_fd #ifdef ENABLE_HTTP3 - , - quic_ipc_fd, cid_prefixes + , + quic_ipc_fd, std::move(worker_ids), worker_process_seq++ #endif // ENABLE_HTTP3 - )); + )); worker_process_adjust_limit(); @@ -4377,6 +4404,12 @@ int main(int argc, char **argv) { {SHRPX_OPT_REQUIRE_HTTP_SCHEME.c_str(), no_argument, &flag, 191}, {SHRPX_OPT_TLS_KTLS.c_str(), no_argument, &flag, 192}, {SHRPX_OPT_ALPN_LIST.c_str(), required_argument, &flag, 193}, + {SHRPX_OPT_FRONTEND_HEADER_TIMEOUT.c_str(), required_argument, &flag, + 194}, + {SHRPX_OPT_FRONTEND_HTTP2_IDLE_TIMEOUT.c_str(), required_argument, + &flag, 195}, + {SHRPX_OPT_FRONTEND_HTTP3_IDLE_TIMEOUT.c_str(), required_argument, + &flag, 196}, {nullptr, 0, nullptr, 0}}; int option_index = 0; @@ -5294,6 +5327,21 @@ int main(int argc, char **argv) { // --alpn-list cmdcfgs.emplace_back(SHRPX_OPT_ALPN_LIST, StringRef{optarg}); break; + case 194: + // --frontend-header-timeout + cmdcfgs.emplace_back(SHRPX_OPT_FRONTEND_HEADER_TIMEOUT, + StringRef{optarg}); + break; + case 195: + // --frontend-http2-idle-timeout + cmdcfgs.emplace_back(SHRPX_OPT_FRONTEND_HTTP2_IDLE_TIMEOUT, + StringRef{optarg}); + break; + case 196: + // --frontend-http3-idle-timeout + cmdcfgs.emplace_back(SHRPX_OPT_FRONTEND_HTTP3_IDLE_TIMEOUT, + StringRef{optarg}); + break; default: break; } diff --git a/src/shrpx_client_handler.cc b/src/shrpx_client_handler.cc index 1f0c01c..a78b00a 100644 --- a/src/shrpx_client_handler.cc +++ b/src/shrpx_client_handler.cc @@ -444,7 +444,7 @@ ClientHandler::ClientHandler(Worker *worker, int fd, SSL *ssl, rb_(worker->get_mcpool()), conn_(worker->get_loop(), fd, ssl, worker->get_mcpool(), get_config()->conn.upstream.timeout.write, - get_config()->conn.upstream.timeout.read, + get_config()->conn.upstream.timeout.idle, get_config()->conn.upstream.ratelimit.write, get_config()->conn.upstream.ratelimit.read, writecb, readcb, timeoutcb, this, get_config()->tls.dyn_rec.warmup_threshold, @@ -551,7 +551,7 @@ void ClientHandler::setup_http3_upstream( auto config = get_config(); - reset_upstream_read_timeout(config->conn.upstream.timeout.http3_read); + reset_upstream_read_timeout(config->conn.upstream.timeout.http3_idle); } #endif // ENABLE_HTTP3 @@ -591,16 +591,14 @@ struct ev_loop *ClientHandler::get_loop() const { return conn_.loop; } void ClientHandler::reset_upstream_read_timeout(ev_tstamp t) { conn_.rt.repeat = t; - if (ev_is_active(&conn_.rt)) { - ev_timer_again(conn_.loop, &conn_.rt); - } + + ev_timer_again(conn_.loop, &conn_.rt); } void ClientHandler::reset_upstream_write_timeout(ev_tstamp t) { conn_.wt.repeat = t; - if (ev_is_active(&conn_.wt)) { - ev_timer_again(conn_.loop, &conn_.wt); - } + + ev_timer_again(conn_.loop, &conn_.wt); } void ClientHandler::repeat_read_timer() { diff --git a/src/shrpx_config.cc b/src/shrpx_config.cc index 89b3672..d856c95 100644 --- a/src/shrpx_config.cc +++ b/src/shrpx_config.cc @@ -282,9 +282,9 @@ read_quic_secret_file(const StringRef &path) { assert(static_cast<size_t>(p - std::begin(s)) == expectedlen * 2); - qkm.id = qkm.reserved[0] & 0xc0; + qkm.id = qkm.reserved[0] & SHRPX_QUIC_DCID_KM_ID_MASK; - if (kms.size() == 4) { + if (kms.size() == 8) { break; } } @@ -2396,6 +2396,9 @@ int option_lookup_token(const char *name, size_t namelen) { if (util::strieq_l("backend-connect-timeou", name, 22)) { return SHRPX_OPTID_BACKEND_CONNECT_TIMEOUT; } + if (util::strieq_l("frontend-header-timeou", name, 22)) { + return SHRPX_OPTID_FRONTEND_HEADER_TIMEOUT; + } break; } break; @@ -2526,9 +2529,15 @@ int option_lookup_token(const char *name, size_t namelen) { } break; case 't': + if (util::strieq_l("frontend-http2-idle-timeou", name, 26)) { + return SHRPX_OPTID_FRONTEND_HTTP2_IDLE_TIMEOUT; + } if (util::strieq_l("frontend-http2-read-timeou", name, 26)) { return SHRPX_OPTID_FRONTEND_HTTP2_READ_TIMEOUT; } + if (util::strieq_l("frontend-http3-idle-timeou", name, 26)) { + return SHRPX_OPTID_FRONTEND_HTTP3_IDLE_TIMEOUT; + } if (util::strieq_l("frontend-http3-read-timeou", name, 26)) { return SHRPX_OPTID_FRONTEND_HTTP3_READ_TIMEOUT; } @@ -2966,13 +2975,28 @@ int parse_config(Config *config, int optid, const StringRef &opt, return 0; } - case SHRPX_OPTID_WORKERS: + case SHRPX_OPTID_WORKERS: { #ifdef NOTHREADS LOG(WARN) << "Threading disabled at build time, no threads created."; return 0; #else // !NOTHREADS - return parse_uint(&config->num_worker, opt, optarg); + size_t n; + + if (parse_uint(&n, opt, optarg) != 0) { + return -1; + } + + if (n > 65530) { + LOG(ERROR) << opt << ": the number of workers must not exceed 65530"; + + return -1; + } + + config->num_worker = n; + + return 0; #endif // !NOTHREADS + } case SHRPX_OPTID_HTTP2_MAX_CONCURRENT_STREAMS: { LOG(WARN) << opt << ": deprecated. Use " << SHRPX_OPT_FRONTEND_HTTP2_MAX_CONCURRENT_STREAMS << " and " @@ -3028,10 +3052,17 @@ int parse_config(Config *config, int optid, const StringRef &opt, return 0; case SHRPX_OPTID_FRONTEND_HTTP2_READ_TIMEOUT: - return parse_duration(&config->conn.upstream.timeout.http2_read, opt, + LOG(WARN) << opt << ": deprecated. Use frontend-http2-idle-timeout"; + // fall through + case SHRPX_OPTID_FRONTEND_HTTP2_IDLE_TIMEOUT: + return parse_duration(&config->conn.upstream.timeout.http2_idle, opt, optarg); case SHRPX_OPTID_FRONTEND_READ_TIMEOUT: - return parse_duration(&config->conn.upstream.timeout.read, opt, optarg); + LOG(WARN) << opt << ": deprecated. Use frontend-header-timeout"; + + return 0; + case SHRPX_OPTID_FRONTEND_HEADER_TIMEOUT: + return parse_duration(&config->http.timeout.header, opt, optarg); case SHRPX_OPTID_FRONTEND_WRITE_TIMEOUT: return parse_duration(&config->conn.upstream.timeout.write, opt, optarg); case SHRPX_OPTID_BACKEND_READ_TIMEOUT: @@ -3907,8 +3938,7 @@ int parse_config(Config *config, int optid, const StringRef &opt, return 0; } case SHRPX_OPTID_FRONTEND_KEEP_ALIVE_TIMEOUT: - return parse_duration(&config->conn.upstream.timeout.idle_read, opt, - optarg); + return parse_duration(&config->conn.upstream.timeout.idle, opt, optarg); case SHRPX_OPTID_PSK_SECRETS: #ifndef OPENSSL_NO_PSK return parse_psk_secrets(config, optarg); @@ -4032,8 +4062,11 @@ int parse_config(Config *config, int optid, const StringRef &opt, return 0; } case SHRPX_OPTID_FRONTEND_HTTP3_READ_TIMEOUT: + LOG(WARN) << opt << ": deprecated. Use frontend-http3-idle-timeout"; + // fall through + case SHRPX_OPTID_FRONTEND_HTTP3_IDLE_TIMEOUT: #ifdef ENABLE_HTTP3 - return parse_duration(&config->conn.upstream.timeout.http3_read, opt, + return parse_duration(&config->conn.upstream.timeout.http3_idle, opt, optarg); #else // !ENABLE_HTTP3 return 0; @@ -4126,12 +4159,13 @@ int parse_config(Config *config, int optid, const StringRef &opt, return 0; case SHRPX_OPTID_QUIC_SERVER_ID: #ifdef ENABLE_HTTP3 - if (optarg.size() != config->quic.server_id.size() * 2 || + if (optarg.size() != sizeof(config->quic.server_id) * 2 || !util::is_hex_string(optarg)) { LOG(ERROR) << opt << ": must be a hex-string"; return -1; } - util::decode_hex(std::begin(config->quic.server_id), optarg); + util::decode_hex(reinterpret_cast<uint8_t *>(&config->quic.server_id), + optarg); #endif // ENABLE_HTTP3 return 0; @@ -4700,6 +4734,7 @@ int resolve_hostname(Address *addr, const char *hostname, uint16_t port, #ifdef ENABLE_HTTP3 QUICKeyingMaterial::QUICKeyingMaterial(QUICKeyingMaterial &&other) noexcept : cid_encryption_ctx{std::exchange(other.cid_encryption_ctx, nullptr)}, + cid_decryption_ctx{std::exchange(other.cid_decryption_ctx, nullptr)}, reserved{other.reserved}, secret{other.secret}, salt{other.salt}, @@ -4710,11 +4745,16 @@ QUICKeyingMaterial::~QUICKeyingMaterial() noexcept { if (cid_encryption_ctx) { EVP_CIPHER_CTX_free(cid_encryption_ctx); } + + if (cid_decryption_ctx) { + EVP_CIPHER_CTX_free(cid_decryption_ctx); + } } QUICKeyingMaterial & QUICKeyingMaterial::operator=(QUICKeyingMaterial &&other) noexcept { cid_encryption_ctx = std::exchange(other.cid_encryption_ctx, nullptr); + cid_decryption_ctx = std::exchange(other.cid_decryption_ctx, nullptr); reserved = other.reserved; secret = other.secret; salt = other.salt; diff --git a/src/shrpx_config.h b/src/shrpx_config.h index 335b0f9..f264b6a 100644 --- a/src/shrpx_config.h +++ b/src/shrpx_config.h @@ -406,6 +406,12 @@ constexpr auto SHRPX_OPT_REQUIRE_HTTP_SCHEME = StringRef::from_lit("require-http-scheme"); constexpr auto SHRPX_OPT_TLS_KTLS = StringRef::from_lit("tls-ktls"); constexpr auto SHRPX_OPT_ALPN_LIST = StringRef::from_lit("alpn-list"); +constexpr auto SHRPX_OPT_FRONTEND_HEADER_TIMEOUT = + StringRef::from_lit("frontend-header-timeout"); +constexpr auto SHRPX_OPT_FRONTEND_HTTP2_IDLE_TIMEOUT = + StringRef::from_lit("frontend-http2-idle-timeout"); +constexpr auto SHRPX_OPT_FRONTEND_HTTP3_IDLE_TIMEOUT = + StringRef::from_lit("frontend-http3-idle-timeout"); constexpr size_t SHRPX_OBFUSCATED_NODE_LENGTH = 8; @@ -641,6 +647,7 @@ struct QUICKeyingMaterial { ~QUICKeyingMaterial() noexcept; QUICKeyingMaterial &operator=(QUICKeyingMaterial &&other) noexcept; EVP_CIPHER_CTX *cid_encryption_ctx; + EVP_CIPHER_CTX *cid_decryption_ctx; std::array<uint8_t, SHRPX_QUIC_SECRET_RESERVEDLEN> reserved; std::array<uint8_t, SHRPX_QUIC_SECRETLEN> secret; std::array<uint8_t, SHRPX_QUIC_SALTLEN> salt; @@ -815,7 +822,7 @@ struct QUICConfig { StringRef prog_file; bool disabled; } bpf; - std::array<uint8_t, SHRPX_QUIC_SERVER_IDLEN> server_id; + uint32_t server_id; }; struct Http3Config { @@ -864,6 +871,9 @@ struct HttpConfig { struct { bool strip_incoming; } early_data; + struct { + ev_tstamp header; + } timeout; std::vector<AltSvc> altsvcs; // altsvcs serialized in a wire format. StringRef altsvc_header_value; @@ -1048,11 +1058,10 @@ struct ConnectionConfig { struct { struct { - ev_tstamp http2_read; - ev_tstamp http3_read; - ev_tstamp read; + ev_tstamp http2_idle; + ev_tstamp http3_idle; ev_tstamp write; - ev_tstamp idle_read; + ev_tstamp idle; } timeout; struct { RateLimitConfig read; @@ -1249,12 +1258,14 @@ enum { SHRPX_OPTID_FORWARDED_FOR, SHRPX_OPTID_FRONTEND, SHRPX_OPTID_FRONTEND_FRAME_DEBUG, + SHRPX_OPTID_FRONTEND_HEADER_TIMEOUT, SHRPX_OPTID_FRONTEND_HTTP2_CONNECTION_WINDOW_BITS, SHRPX_OPTID_FRONTEND_HTTP2_CONNECTION_WINDOW_SIZE, SHRPX_OPTID_FRONTEND_HTTP2_DECODER_DYNAMIC_TABLE_SIZE, SHRPX_OPTID_FRONTEND_HTTP2_DUMP_REQUEST_HEADER, SHRPX_OPTID_FRONTEND_HTTP2_DUMP_RESPONSE_HEADER, SHRPX_OPTID_FRONTEND_HTTP2_ENCODER_DYNAMIC_TABLE_SIZE, + SHRPX_OPTID_FRONTEND_HTTP2_IDLE_TIMEOUT, SHRPX_OPTID_FRONTEND_HTTP2_MAX_CONCURRENT_STREAMS, SHRPX_OPTID_FRONTEND_HTTP2_OPTIMIZE_WINDOW_SIZE, SHRPX_OPTID_FRONTEND_HTTP2_OPTIMIZE_WRITE_BUFFER_SIZE, @@ -1263,6 +1274,7 @@ enum { SHRPX_OPTID_FRONTEND_HTTP2_WINDOW_BITS, SHRPX_OPTID_FRONTEND_HTTP2_WINDOW_SIZE, SHRPX_OPTID_FRONTEND_HTTP3_CONNECTION_WINDOW_SIZE, + SHRPX_OPTID_FRONTEND_HTTP3_IDLE_TIMEOUT, SHRPX_OPTID_FRONTEND_HTTP3_MAX_CONCURRENT_STREAMS, SHRPX_OPTID_FRONTEND_HTTP3_MAX_CONNECTION_WINDOW_SIZE, SHRPX_OPTID_FRONTEND_HTTP3_MAX_WINDOW_SIZE, diff --git a/src/shrpx_connection_handler.cc b/src/shrpx_connection_handler.cc index af4b8fc..b29ce9a 100644 --- a/src/shrpx_connection_handler.cc +++ b/src/shrpx_connection_handler.cc @@ -278,15 +278,14 @@ int ConnectionHandler::create_single_worker() { #endif // ENABLE_HTTP3 && HAVE_LIBBPF #ifdef ENABLE_HTTP3 - assert(cid_prefixes_.size() == 1); - const auto &cid_prefix = cid_prefixes_[0]; + assert(worker_ids_.size() == 1); + const auto &wid = worker_ids_[0]; #endif // ENABLE_HTTP3 single_worker_ = std::make_unique<Worker>( loop_, sv_ssl_ctx, cl_ssl_ctx, session_cache_ssl_ctx, cert_tree_.get(), #ifdef ENABLE_HTTP3 - quic_sv_ssl_ctx, quic_cert_tree_.get(), cid_prefix.data(), - cid_prefix.size(), + quic_sv_ssl_ctx, quic_cert_tree_.get(), wid, # ifdef HAVE_LIBBPF /* index = */ 0, # endif // HAVE_LIBBPF @@ -376,21 +375,20 @@ int ConnectionHandler::create_worker_thread(size_t num) { } # ifdef ENABLE_HTTP3 - assert(cid_prefixes_.size() == num); + assert(worker_ids_.size() == num); # endif // ENABLE_HTTP3 for (size_t i = 0; i < num; ++i) { auto loop = ev_loop_new(config->ev_loop_flags); # ifdef ENABLE_HTTP3 - const auto &cid_prefix = cid_prefixes_[i]; + const auto &wid = worker_ids_[i]; # endif // ENABLE_HTTP3 auto worker = std::make_unique<Worker>( loop, sv_ssl_ctx, cl_ssl_ctx, session_cache_ssl_ctx, cert_tree_.get(), # ifdef ENABLE_HTTP3 - quic_sv_ssl_ctx, quic_cert_tree_.get(), cid_prefix.data(), - cid_prefix.size(), + quic_sv_ssl_ctx, quic_cert_tree_.get(), wid, # ifdef HAVE_LIBBPF i, # endif // HAVE_LIBBPF @@ -1008,27 +1006,23 @@ void ConnectionHandler::set_enable_acceptor_on_ocsp_completion(bool f) { #ifdef ENABLE_HTTP3 int ConnectionHandler::forward_quic_packet( const UpstreamAddr *faddr, const Address &remote_addr, - const Address &local_addr, const ngtcp2_pkt_info &pi, - const uint8_t *cid_prefix, const uint8_t *data, size_t datalen) { + const Address &local_addr, const ngtcp2_pkt_info &pi, const WorkerID &wid, + const uint8_t *data, size_t datalen) { assert(!get_config()->single_thread); - for (auto &worker : workers_) { - if (!std::equal(cid_prefix, cid_prefix + SHRPX_QUIC_CID_PREFIXLEN, - worker->get_cid_prefix())) { - continue; - } - - WorkerEvent wev{}; - wev.type = WorkerEventType::QUIC_PKT_FORWARD; - wev.quic_pkt = std::make_unique<QUICPacket>(faddr->index, remote_addr, - local_addr, pi, data, datalen); + auto worker = find_worker(wid); + if (worker == nullptr) { + return -1; + } - worker->send(std::move(wev)); + WorkerEvent wev{}; + wev.type = WorkerEventType::QUIC_PKT_FORWARD; + wev.quic_pkt = std::make_unique<QUICPacket>(faddr->index, remote_addr, + local_addr, pi, data, datalen); - return 0; - } + worker->send(std::move(wev)); - return -1; + return 0; } void ConnectionHandler::set_quic_keying_materials( @@ -1041,22 +1035,40 @@ ConnectionHandler::get_quic_keying_materials() const { return quic_keying_materials_; } -void ConnectionHandler::set_cid_prefixes( - const std::vector<std::array<uint8_t, SHRPX_QUIC_CID_PREFIXLEN>> - &cid_prefixes) { - cid_prefixes_ = cid_prefixes; +void ConnectionHandler::set_worker_ids(std::vector<WorkerID> worker_ids) { + worker_ids_ = std::move(worker_ids); } -QUICLingeringWorkerProcess * -ConnectionHandler::match_quic_lingering_worker_process_cid_prefix( - const uint8_t *dcid, size_t dcidlen) { - assert(dcidlen >= SHRPX_QUIC_CID_PREFIXLEN); +namespace { +ssize_t find_worker_index(const std::vector<WorkerID> &worker_ids, + const WorkerID &wid) { + assert(!worker_ids.empty()); + if (wid.server != worker_ids[0].server || + wid.worker_process != worker_ids[0].worker_process || + wid.thread >= worker_ids.size()) { + return -1; + } + + return wid.thread; +} +} // namespace + +Worker *ConnectionHandler::find_worker(const WorkerID &wid) const { + auto idx = find_worker_index(worker_ids_, wid); + if (idx == -1) { + return nullptr; + } + + return workers_[idx].get(); +} + +QUICLingeringWorkerProcess * +ConnectionHandler::match_quic_lingering_worker_process_worker_id( + const WorkerID &wid) { for (auto &lwps : quic_lingering_worker_processes_) { - for (auto &cid_prefix : lwps.cid_prefixes) { - if (std::equal(std::begin(cid_prefix), std::end(cid_prefix), dcid)) { - return &lwps; - } + if (find_worker_index(lwps.worker_ids, wid) != -1) { + return &lwps; } } @@ -1275,33 +1287,29 @@ int ConnectionHandler::quic_ipc_read() { auto &qkm = quic_keying_materials_->keying_materials.front(); - std::array<uint8_t, SHRPX_QUIC_DECRYPTED_DCIDLEN> decrypted_dcid; + ConnectionID decrypted_dcid; - if (decrypt_quic_connection_id(decrypted_dcid.data(), - vc.dcid + SHRPX_QUIC_CID_PREFIX_OFFSET, - qkm.cid_encryption_ctx) != 0) { + if (decrypt_quic_connection_id(decrypted_dcid, + vc.dcid + SHRPX_QUIC_CID_WORKER_ID_OFFSET, + qkm.cid_decryption_ctx) != 0) { return -1; } - for (auto &worker : workers_) { - if (!std::equal(std::begin(decrypted_dcid), - std::begin(decrypted_dcid) + SHRPX_QUIC_CID_PREFIXLEN, - worker->get_cid_prefix())) { - continue; + auto worker = find_worker(decrypted_dcid.worker); + if (worker == nullptr) { + if (LOG_ENABLED(INFO)) { + LOG(INFO) << "No worker to match Worker ID"; } - WorkerEvent wev{ - .type = WorkerEventType::QUIC_PKT_FORWARD, - .quic_pkt = std::move(pkt), - }; - worker->send(std::move(wev)); - return 0; } - if (LOG_ENABLED(INFO)) { - LOG(INFO) << "No worker to match CID prefix"; - } + WorkerEvent wev{ + .type = WorkerEventType::QUIC_PKT_FORWARD, + .quic_pkt = std::move(pkt), + }; + + worker->send(std::move(wev)); return 0; } diff --git a/src/shrpx_connection_handler.h b/src/shrpx_connection_handler.h index f3748ab..47ec209 100644 --- a/src/shrpx_connection_handler.h +++ b/src/shrpx_connection_handler.h @@ -108,7 +108,7 @@ struct SerialEvent { struct BPFRef { bpf_object *obj; bpf_map *reuseport_array; - bpf_map *cid_prefix_map; + bpf_map *worker_id_map; }; # endif // HAVE_LIBBPF @@ -121,12 +121,10 @@ enum class QUICIPCType { // WorkerProcesses which are in graceful shutdown period. struct QUICLingeringWorkerProcess { - QUICLingeringWorkerProcess( - std::vector<std::array<uint8_t, SHRPX_QUIC_CID_PREFIXLEN>> cid_prefixes, - int quic_ipc_fd) - : cid_prefixes{std::move(cid_prefixes)}, quic_ipc_fd{quic_ipc_fd} {} + QUICLingeringWorkerProcess(std::vector<WorkerID> worker_ids, int quic_ipc_fd) + : worker_ids{std::move(worker_ids)}, quic_ipc_fd{quic_ipc_fd} {} - std::vector<std::array<uint8_t, SHRPX_QUIC_CID_PREFIXLEN>> cid_prefixes; + std::vector<WorkerID> worker_ids; // Socket to send QUIC IPC message to this worker process. int quic_ipc_fd; }; @@ -197,25 +195,23 @@ public: int forward_quic_packet(const UpstreamAddr *faddr, const Address &remote_addr, const Address &local_addr, const ngtcp2_pkt_info &pi, - const uint8_t *cid_prefix, const uint8_t *data, + const WorkerID &wid, const uint8_t *data, size_t datalen); void set_quic_keying_materials(std::shared_ptr<QUICKeyingMaterials> qkms); const std::shared_ptr<QUICKeyingMaterials> &get_quic_keying_materials() const; - void set_cid_prefixes( - const std::vector<std::array<uint8_t, SHRPX_QUIC_CID_PREFIXLEN>> - &cid_prefixes); + void set_worker_ids(std::vector<WorkerID> worker_ids); + Worker *find_worker(const WorkerID &wid) const; void set_quic_lingering_worker_processes( const std::vector<QUICLingeringWorkerProcess> &quic_lwps); - // Return matching QUICLingeringWorkerProcess which has a CID prefix + // Return matching QUICLingeringWorkerProcess which has a Worker ID // such that |dcid| starts with it. If no such // QUICLingeringWorkerProcess, it returns nullptr. QUICLingeringWorkerProcess * - match_quic_lingering_worker_process_cid_prefix(const uint8_t *dcid, - size_t dcidlen); + match_quic_lingering_worker_process_worker_id(const WorkerID &wid); int forward_quic_packet_to_lingering_worker_process( QUICLingeringWorkerProcess *quic_lwp, const Address &remote_addr, @@ -260,9 +256,8 @@ private: // and signature algorithm presented by client. std::vector<std::vector<SSL_CTX *>> indexed_ssl_ctx_; #ifdef ENABLE_HTTP3 - std::vector<std::array<uint8_t, SHRPX_QUIC_CID_PREFIXLEN>> cid_prefixes_; - std::vector<std::array<uint8_t, SHRPX_QUIC_CID_PREFIXLEN>> - lingering_cid_prefixes_; + std::vector<WorkerID> worker_ids_; + std::vector<WorkerID> lingering_worker_ids_; int quic_ipc_fd_; std::vector<QUICLingeringWorkerProcess> quic_lingering_worker_processes_; # ifdef HAVE_LIBBPF diff --git a/src/shrpx_dns_resolver.cc b/src/shrpx_dns_resolver.cc index f83ecb7..8253942 100644 --- a/src/shrpx_dns_resolver.cc +++ b/src/shrpx_dns_resolver.cc @@ -55,9 +55,11 @@ void sock_state_cb(void *data, int s, int read, int write) { } // namespace namespace { -void host_cb(void *arg, int status, int timeouts, hostent *hostent) { +void addrinfo_cb(void *arg, int status, int timeouts, ares_addrinfo *result) { auto resolv = static_cast<DNSResolver *>(arg); - resolv->on_result(status, hostent); + resolv->on_result(status, result); + + ares_freeaddrinfo(result); } } // namespace @@ -173,7 +175,10 @@ int DNSResolver::resolve(const StringRef &name, int family) { channel_ = chan; status_ = DNSResolverStatus::RUNNING; - ares_gethostbyname(channel_, name_.c_str(), family_, host_cb, this); + ares_addrinfo_hints hints{}; + hints.ai_family = family_; + + ares_getaddrinfo(channel_, name_.c_str(), nullptr, &hints, addrinfo_cb, this); reset_timeout(); return 0; @@ -285,7 +290,7 @@ void DNSResolver::start_wev(int fd) { void DNSResolver::stop_wev(int fd) { stop_ev(wevs_, loop_, fd, EV_WRITE); } -void DNSResolver::on_result(int status, hostent *hostent) { +void DNSResolver::on_result(int status, ares_addrinfo *ai) { stop_ev(loop_, revs_); stop_ev(loop_, wevs_); ev_timer_stop(loop_, &timer_); @@ -299,40 +304,44 @@ void DNSResolver::on_result(int status, hostent *hostent) { return; } - auto ap = *hostent->h_addr_list; + auto ap = ai->nodes; + + for (; ap; ap = ap->ai_next) { + switch (ap->ai_family) { + case AF_INET: + status_ = DNSResolverStatus::OK; + result_.len = sizeof(result_.su.in); + + assert(sizeof(result_.su.in) == ap->ai_addrlen); + + memcpy(&result_.su.in, ap->ai_addr, sizeof(result_.su.in)); + + break; + case AF_INET6: + status_ = DNSResolverStatus::OK; + result_.len = sizeof(result_.su.in6); + + assert(sizeof(result_.su.in6) == ap->ai_addrlen); + + memcpy(&result_.su.in6, ap->ai_addr, sizeof(result_.su.in6)); + + break; + default: + continue; + } + + break; + } + if (!ap) { if (LOG_ENABLED(INFO)) { - LOG(INFO) << "Name lookup for " << name_ << "failed: no address returned"; + LOG(INFO) << "Name lookup for " << name_ + << " failed: no address returned"; } status_ = DNSResolverStatus::ERROR; return; } - switch (hostent->h_addrtype) { - case AF_INET: - status_ = DNSResolverStatus::OK; - result_.len = sizeof(result_.su.in); - result_.su.in = {}; - result_.su.in.sin_family = AF_INET; -#ifdef HAVE_SOCKADDR_IN_SIN_LEN - result_.su.in.sin_len = sizeof(result_.su.in); -#endif // HAVE_SOCKADDR_IN_SIN_LEN - memcpy(&result_.su.in.sin_addr, ap, sizeof(result_.su.in.sin_addr)); - break; - case AF_INET6: - status_ = DNSResolverStatus::OK; - result_.len = sizeof(result_.su.in6); - result_.su.in6 = {}; - result_.su.in6.sin6_family = AF_INET6; -#ifdef HAVE_SOCKADDR_IN6_SIN6_LEN - result_.su.in6.sin6_len = sizeof(result_.su.in6); -#endif // HAVE_SOCKADDR_IN6_SIN6_LEN - memcpy(&result_.su.in6.sin6_addr, ap, sizeof(result_.su.in6.sin6_addr)); - break; - default: - assert(0); - } - if (status_ == DNSResolverStatus::OK) { if (LOG_ENABLED(INFO)) { LOG(INFO) << "Name lookup succeeded: " << name_ << " -> " diff --git a/src/shrpx_dns_resolver.h b/src/shrpx_dns_resolver.h index e622f99..4d68273 100644 --- a/src/shrpx_dns_resolver.h +++ b/src/shrpx_dns_resolver.h @@ -88,7 +88,7 @@ public: int on_write(int fd); int on_timeout(); // Calls this function when DNS query finished. - void on_result(int status, hostent *hostent); + void on_result(int status, ares_addrinfo *result); void reset_timeout(); void start_rev(int fd); diff --git a/src/shrpx_downstream.cc b/src/shrpx_downstream.cc index 9ea52b4..5fd717e 100644 --- a/src/shrpx_downstream.cc +++ b/src/shrpx_downstream.cc @@ -46,6 +46,23 @@ namespace shrpx { namespace { +void header_timeoutcb(struct ev_loop *loop, ev_timer *w, int revents) { + auto downstream = static_cast<Downstream *>(w->data); + auto upstream = downstream->get_upstream(); + + if (LOG_ENABLED(INFO)) { + DLOG(INFO, downstream) << "request header timeout stream_id=" + << downstream->get_stream_id(); + } + + downstream->disable_upstream_rtimer(); + downstream->disable_upstream_wtimer(); + + upstream->on_timeout(downstream); +} +} // namespace + +namespace { void upstream_timeoutcb(struct ev_loop *loop, ev_timer *w, int revents) { auto downstream = static_cast<Downstream *>(w->data); auto upstream = downstream->get_upstream(); @@ -148,7 +165,12 @@ Downstream::Downstream(Upstream *upstream, MemchunkPool *mcpool, expect_100_continue_(false), stop_reading_(false) { - auto &timeoutconf = get_config()->http2.timeout; + auto config = get_config(); + auto &httpconf = config->http; + + ev_timer_init(&header_timer_, header_timeoutcb, 0., httpconf.timeout.header); + + auto &timeoutconf = config->http2.timeout; ev_timer_init(&upstream_rtimer_, &upstream_rtimeoutcb, 0., timeoutconf.stream_read); @@ -159,6 +181,7 @@ Downstream::Downstream(Upstream *upstream, MemchunkPool *mcpool, ev_timer_init(&downstream_wtimer_, &downstream_wtimeoutcb, 0., timeoutconf.stream_write); + header_timer_.data = this; upstream_rtimer_.data = this; upstream_wtimer_.data = this; downstream_rtimer_.data = this; @@ -183,6 +206,7 @@ Downstream::~Downstream() { ev_timer_stop(loop, &upstream_wtimer_); ev_timer_stop(loop, &downstream_rtimer_); ev_timer_stop(loop, &downstream_wtimer_); + ev_timer_stop(loop, &header_timer_); #ifdef HAVE_MRUBY auto handler = upstream_->get_client_handler(); @@ -946,6 +970,18 @@ bool Downstream::expect_response_trailer() const { (resp_.http_major == 3 || resp_.http_major == 2); } +void Downstream::repeat_header_timer() { + auto loop = upstream_->get_client_handler()->get_loop(); + + ev_timer_again(loop, &header_timer_); +} + +void Downstream::stop_header_timer() { + auto loop = upstream_->get_client_handler()->get_loop(); + + ev_timer_stop(loop, &header_timer_); +} + namespace { void reset_timer(struct ev_loop *loop, ev_timer *w) { ev_timer_again(loop, w); } } // namespace diff --git a/src/shrpx_downstream.h b/src/shrpx_downstream.h index 146cae5..15f3a47 100644 --- a/src/shrpx_downstream.h +++ b/src/shrpx_downstream.h @@ -448,6 +448,9 @@ public: // connection. int on_read(); + void repeat_header_timer(); + void stop_header_timer(); + // Resets upstream read timer. If it is active, timeout value is // reset. If it is not active, timer will be started. void reset_upstream_rtimer(); @@ -562,6 +565,8 @@ private: // if frontend uses RFC 8441 WebSocket bootstrapping via HTTP/2. StringRef ws_key_; + ev_timer header_timer_; + ev_timer upstream_rtimer_; ev_timer upstream_wtimer_; diff --git a/src/shrpx_http2_upstream.cc b/src/shrpx_http2_upstream.cc index 2cb5436..7816f5f 100644 --- a/src/shrpx_http2_upstream.cc +++ b/src/shrpx_http2_upstream.cc @@ -285,7 +285,10 @@ void Http2Upstream::on_start_request(const nghttp2_frame *frame) { downstream->reset_upstream_rtimer(); - handler_->repeat_read_timer(); + auto config = get_config(); + auto &httpconf = config->http; + + handler_->reset_upstream_read_timeout(httpconf.timeout.header); auto &req = downstream->request(); @@ -298,8 +301,6 @@ void Http2Upstream::on_start_request(const nghttp2_frame *frame) { ++num_requests_; - auto config = get_config(); - auto &httpconf = config->http; if (httpconf.max_requests <= num_requests_) { start_graceful_shutdown(); } @@ -1132,7 +1133,7 @@ Http2Upstream::Http2Upstream(ClientHandler *handler) #endif // defined(TCP_INFO) && defined(TCP_NOTSENT_LOWAT) handler_->reset_upstream_read_timeout( - config->conn.upstream.timeout.http2_read); + config->conn.upstream.timeout.http2_idle); handler_->signal_write(); } @@ -1640,7 +1641,10 @@ void Http2Upstream::remove_downstream(Downstream *downstream) { if (downstream_queue_.get_downstreams() == nullptr) { // There is no downstream at the moment. Start idle timer now. - handler_->repeat_read_timer(); + auto config = get_config(); + auto &upstreamconf = config->conn.upstream; + + handler_->reset_upstream_read_timeout(upstreamconf.timeout.http2_idle); } } diff --git a/src/shrpx_http3_upstream.cc b/src/shrpx_http3_upstream.cc index b8667a3..d12d2da 100644 --- a/src/shrpx_http3_upstream.cc +++ b/src/shrpx_http3_upstream.cc @@ -118,7 +118,6 @@ Http3Upstream::Http3Upstream(ClientHandler *handler) httpconn_{nullptr}, downstream_queue_{downstream_queue_size(handler->get_worker()), !get_config()->http2_proxy}, - retry_close_{false}, tx_{ .data = std::unique_ptr<uint8_t[]>(new uint8_t[64_k]), } { @@ -212,8 +211,10 @@ int get_new_connection_id(ngtcp2_conn *conn, ngtcp2_cid *cid, uint8_t *token, auto &qkms = conn_handler->get_quic_keying_materials(); auto &qkm = qkms->keying_materials.front(); - if (generate_quic_connection_id(*cid, cidlen, worker->get_cid_prefix(), - qkm.id, qkm.cid_encryption_ctx) != 0) { + assert(SHRPX_QUIC_SCIDLEN == cidlen); + + if (generate_quic_connection_id(*cid, worker->get_worker_id(), qkm.id, + qkm.cid_encryption_ctx) != 0) { return NGTCP2_ERR_CALLBACK_FAILURE; } @@ -250,8 +251,9 @@ void Http3Upstream::http_begin_request_headers(int64_t stream_id) { nghttp3_conn_set_stream_user_data(httpconn_, stream_id, downstream.get()); downstream->reset_upstream_rtimer(); + downstream->repeat_header_timer(); - handler_->repeat_read_timer(); + handler_->stop_read_timer(); auto &req = downstream->request(); req.http_major = 3; @@ -609,8 +611,7 @@ int Http3Upstream::init(const UpstreamAddr *faddr, const Address &remote_addr, ngtcp2_cid scid; - if (generate_quic_connection_id(scid, SHRPX_QUIC_SCIDLEN, - worker->get_cid_prefix(), qkm.id, + if (generate_quic_connection_id(scid, worker->get_worker_id(), qkm.id, qkm.cid_encryption_ctx) != 0) { return -1; } @@ -997,7 +998,18 @@ int Http3Upstream::write_streams() { return 0; } -int Http3Upstream::on_timeout(Downstream *downstream) { return 0; } +int Http3Upstream::on_timeout(Downstream *downstream) { + if (LOG_ENABLED(INFO)) { + ULOG(INFO, this) << "Stream timeout stream_id=" + << downstream->get_stream_id(); + } + + shutdown_stream(downstream, NGHTTP3_H3_INTERNAL_ERROR); + + handler_->signal_write(); + + return 0; +} int Http3Upstream::on_downstream_abort_request(Downstream *downstream, unsigned int status_code) { @@ -1528,8 +1540,13 @@ void Http3Upstream::on_handler_delete() { quic_conn_handler->remove_connection_id(cid); } - if (retry_close_ || last_error_.type == NGTCP2_CCERR_TYPE_IDLE_CLOSE) { + switch (last_error_.type) { + case NGTCP2_CCERR_TYPE_IDLE_CLOSE: + case NGTCP2_CCERR_TYPE_DROP_CONN: + case NGTCP2_CCERR_TYPE_RETRY: return; + default: + break; } // If this is not idle close, send CONNECTION_CLOSE. @@ -1823,7 +1840,8 @@ int Http3Upstream::on_read(const UpstreamAddr *faddr, return -1; } - retry_close_ = true; + // Overwrite error if any is set + ngtcp2_ccerr_set_liberr(&last_error_, rv, nullptr, 0); quic_conn_handler->send_retry(handler_->get_upstream_addr(), vc.version, vc.dcid, vc.dcidlen, vc.scid, vc.scidlen, @@ -1838,6 +1856,9 @@ int Http3Upstream::on_read(const UpstreamAddr *faddr, } break; case NGTCP2_ERR_DROP_CONN: + // Overwrite error if any is set + ngtcp2_ccerr_set_liberr(&last_error_, rv, nullptr, 0); + return -1; default: if (!last_error_.error_code) { @@ -2149,6 +2170,11 @@ int Http3Upstream::http_recv_request_header(Downstream *downstream, // just ignore if this is a trailer part. if (trailer) { + if (shutdown_stream_read(downstream->get_stream_id(), + NGHTTP3_H3_NO_ERROR) != 0) { + return -1; + } + return 0; } @@ -2182,7 +2208,6 @@ namespace { int http_end_request_headers(nghttp3_conn *conn, int64_t stream_id, int fin, void *user_data, void *stream_user_data) { auto upstream = static_cast<Http3Upstream *>(user_data); - auto handler = upstream->get_client_handler(); auto downstream = static_cast<Downstream *>(stream_user_data); if (!downstream || downstream->get_stop_reading()) { @@ -2194,7 +2219,7 @@ int http_end_request_headers(nghttp3_conn *conn, int64_t stream_id, int fin, } downstream->reset_upstream_rtimer(); - handler->stop_read_timer(); + downstream->stop_header_timer(); return 0; } diff --git a/src/shrpx_http3_upstream.h b/src/shrpx_http3_upstream.h index 89dfc17..53c73ae 100644 --- a/src/shrpx_http3_upstream.h +++ b/src/shrpx_http3_upstream.h @@ -167,7 +167,6 @@ private: ngtcp2_ccerr last_error_; nghttp3_conn *httpconn_; DownstreamQueue downstream_queue_; - bool retry_close_; std::vector<uint8_t> conn_close_; struct { diff --git a/src/shrpx_https_upstream.cc b/src/shrpx_https_upstream.cc index 49d2088..0412384 100644 --- a/src/shrpx_https_upstream.cc +++ b/src/shrpx_https_upstream.cc @@ -115,12 +115,9 @@ void HttpsUpstream::on_start_request() { attach_downstream(std::move(downstream)); - auto conn = handler_->get_connection(); - auto &upstreamconf = get_config()->conn.upstream; - - conn->rt.repeat = upstreamconf.timeout.read; + auto &httpconf = get_config()->http; - handler_->repeat_read_timer(); + handler_->reset_upstream_read_timeout(httpconf.timeout.header); ++num_requests_; } @@ -795,12 +792,9 @@ int HttpsUpstream::on_write() { return 0; } - auto conn = handler_->get_connection(); auto &upstreamconf = get_config()->conn.upstream; - conn->rt.repeat = upstreamconf.timeout.idle_read; - - handler_->repeat_read_timer(); + handler_->reset_upstream_read_timeout(upstreamconf.timeout.idle); return resume_read(SHRPX_NO_BUFFER, nullptr, 0); } else { diff --git a/src/shrpx_quic.cc b/src/shrpx_quic.cc index a6d4dfa..c52eee4 100644 --- a/src/shrpx_quic.cc +++ b/src/shrpx_quic.cc @@ -173,42 +173,34 @@ int quic_send_packet(const UpstreamAddr *faddr, const sockaddr *remote_sa, return 0; } -int generate_quic_retry_connection_id(ngtcp2_cid &cid, size_t cidlen, - const uint8_t *server_id, uint8_t km_id, - EVP_CIPHER_CTX *ctx) { - assert(cidlen == SHRPX_QUIC_SCIDLEN); - - if (RAND_bytes(cid.data, cidlen) != 1) { +int generate_quic_retry_connection_id(ngtcp2_cid &cid, uint32_t server_id, + uint8_t km_id, EVP_CIPHER_CTX *ctx) { + if (RAND_bytes(cid.data, SHRPX_QUIC_SCIDLEN) != 1) { return -1; } - cid.datalen = cidlen; - - cid.data[0] = (cid.data[0] & 0x3f) | km_id; + cid.datalen = SHRPX_QUIC_SCIDLEN; + cid.data[0] = (cid.data[0] & (~SHRPX_QUIC_DCID_KM_ID_MASK)) | km_id; - auto p = cid.data + SHRPX_QUIC_CID_PREFIX_OFFSET; + auto p = cid.data + SHRPX_QUIC_CID_WORKER_ID_OFFSET; - std::copy_n(server_id, SHRPX_QUIC_SERVER_IDLEN, p); + std::copy_n(reinterpret_cast<uint8_t *>(&server_id), sizeof(server_id), p); return encrypt_quic_connection_id(p, p, ctx); } -int generate_quic_connection_id(ngtcp2_cid &cid, size_t cidlen, - const uint8_t *cid_prefix, uint8_t km_id, - EVP_CIPHER_CTX *ctx) { - assert(cidlen == SHRPX_QUIC_SCIDLEN); - - if (RAND_bytes(cid.data, cidlen) != 1) { +int generate_quic_connection_id(ngtcp2_cid &cid, const WorkerID &wid, + uint8_t km_id, EVP_CIPHER_CTX *ctx) { + if (RAND_bytes(cid.data, SHRPX_QUIC_SCIDLEN) != 1) { return -1; } - cid.datalen = cidlen; + cid.datalen = SHRPX_QUIC_SCIDLEN; + cid.data[0] = (cid.data[0] & (~SHRPX_QUIC_DCID_KM_ID_MASK)) | km_id; - cid.data[0] = (cid.data[0] & 0x3f) | km_id; + auto p = cid.data + SHRPX_QUIC_CID_WORKER_ID_OFFSET; - auto p = cid.data + SHRPX_QUIC_CID_PREFIX_OFFSET; - - std::copy_n(cid_prefix, SHRPX_QUIC_CID_PREFIXLEN, p); + std::copy_n(reinterpret_cast<const uint8_t *>(&wid), sizeof(wid), p); return encrypt_quic_connection_id(p, p, ctx); } @@ -225,12 +217,13 @@ int encrypt_quic_connection_id(uint8_t *dest, const uint8_t *src, return 0; } -int decrypt_quic_connection_id(uint8_t *dest, const uint8_t *src, +int decrypt_quic_connection_id(ConnectionID &dest, const uint8_t *src, EVP_CIPHER_CTX *ctx) { int len; + auto p = reinterpret_cast<uint8_t *>(&dest); - if (!EVP_EncryptUpdate(ctx, dest, &len, src, SHRPX_QUIC_DECRYPTED_DCIDLEN) || - !EVP_EncryptFinal_ex(ctx, dest + len, &len)) { + if (!EVP_DecryptUpdate(ctx, p, &len, src, SHRPX_QUIC_DECRYPTED_DCIDLEN) || + !EVP_DecryptFinal_ex(ctx, p + len, &len)) { return -1; } diff --git a/src/shrpx_quic.h b/src/shrpx_quic.h index 88388e9..dae6e31 100644 --- a/src/shrpx_quic.h +++ b/src/shrpx_quic.h @@ -65,19 +65,50 @@ struct UpstreamAddr; struct QUICKeyingMaterials; struct QUICKeyingMaterial; -constexpr size_t SHRPX_QUIC_SCIDLEN = 20; +constexpr size_t SHRPX_QUIC_CID_WORKER_ID_OFFSET = 1; constexpr size_t SHRPX_QUIC_SERVER_IDLEN = 4; -// SHRPX_QUIC_CID_PREFIXLEN includes SHRPX_QUIC_SERVER_IDLEN. -constexpr size_t SHRPX_QUIC_CID_PREFIXLEN = 8; -constexpr size_t SHRPX_QUIC_CID_PREFIX_OFFSET = 1; -constexpr size_t SHRPX_QUIC_DECRYPTED_DCIDLEN = 16; +constexpr size_t SHRPX_QUIC_SOCK_IDLEN = 4; +constexpr size_t SHRPX_QUIC_WORKER_IDLEN = + SHRPX_QUIC_SERVER_IDLEN + SHRPX_QUIC_SOCK_IDLEN; +constexpr size_t SHRPX_QUIC_CLIENT_IDLEN = 8; +constexpr size_t SHRPX_QUIC_DECRYPTED_DCIDLEN = + SHRPX_QUIC_WORKER_IDLEN + SHRPX_QUIC_CLIENT_IDLEN; +constexpr size_t SHRPX_QUIC_SCIDLEN = + SHRPX_QUIC_CID_WORKER_ID_OFFSET + SHRPX_QUIC_DECRYPTED_DCIDLEN; constexpr size_t SHRPX_QUIC_CID_ENCRYPTION_KEYLEN = 16; constexpr size_t SHRPX_QUIC_CONN_CLOSE_PKTLEN = 256; constexpr size_t SHRPX_QUIC_STATELESS_RESET_BURST = 100; constexpr size_t SHRPX_QUIC_SECRET_RESERVEDLEN = 4; constexpr size_t SHRPX_QUIC_SECRETLEN = 32; constexpr size_t SHRPX_QUIC_SALTLEN = 32; -constexpr uint8_t SHRPX_QUIC_DCID_KM_ID_MASK = 0xc0; +constexpr uint8_t SHRPX_QUIC_DCID_KM_ID_MASK = 0xe0; + +struct WorkerID { + union { + struct { + uint32_t server; + uint16_t worker_process; + uint16_t thread; + }; + uint64_t worker; + }; +}; + +static_assert(sizeof(WorkerID) == SHRPX_QUIC_WORKER_IDLEN, + "WorkerID length assertion failure"); + +inline bool operator==(const WorkerID &lhd, const WorkerID &rhd) { + return lhd.worker == rhd.worker; +} + +inline bool operator!=(const WorkerID &lhd, const WorkerID &rhd) { + return lhd.worker != rhd.worker; +} + +struct ConnectionID { + WorkerID worker; + uint64_t client; +}; ngtcp2_tstamp quic_timestamp(); @@ -86,18 +117,16 @@ int quic_send_packet(const UpstreamAddr *faddr, const sockaddr *remote_sa, size_t local_salen, const ngtcp2_pkt_info &pi, const uint8_t *data, size_t datalen, size_t gso_size); -int generate_quic_retry_connection_id(ngtcp2_cid &cid, size_t cidlen, - const uint8_t *server_id, uint8_t km_id, - EVP_CIPHER_CTX *ctx); +int generate_quic_retry_connection_id(ngtcp2_cid &cid, uint32_t server_id, + uint8_t km_id, EVP_CIPHER_CTX *ctx); -int generate_quic_connection_id(ngtcp2_cid &cid, size_t cidlen, - const uint8_t *cid_prefix, uint8_t km_id, - EVP_CIPHER_CTX *ctx); +int generate_quic_connection_id(ngtcp2_cid &cid, const WorkerID &wid, + uint8_t km_id, EVP_CIPHER_CTX *ctx); int encrypt_quic_connection_id(uint8_t *dest, const uint8_t *src, EVP_CIPHER_CTX *ctx); -int decrypt_quic_connection_id(uint8_t *dest, const uint8_t *src, +int decrypt_quic_connection_id(ConnectionID &dest, const uint8_t *src, EVP_CIPHER_CTX *ctx); int generate_quic_hashed_connection_id(ngtcp2_cid &dest, diff --git a/src/shrpx_quic_connection_handler.cc b/src/shrpx_quic_connection_handler.cc index 13f710b..b810aa6 100644 --- a/src/shrpx_quic_connection_handler.cc +++ b/src/shrpx_quic_connection_handler.cc @@ -123,7 +123,7 @@ int QUICConnectionHandler::handle_packet(const UpstreamAddr *faddr, } if (it == std::end(connections_)) { - std::array<uint8_t, SHRPX_QUIC_DECRYPTED_DCIDLEN> decrypted_dcid; + ConnectionID decrypted_dcid; auto &qkms = conn_handler->get_quic_keying_materials(); const QUICKeyingMaterial *qkm = nullptr; @@ -132,19 +132,17 @@ int QUICConnectionHandler::handle_packet(const UpstreamAddr *faddr, qkm = select_quic_keying_material( *qkms.get(), vc.dcid[0] & SHRPX_QUIC_DCID_KM_ID_MASK); - if (decrypt_quic_connection_id(decrypted_dcid.data(), - vc.dcid + SHRPX_QUIC_CID_PREFIX_OFFSET, - qkm->cid_encryption_ctx) != 0) { + if (decrypt_quic_connection_id(decrypted_dcid, + vc.dcid + SHRPX_QUIC_CID_WORKER_ID_OFFSET, + qkm->cid_decryption_ctx) != 0) { return 0; } if (qkm != &qkms->keying_materials.front() || - !std::equal(std::begin(decrypted_dcid), - std::begin(decrypted_dcid) + SHRPX_QUIC_CID_PREFIXLEN, - worker_->get_cid_prefix())) { + decrypted_dcid.worker != worker_->get_worker_id()) { auto quic_lwp = - conn_handler->match_quic_lingering_worker_process_cid_prefix( - decrypted_dcid.data(), decrypted_dcid.size()); + conn_handler->match_quic_lingering_worker_process_worker_id( + decrypted_dcid.worker); if (quic_lwp) { if (conn_handler->forward_quic_packet_to_lingering_worker_process( quic_lwp, remote_addr, local_addr, pi, data, datalen) == 0) { @@ -177,23 +175,21 @@ int QUICConnectionHandler::handle_packet(const UpstreamAddr *faddr, switch (ngtcp2_accept(&hd, data, datalen)) { case 0: { - // If we get Initial and it has the CID prefix of this worker, - // it is likely that client is intentionally use the prefix. - // Just drop it. + // If we get Initial and it has the Worker ID of this worker, it + // is likely that client is intentionally use the prefix. Just + // drop it. if (vc.dcidlen == SHRPX_QUIC_SCIDLEN) { if (qkm != &qkms->keying_materials.front()) { qkm = &qkms->keying_materials.front(); - if (decrypt_quic_connection_id(decrypted_dcid.data(), - vc.dcid + SHRPX_QUIC_CID_PREFIX_OFFSET, - qkm->cid_encryption_ctx) != 0) { + if (decrypt_quic_connection_id( + decrypted_dcid, vc.dcid + SHRPX_QUIC_CID_WORKER_ID_OFFSET, + qkm->cid_decryption_ctx) != 0) { return 0; } } - if (std::equal(std::begin(decrypted_dcid), - std::begin(decrypted_dcid) + SHRPX_QUIC_CID_PREFIXLEN, - worker_->get_cid_prefix())) { + if (decrypted_dcid.worker == worker_->get_worker_id()) { return 0; } } @@ -324,22 +320,19 @@ int QUICConnectionHandler::handle_packet(const UpstreamAddr *faddr, break; } default: - if (!config->single_thread && !(data[0] & 0x80) && - vc.dcidlen == SHRPX_QUIC_SCIDLEN && - !std::equal(std::begin(decrypted_dcid), - std::begin(decrypted_dcid) + SHRPX_QUIC_CID_PREFIXLEN, - worker_->get_cid_prefix())) { - if (conn_handler->forward_quic_packet(faddr, remote_addr, local_addr, - pi, decrypted_dcid.data(), data, + if (!(data[0] & 0x80) && vc.dcidlen == SHRPX_QUIC_SCIDLEN && + decrypted_dcid.worker != worker_->get_worker_id()) { + if (!config->single_thread && + conn_handler->forward_quic_packet(faddr, remote_addr, local_addr, + pi, decrypted_dcid.worker, data, datalen) == 0) { return 0; } - } - if (!(data[0] & 0x80)) { - // TODO Must be rate limited - send_stateless_reset(faddr, vc.dcid, vc.dcidlen, remote_addr, - local_addr); + if (datalen >= SHRPX_QUIC_SCIDLEN + 22) { + send_stateless_reset(faddr, datalen, vc.dcid, vc.dcidlen, remote_addr, + local_addr); + } } return 0; @@ -478,8 +471,7 @@ int QUICConnectionHandler::send_retry( ngtcp2_cid retry_scid; - if (generate_quic_retry_connection_id(retry_scid, SHRPX_QUIC_SCIDLEN, - quicconf.server_id.data(), qkm.id, + if (generate_quic_retry_connection_id(retry_scid, quicconf.server_id, qkm.id, qkm.cid_encryption_ctx) != 0) { return -1; } @@ -563,11 +555,9 @@ int QUICConnectionHandler::send_version_negotiation( buf.data(), nwrite, 0); } -int QUICConnectionHandler::send_stateless_reset(const UpstreamAddr *faddr, - const uint8_t *dcid, - size_t dcidlen, - const Address &remote_addr, - const Address &local_addr) { +int QUICConnectionHandler::send_stateless_reset( + const UpstreamAddr *faddr, size_t pktlen, const uint8_t *dcid, + size_t dcidlen, const Address &remote_addr, const Address &local_addr) { if (stateless_reset_bucket_ == 0) { if (LOG_ENABLED(INFO)) { LOG(INFO) << "Stateless Reset bucket has been depleted"; @@ -598,17 +588,30 @@ int QUICConnectionHandler::send_stateless_reset(const UpstreamAddr *faddr, return -1; } - std::array<uint8_t, NGTCP2_MIN_STATELESS_RESET_RANDLEN> rand_bytes; + // SCID + minimum expansion - NGTCP2_STATELESS_RESET_TOKENLEN + constexpr size_t max_rand_byteslen = + SHRPX_QUIC_SCIDLEN + 22 - NGTCP2_STATELESS_RESET_TOKENLEN; + + size_t rand_byteslen; + + if (pktlen <= 43) { + // As per + // https://datatracker.ietf.org/doc/html/rfc9000#section-10.3 + rand_byteslen = pktlen - NGTCP2_STATELESS_RESET_TOKENLEN - 1; + } else { + rand_byteslen = max_rand_byteslen; + } + + std::array<uint8_t, max_rand_byteslen> rand_bytes; - if (RAND_bytes(rand_bytes.data(), rand_bytes.size()) != 1) { + if (RAND_bytes(rand_bytes.data(), rand_byteslen) != 1) { return -1; } std::array<uint8_t, NGTCP2_MAX_UDP_PAYLOAD_SIZE> buf; - auto nwrite = - ngtcp2_pkt_write_stateless_reset(buf.data(), buf.size(), token.data(), - rand_bytes.data(), rand_bytes.size()); + auto nwrite = ngtcp2_pkt_write_stateless_reset( + buf.data(), buf.size(), token.data(), rand_bytes.data(), rand_byteslen); if (nwrite < 0) { LOG(ERROR) << "ngtcp2_pkt_write_stateless_reset: " << ngtcp2_strerror(nwrite); diff --git a/src/shrpx_quic_connection_handler.h b/src/shrpx_quic_connection_handler.h index 29e73a4..7f65370 100644 --- a/src/shrpx_quic_connection_handler.h +++ b/src/shrpx_quic_connection_handler.h @@ -103,8 +103,9 @@ public: const uint8_t *ini_scid, size_t ini_scidlen, const Address &remote_addr, const Address &local_addr); - int send_stateless_reset(const UpstreamAddr *faddr, const uint8_t *dcid, - size_t dcidlen, const Address &remote_addr, + int send_stateless_reset(const UpstreamAddr *faddr, size_t pktlen, + const uint8_t *dcid, size_t dcidlen, + const Address &remote_addr, const Address &local_addr); // Send Initial CONNECTION_CLOSE. |ini_dcid| is the destination // Connection ID which appeared in Client Initial packet. diff --git a/src/shrpx_quic_listener.cc b/src/shrpx_quic_listener.cc index 9b9f120..681f605 100644 --- a/src/shrpx_quic_listener.cc +++ b/src/shrpx_quic_listener.cc @@ -74,6 +74,19 @@ void QUICListener::on_read() { return; } + // Packets less than 22 bytes never be a valid QUIC packet. + if (nread < 22) { + ++pktcnt; + + continue; + } + + if (util::quic_prohibited_port(util::get_port(&su))) { + ++pktcnt; + + continue; + } + Address local_addr{}; if (util::msghdr_get_local_addr(local_addr, &msg, su.storage.ss_family) != 0) { @@ -108,7 +121,8 @@ void QUICListener::on_read() { << " bytes"; } - if (datalen == 0) { + // Packets less than 22 bytes never be a valid QUIC packet. + if (datalen < 22) { break; } diff --git a/src/shrpx_worker.cc b/src/shrpx_worker.cc index e7d6740..9f5911f 100644 --- a/src/shrpx_worker.cc +++ b/src/shrpx_worker.cc @@ -148,7 +148,7 @@ Worker::Worker(struct ev_loop *loop, SSL_CTX *sv_ssl_ctx, SSL_CTX *cl_ssl_ctx, tls::CertLookupTree *cert_tree, #ifdef ENABLE_HTTP3 SSL_CTX *quic_sv_ssl_ctx, tls::CertLookupTree *quic_cert_tree, - const uint8_t *cid_prefix, size_t cid_prefixlen, + WorkerID wid, # ifdef HAVE_LIBBPF size_t index, # endif // HAVE_LIBBPF @@ -164,6 +164,7 @@ Worker::Worker(struct ev_loop *loop, SSL_CTX *sv_ssl_ctx, SSL_CTX *cl_ssl_ctx, worker_stat_{}, dns_tracker_(loop, get_config()->conn.downstream->family), #ifdef ENABLE_HTTP3 + worker_id_{std::move(wid)}, quic_upstream_addrs_{get_config()->conn.quic_listener.addrs}, #endif // ENABLE_HTTP3 loop_(loop), @@ -180,10 +181,6 @@ Worker::Worker(struct ev_loop *loop, SSL_CTX *sv_ssl_ctx, SSL_CTX *cl_ssl_ctx, connect_blocker_( std::make_unique<ConnectBlocker>(randgen_, loop_, nullptr, nullptr)), graceful_shutdown_(false) { -#ifdef ENABLE_HTTP3 - std::copy_n(cid_prefix, cid_prefixlen, std::begin(cid_prefix_)); -#endif // ENABLE_HTTP3 - ev_async_init(&w_, eventcb); w_.data = this; ev_async_start(loop_, &w_); @@ -1071,10 +1068,10 @@ int Worker::create_quic_server_socket(UpstreamAddr &faddr) { return -1; } - ref.cid_prefix_map = bpf_object__find_map_by_name(obj, "cid_prefix_map"); - if (!ref.cid_prefix_map) { + ref.worker_id_map = bpf_object__find_map_by_name(obj, "worker_id_map"); + if (!ref.worker_id_map) { auto error = errno; - LOG(FATAL) << "Failed to get cid_prefix_map: " + LOG(FATAL) << "Failed to get worker_id_map: " << xsi_strerror(error, errbuf.data(), errbuf.size()); close(fd); return -1; @@ -1155,12 +1152,12 @@ int Worker::create_quic_server_socket(UpstreamAddr &faddr) { return -1; } - rv = bpf_map__update_elem(ref.cid_prefix_map, cid_prefix_.data(), - cid_prefix_.size(), &sk_index, sizeof(sk_index), + rv = bpf_map__update_elem(ref.worker_id_map, &worker_id_, + sizeof(worker_id_), &sk_index, sizeof(sk_index), BPF_NOEXIST); if (rv != 0) { auto error = errno; - LOG(FATAL) << "Failed to update cid_prefix_map: " + LOG(FATAL) << "Failed to update worker_id_map: " << xsi_strerror(error, errbuf.data(), errbuf.size()); close(fd); return -1; @@ -1187,7 +1184,7 @@ int Worker::create_quic_server_socket(UpstreamAddr &faddr) { return 0; } -const uint8_t *Worker::get_cid_prefix() const { return cid_prefix_.data(); } +const WorkerID &Worker::get_worker_id() const { return worker_id_; } const UpstreamAddr *Worker::find_quic_upstream_addr(const Address &local_addr) { std::array<char, NI_MAXHOST> host; @@ -1444,16 +1441,4 @@ void downstream_failure(DownstreamAddr *addr, const Address *raddr) { } } -#ifdef ENABLE_HTTP3 -int create_cid_prefix(uint8_t *cid_prefix, const uint8_t *server_id) { - auto p = std::copy_n(server_id, SHRPX_QUIC_SERVER_IDLEN, cid_prefix); - - if (RAND_bytes(p, SHRPX_QUIC_CID_PREFIXLEN - SHRPX_QUIC_SERVER_IDLEN) != 1) { - return -1; - } - - return 0; -} -#endif // ENABLE_HTTP3 - } // namespace shrpx diff --git a/src/shrpx_worker.h b/src/shrpx_worker.h index 3cc7b57..f8a2d84 100644 --- a/src/shrpx_worker.h +++ b/src/shrpx_worker.h @@ -312,7 +312,7 @@ public: tls::CertLookupTree *cert_tree, #ifdef ENABLE_HTTP3 SSL_CTX *quic_sv_ssl_ctx, tls::CertLookupTree *quic_cert_tree, - const uint8_t *cid_prefix, size_t cid_prefixlen, + WorkerID wid, # ifdef HAVE_LIBBPF size_t index, # endif // HAVE_LIBBPF @@ -377,7 +377,7 @@ public: int setup_quic_server_socket(); - const uint8_t *get_cid_prefix() const; + const WorkerID &get_worker_id() const; # ifdef HAVE_LIBBPF bool should_attach_bpf() const; @@ -414,7 +414,7 @@ private: DNSTracker dns_tracker_; #ifdef ENABLE_HTTP3 - std::array<uint8_t, SHRPX_QUIC_CID_PREFIXLEN> cid_prefix_; + WorkerID worker_id_; std::vector<UpstreamAddr> quic_upstream_addrs_; std::vector<std::unique_ptr<QUICListener>> quic_listeners_; #endif // ENABLE_HTTP3 @@ -468,13 +468,6 @@ size_t match_downstream_addr_group( // nullptr. This function may schedule live check. void downstream_failure(DownstreamAddr *addr, const Address *raddr); -#ifdef ENABLE_HTTP3 -// Creates unpredictable SHRPX_QUIC_CID_PREFIXLEN bytes sequence which -// is used as a prefix of QUIC Connection ID. This function returns -// -1 on failure. |server_id| must be 2 bytes long. -int create_cid_prefix(uint8_t *cid_prefix, const uint8_t *server_id); -#endif // ENABLE_HTTP3 - } // namespace shrpx #endif // SHRPX_WORKER_H diff --git a/src/shrpx_worker_process.cc b/src/shrpx_worker_process.cc index e3f7dae..6591e9b 100644 --- a/src/shrpx_worker_process.cc +++ b/src/shrpx_worker_process.cc @@ -593,11 +593,21 @@ int worker_process_event_loop(WorkerProcessConfig *wpconf) { } EVP_CIPHER_CTX_set_padding(qkm.cid_encryption_ctx, 0); + + qkm.cid_decryption_ctx = EVP_CIPHER_CTX_new(); + if (!EVP_DecryptInit_ex(qkm.cid_decryption_ctx, EVP_aes_128_ecb(), nullptr, + qkm.cid_encryption_key.data(), nullptr)) { + LOG(ERROR) + << "Failed to initialize QUIC Connection ID decryption context"; + return -1; + } + + EVP_CIPHER_CTX_set_padding(qkm.cid_decryption_ctx, 0); } conn_handler->set_quic_keying_materials(std::move(qkms)); - conn_handler->set_cid_prefixes(wpconf->cid_prefixes); + conn_handler->set_worker_ids(wpconf->worker_ids); conn_handler->set_quic_lingering_worker_processes( wpconf->quic_lingering_worker_processes); #endif // ENABLE_HTTP3 diff --git a/src/shrpx_worker_process.h b/src/shrpx_worker_process.h index f432503..155b565 100644 --- a/src/shrpx_worker_process.h +++ b/src/shrpx_worker_process.h @@ -49,8 +49,8 @@ struct WorkerProcessConfig { // IPv6 socket, or -1 if not used int server_fd6; #ifdef ENABLE_HTTP3 - // CID prefixes for the new worker process. - std::vector<std::array<uint8_t, SHRPX_QUIC_CID_PREFIXLEN>> cid_prefixes; + // Worker IDs for the new worker process. + std::vector<WorkerID> worker_ids; // IPC socket to read forwarded QUIC UDP datagram from the current // worker process. int quic_ipc_fd; @@ -25,9 +25,11 @@ #include "tls.h" #include <cassert> +#include <cstring> #include <vector> #include <mutex> #include <iostream> +#include <fstream> #include <openssl/crypto.h> #include <openssl/conf.h> @@ -176,6 +178,32 @@ int cert_decompress(SSL *ssl, CRYPTO_BUFFER **out, size_t uncompressed_len, } #endif // NGHTTP2_OPENSSL_IS_BORINGSSL && HAVE_LIBBROTLI +namespace { +std::ofstream keylog_file; + +void keylog_callback(const SSL *ssl, const char *line) { + keylog_file.write(line, strlen(line)); + keylog_file.put('\n'); + keylog_file.flush(); +} +} // namespace + +int setup_keylog_callback(SSL_CTX *ssl_ctx) { + auto keylog_filename = getenv("SSLKEYLOGFILE"); + if (!keylog_filename) { + return 0; + } + + keylog_file.open(keylog_filename, std::ios_base::app); + if (!keylog_file) { + return -1; + } + + SSL_CTX_set_keylog_callback(ssl_ctx, keylog_callback); + + return 0; +} + } // namespace tls } // namespace nghttp2 @@ -106,6 +106,9 @@ int cert_decompress(SSL *ssl, CRYPTO_BUFFER **out, size_t uncompressed_len, const uint8_t *in, size_t in_len); #endif // NGHTTP2_OPENSSL_IS_BORINGSSL && HAVE_LIBBROTLI +// Setup keylog callback. It returns 0 if it succeeds, or -1. +int setup_keylog_callback(SSL_CTX *ssl_ctx); + } // namespace tls } // namespace nghttp2 diff --git a/src/util.cc b/src/util.cc index 0996c0a..47151b2 100644 --- a/src/util.cc +++ b/src/util.cc @@ -802,6 +802,30 @@ void set_port(Address &addr, uint16_t port) { } } +uint16_t get_port(const sockaddr_union *su) { + switch (su->storage.ss_family) { + case AF_INET: + return ntohs(su->in.sin_port); + case AF_INET6: + return ntohs(su->in6.sin6_port); + default: + return 0; + } +} + +bool quic_prohibited_port(uint16_t port) { + switch (port) { + case 1900: + case 5353: + case 11211: + case 20800: + case 27015: + return true; + default: + return port < 1024; + } +} + std::string ascii_dump(const uint8_t *data, size_t len) { std::string res; @@ -1348,66 +1372,166 @@ StringRef make_hostport(BlockAllocator &balloc, const StringRef &host, } namespace { -void hexdump8(FILE *out, const uint8_t *first, const uint8_t *last) { - auto stop = std::min(first + 8, last); - for (auto k = first; k != stop; ++k) { - fprintf(out, "%02x ", *k); +uint8_t *hexdump_addr(uint8_t *dest, size_t addr) { + // Lower 32 bits are displayed. + for (size_t i = 0; i < 4; ++i) { + auto a = (addr >> (3 - i) * 8) & 0xff; + + *dest++ = LOWER_XDIGITS[a >> 4]; + *dest++ = LOWER_XDIGITS[a & 0xf]; + } + + return dest; +} +} // namespace + +namespace { +uint8_t *hexdump_ascii(uint8_t *dest, const uint8_t *data, size_t datalen) { + *dest++ = '|'; + + for (size_t i = 0; i < datalen; ++i) { + if (0x20 <= data[i] && data[i] <= 0x7e) { + *dest++ = data[i]; + } else { + *dest++ = '.'; + } + } + + *dest++ = '|'; + + return dest; +} +} // namespace + +namespace { +uint8_t *hexdump8(uint8_t *dest, const uint8_t *data, size_t datalen) { + size_t i; + + for (i = 0; i < datalen; ++i) { + *dest++ = LOWER_XDIGITS[data[i] >> 4]; + *dest++ = LOWER_XDIGITS[data[i] & 0xf]; + *dest++ = ' '; } - // each byte needs 3 spaces (2 hex value and space) - for (; stop != first + 8; ++stop) { - fputs(" ", out); + + for (; i < 8; ++i) { + *dest++ = ' '; + *dest++ = ' '; + *dest++ = ' '; } - // we have extra space after 8 bytes - fputc(' ', out); + + return dest; } } // namespace -void hexdump(FILE *out, const uint8_t *src, size_t len) { - if (len == 0) { - return; +namespace { +uint8_t *hexdump16(uint8_t *dest, const uint8_t *data, size_t datalen) { + if (datalen > 8) { + dest = hexdump8(dest, data, 8); + *dest++ = ' '; + dest = hexdump8(dest, data + 8, datalen - 8); + *dest++ = ' '; + } else { + dest = hexdump8(dest, data, datalen); + *dest++ = ' '; + dest = hexdump8(dest, nullptr, 0); + *dest++ = ' '; } - size_t buflen = 0; + + return dest; +} +} // namespace + +namespace { +uint8_t *hexdump_line(uint8_t *dest, const uint8_t *data, size_t datalen, + size_t addr) { + dest = hexdump_addr(dest, addr); + *dest++ = ' '; + *dest++ = ' '; + + dest = hexdump16(dest, data, datalen); + + return hexdump_ascii(dest, data, datalen); +} +} // namespace + +namespace { +int hexdump_write(int fd, const uint8_t *data, size_t datalen) { + ssize_t nwrite; + + for (; (nwrite = write(fd, data, datalen)) == -1 && errno == EINTR;) + ; + if (nwrite == -1) { + return -1; + } + + return 0; +} +} // namespace + +int hexdump(FILE *out, const void *data, size_t datalen) { + if (datalen == 0) { + return 0; + } + + // min_space is the additional minimum space that the buffer must + // accept, which is the size of a single full line output + one + // repeat line marker ("*\n"). If the remaining buffer size is less + // than that, flush the buffer and reset. + constexpr size_t min_space = 79 + 2; + + auto fd = fileno(out); + std::array<uint8_t, 4096> buf; + auto last = buf.data(); + auto in = reinterpret_cast<const uint8_t *>(data); auto repeated = false; - std::array<uint8_t, 16> buf{}; - auto end = src + len; - auto i = src; - for (;;) { - auto nextlen = - std::min(static_cast<size_t>(16), static_cast<size_t>(end - i)); - if (nextlen == buflen && - std::equal(std::begin(buf), std::begin(buf) + buflen, i)) { - // as long as adjacent 16 bytes block are the same, we just - // print single '*'. - if (!repeated) { - repeated = true; - fputs("*\n", out); + + for (size_t offset = 0; offset < datalen; offset += 16) { + auto n = datalen - offset; + auto s = in + offset; + + if (n >= 16) { + n = 16; + + if (offset > 0) { + if (std::equal(s - 16, s, s)) { + if (repeated) { + continue; + } + + repeated = true; + + *last++ = '*'; + *last++ = '\n'; + + continue; + } + + repeated = false; } - i += nextlen; - continue; } - repeated = false; - fprintf(out, "%08lx", static_cast<unsigned long>(i - src)); - if (i == end) { - fputc('\n', out); - break; - } - fputs(" ", out); - hexdump8(out, i, end); - hexdump8(out, i + 8, std::max(i + 8, end)); - fputc('|', out); - auto stop = std::min(i + 16, end); - buflen = stop - i; - auto p = buf.data(); - for (; i != stop; ++i) { - *p++ = *i; - if (0x20 <= *i && *i <= 0x7e) { - fputc(*i, out); - } else { - fputc('.', out); + + last = hexdump_line(last, s, n, offset); + *last++ = '\n'; + + auto len = static_cast<size_t>(last - buf.data()); + if (len + min_space > buf.size()) { + if (hexdump_write(fd, buf.data(), len) != 0) { + return -1; } + + last = buf.data(); } - fputs("|\n", out); } + + last = hexdump_addr(last, datalen); + *last++ = '\n'; + + auto len = static_cast<size_t>(last - buf.data()); + if (len) { + return hexdump_write(fd, buf.data(), len); + } + + return 0; } void put_uint16be(uint8_t *buf, uint16_t n) { @@ -567,6 +567,12 @@ std::string to_numeric_addr(const struct sockaddr *sa, socklen_t salen); // Sets |port| to |addr|. void set_port(Address &addr, uint16_t port); +// Get port from |su|. +uint16_t get_port(const sockaddr_union *su); + +// Returns true if |port| is prohibited as a QUIC client port. +bool quic_prohibited_port(uint16_t port); + // Returns ASCII dump of |data| of length |len|. Only ASCII printable // characters are preserved. Other characters are replaced with ".". std::string ascii_dump(const uint8_t *data, size_t len); @@ -840,8 +846,10 @@ StringRef make_http_hostport(OutputIt first, const StringRef &host, return StringRef{first, p}; } -// Dumps |src| of length |len| in the format similar to `hexdump -C`. -void hexdump(FILE *out, const uint8_t *src, size_t len); +// hexdump dumps |data| of length |datalen| in the format similar to +// hexdump(1) with -C option. This function returns 0 if it succeeds, +// or -1. +int hexdump(FILE *out, const void *data, size_t datalen); // Copies 2 byte unsigned integer |n| in host byte order to |buf| in // network byte order. |