diff options
Diffstat (limited to 'src/shrpx_worker_process.cc')
| -rw-r--r-- | src/shrpx_worker_process.cc | 701 |
1 files changed, 701 insertions, 0 deletions
diff --git a/src/shrpx_worker_process.cc b/src/shrpx_worker_process.cc new file mode 100644 index 0000000..05eac02 --- /dev/null +++ b/src/shrpx_worker_process.cc @@ -0,0 +1,701 @@ +/* + * nghttp2 - HTTP/2 C Library + * + * Copyright (c) 2015 Tatsuhiro Tsujikawa + * + * Permission is hereby granted, free of charge, to any person obtaining + * a copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE + * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION + * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +#include "shrpx_worker_process.h" + +#include <sys/types.h> +#ifdef HAVE_UNISTD_H +# include <unistd.h> +#endif // HAVE_UNISTD_H +#include <sys/resource.h> +#include <sys/wait.h> +#include <grp.h> + +#include <cinttypes> +#include <cstdlib> + +#include <openssl/rand.h> + +#include <ev.h> + +#include <ares.h> + +#include "shrpx_config.h" +#include "shrpx_connection_handler.h" +#include "shrpx_log_config.h" +#include "shrpx_worker.h" +#include "shrpx_accept_handler.h" +#include "shrpx_http2_upstream.h" +#include "shrpx_http2_session.h" +#include "shrpx_memcached_dispatcher.h" +#include "shrpx_memcached_request.h" +#include "shrpx_process.h" +#include "shrpx_tls.h" +#include "shrpx_log.h" +#include "util.h" +#include "app_helper.h" +#include "template.h" +#include "xsi_strerror.h" + +using namespace nghttp2; + +namespace shrpx { + +namespace { +void drop_privileges( +#ifdef HAVE_NEVERBLEED + neverbleed_t *nb +#endif // HAVE_NEVERBLEED +) { + std::array<char, STRERROR_BUFSIZE> errbuf; + auto config = get_config(); + + if (getuid() == 0 && config->uid != 0) { +#ifdef HAVE_NEVERBLEED + if (nb) { + neverbleed_setuidgid(nb, config->user.c_str(), 1); + } +#endif // HAVE_NEVERBLEED + + if (initgroups(config->user.c_str(), config->gid) != 0) { + auto error = errno; + LOG(FATAL) << "Could not change supplementary groups: " + << xsi_strerror(error, errbuf.data(), errbuf.size()); + exit(EXIT_FAILURE); + } + if (setgid(config->gid) != 0) { + auto error = errno; + LOG(FATAL) << "Could not change gid: " + << xsi_strerror(error, errbuf.data(), errbuf.size()); + exit(EXIT_FAILURE); + } + if (setuid(config->uid) != 0) { + auto error = errno; + LOG(FATAL) << "Could not change uid: " + << xsi_strerror(error, errbuf.data(), errbuf.size()); + exit(EXIT_FAILURE); + } + if (setuid(0) != -1) { + LOG(FATAL) << "Still have root privileges?"; + exit(EXIT_FAILURE); + } + } +} +} // namespace + +namespace { +void graceful_shutdown(ConnectionHandler *conn_handler) { + if (conn_handler->get_graceful_shutdown()) { + return; + } + + LOG(NOTICE) << "Graceful shutdown signal received"; + + conn_handler->set_graceful_shutdown(true); + + // TODO What happens for the connections not established in the + // kernel? + conn_handler->accept_pending_connection(); + conn_handler->delete_acceptor(); + + conn_handler->graceful_shutdown_worker(); + + auto single_worker = conn_handler->get_single_worker(); + if (single_worker) { + auto worker_stat = single_worker->get_worker_stat(); + if (worker_stat->num_connections == 0 && + worker_stat->num_close_waits == 0) { + ev_break(conn_handler->get_loop()); + } + + return; + } +} +} // namespace + +namespace { +void reopen_log(ConnectionHandler *conn_handler) { + LOG(NOTICE) << "Reopening log files: worker process (thread main)"; + + auto config = get_config(); + auto &loggingconf = config->logging; + + (void)reopen_log_files(loggingconf); + redirect_stderr_to_errorlog(loggingconf); + + conn_handler->worker_reopen_log_files(); +} +} // namespace + +namespace { +void ipc_readcb(struct ev_loop *loop, ev_io *w, int revents) { + auto conn_handler = static_cast<ConnectionHandler *>(w->data); + std::array<uint8_t, 1024> buf; + ssize_t nread; + while ((nread = read(w->fd, buf.data(), buf.size())) == -1 && errno == EINTR) + ; + if (nread == -1) { + auto error = errno; + LOG(ERROR) << "Failed to read data from ipc channel: errno=" << error; + return; + } + + if (nread == 0) { + // IPC socket closed. Perform immediate shutdown. + LOG(FATAL) << "IPC socket is closed. Perform immediate shutdown."; + nghttp2_Exit(EXIT_FAILURE); + } + + for (ssize_t i = 0; i < nread; ++i) { + switch (buf[i]) { + case SHRPX_IPC_GRACEFUL_SHUTDOWN: + graceful_shutdown(conn_handler); + break; + case SHRPX_IPC_REOPEN_LOG: + reopen_log(conn_handler); + break; + } + } +} +} // namespace + +#ifdef ENABLE_HTTP3 +namespace { +void quic_ipc_readcb(struct ev_loop *loop, ev_io *w, int revents) { + auto conn_handler = static_cast<ConnectionHandler *>(w->data); + + if (conn_handler->quic_ipc_read() != 0) { + LOG(ERROR) << "Failed to read data from QUIC IPC channel"; + + return; + } +} +} // namespace +#endif // ENABLE_HTTP3 + +namespace { +int generate_ticket_key(TicketKey &ticket_key) { + ticket_key.cipher = get_config()->tls.ticket.cipher; + ticket_key.hmac = EVP_sha256(); + ticket_key.hmac_keylen = EVP_MD_size(ticket_key.hmac); + + assert(static_cast<size_t>(EVP_CIPHER_key_length(ticket_key.cipher)) <= + ticket_key.data.enc_key.size()); + assert(ticket_key.hmac_keylen <= ticket_key.data.hmac_key.size()); + + if (LOG_ENABLED(INFO)) { + LOG(INFO) << "enc_keylen=" << EVP_CIPHER_key_length(ticket_key.cipher) + << ", hmac_keylen=" << ticket_key.hmac_keylen; + } + + if (RAND_bytes(reinterpret_cast<unsigned char *>(&ticket_key.data), + sizeof(ticket_key.data)) == 0) { + return -1; + } + + return 0; +} +} // namespace + +namespace { +void renew_ticket_key_cb(struct ev_loop *loop, ev_timer *w, int revents) { + auto conn_handler = static_cast<ConnectionHandler *>(w->data); + const auto &old_ticket_keys = conn_handler->get_ticket_keys(); + + auto ticket_keys = std::make_shared<TicketKeys>(); + LOG(NOTICE) << "Renew new ticket keys"; + + // If old_ticket_keys is not empty, it should contain at least 2 + // keys: one for encryption, and last one for the next encryption + // key but decryption only. The keys in between are old keys and + // decryption only. The next key is provided to ensure to mitigate + // possible problem when one worker encrypt new key, but one worker, + // which did not take the that key yet, and cannot decrypt it. + // + // We keep keys for get_config()->tls_session_timeout seconds. The + // default is 12 hours. Thus the maximum ticket vector size is 12. + if (old_ticket_keys) { + auto &old_keys = old_ticket_keys->keys; + auto &new_keys = ticket_keys->keys; + + assert(!old_keys.empty()); + + auto max_tickets = + static_cast<size_t>(std::chrono::duration_cast<std::chrono::hours>( + get_config()->tls.session_timeout) + .count()); + + new_keys.resize(std::min(max_tickets, old_keys.size() + 1)); + std::copy_n(std::begin(old_keys), new_keys.size() - 1, + std::begin(new_keys) + 1); + } else { + ticket_keys->keys.resize(1); + } + + auto &new_key = ticket_keys->keys[0]; + + if (generate_ticket_key(new_key) != 0) { + if (LOG_ENABLED(INFO)) { + LOG(INFO) << "failed to generate ticket key"; + } + conn_handler->set_ticket_keys(nullptr); + conn_handler->set_ticket_keys_to_worker(nullptr); + return; + } + + if (LOG_ENABLED(INFO)) { + LOG(INFO) << "ticket keys generation done"; + assert(ticket_keys->keys.size() >= 1); + LOG(INFO) << 0 << " enc+dec: " + << util::format_hex(ticket_keys->keys[0].data.name); + for (size_t i = 1; i < ticket_keys->keys.size(); ++i) { + auto &key = ticket_keys->keys[i]; + LOG(INFO) << i << " dec: " << util::format_hex(key.data.name); + } + } + + conn_handler->set_ticket_keys(ticket_keys); + conn_handler->set_ticket_keys_to_worker(ticket_keys); +} +} // namespace + +namespace { +void memcached_get_ticket_key_cb(struct ev_loop *loop, ev_timer *w, + int revents) { + auto conn_handler = static_cast<ConnectionHandler *>(w->data); + auto dispatcher = conn_handler->get_tls_ticket_key_memcached_dispatcher(); + + auto req = std::make_unique<MemcachedRequest>(); + req->key = "nghttpx:tls-ticket-key"; + req->op = MemcachedOp::GET; + req->cb = [conn_handler, w](MemcachedRequest *req, MemcachedResult res) { + switch (res.status_code) { + case MemcachedStatusCode::NO_ERROR: + break; + case MemcachedStatusCode::EXT_NETWORK_ERROR: + conn_handler->on_tls_ticket_key_network_error(w); + return; + default: + conn_handler->on_tls_ticket_key_not_found(w); + return; + } + + // |version (4bytes)|len (2bytes)|key (variable length)|... + // (len, key) pairs are repeated as necessary. + + auto &value = res.value; + if (value.size() < 4) { + LOG(WARN) << "Memcached: tls ticket key value is too small: got " + << value.size(); + conn_handler->on_tls_ticket_key_not_found(w); + return; + } + auto p = value.data(); + auto version = util::get_uint32(p); + // Currently supported version is 1. + if (version != 1) { + LOG(WARN) << "Memcached: tls ticket key version: want 1, got " << version; + conn_handler->on_tls_ticket_key_not_found(w); + return; + } + + auto end = p + value.size(); + p += 4; + + auto &ticketconf = get_config()->tls.ticket; + + size_t expectedlen; + size_t enc_keylen; + size_t hmac_keylen; + if (ticketconf.cipher == EVP_aes_128_cbc()) { + expectedlen = 48; + enc_keylen = 16; + hmac_keylen = 16; + } else if (ticketconf.cipher == EVP_aes_256_cbc()) { + expectedlen = 80; + enc_keylen = 32; + hmac_keylen = 32; + } else { + return; + } + + auto ticket_keys = std::make_shared<TicketKeys>(); + + for (; p != end;) { + if (end - p < 2) { + LOG(WARN) << "Memcached: tls ticket key data is too small"; + conn_handler->on_tls_ticket_key_not_found(w); + return; + } + auto len = util::get_uint16(p); + p += 2; + if (len != expectedlen) { + LOG(WARN) << "Memcached: wrong tls ticket key size: want " + << expectedlen << ", got " << len; + conn_handler->on_tls_ticket_key_not_found(w); + return; + } + if (p + len > end) { + LOG(WARN) << "Memcached: too short tls ticket key payload: want " << len + << ", got " << (end - p); + conn_handler->on_tls_ticket_key_not_found(w); + return; + } + auto key = TicketKey(); + key.cipher = ticketconf.cipher; + key.hmac = EVP_sha256(); + key.hmac_keylen = hmac_keylen; + + std::copy_n(p, key.data.name.size(), std::begin(key.data.name)); + p += key.data.name.size(); + + std::copy_n(p, enc_keylen, std::begin(key.data.enc_key)); + p += enc_keylen; + + std::copy_n(p, hmac_keylen, std::begin(key.data.hmac_key)); + p += hmac_keylen; + + ticket_keys->keys.push_back(std::move(key)); + } + + conn_handler->on_tls_ticket_key_get_success(ticket_keys, w); + }; + + if (LOG_ENABLED(INFO)) { + LOG(INFO) << "Memcached: tls ticket key get request sent"; + } + + dispatcher->add_request(std::move(req)); +} + +} // namespace + +#ifdef HAVE_NEVERBLEED +namespace { +void nb_child_cb(struct ev_loop *loop, ev_child *w, int revents) { + log_chld(w->rpid, w->rstatus, "neverbleed process"); + + ev_child_stop(loop, w); + + LOG(FATAL) << "neverbleed process exited; aborting now"; + + nghttp2_Exit(EXIT_FAILURE); +} +} // namespace +#endif // HAVE_NEVERBLEED + +namespace { +int send_ready_event(int ready_ipc_fd) { + std::array<char, STRERROR_BUFSIZE> errbuf; + auto pid = getpid(); + ssize_t nwrite; + + while ((nwrite = write(ready_ipc_fd, &pid, sizeof(pid))) == -1 && + errno == EINTR) + ; + + if (nwrite < 0) { + auto error = errno; + + LOG(ERROR) << "Writing PID to ready IPC channel failed: " + << xsi_strerror(error, errbuf.data(), errbuf.size()); + + return -1; + } + + return 0; +} +} // namespace + +int worker_process_event_loop(WorkerProcessConfig *wpconf) { + int rv; + std::array<char, STRERROR_BUFSIZE> errbuf; + (void)errbuf; + + auto config = get_config(); + + if (reopen_log_files(config->logging) != 0) { + LOG(FATAL) << "Failed to open log file"; + return -1; + } + + rv = ares_library_init(ARES_LIB_INIT_ALL); + if (rv != 0) { + LOG(FATAL) << "ares_library_init failed: " << ares_strerror(rv); + return -1; + } + + auto loop = EV_DEFAULT; + + auto gen = util::make_mt19937(); + +#ifdef HAVE_NEVERBLEED + std::array<char, NEVERBLEED_ERRBUF_SIZE> nb_errbuf; + auto nb = std::make_unique<neverbleed_t>(); + if (neverbleed_init(nb.get(), nb_errbuf.data()) != 0) { + LOG(FATAL) << "neverbleed_init failed: " << nb_errbuf.data(); + return -1; + } + + LOG(NOTICE) << "neverbleed process [" << nb->daemon_pid << "] spawned"; + + ev_child nb_childev; + + ev_child_init(&nb_childev, nb_child_cb, nb->daemon_pid, 0); + nb_childev.data = nullptr; + ev_child_start(loop, &nb_childev); +#endif // HAVE_NEVERBLEED + + auto conn_handler = std::make_unique<ConnectionHandler>(loop, gen); + +#ifdef HAVE_NEVERBLEED + conn_handler->set_neverbleed(nb.get()); +#endif // HAVE_NEVERBLEED + +#ifdef ENABLE_HTTP3 + conn_handler->set_quic_ipc_fd(wpconf->quic_ipc_fd); + conn_handler->set_quic_lingering_worker_processes( + wpconf->quic_lingering_worker_processes); +#endif // ENABLE_HTTP3 + + for (auto &addr : config->conn.listener.addrs) { + conn_handler->add_acceptor( + std::make_unique<AcceptHandler>(&addr, conn_handler.get())); + } + + MemchunkPool mcpool; + + ev_timer renew_ticket_key_timer; + if (tls::upstream_tls_enabled(config->conn)) { + auto &ticketconf = config->tls.ticket; + auto &memcachedconf = ticketconf.memcached; + + if (!memcachedconf.host.empty()) { + SSL_CTX *ssl_ctx = nullptr; + + if (memcachedconf.tls) { + ssl_ctx = conn_handler->create_tls_ticket_key_memcached_ssl_ctx(); + } + + conn_handler->set_tls_ticket_key_memcached_dispatcher( + std::make_unique<MemcachedDispatcher>( + &ticketconf.memcached.addr, loop, ssl_ctx, + StringRef{memcachedconf.host}, &mcpool, gen)); + + ev_timer_init(&renew_ticket_key_timer, memcached_get_ticket_key_cb, 0., + 0.); + renew_ticket_key_timer.data = conn_handler.get(); + // Get first ticket keys. + memcached_get_ticket_key_cb(loop, &renew_ticket_key_timer, 0); + } else { + bool auto_tls_ticket_key = true; + if (!ticketconf.files.empty()) { + if (!ticketconf.cipher_given) { + LOG(WARN) + << "It is strongly recommended to specify " + "--tls-ticket-key-cipher=aes-128-cbc (or " + "tls-ticket-key-cipher=aes-128-cbc in configuration file) " + "when --tls-ticket-key-file is used for the smooth " + "transition when the default value of --tls-ticket-key-cipher " + "becomes aes-256-cbc"; + } + auto ticket_keys = read_tls_ticket_key_file( + ticketconf.files, ticketconf.cipher, EVP_sha256()); + if (!ticket_keys) { + LOG(WARN) << "Use internal session ticket key generator"; + } else { + conn_handler->set_ticket_keys(std::move(ticket_keys)); + auto_tls_ticket_key = false; + } + } + if (auto_tls_ticket_key) { + // Generate new ticket key every 1hr. + ev_timer_init(&renew_ticket_key_timer, renew_ticket_key_cb, 0., 1_h); + renew_ticket_key_timer.data = conn_handler.get(); + ev_timer_again(loop, &renew_ticket_key_timer); + + // Generate first session ticket key before running workers. + renew_ticket_key_cb(loop, &renew_ticket_key_timer, 0); + } + } + } + +#ifdef ENABLE_HTTP3 + auto &quicconf = config->quic; + + std::shared_ptr<QUICKeyingMaterials> qkms; + + if (!quicconf.upstream.secret_file.empty()) { + qkms = read_quic_secret_file(quicconf.upstream.secret_file); + if (!qkms) { + LOG(WARN) << "Use QUIC keying materials generated internally"; + } + } + + if (!qkms) { + qkms = std::make_shared<QUICKeyingMaterials>(); + qkms->keying_materials.resize(1); + + auto &qkm = qkms->keying_materials.front(); + + if (RAND_bytes(qkm.reserved.data(), qkm.reserved.size()) != 1) { + LOG(ERROR) << "Failed to generate QUIC secret reserved data"; + return -1; + } + + if (RAND_bytes(qkm.secret.data(), qkm.secret.size()) != 1) { + LOG(ERROR) << "Failed to generate QUIC secret"; + return -1; + } + + if (RAND_bytes(qkm.salt.data(), qkm.salt.size()) != 1) { + LOG(ERROR) << "Failed to generate QUIC salt"; + return -1; + } + } + + for (auto &qkm : qkms->keying_materials) { + if (generate_quic_connection_id_encryption_key( + qkm.cid_encryption_key.data(), qkm.cid_encryption_key.size(), + qkm.secret.data(), qkm.secret.size(), qkm.salt.data(), + qkm.salt.size()) != 0) { + LOG(ERROR) << "Failed to generate QUIC Connection ID encryption key"; + return -1; + } + } + + conn_handler->set_quic_keying_materials(std::move(qkms)); + + conn_handler->set_cid_prefixes(wpconf->cid_prefixes); + conn_handler->set_quic_lingering_worker_processes( + wpconf->quic_lingering_worker_processes); +#endif // ENABLE_HTTP3 + + if (config->single_thread) { + rv = conn_handler->create_single_worker(); + if (rv != 0) { + return -1; + } + } else { +#ifndef NOTHREADS + sigset_t set; + sigemptyset(&set); + sigaddset(&set, SIGCHLD); + + rv = pthread_sigmask(SIG_BLOCK, &set, nullptr); + if (rv != 0) { + LOG(ERROR) << "Blocking SIGCHLD failed: " + << xsi_strerror(rv, errbuf.data(), errbuf.size()); + return -1; + } +#endif // !NOTHREADS + + rv = conn_handler->create_worker_thread(config->num_worker); + if (rv != 0) { + return -1; + } + +#ifndef NOTHREADS + rv = pthread_sigmask(SIG_UNBLOCK, &set, nullptr); + if (rv != 0) { + LOG(ERROR) << "Unblocking SIGCHLD failed: " + << xsi_strerror(rv, errbuf.data(), errbuf.size()); + return -1; + } +#endif // !NOTHREADS + } + +#if defined(ENABLE_HTTP3) && defined(HAVE_LIBBPF) + conn_handler->unload_bpf_objects(); +#endif // defined(ENABLE_HTTP3) && defined(HAVE_LIBBPF) + + drop_privileges( +#ifdef HAVE_NEVERBLEED + nb.get() +#endif // HAVE_NEVERBLEED + ); + + ev_io ipcev; + ev_io_init(&ipcev, ipc_readcb, wpconf->ipc_fd, EV_READ); + ipcev.data = conn_handler.get(); + ev_io_start(loop, &ipcev); + +#ifdef ENABLE_HTTP3 + ev_io quic_ipcev; + ev_io_init(&quic_ipcev, quic_ipc_readcb, wpconf->quic_ipc_fd, EV_READ); + quic_ipcev.data = conn_handler.get(); + ev_io_start(loop, &quic_ipcev); +#endif // ENABLE_HTTP3 + + if (tls::upstream_tls_enabled(config->conn) && !config->tls.ocsp.disabled) { + if (config->tls.ocsp.startup) { + conn_handler->set_enable_acceptor_on_ocsp_completion(true); + conn_handler->disable_acceptor(); + } + + conn_handler->proceed_next_cert_ocsp(); + } + + if (LOG_ENABLED(INFO)) { + LOG(INFO) << "Entering event loop"; + } + + if (send_ready_event(wpconf->ready_ipc_fd) != 0) { + return -1; + } + + ev_run(loop, 0); + + conn_handler->cancel_ocsp_update(); + + // Destroy SSL_CTX held in conn_handler before killing neverbleed + // daemon. Otherwise priv_rsa_finish yields "write error" and + // worker process aborts. + conn_handler.reset(); + +#ifdef HAVE_NEVERBLEED + assert(nb->daemon_pid > 0); + + rv = kill(nb->daemon_pid, SIGTERM); + if (rv != 0) { + auto error = errno; + LOG(ERROR) << "Could not send signal to neverbleed daemon: errno=" << error; + } + + while ((rv = waitpid(nb->daemon_pid, nullptr, 0)) == -1 && errno == EINTR) + ; + if (rv == -1) { + auto error = errno; + LOG(ERROR) << "Error occurred while we were waiting for the completion " + "of neverbleed process: errno=" + << error; + } +#endif // HAVE_NEVERBLEED + + ares_library_cleanup(); + + return 0; +} + +} // namespace shrpx |