diff options
Diffstat (limited to '')
| -rw-r--r-- | doh.cc | 1707 |
1 files changed, 1707 insertions, 0 deletions
@@ -0,0 +1,1707 @@ +#include "config.h" +#include "doh.hh" + +#ifdef HAVE_DNS_OVER_HTTPS +#define H2O_USE_EPOLL 1 + +#include <errno.h> +#include <iostream> +#include <thread> + +#include <boost/algorithm/string.hpp> +#include <h2o.h> +//#include <h2o/http1.h> +#include <h2o/http2.h> + +#include <openssl/err.h> +#include <openssl/ssl.h> + +#include "base64.hh" +#include "dnsname.hh" +#undef CERT +#include "dnsdist.hh" +#include "dnsdist-tcp.hh" +#include "misc.hh" +#include "dns.hh" +#include "dolog.hh" +#include "dnsdist-ecs.hh" +#include "dnsdist-proxy-protocol.hh" +#include "dnsdist-rules.hh" +#include "dnsdist-xpf.hh" +#include "libssl.hh" +#include "threadname.hh" + +/* So, how does this work. We use h2o for our http2 and TLS needs. + If the operator has configured multiple IP addresses to listen on, + we launch multiple h2o listener threads. We can hook in to multiple + URLs though on the same IP. There is no SNI yet (I think). + + h2o is event driven, so we get callbacks if a new DNS query arrived. + When it does, we do some minimal parsing on it, and send it on to the + dnsdist worker thread which we also launched. + + This dnsdist worker thread injects the query into the normal dnsdist flow + (over a pipe). The response also goes back over a (different) pipe, + where we pick it up and deliver it back to h2o. + + For coordination, we use the h2o socket multiplexer, which is sensitive to our + pipe too. +*/ + +/* h2o notes. + Paths and parameters etc just *happen* to be null-terminated in HTTP2. + They are not in HTTP1. So you MUST use the length field! +*/ + +/* 'Intermediate' compatibility from https://wiki.mozilla.org/Security/Server_Side_TLS#Intermediate_compatibility_.28default.29 */ +#define DOH_DEFAULT_CIPHERS "ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA:ECDHE-RSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-RSA-AES256-SHA256:DHE-RSA-AES256-SHA:ECDHE-ECDSA-DES-CBC3-SHA:ECDHE-RSA-DES-CBC3-SHA:EDH-RSA-DES-CBC3-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:DES-CBC3-SHA:!DSS" + +class DOHAcceptContext +{ +public: + DOHAcceptContext() + { + memset(&d_h2o_accept_ctx, 0, sizeof(d_h2o_accept_ctx)); + d_rotatingTicketsKey.clear(); + } + DOHAcceptContext(const DOHAcceptContext&) = delete; + DOHAcceptContext& operator=(const DOHAcceptContext&) = delete; + + h2o_accept_ctx_t* get() + { + return &d_h2o_accept_ctx; + } + + ~DOHAcceptContext() + { + SSL_CTX_free(d_h2o_accept_ctx.ssl_ctx); + d_h2o_accept_ctx.ssl_ctx = nullptr; + } + + void decrementConcurrentConnections() + { + if (d_cs != nullptr) { + --d_cs->tcpCurrentConnections; + } + } + + time_t getNextTicketsKeyRotation() const + { + return d_ticketsKeyNextRotation; + } + + size_t getTicketsKeysCount() const + { + size_t res = 0; + if (d_ticketKeys) { + res = d_ticketKeys->getKeysCount(); + } + return res; + } + + void rotateTicketsKey(time_t now) + { + if (!d_ticketKeys) { + return; + } + + d_ticketKeys->rotateTicketsKey(now); + + if (d_ticketsKeyRotationDelay > 0) { + d_ticketsKeyNextRotation = now + d_ticketsKeyRotationDelay; + } + } + + void loadTicketsKeys(const std::string& keyFile) + { + if (!d_ticketKeys) { + return; + } + d_ticketKeys->loadTicketsKeys(keyFile); + + if (d_ticketsKeyRotationDelay > 0) { + d_ticketsKeyNextRotation = time(nullptr) + d_ticketsKeyRotationDelay; + } + } + + void handleTicketsKeyRotation() + { + if (d_ticketsKeyRotationDelay == 0) { + return; + } + + time_t now = time(nullptr); + if (now > d_ticketsKeyNextRotation) { + if (d_rotatingTicketsKey.test_and_set()) { + /* someone is already rotating */ + return; + } + try { + rotateTicketsKey(now); + + d_rotatingTicketsKey.clear(); + } + catch(const std::runtime_error& e) { + d_rotatingTicketsKey.clear(); + throw std::runtime_error(std::string("Error generating a new tickets key for TLS context:") + e.what()); + } + catch(...) { + d_rotatingTicketsKey.clear(); + throw; + } + } + } + + std::map<int, std::string> d_ocspResponses; + std::unique_ptr<OpenSSLTLSTicketKeysRing> d_ticketKeys{nullptr}; + std::unique_ptr<FILE, int(*)(FILE*)> d_keyLogFile{nullptr, fclose}; + ClientState* d_cs{nullptr}; + time_t d_ticketsKeyRotationDelay{0}; + +private: + h2o_accept_ctx_t d_h2o_accept_ctx; + std::atomic<uint64_t> d_refcnt{1}; + time_t d_ticketsKeyNextRotation{0}; + std::atomic_flag d_rotatingTicketsKey; +}; + +// we create one of these per thread, and pass around a pointer to it +// through the bowels of h2o +struct DOHServerConfig +{ + DOHServerConfig(uint32_t idleTimeout, uint32_t internalPipeBufferSize): accept_ctx(std::make_shared<DOHAcceptContext>()) + { + int fd[2]; + if (pipe(fd) < 0) { + unixDie("Creating a pipe for DNS over HTTPS"); + } + dohquerypair[0] = fd[1]; + dohquerypair[1] = fd[0]; + + setNonBlocking(dohquerypair[0]); + if (internalPipeBufferSize > 0) { + setPipeBufferSize(dohquerypair[0], internalPipeBufferSize); + } + + if (pipe(fd) < 0) { + close(dohquerypair[0]); + close(dohquerypair[1]); + unixDie("Creating a pipe for DNS over HTTPS"); + } + + dohresponsepair[0] = fd[1]; + dohresponsepair[1] = fd[0]; + + setNonBlocking(dohresponsepair[0]); + if (internalPipeBufferSize > 0) { + setPipeBufferSize(dohresponsepair[0], internalPipeBufferSize); + } + + setNonBlocking(dohresponsepair[1]); + + h2o_config_init(&h2o_config); + h2o_config.http2.idle_timeout = idleTimeout * 1000; + /* if you came here for a way to make the number of concurrent streams (concurrent requests per connection) + configurable, or even just bigger, I have bad news for you. + h2o_config.http2.max_concurrent_requests_per_connection (default of 100) is capped by + H2O_HTTP2_SETTINGS_HOST.max_concurrent_streams which is not configurable. Even if decided to change the + hard-coded value, libh2o's author warns that there might be parts of the code where the stream ID is stored + in 8 bits, making 256 a hard value: https://github.com/h2o/h2o/issues/805 + */ + } + DOHServerConfig(const DOHServerConfig&) = delete; + DOHServerConfig& operator=(const DOHServerConfig&) = delete; + + LocalHolders holders; + std::set<std::string> paths; + h2o_globalconf_t h2o_config; + h2o_context_t h2o_ctx; + std::shared_ptr<DOHAcceptContext> accept_ctx{nullptr}; + ClientState* cs{nullptr}; + std::shared_ptr<DOHFrontend> df{nullptr}; + int dohquerypair[2]{-1,-1}; + int dohresponsepair[2]{-1,-1}; +}; + +/* This internal function sends back the object to the main thread to send a reply. + The caller should NOT release or touch the unit after calling this function */ +static void sendDoHUnitToTheMainThread(DOHUnitUniquePtr&& du, const char* description) +{ + /* taking a naked pointer since we are about to send that pointer over a pipe */ + auto ptr = du.release(); + /* increasing the reference counter. This should not be strictly needed because + we already hold a reference and will only release it if we failed to send the + pointer over the pipe, but TSAN seems confused when the responder thread gets + a reply from a backend before the send() syscall sending the corresponding query + to that backend has returned in the initial thread. + The memory barrier needed to increase that counter seems to work around that. + */ + ptr->get(); + static_assert(sizeof(ptr) <= PIPE_BUF, "Writes up to PIPE_BUF are guaranteed not to be interleaved and to either fully succeed or fail"); + + ssize_t sent = write(ptr->rsock, &ptr, sizeof(ptr)); + if (sent != sizeof(ptr)) { + if (errno == EAGAIN || errno == EWOULDBLOCK) { + ++g_stats.dohResponsePipeFull; + vinfolog("Unable to pass a %s to the DoH worker thread because the pipe is full", description); + } + else { + vinfolog("Unable to pass a %s to the DoH worker thread because we couldn't write to the pipe: %s", description, stringerror()); + } + + /* we fail to write over the pipe so we do not need to hold to that ref anymore */ + ptr->release(); + } + /* we decrement the counter incremented above at the beginning of that function */ + ptr->release(); +} + +/* This function is called from other threads than the main DoH one, + instructing it to send a 502 error to the client. + It takes ownership of the unit. */ +void handleDOHTimeout(DOHUnitUniquePtr&& oldDU) +{ + if (oldDU == nullptr) { + return; + } + +/* we are about to erase an existing DU */ + oldDU->status_code = 502; + + sendDoHUnitToTheMainThread(std::move(oldDU), "DoH timeout"); +} + +struct DOHConnection +{ + std::shared_ptr<DOHAcceptContext> d_acceptCtx{nullptr}; + struct timeval d_connectionStartTime{0, 0}; + size_t d_nbQueries{0}; + int d_desc{-1}; +}; + +static thread_local std::unordered_map<int, DOHConnection> t_conns; + +static void on_socketclose(void *data) +{ + auto conn = reinterpret_cast<DOHConnection*>(data); + if (conn != nullptr) { + if (conn->d_acceptCtx) { + struct timeval now; + gettimeofday(&now, nullptr); + + auto diff = now - conn->d_connectionStartTime; + + conn->d_acceptCtx->decrementConcurrentConnections(); + conn->d_acceptCtx->d_cs->updateTCPMetrics(conn->d_nbQueries, diff.tv_sec * 1000 + diff.tv_usec / 1000); + } + + t_conns.erase(conn->d_desc); + } +} + +static const std::string& getReasonFromStatusCode(uint16_t statusCode) +{ + /* no need to care too much about this, HTTP/2 has no 'reason' anyway */ + static const std::unordered_map<uint16_t, std::string> reasons = { + { 200, "OK" }, + { 301, "Moved Permanently" }, + { 302, "Found" }, + { 303, "See Other" }, + { 304, "Not Modified" }, + { 305, "Use Proxy" }, + { 306, "Switch Proxy" }, + { 307, "Temporary Redirect" }, + { 308, "Permanent Redirect" }, + { 400, "Bad Request" }, + { 401, "Unauthorized" }, + { 402, "Payment Required" }, + { 403, "Forbidden" }, + { 404, "Not Found" }, + { 405, "Method Not Allowed" }, + { 406, "Not Acceptable" }, + { 407, "Proxy Authentication Required" }, + { 408, "Request Timeout" }, + { 409, "Conflict" }, + { 410, "Gone" }, + { 411, "Length Required" }, + { 412, "Precondition Failed" }, + { 413, "Payload Too Large" }, + { 414, "URI Too Long" }, + { 415, "Unsupported Media Type" }, + { 416, "Range Not Satisfiable" }, + { 417, "Expectation Failed" }, + { 418, "I'm a teapot" }, + { 451, "Unavailable For Legal Reasons" }, + { 500, "Internal Server Error" }, + { 501, "Not Implemented" }, + { 502, "Bad Gateway" }, + { 503, "Service Unavailable" }, + { 504, "Gateway Timeout" }, + { 505, "HTTP Version Not Supported" } + }; + static const std::string unknown = "Unknown"; + + const auto it = reasons.find(statusCode); + if (it == reasons.end()) { + return unknown; + } + else { + return it->second; + } +} + +/* Always called from the main DoH thread */ +static void handleResponse(DOHFrontend& df, st_h2o_req_t* req, uint16_t statusCode, const PacketBuffer& response, const std::vector<std::pair<std::string, std::string>>& customResponseHeaders, const std::string& contentType, bool addContentType) +{ + constexpr int overwrite_if_exists = 1; + constexpr int maybe_token = 1; + for (auto const& headerPair : customResponseHeaders) { + h2o_set_header_by_str(&req->pool, &req->res.headers, headerPair.first.c_str(), headerPair.first.size(), maybe_token, headerPair.second.c_str(), headerPair.second.size(), overwrite_if_exists); + } + + if (statusCode == 200) { + ++df.d_validresponses; + req->res.status = 200; + + if (addContentType) { + if (contentType.empty()) { + h2o_add_header(&req->pool, &req->res.headers, H2O_TOKEN_CONTENT_TYPE, nullptr, H2O_STRLIT("application/dns-message")); + } + else { + /* we need to duplicate the header content because h2o keeps a pointer and we will be deleted before the response has been sent */ + h2o_iovec_t ct = h2o_strdup(&req->pool, contentType.c_str(), contentType.size()); + h2o_add_header(&req->pool, &req->res.headers, H2O_TOKEN_CONTENT_TYPE, nullptr, ct.base, ct.len); + } + } + + if (df.d_sendCacheControlHeaders && response.size() > sizeof(dnsheader)) { + uint32_t minTTL = getDNSPacketMinTTL(reinterpret_cast<const char*>(response.data()), response.size()); + if (minTTL != std::numeric_limits<uint32_t>::max()) { + std::string cacheControlValue = "max-age=" + std::to_string(minTTL); + /* we need to duplicate the header content because h2o keeps a pointer and we will be deleted before the response has been sent */ + h2o_iovec_t ccv = h2o_strdup(&req->pool, cacheControlValue.c_str(), cacheControlValue.size()); + h2o_add_header(&req->pool, &req->res.headers, H2O_TOKEN_CACHE_CONTROL, nullptr, ccv.base, ccv.len); + } + } + + req->res.content_length = response.size(); + h2o_send_inline(req, reinterpret_cast<const char*>(response.data()), response.size()); + } + else if (statusCode >= 300 && statusCode < 400) { + /* in that case the response is actually a URL */ + /* we need to duplicate the URL because h2o uses it for the location header, keeping a pointer, and we will be deleted before the response has been sent */ + h2o_iovec_t url = h2o_strdup(&req->pool, reinterpret_cast<const char*>(response.data()), response.size()); + h2o_send_redirect(req, statusCode, getReasonFromStatusCode(statusCode).c_str(), url.base, url.len); + ++df.d_redirectresponses; + } + else { + // we need to make sure it's null-terminated */ + if (!response.empty() && response.at(response.size() - 1) == 0) { + h2o_send_error_generic(req, statusCode, getReasonFromStatusCode(statusCode).c_str(), reinterpret_cast<const char*>(response.data()), H2O_SEND_ERROR_KEEP_HEADERS); + } + else { + switch(statusCode) { + case 400: + h2o_send_error_400(req, getReasonFromStatusCode(statusCode).c_str(), "invalid DNS query" , 0); + break; + case 403: + h2o_send_error_403(req, getReasonFromStatusCode(statusCode).c_str(), "dns query not allowed", 0); + break; + case 502: + h2o_send_error_502(req, getReasonFromStatusCode(statusCode).c_str(), "no downstream server available", 0); + break; + case 500: + /* fall-through */ + default: + h2o_send_error_500(req, getReasonFromStatusCode(statusCode).c_str(), "Internal Server Error", 0); + break; + } + } + + ++df.d_errorresponses; + } +} + +class DoHTCPCrossQuerySender : public TCPQuerySender +{ +public: + DoHTCPCrossQuerySender(DOHUnitUniquePtr&& du_): du(std::move(du_)) + { + } + + bool active() const override + { + return true; + } + + const ClientState* getClientState() const override + { + if (!du || !du->dsc || !du->dsc->cs) { + throw std::runtime_error("No query associated to this DoHTCPCrossQuerySender"); + } + + return du->dsc->cs; + } + + void handleResponse(const struct timeval& now, TCPResponse&& response) override + { + if (!du) { + return; + } + + if (du->rsock == -1) { + return; + } + + du->response = std::move(response.d_buffer); + du->ids = std::move(response.d_idstate); + + thread_local LocalStateHolder<vector<DNSDistResponseRuleAction>> localRespRuleActions = g_respruleactions.getLocal(); + DNSResponse dr = makeDNSResponseFromIDState(du->ids, du->response); + dnsheader cleartextDH; + memcpy(&cleartextDH, dr.getHeader(), sizeof(cleartextDH)); + + if (!processResponse(du->response, localRespRuleActions, dr, false, false)) { + du.reset(); + return; + } + + double udiff = du->ids.sentTime.udiff(); + vinfolog("Got answer from %s, relayed to %s (https), took %f usec", du->downstream->remote.toStringWithPort(), du->ids.origRemote.toStringWithPort(), udiff); + + auto backendProtocol = du->downstream->getProtocol(); + if (backendProtocol == dnsdist::Protocol::DoUDP && du->tcp) { + backendProtocol = dnsdist::Protocol::DoTCP; + } + handleResponseSent(du->ids, udiff, *dr.remote, du->downstream->remote, du->response.size(), cleartextDH, backendProtocol); + + ++g_stats.responses; + if (du->ids.cs) { + ++du->ids.cs->responses; + } + + sendDoHUnitToTheMainThread(std::move(du), "cross-protocol response"); + } + + void handleXFRResponse(const struct timeval& now, TCPResponse&& response) override + { + return handleResponse(now, std::move(response)); + } + + void notifyIOError(IDState&& query, const struct timeval& now) override + { + if (!du) { + return; + } + + if (du->rsock == -1) { + return; + } + + du->ids = std::move(query); + du->status_code = 502; + sendDoHUnitToTheMainThread(std::move(du), "cross-protocol error response"); + } + +private: + DOHUnitUniquePtr du; +}; + +class DoHCrossProtocolQuery : public CrossProtocolQuery +{ +public: + DoHCrossProtocolQuery(DOHUnitUniquePtr&& du_): du(std::move(du_)) + { + query = InternalQuery(std::move(du->query), std::move(du->ids)); + /* we _could_ remove it from the query buffer and put in query's d_proxyProtocolPayload, + clearing query.d_proxyProtocolPayloadAdded and du->proxyProtocolPayloadSize. + Leave it for now because we know that the onky case where the payload has been + added is when we tried over UDP, got a TC=1 answer and retried over TCP/DoT, + and we know the TCP/DoT code can handle it. */ + query.d_proxyProtocolPayloadAdded = du->proxyProtocolPayloadSize > 0; + downstream = du->downstream; + proxyProtocolPayloadSize = du->proxyProtocolPayloadSize; + } + + void handleInternalError() + { + du->status_code = 502; + sendDoHUnitToTheMainThread(std::move(du), "DoH internal error"); + } + + std::shared_ptr<TCPQuerySender> getTCPQuerySender() override + { + auto sender = std::make_shared<DoHTCPCrossQuerySender>(std::move(du)); + return sender; + } + +private: + DOHUnitUniquePtr du; +}; + +/* + We are not in the main DoH thread but in the DoH 'client' thread. +*/ +static void processDOHQuery(DOHUnitUniquePtr&& du) +{ + uint16_t queryId = 0; + ComboAddress remote; + bool duRefCountIncremented = false; + try { + if (!du->req) { + // we got closed meanwhile. XXX small race condition here + // but we should be fine as long as we don't touch du->req + // outside of the main DoH thread + du->status_code = 500; + sendDoHUnitToTheMainThread(std::move(du), "DoH killed in flight"); + return; + } + remote = du->ids.origRemote; + DOHServerConfig* dsc = du->dsc; + auto& holders = dsc->holders; + ClientState& cs = *dsc->cs; + + if (du->query.size() < sizeof(dnsheader)) { + ++g_stats.nonCompliantQueries; + du->status_code = 400; + sendDoHUnitToTheMainThread(std::move(du), "DoH non-compliant query"); + return; + } + + ++cs.queries; + ++g_stats.queries; + + /* we need an accurate ("real") value for the response and + to store into the IDS, but not for insertion into the + rings for example */ + struct timespec queryRealTime; + gettime(&queryRealTime, true); + + { + /* don't keep that pointer around, it will be invalidated if the buffer is ever resized */ + struct dnsheader* dh = reinterpret_cast<struct dnsheader*>(du->query.data()); + + if (!checkQueryHeaders(dh)) { + du->status_code = 400; + sendDoHUnitToTheMainThread(std::move(du), "DoH invalid headers"); + return; + } + + if (dh->qdcount == 0) { + dh->rcode = RCode::NotImp; + dh->qr = true; + du->response = std::move(du->query); + + sendDoHUnitToTheMainThread(std::move(du), "DoH empty query"); + return; + } + + queryId = ntohs(dh->id); + } + + uint16_t qtype, qclass; + unsigned int qnameWireLength = 0; + DNSName qname(reinterpret_cast<const char*>(du->query.data()), du->query.size(), sizeof(dnsheader), false, &qtype, &qclass, &qnameWireLength); + DNSQuestion dq(&qname, qtype, qclass, &du->ids.origDest, &du->ids.origRemote, du->query, dnsdist::Protocol::DoH, &queryRealTime); + dq.ednsAdded = du->ids.ednsAdded; + /* store the raw pointer */ + dq.du = du.get(); + dq.sni = std::move(du->sni); + + auto result = processQuery(dq, cs, holders, du->downstream); + + if (result == ProcessQueryResult::Drop) { + du->status_code = 403; + sendDoHUnitToTheMainThread(std::move(du), "DoH dropped query"); + return; + } + + if (result == ProcessQueryResult::SendAnswer) { + if (du->response.empty()) { + du->response = std::move(du->query); + } + sendDoHUnitToTheMainThread(std::move(du), "DoH self-answered response"); + return; + } + + if (result != ProcessQueryResult::PassToBackend) { + du->status_code = 500; + sendDoHUnitToTheMainThread(std::move(du), "DoH no backend available"); + return; + } + + if (du->downstream == nullptr) { + du->status_code = 502; + sendDoHUnitToTheMainThread(std::move(du), "DoH no backend available"); + return; + } + + if (du->downstream->isTCPOnly()) { + std::string proxyProtocolPayload; + /* we need to do this _before_ creating the cross protocol query because + after that the buffer will have been moved */ + if (du->downstream->useProxyProtocol) { + proxyProtocolPayload = getProxyProtocolPayload(dq); + } + + du->ids.origID = htons(queryId); + du->ids.cs = &cs; + setIDStateFromDNSQuestion(du->ids, dq, std::move(qname)); + + du->tcp = true; + std::shared_ptr<DownstreamState>& downstream = du->downstream; + + /* this moves du->ids, careful! */ + auto cpq = std::make_unique<DoHCrossProtocolQuery>(std::move(du)); + cpq->query.d_proxyProtocolPayload = std::move(proxyProtocolPayload); + + if (downstream->passCrossProtocolQuery(std::move(cpq))) { + return; + } + else { + cpq->handleInternalError(); + return; + } + } + + ComboAddress dest = du->ids.origDest; + unsigned int idOffset = (du->downstream->idOffset++) % du->downstream->idStates.size(); + IDState* ids = &du->downstream->idStates[idOffset]; + ids->age = 0; + DOHUnit* oldDU = nullptr; + if (ids->isInUse()) { + /* that means that the state was in use, possibly with an allocated + DOHUnit that we will need to handle, but we can't touch it before + confirming that we now own this state */ + oldDU = ids->du; + } + + /* we atomically replace the value, we now own this state */ + int64_t generation = ids->generation++; + if (!ids->markAsUsed(generation)) { + /* the state was not in use. + we reset 'oldDU' because it might have still been in use when we read it. */ + oldDU = nullptr; + ++du->downstream->outstanding; + } + else { + ids->du = nullptr; + /* we are reusing a state, no change in outstanding but if there was an existing DOHUnit we need + to handle it because it's about to be overwritten. */ + ++du->downstream->reuseds; + ++g_stats.downstreamTimeouts; + handleDOHTimeout(DOHUnitUniquePtr(oldDU, DOHUnit::release)); + } + + ids->origFD = 0; + /* increase the ref count since we are about to store the pointer */ + du->get(); + duRefCountIncremented = true; + /* store the raw pointer */ + ids->du = du.get(); + + ids->cs = &cs; + ids->origID = htons(queryId); + setIDStateFromDNSQuestion(*ids, dq, std::move(qname)); + + dq.getHeader()->id = idOffset; + + /* If we couldn't harvest the real dest addr, still + write down the listening addr since it will be useful + (especially if it's not an 'any' one). + We need to keep track of which one it is since we may + want to use the real but not the listening addr to reply. + */ + if (dest.sin4.sin_family != 0) { + ids->origDest = dest; + ids->destHarvested = true; + } + else { + ids->origDest = cs.local; + ids->destHarvested = false; + } + + bool failed = false; + if (du->downstream->useProxyProtocol) { + try { + size_t payloadSize = 0; + if (addProxyProtocol(dq, &payloadSize)) { + du->proxyProtocolPayloadSize = payloadSize; + } + } + catch (const std::exception& e) { + vinfolog("Adding proxy protocol payload to DoH query from %s failed: %s", ids->origDest.toStringWithPort(), e.what()); + failed = true; + } + } + + try { + if (!failed) { + int fd = pickBackendSocketForSending(du->downstream); + /* you can't touch du after this line, unless the call returned a non-negative value, + because it might already have been freed */ + ssize_t ret = udpClientSendRequestToBackend(du->downstream, fd, du->query); + + if (ret < 0) { + failed = true; + } + } + + if (failed) { + /* we are about to handle the error, make sure that + this pointer is not accessed when the state is cleaned, + but first check that it still belongs to us */ + if (ids->tryMarkUnused(generation)) { + ids->du = nullptr; + du->release(); + duRefCountIncremented = false; + --du->downstream->outstanding; + } + ++du->downstream->sendErrors; + ++g_stats.downstreamSendErrors; + du->status_code = 502; + sendDoHUnitToTheMainThread(std::move(du), "DoH internal error"); + return; + } + } + catch (const std::exception& e) { + if (duRefCountIncremented) { + du->release(); + } + throw; + } + + vinfolog("Got query for %s|%s from %s (https), relayed to %s", ids->qname.toString(), QType(ids->qtype).toString(), remote.toStringWithPort(), du->downstream->getName()); + } + catch (const std::exception& e) { + vinfolog("Got an error in DOH question thread while parsing a query from %s, id %d: %s", remote.toStringWithPort(), queryId, e.what()); + du->status_code = 500; + sendDoHUnitToTheMainThread(std::move(du), "DoH internal error"); + return; + } + + return; +} + +/* called when a HTTP response is about to be sent, from the main DoH thread */ +static void on_response_ready_cb(struct st_h2o_filter_t *self, h2o_req_t *req, h2o_ostream_t **slot) +{ + if (req == nullptr) { + return; + } + + DOHServerConfig* dsc = reinterpret_cast<DOHServerConfig*>(req->conn->ctx->storage.entries[0].data); + + DOHFrontend::HTTPVersionStats* stats = nullptr; + if (req->version < 0x200) { + /* HTTP 1.x */ + stats = &dsc->df->d_http1Stats; + } + else { + /* HTTP 2.0 */ + stats = &dsc->df->d_http2Stats; + } + + switch (req->res.status) { + case 200: + ++stats->d_nb200Responses; + break; + case 400: + ++stats->d_nb400Responses; + break; + case 403: + ++stats->d_nb403Responses; + break; + case 500: + ++stats->d_nb500Responses; + break; + case 502: + ++stats->d_nb502Responses; + break; + default: + ++stats->d_nbOtherResponses; + break; + } + + h2o_setup_next_ostream(req, slot); +} + +/* this is called by h2o when our request dies. + We use this to signal to the 'du' that this req is no longer alive */ +static void on_generator_dispose(void *_self) +{ + DOHUnit** du = reinterpret_cast<DOHUnit**>(_self); + if (*du) { // if 0, on_dnsdist cleaned up du already + (*du)->self = nullptr; + (*du)->req = nullptr; + } +} + +/* This executes in the main DoH thread. + We allocate a DOHUnit and send it to dnsdistclient() function in the doh client thread + via a pipe */ +static void doh_dispatch_query(DOHServerConfig* dsc, h2o_handler_t* self, h2o_req_t* req, PacketBuffer&& query, const ComboAddress& local, const ComboAddress& remote, std::string&& path) +{ + try { + /* we only parse it there as a sanity check, we will parse it again later */ + uint16_t qtype; + DNSName qname(reinterpret_cast<const char*>(query.data()), query.size(), sizeof(dnsheader), false, &qtype); + + auto du = std::make_unique<DOHUnit>(); + du->dsc = dsc; + du->req = req; + du->ids.origDest = local; + du->ids.origRemote = remote; + du->rsock = dsc->dohresponsepair[0]; + du->query = std::move(query); + du->path = std::move(path); + /* we are doing quite some copies here, sorry about that, + but we can't keep accessing the req object once we are in a different thread + because the request might get killed by h2o at pretty much any time */ + if (req->scheme != nullptr) { + du->scheme = std::string(req->scheme->name.base, req->scheme->name.len); + } + du->host = std::string(req->authority.base, req->authority.len); + du->query_at = req->query_at; + du->headers.reserve(req->headers.size); + for (size_t i = 0; i < req->headers.size; ++i) { + du->headers.emplace_back(std::string(req->headers.entries[i].name->base, req->headers.entries[i].name->len), + std::string(req->headers.entries[i].value.base, req->headers.entries[i].value.len)); + } + +#ifdef HAVE_H2O_SOCKET_GET_SSL_SERVER_NAME + h2o_socket_t* sock = req->conn->callbacks->get_socket(req->conn); + const char * sni = h2o_socket_get_ssl_server_name(sock); + if (sni != nullptr) { + du->sni = sni; + } +#endif /* HAVE_H2O_SOCKET_GET_SSL_SERVER_NAME */ + + du->self = reinterpret_cast<DOHUnit**>(h2o_mem_alloc_shared(&req->pool, sizeof(*self), on_generator_dispose)); + auto ptr = du.release(); + *(ptr->self) = ptr; + try { + static_assert(sizeof(ptr) <= PIPE_BUF, "Writes up to PIPE_BUF are guaranteed not to be interleaved and to either fully succeed or fail"); + ssize_t sent = write(dsc->dohquerypair[0], &ptr, sizeof(ptr)); + if (sent != sizeof(ptr)) { + if (errno == EAGAIN || errno == EWOULDBLOCK) { + ++g_stats.dohQueryPipeFull; + vinfolog("Unable to pass a DoH query to the DoH worker thread because the pipe is full"); + } + else { + vinfolog("Unable to pass a DoH query to the DoH worker thread because we couldn't write to the pipe: %s", stringerror()); + } + ptr->release(); + ptr = nullptr; + h2o_send_error_500(req, "Internal Server Error", "Internal Server Error", 0); + } + } + catch (...) { + if (ptr != nullptr) { + ptr->release(); + } + } + } + catch(const std::exception& e) { + vinfolog("Had error parsing DoH DNS packet from %s: %s", remote.toStringWithPort(), e.what()); + h2o_send_error_400(req, "Bad Request", "The DNS query could not be parsed", 0); + } +} + +/* can only be called from the main DoH thread */ +static bool getHTTPHeaderValue(const h2o_req_t* req, const std::string& headerName, pdns_string_view& value) +{ + bool found = false; + /* early versions of boost::string_ref didn't have the ability to compare to string */ + pdns_string_view headerNameView(headerName); + + for (size_t i = 0; i < req->headers.size; ++i) { + if (pdns_string_view(req->headers.entries[i].name->base, req->headers.entries[i].name->len) == headerNameView) { + value = pdns_string_view(req->headers.entries[i].value.base, req->headers.entries[i].value.len); + /* don't stop there, we might have more than one header with the same name, and we want the last one */ + found = true; + } + } + + return found; +} + +/* can only be called from the main DoH thread */ +static void processForwardedForHeader(const h2o_req_t* req, ComboAddress& remote) +{ + static const std::string headerName = "x-forwarded-for"; + pdns_string_view value; + + if (getHTTPHeaderValue(req, headerName, value)) { + try { + auto pos = value.rfind(','); + if (pos != pdns_string_view::npos) { + ++pos; + for (; pos < value.size() && value[pos] == ' '; ++pos) + { + } + + if (pos < value.size()) { + value = value.substr(pos); + } + } + auto newRemote = ComboAddress(std::string(value)); + remote = newRemote; + } + catch (const std::exception& e) { + vinfolog("Invalid X-Forwarded-For header ('%s') received from %s : %s", std::string(value), remote.toStringWithPort(), e.what()); + } + catch (const PDNSException& e) { + vinfolog("Invalid X-Forwarded-For header ('%s') received from %s : %s", std::string(value), remote.toStringWithPort(), e.reason); + } + } +} + +/* + A query has been parsed by h2o, this executes in the main DoH thread. + For GET, the base64url-encoded payload is in the 'dns' parameter, which might be the first parameter, or not. + For POST, the payload is the payload. + */ +static int doh_handler(h2o_handler_t *self, h2o_req_t *req) +{ + try { + if (!req->conn->ctx->storage.size) { + return 0; // although we might was well crash on this + } + h2o_socket_t* sock = req->conn->callbacks->get_socket(req->conn); + ComboAddress remote; + ComboAddress local; + + if (h2o_socket_getpeername(sock, reinterpret_cast<struct sockaddr*>(&remote)) == 0) { + /* getpeername failed, likely because the connection has already been closed, + but anyway that means we can't get the remote address, which could allow an ACL bypass */ + h2o_send_error_500(req, getReasonFromStatusCode(500).c_str(), "Internal Server Error - Unable to get remote address", 0); + return 0; + } + + h2o_socket_getsockname(sock, reinterpret_cast<struct sockaddr*>(&local)); + DOHServerConfig* dsc = reinterpret_cast<DOHServerConfig*>(req->conn->ctx->storage.entries[0].data); + + if (dsc->df->d_trustForwardedForHeader) { + processForwardedForHeader(req, remote); + } + + auto& holders = dsc->holders; + if (!holders.acl->match(remote)) { + ++g_stats.aclDrops; + vinfolog("Query from %s (DoH) dropped because of ACL", remote.toStringWithPort()); + h2o_send_error_403(req, "Forbidden", "dns query not allowed because of ACL", 0); + return 0; + } + + const int descriptor = h2o_socket_get_fd(sock); + if (descriptor != -1) { + auto& conn = t_conns.at(descriptor); + ++conn.d_nbQueries; + if (conn.d_nbQueries == 1) { + if (h2o_socket_get_ssl_session_reused(sock) == 0) { + ++dsc->cs->tlsNewSessions; + } + else { + ++dsc->cs->tlsResumptions; + } + } + } + + if (auto tlsversion = h2o_socket_get_ssl_protocol_version(sock)) { + if(!strcmp(tlsversion, "TLSv1.0")) + ++dsc->cs->tls10queries; + else if(!strcmp(tlsversion, "TLSv1.1")) + ++dsc->cs->tls11queries; + else if(!strcmp(tlsversion, "TLSv1.2")) + ++dsc->cs->tls12queries; + else if(!strcmp(tlsversion, "TLSv1.3")) + ++dsc->cs->tls13queries; + else + ++dsc->cs->tlsUnknownqueries; + } + + if (dsc->df->d_exactPathMatching) { + // would be nice to be able to use a pdns_string_view there, but we would need heterogeneous lookups + // (having string in the set and compare them to string_view, for example. Note that comparing + // two boost::string_view uses the pointer, not the content). + const std::string pathOnly(req->path_normalized.base, req->path_normalized.len); + if (dsc->paths.count(pathOnly) == 0) { + h2o_send_error_404(req, "Not Found", "there is no endpoint configured for this path", 0); + return 0; + } + } + + // would be nice to be able to use a pdns_string_view there, + // but regex (called by matches() internally) requires a null-terminated string + string path(req->path.base, req->path.len); + /* the responses map can be updated at runtime, so we need to take a copy of + the shared pointer, increasing the reference counter */ + auto responsesMap = dsc->df->d_responsesMap; + /* 1 byte for the root label, 2 type, 2 class, 4 TTL (fake), 2 record length, 2 option length, 2 option code, 2 family, 1 source, 1 scope, 16 max for a full v6 */ + const size_t maxAdditionalSizeForEDNS = 35U; + if (responsesMap) { + for (const auto& entry : *responsesMap) { + if (entry->matches(path)) { + const auto& customHeaders = entry->getHeaders(); + handleResponse(*dsc->df, req, entry->getStatusCode(), entry->getContent(), customHeaders ? *customHeaders : dsc->df->d_customResponseHeaders, std::string(), false); + return 0; + } + } + } + + if (h2o_memis(req->method.base, req->method.len, H2O_STRLIT("POST"))) { + ++dsc->df->d_postqueries; + if(req->version >= 0x0200) + ++dsc->df->d_http2Stats.d_nbQueries; + else + ++dsc->df->d_http1Stats.d_nbQueries; + + PacketBuffer query; + /* We reserve a few additional bytes to be able to add EDNS later */ + query.reserve(req->entity.len + maxAdditionalSizeForEDNS); + query.resize(req->entity.len); + memcpy(query.data(), req->entity.base, req->entity.len); + doh_dispatch_query(dsc, self, req, std::move(query), local, remote, std::move(path)); + } + else if(req->query_at != SIZE_MAX && (req->path.len - req->query_at > 5)) { + auto pos = path.find("?dns="); + if(pos == string::npos) + pos = path.find("&dns="); + if(pos != string::npos) { + // need to base64url decode this + string sdns(path.substr(pos+5)); + boost::replace_all(sdns,"-", "+"); + boost::replace_all(sdns,"_", "/"); + // re-add padding that may have been missing + switch (sdns.size() % 4) { + case 2: + sdns.append(2, '='); + break; + case 3: + sdns.append(1, '='); + break; + } + + PacketBuffer decoded; + + /* rough estimate so we hopefully don't need a new allocation later */ + /* We reserve at few additional bytes to be able to add EDNS later */ + const size_t estimate = ((sdns.size() * 3) / 4); + decoded.reserve(estimate + maxAdditionalSizeForEDNS); + if(B64Decode(sdns, decoded) < 0) { + h2o_send_error_400(req, "Bad Request", "Unable to decode BASE64-URL", 0); + ++dsc->df->d_badrequests; + return 0; + } + else { + ++dsc->df->d_getqueries; + if(req->version >= 0x0200) + ++dsc->df->d_http2Stats.d_nbQueries; + else + ++dsc->df->d_http1Stats.d_nbQueries; + + doh_dispatch_query(dsc, self, req, std::move(decoded), local, remote, std::move(path)); + } + } + else + { + vinfolog("HTTP request without DNS parameter: %s", req->path.base); + h2o_send_error_400(req, "Bad Request", "Unable to find the DNS parameter", 0); + ++dsc->df->d_badrequests; + return 0; + } + } + else { + h2o_send_error_400(req, "Bad Request", "Unable to parse the request", 0); + ++dsc->df->d_badrequests; + } + return 0; + } + catch(const std::exception& e) + { + errlog("DOH Handler function failed with error %s", e.what()); + return 0; + } +} + +HTTPHeaderRule::HTTPHeaderRule(const std::string& header, const std::string& regex) + : d_header(toLower(header)), d_regex(regex), d_visual("http[" + header+ "] ~ " + regex) +{ +} + +bool HTTPHeaderRule::matches(const DNSQuestion* dq) const +{ + if (!dq->du) { + return false; + } + + for (const auto& header : dq->du->headers) { + if (header.first == d_header) { + return d_regex.match(header.second); + } + } + return false; +} + +string HTTPHeaderRule::toString() const +{ + return d_visual; +} + +HTTPPathRule::HTTPPathRule(const std::string& path) + : d_path(path) +{ + +} + +bool HTTPPathRule::matches(const DNSQuestion* dq) const +{ + if (!dq->du) { + return false; + } + + if (dq->du->query_at == SIZE_MAX) { + return dq->du->path == d_path; + } + else { + return d_path.compare(0, d_path.size(), dq->du->path, 0, dq->du->query_at) == 0; + } +} + +string HTTPPathRule::toString() const +{ + return "url path == " + d_path; +} + +HTTPPathRegexRule::HTTPPathRegexRule(const std::string& regex): d_regex(regex), d_visual("http path ~ " + regex) +{ +} + +bool HTTPPathRegexRule::matches(const DNSQuestion* dq) const +{ + if (!dq->du) { + return false; + } + + return d_regex.match(dq->du->getHTTPPath()); +} + +string HTTPPathRegexRule::toString() const +{ + return d_visual; +} + +std::unordered_map<std::string, std::string> DOHUnit::getHTTPHeaders() const +{ + std::unordered_map<std::string, std::string> results; + results.reserve(headers.size()); + + for (const auto& header : headers) { + results.insert(header); + } + + return results; +} + +std::string DOHUnit::getHTTPPath() const +{ + if (query_at == SIZE_MAX) { + return path; + } + else { + return std::string(path, 0, query_at); + } +} + +std::string DOHUnit::getHTTPHost() const +{ + return host; +} + +std::string DOHUnit::getHTTPScheme() const +{ + return scheme; +} + +std::string DOHUnit::getHTTPQueryString() const +{ + if (query_at == SIZE_MAX) { + return std::string(); + } + else { + return path.substr(query_at); + } +} + +void DOHUnit::setHTTPResponse(uint16_t statusCode, PacketBuffer&& body_, const std::string& contentType_) +{ + status_code = statusCode; + response = std::move(body_); + if (!response.empty() && statusCode >= 400) { + // we need to make sure it's null-terminated */ + if (response.at(response.size() - 1) != 0) { + response.push_back(0); + } + } + + contentType = contentType_; +} + +/* query has been parsed by h2o, which called doh_handler() in the main DoH thread. + In order not to blockfor long, doh_handler() called doh_dispatch_query() which allocated + a DOHUnit object and passed it to us */ +static void dnsdistclient(int qsock) +{ + setThreadName("dnsdist/doh-cli"); + + for(;;) { + try { + DOHUnit* ptr = nullptr; + ssize_t got = read(qsock, &ptr, sizeof(ptr)); + if (got < 0) { + warnlog("Error receiving internal DoH query: %s", strerror(errno)); + continue; + } + else if (static_cast<size_t>(got) < sizeof(ptr)) { + continue; + } + + DOHUnitUniquePtr du(ptr, DOHUnit::release); + /* we are not in the main DoH thread anymore, so there is a real risk of + a race condition where h2o kills the query while we are processing it, + so we can't touch the content of du->req until we are back into the + main DoH thread */ + if (!du->req) { + // it got killed in flight already + du->self = nullptr; + continue; + } + + // if there was no EDNS, we add it with a large buffer size + // so we can use UDP to talk to the backend. + auto dh = const_cast<struct dnsheader*>(reinterpret_cast<const struct dnsheader*>(du->query.data())); + + if (!dh->arcount) { + if (generateOptRR(std::string(), du->query, 4096, 4096, 0, false)) { + dh = const_cast<struct dnsheader*>(reinterpret_cast<const struct dnsheader*>(du->query.data())); // may have reallocated + dh->arcount = htons(1); + du->ids.ednsAdded = true; + } + } + else { + // we leave existing EDNS in place + } + + processDOHQuery(std::move(du)); + } + catch (const std::exception& e) { + errlog("Error while processing query received over DoH: %s", e.what()); + } + catch (...) { + errlog("Unspecified error while processing query received over DoH"); + } + } +} + +/* Called in the main DoH thread if h2o finds that dnsdist gave us an answer by writing into + the dohresponsepair[0] side of the pipe so from: + - handleDOHTimeout() when we did not get a response fast enough (called + either from the health check thread (active) or from the frontend ones (reused)) + - dnsdistclient (error 500 because processDOHQuery() returned a negative value) + - processDOHQuery (self-answered queries) + */ +static void on_dnsdist(h2o_socket_t *listener, const char *err) +{ + /* we want to read as many responses from the pipe as possible before + giving up. Even if we are overloaded and fighting with the DoH connections + for the CPU, the first thing we need to do is to send responses to free slots + anyway, otherwise queries and responses are piling up in our pipes, consuming + memory and likely coming up too late after the client has gone away */ + while (true) { + DOHUnit *ptr = nullptr; + DOHServerConfig* dsc = reinterpret_cast<DOHServerConfig*>(listener->data); + ssize_t got = read(dsc->dohresponsepair[1], &ptr, sizeof(ptr)); + + if (got < 0) { + if (errno != EWOULDBLOCK && errno != EAGAIN) { + errlog("Error reading a DOH internal response: %s", strerror(errno)); + } + return; + } + else if (static_cast<size_t>(got) != sizeof(ptr)) { + errlog("Error reading a DoH internal response, got %d bytes instead of the expected %d", got, sizeof(ptr)); + return; + } + + DOHUnitUniquePtr du(ptr, DOHUnit::release); + if (!du->req) { // it got killed in flight + du->self = nullptr; + continue; + } + + if (!du->tcp && + du->truncated && + du->query.size() > du->proxyProtocolPayloadSize && + (du->query.size() - du->proxyProtocolPayloadSize) > sizeof(dnsheader)) { + /* restoring the original ID */ + dnsheader* queryDH = reinterpret_cast<struct dnsheader*>(du->query.data() + du->proxyProtocolPayloadSize); + queryDH->id = du->ids.origID; + + du->tcp = true; + du->truncated = false; + auto cpq = std::make_unique<DoHCrossProtocolQuery>(std::move(du)); + + if (g_tcpclientthreads && g_tcpclientthreads->passCrossProtocolQueryToThread(std::move(cpq))) { + continue; + } + else { + vinfolog("Unable to pass DoH query to a TCP worker thread after getting a TC response over UDP"); + continue; + } + } + + if (du->self) { + // we are back in the h2o main thread now, so we don't risk + // a race (h2o killing the query) when accessing du->req anymore + *du->self = nullptr; // so we don't clean up again in on_generator_dispose + du->self = nullptr; + } + + handleResponse(*dsc->df, du->req, du->status_code, du->response, dsc->df->d_customResponseHeaders, du->contentType, true); + } +} + +/* called when a TCP connection has been accepted, the TLS session has not been established */ +static void on_accept(h2o_socket_t *listener, const char *err) +{ + DOHServerConfig* dsc = reinterpret_cast<DOHServerConfig*>(listener->data); + h2o_socket_t *sock = nullptr; + + if (err != nullptr) { + return; + } + + if ((sock = h2o_evloop_socket_accept(listener)) == nullptr) { + return; + } + + const int descriptor = h2o_socket_get_fd(sock); + if (descriptor == -1) { + h2o_socket_close(sock); + return; + } + + auto concurrentConnections = ++dsc->cs->tcpCurrentConnections; + if (dsc->cs->d_tcpConcurrentConnectionsLimit > 0 && concurrentConnections > dsc->cs->d_tcpConcurrentConnectionsLimit) { + --dsc->cs->tcpCurrentConnections; + h2o_socket_close(sock); + return; + } + + if (concurrentConnections > dsc->cs->tcpMaxConcurrentConnections.load()) { + dsc->cs->tcpMaxConcurrentConnections.store(concurrentConnections); + } + + auto& conn = t_conns[descriptor]; + + gettimeofday(&conn.d_connectionStartTime, nullptr); + conn.d_nbQueries = 0; + conn.d_acceptCtx = std::atomic_load_explicit(&dsc->accept_ctx, std::memory_order_acquire); + conn.d_desc = descriptor; + + sock->on_close.cb = on_socketclose; + sock->on_close.data = &conn; + sock->data = dsc; + + ++dsc->df->d_httpconnects; + + h2o_accept(conn.d_acceptCtx->get(), sock); +} + +static int create_listener(const ComboAddress& addr, std::shared_ptr<DOHServerConfig>& dsc, int fd) +{ + auto sock = h2o_evloop_socket_create(dsc->h2o_ctx.loop, fd, H2O_SOCKET_FLAG_DONT_READ); + sock->data = dsc.get(); + h2o_socket_read_start(sock, on_accept); + + return 0; +} + +static int ocsp_stapling_callback(SSL* ssl, void* arg) +{ + if (ssl == nullptr || arg == nullptr) { + return SSL_TLSEXT_ERR_NOACK; + } + const auto ocspMap = reinterpret_cast<std::map<int, std::string>*>(arg); + return libssl_ocsp_stapling_callback(ssl, *ocspMap); +} + +static int ticket_key_callback(SSL *s, unsigned char keyName[TLS_TICKETS_KEY_NAME_SIZE], unsigned char *iv, EVP_CIPHER_CTX *ectx, HMAC_CTX *hctx, int enc) +{ + DOHAcceptContext* ctx = reinterpret_cast<DOHAcceptContext*>(libssl_get_ticket_key_callback_data(s)); + if (ctx == nullptr || !ctx->d_ticketKeys) { + return -1; + } + + ctx->handleTicketsKeyRotation(); + + auto ret = libssl_ticket_key_callback(s, *ctx->d_ticketKeys, keyName, iv, ectx, hctx, enc); + if (enc == 0) { + if (ret == 0) { + ++ctx->d_cs->tlsUnknownTicketKey; + } + else if (ret == 2) { + ++ctx->d_cs->tlsInactiveTicketKey; + } + } + + return ret; +} + +static void setupTLSContext(DOHAcceptContext& acceptCtx, + TLSConfig& tlsConfig, + TLSErrorCounters& counters) +{ + if (tlsConfig.d_ciphers.empty()) { + tlsConfig.d_ciphers = DOH_DEFAULT_CIPHERS; + } + + auto ctx = libssl_init_server_context(tlsConfig, acceptCtx.d_ocspResponses); + + if (tlsConfig.d_enableTickets && tlsConfig.d_numberOfTicketsKeys > 0) { + acceptCtx.d_ticketKeys = std::make_unique<OpenSSLTLSTicketKeysRing>(tlsConfig.d_numberOfTicketsKeys); + SSL_CTX_set_tlsext_ticket_key_cb(ctx.get(), &ticket_key_callback); + libssl_set_ticket_key_callback_data(ctx.get(), &acceptCtx); + } + + if (!acceptCtx.d_ocspResponses.empty()) { + SSL_CTX_set_tlsext_status_cb(ctx.get(), &ocsp_stapling_callback); + SSL_CTX_set_tlsext_status_arg(ctx.get(), &acceptCtx.d_ocspResponses); + } + + libssl_set_error_counters_callback(ctx, &counters); + + if (!tlsConfig.d_keyLogFile.empty()) { + acceptCtx.d_keyLogFile = libssl_set_key_log_file(ctx, tlsConfig.d_keyLogFile); + } + + h2o_ssl_register_alpn_protocols(ctx.get(), h2o_http2_alpn_protocols); + + acceptCtx.d_ticketsKeyRotationDelay = tlsConfig.d_ticketsKeyRotationDelay; + if (tlsConfig.d_ticketKeyFile.empty()) { + acceptCtx.handleTicketsKeyRotation(); + } + else { + acceptCtx.loadTicketsKeys(tlsConfig.d_ticketKeyFile); + } + + auto nativeCtx = acceptCtx.get(); + nativeCtx->ssl_ctx = ctx.release(); +} + +static void setupAcceptContext(DOHAcceptContext& ctx, DOHServerConfig& dsc, bool setupTLS) +{ + auto nativeCtx = ctx.get(); + nativeCtx->ctx = &dsc.h2o_ctx; + nativeCtx->hosts = dsc.h2o_config.hosts; + ctx.d_ticketsKeyRotationDelay = dsc.df->d_tlsConfig.d_ticketsKeyRotationDelay; + + if (setupTLS && dsc.df->isHTTPS()) { + try { + setupTLSContext(ctx, + dsc.df->d_tlsConfig, + dsc.df->d_tlsCounters); + } + catch (const std::runtime_error& e) { + throw std::runtime_error("Error setting up TLS context for DoH listener on '" + dsc.df->d_local.toStringWithPort() + "': " + e.what()); + } + } + ctx.d_cs = dsc.cs; +} + +void DOHFrontend::rotateTicketsKey(time_t now) +{ + if (d_dsc && d_dsc->accept_ctx) { + d_dsc->accept_ctx->rotateTicketsKey(now); + } +} + +void DOHFrontend::loadTicketsKeys(const std::string& keyFile) +{ + if (d_dsc && d_dsc->accept_ctx) { + d_dsc->accept_ctx->loadTicketsKeys(keyFile); + } +} + +void DOHFrontend::handleTicketsKeyRotation() +{ + if (d_dsc && d_dsc->accept_ctx) { + d_dsc->accept_ctx->handleTicketsKeyRotation(); + } +} + +time_t DOHFrontend::getNextTicketsKeyRotation() const +{ + if (d_dsc && d_dsc->accept_ctx) { + return d_dsc->accept_ctx->getNextTicketsKeyRotation(); + } + return 0; +} + +size_t DOHFrontend::getTicketsKeysCount() const +{ + size_t res = 0; + if (d_dsc && d_dsc->accept_ctx) { + res = d_dsc->accept_ctx->getTicketsKeysCount(); + } + return res; +} + +void DOHFrontend::reloadCertificates() +{ + auto newAcceptContext = std::make_shared<DOHAcceptContext>(); + setupAcceptContext(*newAcceptContext, *d_dsc, true); + std::atomic_store_explicit(&d_dsc->accept_ctx, newAcceptContext, std::memory_order_release); +} + +void DOHFrontend::setup() +{ + registerOpenSSLUser(); + + d_dsc = std::make_shared<DOHServerConfig>(d_idleTimeout, d_internalPipeBufferSize); + + if (isHTTPS()) { + try { + setupTLSContext(*d_dsc->accept_ctx, + d_tlsConfig, + d_tlsCounters); + } + catch (const std::runtime_error& e) { + throw std::runtime_error("Error setting up TLS context for DoH listener on '" + d_local.toStringWithPort() + "': " + e.what()); + } + } +} + +static h2o_pathconf_t *register_handler(h2o_hostconf_t *hostconf, const char *path, int (*on_req)(h2o_handler_t *, h2o_req_t *)) +{ + h2o_pathconf_t *pathconf = h2o_config_register_path(hostconf, path, 0); + if (pathconf == nullptr) { + return pathconf; + } + h2o_filter_t *filter = h2o_create_filter(pathconf, sizeof(*filter)); + if (filter) { + filter->on_setup_ostream = on_response_ready_cb; + } + + h2o_handler_t *handler = h2o_create_handler(pathconf, sizeof(*handler)); + if (handler != nullptr) { + handler->on_req = on_req; + } + + return pathconf; +} + +// this is the entrypoint from dnsdist.cc +void dohThread(ClientState* cs) +{ + try { + std::shared_ptr<DOHFrontend>& df = cs->dohFrontend; + auto& dsc = df->d_dsc; + dsc->cs = cs; + dsc->df = cs->dohFrontend; + dsc->h2o_config.server_name = h2o_iovec_init(df->d_serverTokens.c_str(), df->d_serverTokens.size()); + + std::thread dnsdistThread(dnsdistclient, dsc->dohquerypair[1]); + dnsdistThread.detach(); // gets us better error reporting + + setThreadName("dnsdist/doh"); + // I wonder if this registers an IP address.. I think it does + // this may mean we need to actually register a site "name" here and not the IP address + h2o_hostconf_t *hostconf = h2o_config_register_host(&dsc->h2o_config, h2o_iovec_init(df->d_local.toString().c_str(), df->d_local.toString().size()), 65535); + + for(const auto& url : df->d_urls) { + register_handler(hostconf, url.c_str(), doh_handler); + dsc->paths.insert(url); + } + + h2o_context_init(&dsc->h2o_ctx, h2o_evloop_create(), &dsc->h2o_config); + + // in this complicated way we insert the DOHServerConfig pointer in there + h2o_vector_reserve(nullptr, &dsc->h2o_ctx.storage, 1); + dsc->h2o_ctx.storage.entries[0].data = dsc.get(); + ++dsc->h2o_ctx.storage.size; + + auto sock = h2o_evloop_socket_create(dsc->h2o_ctx.loop, dsc->dohresponsepair[1], H2O_SOCKET_FLAG_DONT_READ); + sock->data = dsc.get(); + + // this listens to responses from dnsdist to turn into http responses + h2o_socket_read_start(sock, on_dnsdist); + + setupAcceptContext(*dsc->accept_ctx, *dsc, false); + + if (create_listener(df->d_local, dsc, cs->tcpFD) != 0) { + throw std::runtime_error("DOH server failed to listen on " + df->d_local.toStringWithPort() + ": " + strerror(errno)); + } + + bool stop = false; + do { + int result = h2o_evloop_run(dsc->h2o_ctx.loop, INT32_MAX); + if (result == -1) { + if (errno != EINTR) { + errlog("Error in the DoH event loop: %s", strerror(errno)); + stop = true; + } + } + } + while (stop == false); + + } + catch (const std::exception& e) { + throw runtime_error("DOH thread failed to launch: " + std::string(e.what())); + } + catch (...) { + throw runtime_error("DOH thread failed to launch"); + } +} + +void handleUDPResponseForDoH(DOHUnitUniquePtr&& du, PacketBuffer&& udpResponse, IDState&& state) +{ + du->response = std::move(udpResponse); + du->ids = std::move(state); + + const dnsheader* dh = reinterpret_cast<const struct dnsheader*>(du->response.data()); + if (!dh->tc) { + thread_local LocalStateHolder<vector<DNSDistResponseRuleAction>> localRespRuleActions = g_respruleactions.getLocal(); + DNSResponse dr = makeDNSResponseFromIDState(du->ids, du->response); + dnsheader cleartextDH; + memcpy(&cleartextDH, dr.getHeader(), sizeof(cleartextDH)); + + if (!processResponse(du->response, localRespRuleActions, dr, false, true)) { + return; + } + + double udiff = du->ids.sentTime.udiff(); + vinfolog("Got answer from %s, relayed to %s (https), took %f usec", du->downstream->remote.toStringWithPort(), du->ids.origRemote.toStringWithPort(), udiff); + + handleResponseSent(du->ids, udiff, *dr.remote, du->downstream->remote, du->response.size(), cleartextDH, du->downstream->getProtocol()); + + ++g_stats.responses; + if (du->ids.cs) { + ++du->ids.cs->responses; + } + } + else { + du->truncated = true; + } + + sendDoHUnitToTheMainThread(std::move(du), "DoH response"); +} + +#else /* HAVE_DNS_OVER_HTTPS */ + +void handleDOHTimeout(DOHUnitUniquePtr&& oldDU) +{ +} + +#endif /* HAVE_DNS_OVER_HTTPS */ |