diff options
60 files changed, 1250 insertions, 612 deletions
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 7cfb224..7728f0b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -8,10 +8,10 @@ env: LIBBPF_VERSION: v1.3.0 OPENSSL1_VERSION: 1_1_1w+quic OPENSSL3_VERSION: 3.1.5+quic - BORINGSSL_VERSION: 8e6a26d128484b886e6dcbfa558b993d38950bb5 - AWSLC_VERSION: v1.21.0 + BORINGSSL_VERSION: fae0964b3d44e94ca2a2d21f86e61dabe683d130 + AWSLC_VERSION: v1.23.0 NGHTTP3_VERSION: v1.2.0 - NGTCP2_VERSION: v1.3.0 + NGTCP2_VERSION: v1.4.0 jobs: build-cache: @@ -22,7 +22,8 @@ jobs: runs-on: ${{ matrix.os }} steps: - - uses: actions/checkout@v4 + - name: Checkout + uses: actions/checkout@v4 - name: Restore libbpf cache id: cache-libbpf uses: actions/cache@v4 @@ -116,13 +117,13 @@ jobs: - name: Build libbpf if: steps.cache-libbpf.outputs.cache-hit != 'true' && runner.os == 'Linux' run: | - git clone -b ${{ env.LIBBPF_VERSION }} https://github.com/libbpf/libbpf + git clone --recursive -b ${{ env.LIBBPF_VERSION }} https://github.com/libbpf/libbpf cd libbpf make -C src install PREFIX=$PWD/build - name: Build quictls/openssl v1.1.1 if: steps.cache-openssl1.outputs.cache-hit != 'true' run: | - git clone --depth 1 -b OpenSSL_${{ env.OPENSSL1_VERSION }} https://github.com/quictls/openssl openssl1 + git clone --recursive --depth 1 -b OpenSSL_${{ env.OPENSSL1_VERSION }} https://github.com/quictls/openssl openssl1 cd openssl1 ./config --prefix=$PWD/build make -j"$(nproc 2> /dev/null || sysctl -n hw.ncpu)" @@ -130,7 +131,7 @@ jobs: - name: Build quictls/openssl v3.x if: steps.cache-openssl3.outputs.cache-hit != 'true' run: | - git clone --depth 1 -b openssl-${{ env.OPENSSL3_VERSION }} https://github.com/quictls/openssl openssl3 + git clone --recursive --depth 1 -b openssl-${{ env.OPENSSL3_VERSION }} https://github.com/quictls/openssl openssl3 cd openssl3 ./config enable-ktls --prefix=$PWD/build --libdir=$PWD/build/lib make -j"$(nproc 2> /dev/null || sysctl -n hw.ncpu)" @@ -151,16 +152,15 @@ jobs: - name: Build aws-lc if: steps.cache-awslc.outputs.cache-hit != 'true' run: | - git clone --depth 1 -b "${AWSLC_VERSION}" https://github.com/aws/aws-lc + git clone --recursive --depth 1 -b "${AWSLC_VERSION}" https://github.com/aws/aws-lc cd aws-lc cmake -B build -DDISABLE_GO=ON make -j"$(nproc 2> /dev/null || sysctl -n hw.ncpu)" -C build - name: Build nghttp3 if: steps.cache-nghttp3.outputs.cache-hit != 'true' run: | - git clone --depth 1 -b ${{ env.NGHTTP3_VERSION}} https://github.com/ngtcp2/nghttp3 + git clone --recursive --depth 1 -b ${{ env.NGHTTP3_VERSION}} https://github.com/ngtcp2/nghttp3 cd nghttp3 - git submodule update --init --depth 1 autoreconf -i ./configure --prefix=$PWD/build --enable-lib-only make -j"$(nproc 2> /dev/null || sysctl -n hw.ncpu)" check @@ -168,9 +168,8 @@ jobs: - name: Build ngtcp2 + quictls/openssl v1.1.1 + BoringSSL if: steps.cache-ngtcp2-openssl1.outputs.cache-hit != 'true' run: | - git clone --depth 1 -b ${{ env.NGTCP2_VERSION }} https://github.com/ngtcp2/ngtcp2 ngtcp2-openssl1 + git clone --recursive --depth 1 -b ${{ env.NGTCP2_VERSION }} https://github.com/ngtcp2/ngtcp2 ngtcp2-openssl1 cd ngtcp2-openssl1 - git submodule update --init --depth 1 autoreconf -i ./configure --prefix=$PWD/build --enable-lib-only \ PKG_CONFIG_PATH="../openssl1/build/lib/pkgconfig" \ @@ -182,9 +181,8 @@ jobs: - name: Build ngtcp2 + quictls/openssl v3.x + aws-lc if: steps.cache-ngtcp2-openssl3.outputs.cache-hit != 'true' run: | - git clone --depth 1 -b ${{ env.NGTCP2_VERSION }} https://github.com/ngtcp2/ngtcp2 ngtcp2-openssl3 + git clone --recursive --depth 1 -b ${{ env.NGTCP2_VERSION }} https://github.com/ngtcp2/ngtcp2 ngtcp2-openssl3 cd ngtcp2-openssl3 - git submodule update --init --depth 1 autoreconf -i ./configure --prefix=$PWD/build --enable-lib-only \ PKG_CONFIG_PATH="../openssl3/build/lib/pkgconfig" \ @@ -231,7 +229,10 @@ jobs: runs-on: ${{ matrix.os }} steps: - - uses: actions/checkout@v4 + - name: Checkout + uses: actions/checkout@v4 + with: + submodules: recursive - name: Linux setup if: runner.os == 'Linux' run: | @@ -258,6 +259,12 @@ jobs: cmake-data echo 'CPPFLAGS=-fsanitize=address,undefined -fno-sanitize-recover=undefined -g' >> $GITHUB_ENV echo 'LDFLAGS=-fsanitize=address,undefined -fno-sanitize-recover=undefined' >> $GITHUB_ENV + + # https://github.com/actions/runner-images/issues/9491#issuecomment-1989718917 + # Asan in llvm 14 provided in ubuntu 22.04 is incompatible with + # high-entropy ASLR in much newer kernels that GitHub runners are + # using leading to random crashes: https://reviews.llvm.org/D148280 + sudo sysctl vm.mmap_rnd_bits=28 - name: MacOS setup if: runner.os == 'macOS' run: | @@ -350,7 +357,7 @@ jobs: OPENSSL_CFLAGS="-I$PWD/include/" OPENSSL_LIBS="-L$PWD/build/ssl -lssl -L$PWD/build/crypto -lcrypto -pthread" - EXTRA_AUTOTOOLS_OPTS="$EXTRA_AUTOTOOLS_OPTS --without-neverbleed --without-jemalloc" + EXTRA_AUTOTOOLS_OPTS="$EXTRA_AUTOTOOLS_OPTS --without-neverbleed --without-jemalloc --disable-examples" echo 'OPENSSL_CFLAGS='"$OPENSSL_CFLAGS" >> $GITHUB_ENV echo 'OPENSSL_LIBS='"$OPENSSL_LIBS" >> $GITHUB_ENV @@ -404,9 +411,6 @@ jobs: echo 'LDFLAGS='"$LDFLAGS" >> $GITHUB_ENV echo 'EXTRA_AUTOTOOLS_OPTS='"$EXTRA_AUTOTOOLS_OPTS" >> $GITHUB_ENV echo 'EXTRA_CMAKE_OPTS='"$EXTRA_CMAKE_OPTS" >> $GITHUB_ENV - - name: Setup git submodules - run: | - git submodule update --init --depth 1 - name: Configure autotools run: | autoreconf -i @@ -420,7 +424,7 @@ jobs: cd nghttp2-$VERSION echo 'NGHTTP2_CMAKE_DIR='"$PWD" >> $GITHUB_ENV - cmake -DENABLE_WERROR=1 -DWITH_MRUBY=1 -DWITH_NEVERBLEED=1 -DENABLE_APP=1 $EXTRA_CMAKE_OPTS -DCPPFLAGS="$CPPFLAGS" -DLDFLAGS="$LDFLAGS" . + cmake -DENABLE_WERROR=1 -DWITH_MRUBY=1 -DWITH_NEVERBLEED=1 -DENABLE_APP=1 $EXTRA_CMAKE_OPTS -DCPPFLAGS="$CPPFLAGS" -DLDFLAGS="$LDFLAGS" -DBUILD_STATIC_LIBS=ON -DBUILD_TESTING=ON . - name: Configure cmake (MacOS) if: matrix.buildtool == 'cmake' && runner.os == 'macOS' run: | @@ -433,7 +437,7 @@ jobs: # This fixes infamous 'stdio.h not found' error. echo 'SDKROOT='"$(xcrun --sdk macosx --show-sdk-path)" >> $GITHUB_ENV - cmake -DENABLE_WERROR=1 -DWITH_MRUBY=1 -DENABLE_APP=1 $EXTRA_CMAKE_OPTS -DCPPFLAGS="$CPPFLAGS" -DLDFLAGS="$LDFLAGS" . + cmake -DENABLE_WERROR=1 -DWITH_MRUBY=1 -DENABLE_APP=1 $EXTRA_CMAKE_OPTS -DCPPFLAGS="$CPPFLAGS" -DLDFLAGS="$LDFLAGS" -DBUILD_STATIC_LIBS=ON -DBUILD_TESTING=ON . - name: Build nghttp2 with autotools (Linux) if: matrix.buildtool == 'autotools' && runner.os == 'Linux' run: | @@ -473,7 +477,10 @@ jobs: HOST: ${{ matrix.host }} steps: - - uses: actions/checkout@v4 + - name: Checkout + uses: actions/checkout@v4 + with: + submodules: recursive - name: Linux setup run: | sudo dpkg --add-architecture i386 @@ -488,7 +495,6 @@ jobs: wine - name: Configure autotools run: | - git submodule update --init --depth 1 autoreconf -i && \ ./configure --enable-werror --enable-lib-only --host="$HOST" \ CFLAGS="-g -O2 -D_WIN32_WINNT=0x0600" LIBS="-pthread" @@ -516,15 +522,84 @@ jobs: runs-on: windows-latest steps: - - uses: actions/checkout@v4 + - name: Checkout + uses: actions/checkout@v4 + with: + submodules: recursive - uses: microsoft/setup-msbuild@v2 - name: Configure cmake - run: | - git submodule update --init --depth 1 - mkdir build - cd build - cmake -DCMAKE_TOOLCHAIN_FILE=C:/vcpkg/scripts/buildsystems/vcpkg.cmake -DCMAKE_GENERATOR_PLATFORM=${{ matrix.platform }} -DVCPKG_TARGET_TRIPLET=${{ matrix.arch}}-windows .. + run: cmake -B build -DCMAKE_TOOLCHAIN_FILE=C:/vcpkg/scripts/buildsystems/vcpkg.cmake -DCMAKE_GENERATOR_PLATFORM=${{ matrix.platform }} -DVCPKG_TARGET_TRIPLET=${{ matrix.arch}}-windows -DBUILD_STATIC_LIBS=ON -DBUILD_TESTING=ON - name: Build nghttp2 run: | cmake --build build cmake --build build --target check + + release: + if: github.ref_type == 'tag' + + needs: + - build + - build-cross + - build-windows + + permissions: + contents: write + + runs-on: ubuntu-22.04 + + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + submodules: recursive + - name: Make artifacts + run: | + ver='${{ github.ref_name }}' + + prev_ver=$(git tag --sort v:refname | grep -v -F "${ver}" | \ + grep 'v[0-9]\+\.[0-9]\+\.0' | tail -n1) + + echo -n "$GPG_KEY" | gpg --batch --pinentry-mode loopback --import + ./makerelease.sh "${ver}" "${prev_ver}" + env: + GPG_KEY: ${{ secrets.GPG_KEY }} + GPG_PASSPHRASE: ${{ secrets.GPG_PASSPHRASE }} + - name: Make release + uses: actions/github-script@v7 + with: + script: | + const fs = require('fs') + + let ver = '${{ github.ref_name }}' + + let {data: release} = await github.rest.repos.createRelease({ + owner: context.repo.owner, + repo: context.repo.repo, + tag_name: ver, + name: `nghttp2 ${ver}`, + draft: true, + generate_release_notes: true, + discussion_category_name: 'Announcements', + }) + + let v = ver.substring(1) + + let files = [ + 'checksums.txt', + `nghttp2-${v}.tar.bz2`, + `nghttp2-${v}.tar.bz2.asc`, + `nghttp2-${v}.tar.gz`, + `nghttp2-${v}.tar.gz.asc`, + `nghttp2-${v}.tar.xz`, + `nghttp2-${v}.tar.xz.asc`, + ] + + await Promise.all(files.map(elem => + github.rest.repos.uploadReleaseAsset({ + owner: context.repo.owner, + repo: context.repo.repo, + release_id: release.id, + name: elem, + data: fs.readFileSync(elem), + }) + )) diff --git a/.github/workflows/fuzz.yml b/.github/workflows/fuzz.yml index 720b25f..b4ced5b 100644 --- a/.github/workflows/fuzz.yml +++ b/.github/workflows/fuzz.yml @@ -5,6 +5,13 @@ jobs: Fuzzing: runs-on: ubuntu-latest steps: + - name: LLVM workaround + run: | + # https://github.com/actions/runner-images/issues/9491#issuecomment-1989718917 + # Asan in llvm 14 provided in ubuntu 22.04 is incompatible with + # high-entropy ASLR in much newer kernels that GitHub runners are + # using leading to random crashes: https://reviews.llvm.org/D148280 + sudo sysctl vm.mmap_rnd_bits=28 - name: Build Fuzzers uses: google/oss-fuzz/infra/cifuzz/actions/build_fuzzers@master with: diff --git a/.github/workflows/stale.yaml b/.github/workflows/stale.yaml new file mode 100644 index 0000000..2c7841b --- /dev/null +++ b/.github/workflows/stale.yaml @@ -0,0 +1,20 @@ +name: 'Close stale issues' + +on: + schedule: + - cron: '30 1 * * *' + +permissions: + issues: write + +jobs: + stale: + runs-on: ubuntu-22.04 + + steps: + - uses: actions/stale@v9 + with: + stale-issue-message: 'This issue is stale because it has been open 30 days with no activity. Remove stale label or comment or this will be closed in 7 days.' + days-before-stale: 30 + days-before-close: 7 + exempt-all-milestones: true @@ -72,6 +72,8 @@ Jay Satiro Jeff 'Raid' Baitis Jianqing Wang Jim Morrison +Jiwoo Park +Jonas Kvinge Josh Braegger José F. Calcerrada Kamil Dudka diff --git a/CMakeLists.txt b/CMakeLists.txt index 58ae48c..d500bb4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -24,13 +24,13 @@ cmake_minimum_required(VERSION 3.14) # XXX using 1.8.90 instead of 1.9.0-DEV -project(nghttp2 VERSION 1.60.0) +project(nghttp2 VERSION 1.61.0) # See versioning rule: # https://www.gnu.org/software/libtool/manual/html_node/Updating-version-info.html -set(LT_CURRENT 41) +set(LT_CURRENT 42) set(LT_REVISION 0) -set(LT_AGE 27) +set(LT_AGE 28) set(CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake" ${CMAKE_MODULE_PATH}) include(Version) @@ -51,6 +51,7 @@ if(NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES) endif() include(GNUInstallDirs) +include(CMakeDependentOption) # For documentation find_package(Python3 COMPONENTS Interpreter) @@ -181,9 +182,11 @@ if(OPENSSL_FOUND) if(WIN32) set(CMAKE_REQUIRED_LIBRARIES "${CMAKE_REQUIRED_LIBRARIES}" "ws2_32" "bcrypt") endif() - check_symbol_exists(SSL_provide_quic_data "openssl/ssl.h" HAVE_SSL_PROVIDE_QUIC_DATA) - if(NOT HAVE_SSL_PROVIDE_QUIC_DATA) - message(WARNING "OpenSSL in ${OPENSSL_LIBRARIES} does not have SSL_provide_quic_data. HTTP/3 support cannot be enabled") + if(ENABLE_HTTP3) + check_symbol_exists(SSL_provide_quic_data "openssl/ssl.h" HAVE_SSL_PROVIDE_QUIC_DATA) + if(NOT HAVE_SSL_PROVIDE_QUIC_DATA) + message(WARNING "OpenSSL in ${OPENSSL_LIBRARIES} does not have SSL_provide_quic_data. HTTP/3 support cannot be enabled") + endif() endif() cmake_pop_check_state() else() @@ -432,9 +435,11 @@ add_subdirectory(lib) add_subdirectory(third-party) add_subdirectory(src) add_subdirectory(examples) -add_subdirectory(tests) -#add_subdirectory(tests/testdata) -add_subdirectory(integration-tests) +if(CMAKE_PROJECT_NAME STREQUAL PROJECT_NAME AND BUILD_TESTING) + add_subdirectory(tests) + #add_subdirectory(tests/testdata) + add_subdirectory(integration-tests) +endif() if(ENABLE_DOC) add_subdirectory(doc) endif() diff --git a/CMakeOptions.txt b/CMakeOptions.txt index 663d8e7..f1b631b 100644 --- a/CMakeOptions.txt +++ b/CMakeOptions.txt @@ -16,6 +16,7 @@ option(BUILD_STATIC_LIBS "Build libnghttp2 in static mode also" OFF) option(ENABLE_STATIC_CRT "Build libnghttp2 against the MS LIBCMT[d]") option(ENABLE_HTTP3 "Enable HTTP/3 support" OFF) option(ENABLE_DOC "Build documentation" ON) +cmake_dependent_option(BUILD_TESTING "Enable tests" ON "BUILD_STATIC_LIBS" OFF) option(WITH_LIBXML2 "Use libxml2" ${WITH_LIBXML2_DEFAULT}) @@ -127,8 +127,8 @@ following libraries are required: <https://github.com/quictls/openssl/tree/OpenSSL_1_1_1w+quic>`_; or LibreSSL (does not support 0RTT); or aws-lc; or `BoringSSL <https://boringssl.googlesource.com/boringssl/>`_ (commit - 8e6a26d128484b886e6dcbfa558b993d38950bb5) -* `ngtcp2 <https://github.com/ngtcp2/ngtcp2>`_ >= 1.0.0 + fae0964b3d44e94ca2a2d21f86e61dabe683d130) +* `ngtcp2 <https://github.com/ngtcp2/ngtcp2>`_ >= 1.4.0 * `nghttp3 <https://github.com/ngtcp2/nghttp3>`_ >= 1.1.0 Use ``--enable-http3`` configure option to enable HTTP/3 feature for @@ -341,7 +341,7 @@ Build aws-lc: .. code-block:: text - $ git clone --depth 1 -b v1.21.0 https://github.com/aws/aws-lc + $ git clone --depth 1 -b v1.23.0 https://github.com/aws/aws-lc $ cd aws-lc $ cmake -B build -DDISABLE_GO=ON --install-prefix=$PWD/opt $ make -j$(nproc) -C build @@ -365,7 +365,7 @@ Build ngtcp2: .. code-block:: text - $ git clone --depth 1 -b v1.3.0 https://github.com/ngtcp2/ngtcp2 + $ git clone --depth 1 -b v1.4.0 https://github.com/ngtcp2/ngtcp2 $ cd ngtcp2 $ git submodule update --init --depth 1 $ autoreconf -i diff --git a/bpf/reuseport_kern.c b/bpf/reuseport_kern.c index a8e65eb..e2c2184 100644 --- a/bpf/reuseport_kern.c +++ b/bpf/reuseport_kern.c @@ -325,7 +325,7 @@ struct { __uint(max_entries, 255); __type(key, __u64); __type(value, __u32); -} cid_prefix_map SEC(".maps"); +} worker_id_map SEC(".maps"); struct { __uint(type, BPF_MAP_TYPE_REUSEPORT_SOCKARRAY); @@ -355,11 +355,11 @@ typedef struct quic_hd { __u8 type; } quic_hd; -#define SV_DCIDLEN 20 +#define SV_DCIDLEN 17 #define MAX_DCIDLEN 20 #define MIN_DCIDLEN 8 -#define CID_PREFIXLEN 8 -#define CID_PREFIX_OFFSET 1 +#define WORKER_IDLEN 8 +#define WORKER_ID_OFFSET 1 enum { NGTCP2_PKT_INITIAL = 0x0, @@ -483,7 +483,33 @@ int select_reuseport(struct sk_reuseport_md *reuse_md) { quic_hd qhd; __u8 qpktbuf[6 + MAX_DCIDLEN]; struct AES_ctx *aes_ctx; - __u8 *cid_prefix; + __u8 *worker_id; + __u16 remote_port; + __u8 *data = reuse_md->data; + + /* Packets less than 22 bytes never be a valid QUIC packet. */ + if (reuse_md->len < sizeof(struct udphdr) + 22) { + return SK_DROP; + } + + if (reuse_md->data + sizeof(struct udphdr) > reuse_md->data_end) { + return SK_DROP; + } + + remote_port = (data[0] << 8) + data[1]; + + switch (remote_port) { + case 1900: + case 5353: + case 11211: + case 20800: + case 27015: + return SK_DROP; + default: + if (remote_port < 1024) { + return SK_DROP; + } + } if (bpf_skb_load_bytes(reuse_md, sizeof(struct udphdr), qpktbuf, sizeof(qpktbuf)) != 0) { @@ -509,10 +535,10 @@ int select_reuseport(struct sk_reuseport_md *reuse_md) { case NGTCP2_PKT_INITIAL: case NGTCP2_PKT_0RTT: if (qhd.dcidlen == SV_DCIDLEN) { - cid_prefix = qhd.dcid + CID_PREFIX_OFFSET; - AES_ECB_decrypt(aes_ctx, cid_prefix); + worker_id = qhd.dcid + WORKER_ID_OFFSET; + AES_ECB_decrypt(aes_ctx, worker_id); - psk_index = bpf_map_lookup_elem(&cid_prefix_map, cid_prefix); + psk_index = bpf_map_lookup_elem(&worker_id_map, worker_id); if (psk_index != NULL) { sk_index = *psk_index; @@ -529,10 +555,10 @@ int select_reuseport(struct sk_reuseport_md *reuse_md) { return SK_DROP; } - cid_prefix = qhd.dcid + CID_PREFIX_OFFSET; - AES_ECB_decrypt(aes_ctx, cid_prefix); + worker_id = qhd.dcid + WORKER_ID_OFFSET; + AES_ECB_decrypt(aes_ctx, worker_id); - psk_index = bpf_map_lookup_elem(&cid_prefix_map, cid_prefix); + psk_index = bpf_map_lookup_elem(&worker_id_map, worker_id); if (psk_index == NULL) { sk_index = sk_index_from_dcid(&qhd, reuse_md, *pnum_socks); diff --git a/configure.ac b/configure.ac index 0fb1ae4..5f9fe43 100644 --- a/configure.ac +++ b/configure.ac @@ -25,7 +25,7 @@ dnl Do not change user variables! dnl https://www.gnu.org/software/automake/manual/html_node/Flag-Variables-Ordering.html AC_PREREQ(2.61) -AC_INIT([nghttp2], [1.60.0], [t-tujikawa@users.sourceforge.net]) +AC_INIT([nghttp2], [1.61.0], [t-tujikawa@users.sourceforge.net]) AC_CONFIG_AUX_DIR([.]) AC_CONFIG_MACRO_DIR([m4]) AC_CONFIG_HEADERS([config.h]) @@ -38,15 +38,15 @@ AC_CANONICAL_BUILD AC_CANONICAL_HOST AC_CANONICAL_TARGET -AM_INIT_AUTOMAKE([subdir-objects]) +AM_INIT_AUTOMAKE([subdir-objects tar-pax]) m4_ifdef([AM_SILENT_RULES], [AM_SILENT_RULES([yes])]) dnl See versioning rule: dnl https://www.gnu.org/software/libtool/manual/html_node/Updating-version-info.html -AC_SUBST(LT_CURRENT, 41) +AC_SUBST(LT_CURRENT, 42) AC_SUBST(LT_REVISION, 0) -AC_SUBST(LT_AGE, 27) +AC_SUBST(LT_AGE, 28) major=`echo $PACKAGE_VERSION |cut -d. -f1 | sed -e "s/[^0-9]//g"` minor=`echo $PACKAGE_VERSION |cut -d. -f2 | sed -e "s/[^0-9]//g"` @@ -420,9 +420,12 @@ if test "x${request_openssl}" != "xno"; then if test "x${have_openssl}" = "xno"; then AC_MSG_NOTICE($OPENSSL_PKG_ERRORS) else - save_CFLAGS="$CFLAGS" + # Use C++ compiler because boringssl needs C++ runtime. + AC_LANG_PUSH(C++) + + save_CXXFLAGS="$CXXFLAGS" save_LIBS="$LIBS" - CFLAGS="$OPENSSL_CFLAGS $CFLAGS" + CXXFLAGS="$OPENSSL_CFLAGS $CXXFLAGS" LIBS="$OPENSSL_LIBS $LIBS" # quictls/openssl has SSL_provide_quic_data. boringssl also has @@ -432,7 +435,7 @@ if test "x${request_openssl}" != "xno"; then AC_LINK_IFELSE([AC_LANG_PROGRAM([[ #include <openssl/ssl.h> ]], [[ - SSL_provide_quic_data(NULL, 0, NULL, 0); + SSL_provide_quic_data(NULL, (ssl_encryption_level_t)0, NULL, 0); ]])], [AC_MSG_RESULT([yes]); have_ssl_provide_quic_data=yes], [AC_MSG_RESULT([no]); have_ssl_provide_quic_data=no]) @@ -448,8 +451,10 @@ if test "x${request_openssl}" != "xno"; then [AC_MSG_RESULT([yes]); have_boringssl_quic=yes], [AC_MSG_RESULT([no]); have_boringssl_quic=no]) - CFLAGS="$save_CFLAGS" + CXXFLAGS="$save_CXXFLAGS" LIBS="$save_LIBS" + + AC_LANG_POP() fi fi @@ -476,7 +481,7 @@ fi # ngtcp2 (for src) have_libngtcp2=no if test "x${request_libngtcp2}" != "xno"; then - PKG_CHECK_MODULES([LIBNGTCP2], [libngtcp2 >= 1.0.0], [have_libngtcp2=yes], + PKG_CHECK_MODULES([LIBNGTCP2], [libngtcp2 >= 1.4.0], [have_libngtcp2=yes], [have_libngtcp2=no]) if test "x${have_libngtcp2}" = "xno"; then AC_MSG_NOTICE($LIBNGTCP2_PKG_ERRORS) diff --git a/doc/Makefile.am b/doc/Makefile.am index 51945e4..50d57b2 100644 --- a/doc/Makefile.am +++ b/doc/Makefile.am @@ -77,6 +77,7 @@ APIDOCS= \ nghttp2_option_set_peer_max_concurrent_streams.rst \ nghttp2_option_set_server_fallback_rfc7540_priorities.rst \ nghttp2_option_set_user_recv_extension_type.rst \ + nghttp2_option_set_max_continuations.rst \ nghttp2_option_set_max_outbound_ack.rst \ nghttp2_option_set_max_settings.rst \ nghttp2_option_set_stream_reset_rate_limit.rst \ diff --git a/doc/bash_completion/nghttpx b/doc/bash_completion/nghttpx index a735cd2..782309c 100644 --- a/doc/bash_completion/nghttpx +++ b/doc/bash_completion/nghttpx @@ -8,7 +8,7 @@ _nghttpx() _get_comp_words_by_ref cur prev case $cur in -*) - COMPREPLY=( $( compgen -W '--backend --frontend --backlog --backend-address-family --backend-http-proxy-uri --workers --single-thread --read-rate --read-burst --write-rate --write-burst --worker-read-rate --worker-read-burst --worker-write-rate --worker-write-burst --worker-frontend-connections --backend-connections-per-host --backend-connections-per-frontend --rlimit-nofile --rlimit-memlock --backend-request-buffer --backend-response-buffer --fastopen --no-kqueue --frontend-http2-read-timeout --frontend-http3-read-timeout --frontend-read-timeout --frontend-write-timeout --frontend-keep-alive-timeout --stream-read-timeout --stream-write-timeout --backend-read-timeout --backend-write-timeout --backend-connect-timeout --backend-keep-alive-timeout --listener-disable-timeout --frontend-http2-setting-timeout --backend-http2-settings-timeout --backend-max-backoff --ciphers --tls13-ciphers --client-ciphers --tls13-client-ciphers --ecdh-curves --insecure --cacert --private-key-passwd-file --subcert --dh-param-file --alpn-list --verify-client --verify-client-cacert --verify-client-tolerate-expired --client-private-key-file --client-cert-file --tls-min-proto-version --tls-max-proto-version --tls-ticket-key-file --tls-ticket-key-memcached --tls-ticket-key-memcached-address-family --tls-ticket-key-memcached-interval --tls-ticket-key-memcached-max-retry --tls-ticket-key-memcached-max-fail --tls-ticket-key-cipher --tls-ticket-key-memcached-cert-file --tls-ticket-key-memcached-private-key-file --fetch-ocsp-response-file --ocsp-update-interval --ocsp-startup --no-verify-ocsp --no-ocsp --tls-session-cache-memcached --tls-session-cache-memcached-address-family --tls-session-cache-memcached-cert-file --tls-session-cache-memcached-private-key-file --tls-dyn-rec-warmup-threshold --tls-dyn-rec-idle-timeout --no-http2-cipher-block-list --client-no-http2-cipher-block-list --tls-sct-dir --psk-secrets --client-psk-secrets --tls-no-postpone-early-data --tls-max-early-data --tls-ktls --frontend-http2-max-concurrent-streams --backend-http2-max-concurrent-streams --frontend-http2-window-size --frontend-http2-connection-window-size --backend-http2-window-size --backend-http2-connection-window-size --http2-no-cookie-crumbling --padding --no-server-push --frontend-http2-optimize-write-buffer-size --frontend-http2-optimize-window-size --frontend-http2-encoder-dynamic-table-size --frontend-http2-decoder-dynamic-table-size --backend-http2-encoder-dynamic-table-size --backend-http2-decoder-dynamic-table-size --http2-proxy --log-level --accesslog-file --accesslog-syslog --accesslog-format --accesslog-write-early --errorlog-file --errorlog-syslog --syslog-facility --add-x-forwarded-for --strip-incoming-x-forwarded-for --no-add-x-forwarded-proto --no-strip-incoming-x-forwarded-proto --add-forwarded --strip-incoming-forwarded --forwarded-by --forwarded-for --no-via --no-strip-incoming-early-data --no-location-rewrite --host-rewrite --altsvc --http2-altsvc --add-request-header --add-response-header --request-header-field-buffer --max-request-header-fields --response-header-field-buffer --max-response-header-fields --error-page --server-name --no-server-rewrite --redirect-https-port --require-http-scheme --api-max-request-body --dns-cache-timeout --dns-lookup-timeout --dns-max-try --frontend-max-requests --frontend-http2-dump-request-header --frontend-http2-dump-response-header --frontend-frame-debug --daemon --pid-file --user --single-process --max-worker-processes --worker-process-grace-shutdown-period --mruby-file --ignore-per-pattern-mruby-error --frontend-quic-idle-timeout --frontend-quic-debug-log --quic-bpf-program-file --frontend-quic-early-data --frontend-quic-qlog-dir --frontend-quic-require-token --frontend-quic-congestion-controller --frontend-quic-secret-file --quic-server-id --frontend-quic-initial-rtt --no-quic-bpf --frontend-http3-window-size --frontend-http3-connection-window-size --frontend-http3-max-window-size --frontend-http3-max-connection-window-size --frontend-http3-max-concurrent-streams --conf --include --version --help ' -- "$cur" ) ) + COMPREPLY=( $( compgen -W '--backend --frontend --backlog --backend-address-family --backend-http-proxy-uri --workers --single-thread --read-rate --read-burst --write-rate --write-burst --worker-read-rate --worker-read-burst --worker-write-rate --worker-write-burst --worker-frontend-connections --backend-connections-per-host --backend-connections-per-frontend --rlimit-nofile --rlimit-memlock --backend-request-buffer --backend-response-buffer --fastopen --no-kqueue --frontend-http2-idle-timeout --frontend-http3-idle-timeout --frontend-write-timeout --frontend-keep-alive-timeout --frontend-header-timeout --stream-read-timeout --stream-write-timeout --backend-read-timeout --backend-write-timeout --backend-connect-timeout --backend-keep-alive-timeout --listener-disable-timeout --frontend-http2-setting-timeout --backend-http2-settings-timeout --backend-max-backoff --ciphers --tls13-ciphers --client-ciphers --tls13-client-ciphers --ecdh-curves --insecure --cacert --private-key-passwd-file --subcert --dh-param-file --alpn-list --verify-client --verify-client-cacert --verify-client-tolerate-expired --client-private-key-file --client-cert-file --tls-min-proto-version --tls-max-proto-version --tls-ticket-key-file --tls-ticket-key-memcached --tls-ticket-key-memcached-address-family --tls-ticket-key-memcached-interval --tls-ticket-key-memcached-max-retry --tls-ticket-key-memcached-max-fail --tls-ticket-key-cipher --tls-ticket-key-memcached-cert-file --tls-ticket-key-memcached-private-key-file --fetch-ocsp-response-file --ocsp-update-interval --ocsp-startup --no-verify-ocsp --no-ocsp --tls-session-cache-memcached --tls-session-cache-memcached-address-family --tls-session-cache-memcached-cert-file --tls-session-cache-memcached-private-key-file --tls-dyn-rec-warmup-threshold --tls-dyn-rec-idle-timeout --no-http2-cipher-block-list --client-no-http2-cipher-block-list --tls-sct-dir --psk-secrets --client-psk-secrets --tls-no-postpone-early-data --tls-max-early-data --tls-ktls --frontend-http2-max-concurrent-streams --backend-http2-max-concurrent-streams --frontend-http2-window-size --frontend-http2-connection-window-size --backend-http2-window-size --backend-http2-connection-window-size --http2-no-cookie-crumbling --padding --no-server-push --frontend-http2-optimize-write-buffer-size --frontend-http2-optimize-window-size --frontend-http2-encoder-dynamic-table-size --frontend-http2-decoder-dynamic-table-size --backend-http2-encoder-dynamic-table-size --backend-http2-decoder-dynamic-table-size --http2-proxy --log-level --accesslog-file --accesslog-syslog --accesslog-format --accesslog-write-early --errorlog-file --errorlog-syslog --syslog-facility --add-x-forwarded-for --strip-incoming-x-forwarded-for --no-add-x-forwarded-proto --no-strip-incoming-x-forwarded-proto --add-forwarded --strip-incoming-forwarded --forwarded-by --forwarded-for --no-via --no-strip-incoming-early-data --no-location-rewrite --host-rewrite --altsvc --http2-altsvc --add-request-header --add-response-header --request-header-field-buffer --max-request-header-fields --response-header-field-buffer --max-response-header-fields --error-page --server-name --no-server-rewrite --redirect-https-port --require-http-scheme --api-max-request-body --dns-cache-timeout --dns-lookup-timeout --dns-max-try --frontend-max-requests --frontend-http2-dump-request-header --frontend-http2-dump-response-header --frontend-frame-debug --daemon --pid-file --user --single-process --max-worker-processes --worker-process-grace-shutdown-period --mruby-file --ignore-per-pattern-mruby-error --frontend-quic-idle-timeout --frontend-quic-debug-log --quic-bpf-program-file --frontend-quic-early-data --frontend-quic-qlog-dir --frontend-quic-require-token --frontend-quic-congestion-controller --frontend-quic-secret-file --quic-server-id --frontend-quic-initial-rtt --no-quic-bpf --frontend-http3-window-size --frontend-http3-connection-window-size --frontend-http3-max-window-size --frontend-http3-max-connection-window-size --frontend-http3-max-concurrent-streams --conf --include --version --help ' -- "$cur" ) ) ;; *) _filedir diff --git a/doc/h2load.1 b/doc/h2load.1 index 09cdcf3..79f6e8a 100644 --- a/doc/h2load.1 +++ b/doc/h2load.1 @@ -27,7 +27,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] .in \\n[rst2man-indent\\n[rst2man-indent-level]]u .. -.TH "H2LOAD" "1" "Mar 01, 2024" "1.60.0" "nghttp2" +.TH "H2LOAD" "1" "Apr 04, 2024" "1.61.0" "nghttp2" .SH NAME h2load \- HTTP/2 benchmarking tool .SH SYNOPSIS diff --git a/doc/nghttp.1 b/doc/nghttp.1 index 231e5a4..0709cc9 100644 --- a/doc/nghttp.1 +++ b/doc/nghttp.1 @@ -27,7 +27,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] .in \\n[rst2man-indent\\n[rst2man-indent-level]]u .. -.TH "NGHTTP" "1" "Mar 01, 2024" "1.60.0" "nghttp2" +.TH "NGHTTP" "1" "Apr 04, 2024" "1.61.0" "nghttp2" .SH NAME nghttp \- HTTP/2 client .SH SYNOPSIS diff --git a/doc/nghttpd.1 b/doc/nghttpd.1 index 93a990d..57a46f2 100644 --- a/doc/nghttpd.1 +++ b/doc/nghttpd.1 @@ -27,7 +27,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] .in \\n[rst2man-indent\\n[rst2man-indent-level]]u .. -.TH "NGHTTPD" "1" "Mar 01, 2024" "1.60.0" "nghttp2" +.TH "NGHTTPD" "1" "Apr 04, 2024" "1.61.0" "nghttp2" .SH NAME nghttpd \- HTTP/2 server .SH SYNOPSIS diff --git a/doc/nghttpx.1 b/doc/nghttpx.1 index ba40059..6b9f54b 100644 --- a/doc/nghttpx.1 +++ b/doc/nghttpx.1 @@ -27,7 +27,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] .in \\n[rst2man-indent\\n[rst2man-indent-level]]u .. -.TH "NGHTTPX" "1" "Mar 01, 2024" "1.60.0" "nghttp2" +.TH "NGHTTPX" "1" "Apr 04, 2024" "1.61.0" "nghttp2" .SH NAME nghttpx \- HTTP/2 proxy .SH SYNOPSIS @@ -555,27 +555,24 @@ this option will be simply ignored. .SS Timeout .INDENT 0.0 .TP -.B \-\-frontend\-http2\-read\-timeout=<DURATION> -Specify read timeout for HTTP/2 frontend connection. +.B \-\-frontend\-http2\-idle\-timeout=<DURATION> +Specify idle timeout for HTTP/2 frontend connection. If +no active streams exist for this duration, connection is +closed. .sp Default: \fB3m\fP .UNINDENT .INDENT 0.0 .TP -.B \-\-frontend\-http3\-read\-timeout=<DURATION> -Specify read timeout for HTTP/3 frontend connection. +.B \-\-frontend\-http3\-idle\-timeout=<DURATION> +Specify idle timeout for HTTP/3 frontend connection. If +no active streams exist for this duration, connection is +closed. .sp Default: \fB3m\fP .UNINDENT .INDENT 0.0 .TP -.B \-\-frontend\-read\-timeout=<DURATION> -Specify read timeout for HTTP/1.1 frontend connection. -.sp -Default: \fB1m\fP -.UNINDENT -.INDENT 0.0 -.TP .B \-\-frontend\-write\-timeout=<DURATION> Specify write timeout for all frontend connections. .sp @@ -591,6 +588,17 @@ Default: \fB1m\fP .UNINDENT .INDENT 0.0 .TP +.B \-\-frontend\-header\-timeout=<DURATION> +Specify duration that the server waits for an HTTP +request header fields to be received completely. On +timeout, HTTP/1 and HTTP/2 connections are closed. For +HTTP/3, the stream is shutdown, and the connection +itself is left intact. +.sp +Default: \fB1m\fP +.UNINDENT +.INDENT 0.0 +.TP .B \-\-stream\-read\-timeout=<DURATION> Specify read timeout for HTTP/2 streams. 0 means no timeout. @@ -1846,12 +1854,12 @@ as QUIC keying materials. It is used to derive keys for encrypting tokens and Connection IDs. It is not used to encrypt QUIC packets. Each line of this file must contain exactly 136 bytes hex\-encoded string (when -decoded the byte string is 68 bytes long). The first 2 +decoded the byte string is 68 bytes long). The first 3 bits of decoded byte string are used to identify the keying material. An empty line or a line which starts \(aq#\(aq is ignored. The file can contain more than one -keying materials. Because the identifier is 2 bits, at -most 4 keying materials are read and the remaining data +keying materials. Because the identifier is 3 bits, at +most 8 keying materials are read and the remaining data is discarded. The first keying material in the file is primarily used for encryption and decryption for new connection. The other ones are used to decrypt data for diff --git a/doc/nghttpx.1.rst b/doc/nghttpx.1.rst index 03109e4..cee23f2 100644 --- a/doc/nghttpx.1.rst +++ b/doc/nghttpx.1.rst @@ -522,24 +522,22 @@ Performance Timeout ~~~~~~~ -.. option:: --frontend-http2-read-timeout=<DURATION> +.. option:: --frontend-http2-idle-timeout=<DURATION> - Specify read timeout for HTTP/2 frontend connection. + Specify idle timeout for HTTP/2 frontend connection. If + no active streams exist for this duration, connection is + closed. Default: ``3m`` -.. option:: --frontend-http3-read-timeout=<DURATION> +.. option:: --frontend-http3-idle-timeout=<DURATION> - Specify read timeout for HTTP/3 frontend connection. + Specify idle timeout for HTTP/3 frontend connection. If + no active streams exist for this duration, connection is + closed. Default: ``3m`` -.. option:: --frontend-read-timeout=<DURATION> - - Specify read timeout for HTTP/1.1 frontend connection. - - Default: ``1m`` - .. option:: --frontend-write-timeout=<DURATION> Specify write timeout for all frontend connections. @@ -553,6 +551,16 @@ Timeout Default: ``1m`` +.. option:: --frontend-header-timeout=<DURATION> + + Specify duration that the server waits for an HTTP + request header fields to be received completely. On + timeout, HTTP/1 and HTTP/2 connections are closed. For + HTTP/3, the stream is shutdown, and the connection + itself is left intact. + + Default: ``1m`` + .. option:: --stream-read-timeout=<DURATION> Specify read timeout for HTTP/2 streams. 0 means no @@ -1686,12 +1694,12 @@ HTTP/3 and QUIC encrypting tokens and Connection IDs. It is not used to encrypt QUIC packets. Each line of this file must contain exactly 136 bytes hex-encoded string (when - decoded the byte string is 68 bytes long). The first 2 + decoded the byte string is 68 bytes long). The first 3 bits of decoded byte string are used to identify the keying material. An empty line or a line which starts '#' is ignored. The file can contain more than one - keying materials. Because the identifier is 2 bits, at - most 4 keying materials are read and the remaining data + keying materials. Because the identifier is 3 bits, at + most 8 keying materials are read and the remaining data is discarded. The first keying material in the file is primarily used for encryption and decryption for new connection. The other ones are used to decrypt data for diff --git a/doc/sources/nghttpx-howto.rst b/doc/sources/nghttpx-howto.rst index 50412f7..6f8a71f 100644 --- a/doc/sources/nghttpx-howto.rst +++ b/doc/sources/nghttpx-howto.rst @@ -546,8 +546,8 @@ keys in order to keep the existing connections alive during reload. The construction of Connection ID closely follows Block Cipher CID Algorithm described in `QUIC-LB draft <https://datatracker.ietf.org/doc/html/draft-ietf-quic-load-balancers>`_. -A Connection ID that nghttpx generates is always 20 bytes long. It -uses first 2 bits as a configuration ID. The remaining bits in the +A Connection ID that nghttpx generates is always 17 bytes long. It +uses first 3 bits as a configuration ID. The remaining bits in the first byte are reserved and random. The next 4 bytes are server ID. The next 4 bytes are used to route UDP datagram to a correct ``SO_REUSEPORT`` socket. The remaining bytes are randomly generated. diff --git a/docker/Dockerfile b/docker/Dockerfile index 8426024..cdec4ee 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -9,7 +9,7 @@ RUN apt-get update && \ zlib1g-dev libev-dev libjemalloc-dev ruby-dev libc-ares-dev bison \ libelf-dev libbrotli-dev -RUN git clone --depth 1 -b v1.21.0 https://github.com/aws/aws-lc && \ +RUN git clone --recursive --depth 1 -b v1.23.0 https://github.com/aws/aws-lc && \ cd aws-lc && \ cmake -B build -DDISABLE_GO=ON && \ make -j$(nproc) -C build && \ @@ -17,9 +17,8 @@ RUN git clone --depth 1 -b v1.21.0 https://github.com/aws/aws-lc && \ cd .. && \ rm -rf aws-lc -RUN git clone --depth 1 -b v1.2.0 https://github.com/ngtcp2/nghttp3 && \ +RUN git clone --recursive --depth 1 -b v1.2.0 https://github.com/ngtcp2/nghttp3 && \ cd nghttp3 && \ - git submodule update --init --depth 1 && \ autoreconf -i && \ ./configure --enable-lib-only && \ make -j$(nproc) && \ @@ -27,9 +26,8 @@ RUN git clone --depth 1 -b v1.2.0 https://github.com/ngtcp2/nghttp3 && \ cd .. && \ rm -rf nghttp3 -RUN git clone --depth 1 -b v1.3.0 https://github.com/ngtcp2/ngtcp2 && \ +RUN git clone --recursive --depth 1 -b v1.4.0 https://github.com/ngtcp2/ngtcp2 && \ cd ngtcp2 && \ - git submodule update --init --depth 1 && \ autoreconf -i && \ ./configure --enable-lib-only --with-boringssl \ LIBTOOL_LDFLAGS="-static-libtool-libs" \ @@ -46,9 +44,8 @@ RUN git clone --depth 1 -b v1.3.0 https://github.com/libbpf/libbpf && \ cd .. && \ rm -rf libbpf -RUN git clone --depth 1 -b $NGHTTP2_BRANCH https://github.com/nghttp2/nghttp2 && \ +RUN git clone --recursive --depth 1 -b $NGHTTP2_BRANCH https://github.com/nghttp2/nghttp2 && \ cd nghttp2 && \ - git submodule update --init --depth 1 && \ autoreconf -i && \ ./configure --disable-examples --disable-hpack-tools \ --with-mruby \ @@ -70,16 +67,16 @@ RUN git clone --depth 1 -b $NGHTTP2_BRANCH https://github.com/nghttp2/nghttp2 && cd .. && \ rm -rf nghttp2 -FROM gcr.io/distroless/base-debian12 +FROM gcr.io/distroless/base-nossl-debian12 -COPY --from=build \ +COPY --from=build --link \ /usr/local/share/nghttp2/ \ /usr/local/share/nghttp2/ -COPY --from=build \ +COPY --from=build --link \ /usr/local/bin/h2load \ /usr/local/bin/nghttpx \ /usr/local/bin/nghttp \ /usr/local/bin/nghttpd \ /usr/local/bin/ -COPY --from=build /usr/local/lib/nghttp2/reuseport_kern.o \ +COPY --from=build --link /usr/local/lib/nghttp2/reuseport_kern.o \ /usr/local/lib/nghttp2/ diff --git a/gennghttpxfun.py b/gennghttpxfun.py index 2977407..80058e0 100755 --- a/gennghttpxfun.py +++ b/gennghttpxfun.py @@ -201,6 +201,9 @@ OPTIONS = [ "require-http-scheme", "tls-ktls", "alpn-list", + "frontend-header-timeout", + "frontend-http2-idle-timeout", + "frontend-http3-idle-timeout", ] LOGVARS = [ @@ -4,9 +4,9 @@ go 1.21.1 require ( github.com/bradfitz/gomemcache v0.0.0-20230905024940-24af94b03874 - github.com/quic-go/quic-go v0.41.0 + github.com/quic-go/quic-go v0.42.0 github.com/tatsuhiro-t/go-nghttp2 v0.0.0-20240121064059-46ccb0a462a8 - golang.org/x/net v0.21.0 + golang.org/x/net v0.22.0 ) require ( @@ -14,11 +14,11 @@ require ( github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38 // indirect github.com/onsi/ginkgo/v2 v2.9.5 // indirect github.com/quic-go/qpack v0.4.0 // indirect - go.uber.org/mock v0.3.0 // indirect - golang.org/x/crypto v0.19.0 // indirect + go.uber.org/mock v0.4.0 // indirect + golang.org/x/crypto v0.21.0 // indirect golang.org/x/exp v0.0.0-20221205204356-47842c84f3db // indirect golang.org/x/mod v0.11.0 // indirect - golang.org/x/sys v0.17.0 // indirect + golang.org/x/sys v0.18.0 // indirect golang.org/x/text v0.14.0 // indirect golang.org/x/tools v0.9.1 // indirect ) @@ -25,28 +25,30 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/quic-go/qpack v0.4.0 h1:Cr9BXA1sQS2SmDUWjSofMPNKmvF6IiIfDRmgU0w1ZCo= github.com/quic-go/qpack v0.4.0/go.mod h1:UZVnYIfi5GRk+zI9UMaCPsmZ2xKJP7XBUvVyT1Knj9A= -github.com/quic-go/quic-go v0.41.0 h1:aD8MmHfgqTURWNJy48IYFg2OnxwHT3JL7ahGs73lb4k= -github.com/quic-go/quic-go v0.41.0/go.mod h1:qCkNjqczPEvgsOnxZ0eCD14lv+B2LHlFAB++CNOh9hA= +github.com/quic-go/quic-go v0.42.0 h1:uSfdap0eveIl8KXnipv9K7nlwZ5IqLlYOpJ58u5utpM= +github.com/quic-go/quic-go v0.42.0/go.mod h1:132kz4kL3F9vxhW3CtQJLDVwcFe5wdWeJXXijhsO57M= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/tatsuhiro-t/go-nghttp2 v0.0.0-20240121064059-46ccb0a462a8 h1:zKJxuRe+a0O34V81GAZWOrotuU6mveT30QLjJ7OPMMg= github.com/tatsuhiro-t/go-nghttp2 v0.0.0-20240121064059-46ccb0a462a8/go.mod h1:gTqc3Q4boc+cKRlSFywTYdX9t6VGRcsThlNIWwaL3Dc= -go.uber.org/mock v0.3.0 h1:3mUxI1No2/60yUYax92Pt8eNOEecx2D3lcXZh2NEZJo= -go.uber.org/mock v0.3.0/go.mod h1:a6FSlNadKUHUa9IP5Vyt1zh4fC7uAwxMutEAscFbkZc= -golang.org/x/crypto v0.19.0 h1:ENy+Az/9Y1vSrlrvBSyna3PITt4tiZLf7sgCjZBX7Wo= -golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= +go.uber.org/mock v0.4.0 h1:VcM4ZOtdbR4f6VXfiOpwpVJDL6lCReaZ6mw31wqh7KU= +go.uber.org/mock v0.4.0/go.mod h1:a6FSlNadKUHUa9IP5Vyt1zh4fC7uAwxMutEAscFbkZc= +golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= +golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= golang.org/x/exp v0.0.0-20221205204356-47842c84f3db h1:D/cFflL63o2KSLJIwjlcIt8PR064j/xsmdEJL/YvY/o= golang.org/x/exp v0.0.0-20221205204356-47842c84f3db/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= golang.org/x/mod v0.11.0 h1:bUO06HqtnRcc/7l71XBe4WcqTZ+3AH1J59zWDDwLKgU= golang.org/x/mod v0.11.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4= -golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= +golang.org/x/net v0.22.0 h1:9sGLhx7iRIHEiX0oAJ3MRZMUCElJgy7Br1nO+AMN3Tc= +golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y= -golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= +golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= +golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.9.1 h1:8WMNJAz3zrtPmnYC7ISf5dEn3MT0gY7jBJfw27yrrLo= golang.org/x/tools v0.9.1/go.mod h1:owI94Op576fPu3cIGQeHs3joujW/2Oc6MtlxbF5dfNc= google.golang.org/protobuf v1.28.0 h1:w43yiav+6bVFTBQFZX0r7ipe9JQ1QsbMgHwbBziscLw= diff --git a/lib/CMakeLists.txt b/lib/CMakeLists.txt index 211c8e4..fda8dcb 100644 --- a/lib/CMakeLists.txt +++ b/lib/CMakeLists.txt @@ -34,6 +34,10 @@ set(NGHTTP2_RES "") set(STATIC_LIB "nghttp2_static") set(SHARED_LIB "nghttp2") +if(BUILD_SHARED_LIBS AND BUILD_STATIC_LIBS AND MSVC AND NOT STATIC_LIB_SUFFIX) + set(STATIC_LIB_SUFFIX "_static") +endif() + if(WIN32) configure_file( version.rc.in @@ -66,23 +70,23 @@ if(BUILD_SHARED_LIBS) endif() # Static library (for unittests because of symbol visibility) -add_library(${STATIC_LIB} STATIC ${NGHTTP2_SOURCES}) +if(BUILD_STATIC_LIBS) + add_library(${STATIC_LIB} STATIC ${NGHTTP2_SOURCES}) -set_target_properties(${STATIC_LIB} PROPERTIES - COMPILE_FLAGS "${WARNCFLAGS}" - VERSION ${LT_VERSION} SOVERSION ${LT_SOVERSION} - ARCHIVE_OUTPUT_NAME nghttp2${STATIC_LIB_SUFFIX} -) + set_target_properties(${STATIC_LIB} PROPERTIES + COMPILE_FLAGS "${WARNCFLAGS}" + VERSION ${LT_VERSION} SOVERSION ${LT_SOVERSION} + ARCHIVE_OUTPUT_NAME nghttp2${STATIC_LIB_SUFFIX} + ) -target_include_directories(${STATIC_LIB} INTERFACE - $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/includes> - $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}/includes> - $<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}> -) + target_include_directories(${STATIC_LIB} INTERFACE + $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/includes> + $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}/includes> + $<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}> + ) -target_compile_definitions(${STATIC_LIB} PUBLIC "-DNGHTTP2_STATICLIB") + target_compile_definitions(${STATIC_LIB} PUBLIC "-DNGHTTP2_STATICLIB") -if(BUILD_STATIC_LIBS) install(TARGETS ${STATIC_LIB} EXPORT ${EXPORT_SET}) list(APPEND nghttp2_exports ${STATIC_LIB}) endif() diff --git a/lib/includes/nghttp2/nghttp2.h b/lib/includes/nghttp2/nghttp2.h index 8891760..92c3ccc 100644 --- a/lib/includes/nghttp2/nghttp2.h +++ b/lib/includes/nghttp2/nghttp2.h @@ -466,7 +466,12 @@ typedef enum { * exhaustion on server side to send these frames forever and does * not read network. */ - NGHTTP2_ERR_FLOODED = -904 + NGHTTP2_ERR_FLOODED = -904, + /** + * When a local endpoint receives too many CONTINUATION frames + * following a HEADER frame. + */ + NGHTTP2_ERR_TOO_MANY_CONTINUATIONS = -905, } nghttp2_error; /** @@ -3208,6 +3213,17 @@ nghttp2_option_set_stream_reset_rate_limit(nghttp2_option *option, /** * @function * + * This function sets the maximum number of CONTINUATION frames + * following an incoming HEADER frame. If more than those frames are + * received, the remote endpoint is considered to be misbehaving and + * session will be closed. The default value is 8. + */ +NGHTTP2_EXTERN void nghttp2_option_set_max_continuations(nghttp2_option *option, + size_t val); + +/** + * @function + * * Initializes |*session_ptr| for client use. The all members of * |callbacks| are copied to |*session_ptr|. Therefore |*session_ptr| * does not store |callbacks|. The |user_data| is an arbitrary user diff --git a/lib/nghttp2_helper.c b/lib/nghttp2_helper.c index 93dd475..b3563d9 100644 --- a/lib/nghttp2_helper.c +++ b/lib/nghttp2_helper.c @@ -336,6 +336,8 @@ const char *nghttp2_strerror(int error_code) { "closed"; case NGHTTP2_ERR_TOO_MANY_SETTINGS: return "SETTINGS frame contained more than the maximum allowed entries"; + case NGHTTP2_ERR_TOO_MANY_CONTINUATIONS: + return "Too many CONTINUATION frames following a HEADER frame"; default: return "Unknown error code"; } diff --git a/lib/nghttp2_option.c b/lib/nghttp2_option.c index 43d4e95..53144b9 100644 --- a/lib/nghttp2_option.c +++ b/lib/nghttp2_option.c @@ -150,3 +150,8 @@ void nghttp2_option_set_stream_reset_rate_limit(nghttp2_option *option, option->stream_reset_burst = burst; option->stream_reset_rate = rate; } + +void nghttp2_option_set_max_continuations(nghttp2_option *option, size_t val) { + option->opt_set_mask |= NGHTTP2_OPT_MAX_CONTINUATIONS; + option->max_continuations = val; +} diff --git a/lib/nghttp2_option.h b/lib/nghttp2_option.h index 2259e18..c89cb97 100644 --- a/lib/nghttp2_option.h +++ b/lib/nghttp2_option.h @@ -71,6 +71,7 @@ typedef enum { NGHTTP2_OPT_SERVER_FALLBACK_RFC7540_PRIORITIES = 1 << 13, NGHTTP2_OPT_NO_RFC9113_LEADING_AND_TRAILING_WS_VALIDATION = 1 << 14, NGHTTP2_OPT_STREAM_RESET_RATE_LIMIT = 1 << 15, + NGHTTP2_OPT_MAX_CONTINUATIONS = 1 << 16, } nghttp2_option_flag; /** @@ -99,6 +100,10 @@ struct nghttp2_option { */ size_t max_settings; /** + * NGHTTP2_OPT_MAX_CONTINUATIONS + */ + size_t max_continuations; + /** * Bitwise OR of nghttp2_option_flag to determine that which fields * are specified. */ diff --git a/lib/nghttp2_session.c b/lib/nghttp2_session.c index 226cdd5..004a4df 100644 --- a/lib/nghttp2_session.c +++ b/lib/nghttp2_session.c @@ -497,6 +497,7 @@ static int session_new(nghttp2_session **session_ptr, (*session_ptr)->max_send_header_block_length = NGHTTP2_MAX_HEADERSLEN; (*session_ptr)->max_outbound_ack = NGHTTP2_DEFAULT_MAX_OBQ_FLOOD_ITEM; (*session_ptr)->max_settings = NGHTTP2_DEFAULT_MAX_SETTINGS; + (*session_ptr)->max_continuations = NGHTTP2_DEFAULT_MAX_CONTINUATIONS; if (option) { if ((option->opt_set_mask & NGHTTP2_OPT_NO_AUTO_WINDOW_UPDATE) && @@ -585,6 +586,10 @@ static int session_new(nghttp2_session **session_ptr, option->stream_reset_burst, option->stream_reset_rate); } + + if (option->opt_set_mask & NGHTTP2_OPT_MAX_CONTINUATIONS) { + (*session_ptr)->max_continuations = option->max_continuations; + } } rv = nghttp2_hd_deflate_init2(&(*session_ptr)->hd_deflater, @@ -979,7 +984,14 @@ static int session_attach_stream_item(nghttp2_session *session, return 0; } - return session_ob_data_push(session, stream); + rv = session_ob_data_push(session, stream); + if (rv != 0) { + nghttp2_stream_detach_item(stream); + + return rv; + } + + return 0; } static void session_detach_stream_item(nghttp2_session *session, @@ -1309,9 +1321,11 @@ nghttp2_stream *nghttp2_session_open_stream(nghttp2_session *session, assert((stream->flags & NGHTTP2_STREAM_FLAG_NO_RFC7540_PRIORITIES) || nghttp2_stream_in_dep_tree(stream)); + nghttp2_session_detach_idle_stream(session, stream); + if (nghttp2_stream_in_dep_tree(stream)) { assert(!(stream->flags & NGHTTP2_STREAM_FLAG_NO_RFC7540_PRIORITIES)); - nghttp2_session_detach_idle_stream(session, stream); + rv = nghttp2_stream_dep_remove(stream); if (rv != 0) { return NULL; @@ -1471,6 +1485,21 @@ int nghttp2_session_close_stream(nghttp2_session *session, int32_t stream_id, DEBUGF("stream: stream(%p)=%d close\n", stream, stream->stream_id); + /* We call on_stream_close_callback even if stream->state is + NGHTTP2_STREAM_INITIAL. This will happen while sending request + HEADERS, a local endpoint receives RST_STREAM for that stream. It + may be PROTOCOL_ERROR, but without notifying stream closure will + hang the stream in a local endpoint. + */ + + if (session->callbacks.on_stream_close_callback) { + if (session->callbacks.on_stream_close_callback( + session, stream_id, error_code, session->user_data) != 0) { + + return NGHTTP2_ERR_CALLBACK_FAILURE; + } + } + if (stream->item) { nghttp2_outbound_item *item; @@ -1488,21 +1517,6 @@ int nghttp2_session_close_stream(nghttp2_session *session, int32_t stream_id, } } - /* We call on_stream_close_callback even if stream->state is - NGHTTP2_STREAM_INITIAL. This will happen while sending request - HEADERS, a local endpoint receives RST_STREAM for that stream. It - may be PROTOCOL_ERROR, but without notifying stream closure will - hang the stream in a local endpoint. - */ - - if (session->callbacks.on_stream_close_callback) { - if (session->callbacks.on_stream_close_callback( - session, stream_id, error_code, session->user_data) != 0) { - - return NGHTTP2_ERR_CALLBACK_FAILURE; - } - } - is_my_stream_id = nghttp2_session_is_my_stream_id(session, stream_id); /* pushed streams which is not opened yet is not counted toward max @@ -1559,6 +1573,11 @@ int nghttp2_session_destroy_stream(nghttp2_session *session, } } + if (stream->queued && + (stream->flags & NGHTTP2_STREAM_FLAG_NO_RFC7540_PRIORITIES)) { + session_ob_data_remove(session, stream); + } + nghttp2_map_remove(&session->streams, stream->stream_id); nghttp2_stream_free(stream); nghttp2_mem_free(mem, stream); @@ -6812,6 +6831,8 @@ nghttp2_ssize nghttp2_session_mem_recv2(nghttp2_session *session, } } session_inbound_frame_reset(session); + + session->num_continuations = 0; } break; } @@ -6933,6 +6954,10 @@ nghttp2_ssize nghttp2_session_mem_recv2(nghttp2_session *session, } #endif /* DEBUGBUILD */ + if (++session->num_continuations > session->max_continuations) { + return NGHTTP2_ERR_TOO_MANY_CONTINUATIONS; + } + readlen = inbound_frame_buf_read(iframe, in, last); in += readlen; diff --git a/lib/nghttp2_session.h b/lib/nghttp2_session.h index b119329..ef8f7b2 100644 --- a/lib/nghttp2_session.h +++ b/lib/nghttp2_session.h @@ -110,6 +110,10 @@ typedef struct { #define NGHTTP2_DEFAULT_STREAM_RESET_BURST 1000 #define NGHTTP2_DEFAULT_STREAM_RESET_RATE 33 +/* The default max number of CONTINUATION frames following an incoming + HEADER frame. */ +#define NGHTTP2_DEFAULT_MAX_CONTINUATIONS 8 + /* Internal state when receiving incoming frame */ typedef enum { /* Receiving frame header */ @@ -290,6 +294,12 @@ struct nghttp2_session { size_t max_send_header_block_length; /* The maximum number of settings accepted per SETTINGS frame. */ size_t max_settings; + /* The maximum number of CONTINUATION frames following an incoming + HEADER frame. */ + size_t max_continuations; + /* The number of CONTINUATION frames following an incoming HEADER + frame. This variable is reset when END_HEADERS flag is seen. */ + size_t num_continuations; /* Next Stream ID. Made unsigned int to detect >= (1 << 31). */ uint32_t next_stream_id; /* The last stream ID this session initiated. For client session, diff --git a/makerelease.sh b/makerelease.sh index 34963dc..9e84988 100755 --- a/makerelease.sh +++ b/makerelease.sh @@ -6,18 +6,17 @@ PREV_TAG=$2 git checkout refs/tags/$TAG git log --pretty=fuller --date=short refs/tags/$PREV_TAG..HEAD > ChangeLog -git submodule update --init --depth 1 - autoreconf -i -./configure --with-mruby && \ - make dist-bzip2 && make dist-gzip && make dist-xz || echo "error" +./configure +make dist-bzip2 +make dist-gzip +make dist-xz +make distclean rm -f checksums.txt -VERSION=`echo -n $TAG | sed -E 's|^v([0-9]+\.[0-9]+\.[0-9]+)(-DEV)?$|\1|'` +VERSION=`echo -n $TAG | sed -E 's|^v([0-9]+\.[0-9]+\.[0-9]+(-[^.]+(\.[0-9]+)?)?)$|\1|'` for f in nghttp2-$VERSION.tar.bz2 nghttp2-$VERSION.tar.gz nghttp2-$VERSION.tar.xz; do sha256sum $f >> checksums.txt - gpg --armor --detach-sign $f + echo -n "$GPG_PASSPHRASE" | gpg --batch --passphrase-fd 0 --pinentry-mode loopback --armor --detach-sign $f done - -make distclean diff --git a/src/HttpServer.cc b/src/HttpServer.cc index b59cecd..6b28d1b 100644 --- a/src/HttpServer.cc +++ b/src/HttpServer.cc @@ -750,34 +750,40 @@ int Http2Handler::read_tls() { ERR_clear_error(); - auto rv = SSL_read(ssl_, buf.data(), buf.size()); - - if (rv <= 0) { - auto err = SSL_get_error(ssl_, rv); - switch (err) { - case SSL_ERROR_WANT_READ: - return write_(*this); - case SSL_ERROR_WANT_WRITE: - // renegotiation started - return -1; - default: - return -1; + for (;;) { + auto rv = SSL_read(ssl_, buf.data(), buf.size()); + + if (rv <= 0) { + auto err = SSL_get_error(ssl_, rv); + switch (err) { + case SSL_ERROR_WANT_READ: + return write_(*this); + case SSL_ERROR_WANT_WRITE: + // renegotiation started + return -1; + default: + return -1; + } } - } - auto nread = rv; + auto nread = rv; - if (get_config()->hexdump) { - util::hexdump(stdout, buf.data(), nread); - } + if (get_config()->hexdump) { + util::hexdump(stdout, buf.data(), nread); + } - rv = nghttp2_session_mem_recv2(session_, buf.data(), nread); - if (rv < 0) { - if (rv != NGHTTP2_ERR_BAD_CLIENT_MAGIC) { - std::cerr << "nghttp2_session_mem_recv2() returned error: " - << nghttp2_strerror(rv) << std::endl; + rv = nghttp2_session_mem_recv2(session_, buf.data(), nread); + if (rv < 0) { + if (rv != NGHTTP2_ERR_BAD_CLIENT_MAGIC) { + std::cerr << "nghttp2_session_mem_recv2() returned error: " + << nghttp2_strerror(rv) << std::endl; + } + return -1; + } + + if (SSL_pending(ssl_) == 0) { + break; } - return -1; } return write_(*this); diff --git a/src/h2load.cc b/src/h2load.cc index 8136a9f..4f9f00e 100644 --- a/src/h2load.cc +++ b/src/h2load.cc @@ -87,15 +87,6 @@ bool recorded(const std::chrono::steady_clock::time_point &t) { } } // namespace -namespace { -std::ofstream keylog_file; -void keylog_callback(const SSL *ssl, const char *line) { - keylog_file.write(line, strlen(line)); - keylog_file.put('\n'); - keylog_file.flush(); -} -} // namespace - Config::Config() : ciphers(tls::DEFAULT_CIPHER_LIST), tls13_ciphers("TLS_AES_128_GCM_SHA256:TLS_AES_256_GCM_SHA384:TLS_" @@ -2977,12 +2968,10 @@ int main(int argc, char **argv) { SSL_CTX_set_alpn_protos(ssl_ctx, proto_list.data(), proto_list.size()); - auto keylog_filename = getenv("SSLKEYLOGFILE"); - if (keylog_filename) { - keylog_file.open(keylog_filename, std::ios_base::app); - if (keylog_file) { - SSL_CTX_set_keylog_callback(ssl_ctx, keylog_callback); - } + if (tls::setup_keylog_callback(ssl_ctx) != 0) { + std::cerr << "Failed to setup keylog" << std::endl; + + exit(EXIT_FAILURE); } #if defined(NGHTTP2_OPENSSL_IS_BORINGSSL) && defined(HAVE_LIBBROTLI) @@ -3002,7 +2991,7 @@ int main(int argc, char **argv) { shared_nva.emplace_back("user-agent", user_agent); // list header fields that can be overridden. - auto override_hdrs = make_array<std::string>(":authority", ":host", ":method", + auto override_hdrs = make_array<std::string>(":authority", "host", ":method", ":scheme", "user-agent"); for (auto &kv : config.custom_headers) { @@ -3010,7 +2999,7 @@ int main(int argc, char **argv) { kv.name) != std::end(override_hdrs)) { // override header for (auto &nv : shared_nva) { - if ((nv.name == ":authority" && kv.name == ":host") || + if ((nv.name == ":authority" && kv.name == "host") || (nv.name == kv.name)) { nv.value = kv.value; } diff --git a/src/nghttp.cc b/src/nghttp.cc index f670320..6684374 100644 --- a/src/nghttp.cc +++ b/src/nghttp.cc @@ -2322,6 +2322,14 @@ int communicate( goto fin; } #endif // NGHTTP2_OPENSSL_IS_BORINGSSL && HAVE_LIBBROTLI + + if (tls::setup_keylog_callback(ssl_ctx) != 0) { + std::cerr << "[ERROR] Failed to setup keylog" << std::endl; + + result = -1; + + goto fin; + } } { HttpClient client{callbacks, loop, ssl_ctx}; diff --git a/src/shrpx.cc b/src/shrpx.cc index b42054c..89a7787 100644 --- a/src/shrpx.cc +++ b/src/shrpx.cc @@ -141,11 +141,13 @@ constexpr auto ENV_ACCEPT_PREFIX = StringRef::from_lit("NGHTTPX_ACCEPT_"); constexpr auto ENV_ORIG_PID = StringRef::from_lit("NGHTTPX_ORIG_PID"); // Prefix of environment variables to tell new binary the QUIC IPC -// file descriptor and CID prefix of the lingering worker process. -// The value must be comma separated parameters: -// <FD>,<CID_PREFIX_0>,<CID_PREFIX_1>,... <FD> is the file -// descriptor. <CID_PREFIX_I> is the I-th CID prefix in hex encoded -// string. +// file descriptor and Worker ID of the lingering worker process. The +// value must be comma separated parameters: +// +// <FD>,<WORKER_ID_0>,<WORKER_ID_1>,...,<WORKER_ID_I> +// +// <FD> is the file descriptor. <WORKER_ID_I> is the I-th Worker ID +// in hex encoded string. constexpr auto ENV_QUIC_WORKER_PROCESS_PREFIX = StringRef::from_lit("NGHTTPX_QUIC_WORKER_PROCESS_"); @@ -203,9 +205,7 @@ struct WorkerProcess { WorkerProcess(struct ev_loop *loop, pid_t worker_pid, int ipc_fd #ifdef ENABLE_HTTP3 , - int quic_ipc_fd, - const std::vector<std::array<uint8_t, SHRPX_QUIC_CID_PREFIXLEN>> - &cid_prefixes + int quic_ipc_fd, std::vector<WorkerID> worker_ids, uint16_t seq #endif // ENABLE_HTTP3 ) : loop(loop), @@ -214,7 +214,8 @@ struct WorkerProcess { #ifdef ENABLE_HTTP3 , quic_ipc_fd(quic_ipc_fd), - cid_prefixes(cid_prefixes) + worker_ids(std::move(worker_ids)), + seq(seq) #endif // ENABLE_HTTP3 { ev_child_init(&worker_process_childev, worker_process_child_cb, worker_pid, @@ -245,7 +246,8 @@ struct WorkerProcess { std::chrono::steady_clock::time_point termination_deadline; #ifdef ENABLE_HTTP3 int quic_ipc_fd; - std::vector<std::array<uint8_t, SHRPX_QUIC_CID_PREFIXLEN>> cid_prefixes; + std::vector<WorkerID> worker_ids; + uint16_t seq; #endif // ENABLE_HTTP3 }; @@ -255,6 +257,10 @@ void reload_config(); namespace { std::deque<std::unique_ptr<WorkerProcess>> worker_processes; + +#ifdef ENABLE_HTTP3 +uint16_t worker_process_seq; +#endif // ENABLE_HTTP3 } // namespace namespace { @@ -582,9 +588,10 @@ void exec_binary() { s += util::utos(i + 1); s += '='; s += util::utos(wp->quic_ipc_fd); - for (auto &cid_prefix : wp->cid_prefixes) { + for (auto &wid : wp->worker_ids) { s += ','; - s += util::format_hex(cid_prefix); + s += util::format_hex(reinterpret_cast<const unsigned char *>(&wid), + sizeof(wid)); } quic_lwps.emplace_back(s); @@ -1223,7 +1230,7 @@ std::vector<QUICLingeringWorkerProcess> namespace { std::vector<QUICLingeringWorkerProcess> get_inherited_quic_lingering_worker_process_from_env() { - std::vector<QUICLingeringWorkerProcess> iwps; + std::vector<QUICLingeringWorkerProcess> lwps; for (size_t i = 1;; ++i) { auto name = ENV_QUIC_WORKER_PROCESS_PREFIX.str(); @@ -1258,26 +1265,27 @@ get_inherited_quic_lingering_worker_process_from_env() { util::make_socket_closeonexec(fd); - std::vector<std::array<uint8_t, SHRPX_QUIC_CID_PREFIXLEN>> cid_prefixes; + std::vector<WorkerID> worker_ids; auto p = end_fd + 1; for (;;) { auto end = std::find(p, envend, ','); - auto hex_cid_prefix = StringRef{p, end}; - if (hex_cid_prefix.size() != SHRPX_QUIC_CID_PREFIXLEN * 2 || - !util::is_hex_string(hex_cid_prefix)) { - LOG(WARN) << "Found invalid CID prefix=" << hex_cid_prefix; + auto hex_wid = StringRef{p, end}; + if (hex_wid.size() != SHRPX_QUIC_WORKER_IDLEN * 2 || + !util::is_hex_string(hex_wid)) { + LOG(WARN) << "Found invalid WorkerID=" << hex_wid; break; } if (LOG_ENABLED(INFO)) { - LOG(INFO) << "Inherit worker process CID prefix=" << hex_cid_prefix; + LOG(INFO) << "Inherit worker process WorkerID=" << hex_wid; } - cid_prefixes.emplace_back(); + worker_ids.emplace_back(); - util::decode_hex(std::begin(cid_prefixes.back()), hex_cid_prefix); + util::decode_hex(reinterpret_cast<uint8_t *>(&worker_ids.back()), + hex_wid); if (end == envend) { break; @@ -1286,10 +1294,20 @@ get_inherited_quic_lingering_worker_process_from_env() { p = end + 1; } - iwps.emplace_back(std::move(cid_prefixes), fd); + lwps.emplace_back(std::move(worker_ids), fd); } - return iwps; + if (!lwps.empty()) { + const auto &lwp = lwps.back(); + + if (!lwp.worker_ids.empty() && + worker_process_seq <= lwp.worker_ids[0].worker_process) { + worker_process_seq = lwp.worker_ids[0].worker_process; + ++worker_process_seq; + } + } + + return lwps; } } // namespace #endif // ENABLE_HTTP3 @@ -1418,32 +1436,33 @@ int create_quic_ipc_socket(std::array<int, 2> &quic_ipc_fd) { } // namespace namespace { -int generate_cid_prefix( - std::vector<std::array<uint8_t, SHRPX_QUIC_CID_PREFIXLEN>> &cid_prefixes, - const Config *config) { +int generate_worker_id(std::vector<WorkerID> &worker_ids, uint16_t wp_seq, + const Config *config) { auto &apiconf = config->api; auto &quicconf = config->quic; - size_t num_cid_prefix; + size_t num_wid; if (config->single_thread) { - num_cid_prefix = 1; + num_wid = 1; } else { - num_cid_prefix = config->num_worker; + num_wid = config->num_worker; // API endpoint occupies the one dedicated worker thread. - // Although such worker never gets QUIC traffic, we create CID - // prefix for it to make code a bit simpler. + // Although such worker never gets QUIC traffic, we create Worker + // ID for it to make code a bit simpler. if (apiconf.enabled) { - ++num_cid_prefix; + ++num_wid; } } - cid_prefixes.resize(num_cid_prefix); + worker_ids.resize(num_wid); - for (auto &cid_prefix : cid_prefixes) { - if (create_cid_prefix(cid_prefix.data(), quicconf.server_id.data()) != 0) { - return -1; - } + uint16_t idx = 0; + + for (auto &wid : worker_ids) { + wid.server = quicconf.server_id; + wid.worker_process = wp_seq; + wid.thread = idx++; } return 0; @@ -1458,7 +1477,7 @@ collect_quic_lingering_worker_processes() { std::end(inherited_quic_lingering_worker_processes)}; for (auto &wp : worker_processes) { - quic_lwps.emplace_back(wp->cid_prefixes, wp->quic_ipc_fd); + quic_lwps.emplace_back(wp->worker_ids, wp->quic_ipc_fd); } return quic_lwps; @@ -1596,19 +1615,17 @@ namespace { // |main_ipc_fd|. In child process, we will close file descriptors // which are inherited from previous configuration/process, but not // used in the current configuration. -pid_t fork_worker_process( - int &main_ipc_fd +pid_t fork_worker_process(int &main_ipc_fd #ifdef ENABLE_HTTP3 - , - int &wp_quic_ipc_fd + , + int &wp_quic_ipc_fd #endif // ENABLE_HTTP3 - , - const std::vector<InheritedAddr> &iaddrs + , + const std::vector<InheritedAddr> &iaddrs #ifdef ENABLE_HTTP3 - , - const std::vector<std::array<uint8_t, SHRPX_QUIC_CID_PREFIXLEN>> - &cid_prefixes, - const std::vector<QUICLingeringWorkerProcess> &quic_lwps + , + std::vector<WorkerID> worker_ids, + std::vector<QUICLingeringWorkerProcess> quic_lwps #endif // ENABLE_HTTP3 ) { std::array<char, STRERROR_BUFSIZE> errbuf; @@ -1714,9 +1731,9 @@ pid_t fork_worker_process( .ipc_fd = ipc_fd[0], .ready_ipc_fd = worker_process_ready_ipc_fd[1], #ifdef ENABLE_HTTP3 - .cid_prefixes = cid_prefixes, + .worker_ids = std::move(worker_ids), .quic_ipc_fd = quic_ipc_fd[0], - .quic_lingering_worker_processes = quic_lwps, + .quic_lingering_worker_processes = std::move(quic_lwps), #endif // ENABLE_HTTP3 }; rv = worker_process_event_loop(&wpconf); @@ -1835,9 +1852,9 @@ int event_loop() { auto quic_lwps = collect_quic_lingering_worker_processes(); - std::vector<std::array<uint8_t, SHRPX_QUIC_CID_PREFIXLEN>> cid_prefixes; + std::vector<WorkerID> worker_ids; - if (generate_cid_prefix(cid_prefixes, config) != 0) { + if (generate_worker_id(worker_ids, worker_process_seq, config) != 0) { return -1; } #endif // ENABLE_HTTP3 @@ -1858,7 +1875,7 @@ int event_loop() { {} #ifdef ENABLE_HTTP3 , - cid_prefixes, quic_lwps + worker_ids, std::move(quic_lwps) #endif // ENABLE_HTTP3 ); @@ -1869,12 +1886,13 @@ int event_loop() { ev_timer_init(&worker_process_grace_period_timer, worker_process_grace_period_timercb, 0., 0.); - worker_process_add(std::make_unique<WorkerProcess>(loop, pid, ipc_fd + worker_process_add(std::make_unique<WorkerProcess>( + loop, pid, ipc_fd #ifdef ENABLE_HTTP3 - , - quic_ipc_fd, cid_prefixes + , + quic_ipc_fd, std::move(worker_ids), worker_process_seq++ #endif // ENABLE_HTTP3 - )); + )); // Write PID file when we are ready to accept connection from peer. // This makes easier to write restart script for nghttpx. Because @@ -2006,6 +2024,7 @@ void fill_default_config(Config *config) { httpconf.xfp.add = true; httpconf.xfp.strip_incoming = true; httpconf.early_data.strip_incoming = true; + httpconf.timeout.header = 1_min; auto &http2conf = config->http2; { @@ -2088,7 +2107,8 @@ void fill_default_config(Config *config) { static_cast<ev_tstamp>(NGTCP2_DEFAULT_INITIAL_RTT) / NGTCP2_SECONDS; } - if (RAND_bytes(quicconf.server_id.data(), quicconf.server_id.size()) != 1) { + if (RAND_bytes(reinterpret_cast<unsigned char *>(&quicconf.server_id), + sizeof(quicconf.server_id)) != 1) { assert(0); abort(); } @@ -2132,20 +2152,17 @@ void fill_default_config(Config *config) { auto &upstreamconf = connconf.upstream; { auto &timeoutconf = upstreamconf.timeout; - // Read timeout for HTTP2 upstream connection - timeoutconf.http2_read = 3_min; + // Idle timeout for HTTP2 upstream connection + timeoutconf.http2_idle = 3_min; - // Read timeout for HTTP3 upstream connection - timeoutconf.http3_read = 3_min; - - // Read timeout for non-HTTP2 upstream connection - timeoutconf.read = 1_min; + // Idle timeout for HTTP3 upstream connection + timeoutconf.http3_idle = 3_min; // Write timeout for HTTP2/non-HTTP2 upstream connection timeoutconf.write = 30_s; - // Keep alive timeout for HTTP/1 upstream connection - timeoutconf.idle_read = 1_min; + // Keep alive (idle) timeout for HTTP/1 upstream connection + timeoutconf.idle = 1_min; } } @@ -2644,18 +2661,18 @@ Performance: this option will be simply ignored. Timeout: - --frontend-http2-read-timeout=<DURATION> - Specify read timeout for HTTP/2 frontend connection. - Default: )" - << util::duration_str(config->conn.upstream.timeout.http2_read) << R"( - --frontend-http3-read-timeout=<DURATION> - Specify read timeout for HTTP/3 frontend connection. + --frontend-http2-idle-timeout=<DURATION> + Specify idle timeout for HTTP/2 frontend connection. If + no active streams exist for this duration, connection is + closed. Default: )" - << util::duration_str(config->conn.upstream.timeout.http3_read) << R"( - --frontend-read-timeout=<DURATION> - Specify read timeout for HTTP/1.1 frontend connection. + << util::duration_str(config->conn.upstream.timeout.http2_idle) << R"( + --frontend-http3-idle-timeout=<DURATION> + Specify idle timeout for HTTP/3 frontend connection. If + no active streams exist for this duration, connection is + closed. Default: )" - << util::duration_str(config->conn.upstream.timeout.read) << R"( + << util::duration_str(config->conn.upstream.timeout.http3_idle) << R"( --frontend-write-timeout=<DURATION> Specify write timeout for all frontend connections. Default: )" @@ -2664,7 +2681,15 @@ Timeout: Specify keep-alive timeout for frontend HTTP/1 connection. Default: )" - << util::duration_str(config->conn.upstream.timeout.idle_read) << R"( + << util::duration_str(config->conn.upstream.timeout.idle) << R"( + --frontend-header-timeout=<DURATION> + Specify duration that the server waits for an HTTP + request header fields to be received completely. On + timeout, HTTP/1 and HTTP/2 connections are closed. For + HTTP/3, the stream is shutdown, and the connection + itself is left intact. + Default: )" + << util::duration_str(config->http.timeout.header) << R"( --stream-read-timeout=<DURATION> Specify read timeout for HTTP/2 streams. 0 means no timeout. @@ -3530,12 +3555,12 @@ HTTP/3 and QUIC: encrypting tokens and Connection IDs. It is not used to encrypt QUIC packets. Each line of this file must contain exactly 136 bytes hex-encoded string (when - decoded the byte string is 68 bytes long). The first 2 + decoded the byte string is 68 bytes long). The first 3 bits of decoded byte string are used to identify the keying material. An empty line or a line which starts '#' is ignored. The file can contain more than one - keying materials. Because the identifier is 2 bits, at - most 4 keying materials are read and the remaining data + keying materials. Because the identifier is 3 bits, at + most 8 keying materials are read and the remaining data is discarded. The first keying material in the file is primarily used for encryption and decryption for new connection. The other ones are used to decrypt data for @@ -3997,9 +4022,10 @@ void reload_config() { auto quic_lwps = collect_quic_lingering_worker_processes(); - std::vector<std::array<uint8_t, SHRPX_QUIC_CID_PREFIXLEN>> cid_prefixes; + std::vector<WorkerID> worker_ids; - if (generate_cid_prefix(cid_prefixes, new_config.get()) != 0) { + if (generate_worker_id(worker_ids, worker_process_seq, new_config.get()) != + 0) { close_not_inherited_fd(new_config.get(), iaddrs); return; } @@ -4020,7 +4046,7 @@ void reload_config() { iaddrs #ifdef ENABLE_HTTP3 , - cid_prefixes, quic_lwps + worker_ids, std::move(quic_lwps) #endif // ENABLE_HTTP3 ); @@ -4035,12 +4061,13 @@ void reload_config() { close_unused_inherited_addr(iaddrs); - worker_process_add(std::make_unique<WorkerProcess>(loop, pid, ipc_fd + worker_process_add(std::make_unique<WorkerProcess>( + loop, pid, ipc_fd #ifdef ENABLE_HTTP3 - , - quic_ipc_fd, cid_prefixes + , + quic_ipc_fd, std::move(worker_ids), worker_process_seq++ #endif // ENABLE_HTTP3 - )); + )); worker_process_adjust_limit(); @@ -4377,6 +4404,12 @@ int main(int argc, char **argv) { {SHRPX_OPT_REQUIRE_HTTP_SCHEME.c_str(), no_argument, &flag, 191}, {SHRPX_OPT_TLS_KTLS.c_str(), no_argument, &flag, 192}, {SHRPX_OPT_ALPN_LIST.c_str(), required_argument, &flag, 193}, + {SHRPX_OPT_FRONTEND_HEADER_TIMEOUT.c_str(), required_argument, &flag, + 194}, + {SHRPX_OPT_FRONTEND_HTTP2_IDLE_TIMEOUT.c_str(), required_argument, + &flag, 195}, + {SHRPX_OPT_FRONTEND_HTTP3_IDLE_TIMEOUT.c_str(), required_argument, + &flag, 196}, {nullptr, 0, nullptr, 0}}; int option_index = 0; @@ -5294,6 +5327,21 @@ int main(int argc, char **argv) { // --alpn-list cmdcfgs.emplace_back(SHRPX_OPT_ALPN_LIST, StringRef{optarg}); break; + case 194: + // --frontend-header-timeout + cmdcfgs.emplace_back(SHRPX_OPT_FRONTEND_HEADER_TIMEOUT, + StringRef{optarg}); + break; + case 195: + // --frontend-http2-idle-timeout + cmdcfgs.emplace_back(SHRPX_OPT_FRONTEND_HTTP2_IDLE_TIMEOUT, + StringRef{optarg}); + break; + case 196: + // --frontend-http3-idle-timeout + cmdcfgs.emplace_back(SHRPX_OPT_FRONTEND_HTTP3_IDLE_TIMEOUT, + StringRef{optarg}); + break; default: break; } diff --git a/src/shrpx_client_handler.cc b/src/shrpx_client_handler.cc index 1f0c01c..a78b00a 100644 --- a/src/shrpx_client_handler.cc +++ b/src/shrpx_client_handler.cc @@ -444,7 +444,7 @@ ClientHandler::ClientHandler(Worker *worker, int fd, SSL *ssl, rb_(worker->get_mcpool()), conn_(worker->get_loop(), fd, ssl, worker->get_mcpool(), get_config()->conn.upstream.timeout.write, - get_config()->conn.upstream.timeout.read, + get_config()->conn.upstream.timeout.idle, get_config()->conn.upstream.ratelimit.write, get_config()->conn.upstream.ratelimit.read, writecb, readcb, timeoutcb, this, get_config()->tls.dyn_rec.warmup_threshold, @@ -551,7 +551,7 @@ void ClientHandler::setup_http3_upstream( auto config = get_config(); - reset_upstream_read_timeout(config->conn.upstream.timeout.http3_read); + reset_upstream_read_timeout(config->conn.upstream.timeout.http3_idle); } #endif // ENABLE_HTTP3 @@ -591,16 +591,14 @@ struct ev_loop *ClientHandler::get_loop() const { return conn_.loop; } void ClientHandler::reset_upstream_read_timeout(ev_tstamp t) { conn_.rt.repeat = t; - if (ev_is_active(&conn_.rt)) { - ev_timer_again(conn_.loop, &conn_.rt); - } + + ev_timer_again(conn_.loop, &conn_.rt); } void ClientHandler::reset_upstream_write_timeout(ev_tstamp t) { conn_.wt.repeat = t; - if (ev_is_active(&conn_.wt)) { - ev_timer_again(conn_.loop, &conn_.wt); - } + + ev_timer_again(conn_.loop, &conn_.wt); } void ClientHandler::repeat_read_timer() { diff --git a/src/shrpx_config.cc b/src/shrpx_config.cc index 89b3672..d856c95 100644 --- a/src/shrpx_config.cc +++ b/src/shrpx_config.cc @@ -282,9 +282,9 @@ read_quic_secret_file(const StringRef &path) { assert(static_cast<size_t>(p - std::begin(s)) == expectedlen * 2); - qkm.id = qkm.reserved[0] & 0xc0; + qkm.id = qkm.reserved[0] & SHRPX_QUIC_DCID_KM_ID_MASK; - if (kms.size() == 4) { + if (kms.size() == 8) { break; } } @@ -2396,6 +2396,9 @@ int option_lookup_token(const char *name, size_t namelen) { if (util::strieq_l("backend-connect-timeou", name, 22)) { return SHRPX_OPTID_BACKEND_CONNECT_TIMEOUT; } + if (util::strieq_l("frontend-header-timeou", name, 22)) { + return SHRPX_OPTID_FRONTEND_HEADER_TIMEOUT; + } break; } break; @@ -2526,9 +2529,15 @@ int option_lookup_token(const char *name, size_t namelen) { } break; case 't': + if (util::strieq_l("frontend-http2-idle-timeou", name, 26)) { + return SHRPX_OPTID_FRONTEND_HTTP2_IDLE_TIMEOUT; + } if (util::strieq_l("frontend-http2-read-timeou", name, 26)) { return SHRPX_OPTID_FRONTEND_HTTP2_READ_TIMEOUT; } + if (util::strieq_l("frontend-http3-idle-timeou", name, 26)) { + return SHRPX_OPTID_FRONTEND_HTTP3_IDLE_TIMEOUT; + } if (util::strieq_l("frontend-http3-read-timeou", name, 26)) { return SHRPX_OPTID_FRONTEND_HTTP3_READ_TIMEOUT; } @@ -2966,13 +2975,28 @@ int parse_config(Config *config, int optid, const StringRef &opt, return 0; } - case SHRPX_OPTID_WORKERS: + case SHRPX_OPTID_WORKERS: { #ifdef NOTHREADS LOG(WARN) << "Threading disabled at build time, no threads created."; return 0; #else // !NOTHREADS - return parse_uint(&config->num_worker, opt, optarg); + size_t n; + + if (parse_uint(&n, opt, optarg) != 0) { + return -1; + } + + if (n > 65530) { + LOG(ERROR) << opt << ": the number of workers must not exceed 65530"; + + return -1; + } + + config->num_worker = n; + + return 0; #endif // !NOTHREADS + } case SHRPX_OPTID_HTTP2_MAX_CONCURRENT_STREAMS: { LOG(WARN) << opt << ": deprecated. Use " << SHRPX_OPT_FRONTEND_HTTP2_MAX_CONCURRENT_STREAMS << " and " @@ -3028,10 +3052,17 @@ int parse_config(Config *config, int optid, const StringRef &opt, return 0; case SHRPX_OPTID_FRONTEND_HTTP2_READ_TIMEOUT: - return parse_duration(&config->conn.upstream.timeout.http2_read, opt, + LOG(WARN) << opt << ": deprecated. Use frontend-http2-idle-timeout"; + // fall through + case SHRPX_OPTID_FRONTEND_HTTP2_IDLE_TIMEOUT: + return parse_duration(&config->conn.upstream.timeout.http2_idle, opt, optarg); case SHRPX_OPTID_FRONTEND_READ_TIMEOUT: - return parse_duration(&config->conn.upstream.timeout.read, opt, optarg); + LOG(WARN) << opt << ": deprecated. Use frontend-header-timeout"; + + return 0; + case SHRPX_OPTID_FRONTEND_HEADER_TIMEOUT: + return parse_duration(&config->http.timeout.header, opt, optarg); case SHRPX_OPTID_FRONTEND_WRITE_TIMEOUT: return parse_duration(&config->conn.upstream.timeout.write, opt, optarg); case SHRPX_OPTID_BACKEND_READ_TIMEOUT: @@ -3907,8 +3938,7 @@ int parse_config(Config *config, int optid, const StringRef &opt, return 0; } case SHRPX_OPTID_FRONTEND_KEEP_ALIVE_TIMEOUT: - return parse_duration(&config->conn.upstream.timeout.idle_read, opt, - optarg); + return parse_duration(&config->conn.upstream.timeout.idle, opt, optarg); case SHRPX_OPTID_PSK_SECRETS: #ifndef OPENSSL_NO_PSK return parse_psk_secrets(config, optarg); @@ -4032,8 +4062,11 @@ int parse_config(Config *config, int optid, const StringRef &opt, return 0; } case SHRPX_OPTID_FRONTEND_HTTP3_READ_TIMEOUT: + LOG(WARN) << opt << ": deprecated. Use frontend-http3-idle-timeout"; + // fall through + case SHRPX_OPTID_FRONTEND_HTTP3_IDLE_TIMEOUT: #ifdef ENABLE_HTTP3 - return parse_duration(&config->conn.upstream.timeout.http3_read, opt, + return parse_duration(&config->conn.upstream.timeout.http3_idle, opt, optarg); #else // !ENABLE_HTTP3 return 0; @@ -4126,12 +4159,13 @@ int parse_config(Config *config, int optid, const StringRef &opt, return 0; case SHRPX_OPTID_QUIC_SERVER_ID: #ifdef ENABLE_HTTP3 - if (optarg.size() != config->quic.server_id.size() * 2 || + if (optarg.size() != sizeof(config->quic.server_id) * 2 || !util::is_hex_string(optarg)) { LOG(ERROR) << opt << ": must be a hex-string"; return -1; } - util::decode_hex(std::begin(config->quic.server_id), optarg); + util::decode_hex(reinterpret_cast<uint8_t *>(&config->quic.server_id), + optarg); #endif // ENABLE_HTTP3 return 0; @@ -4700,6 +4734,7 @@ int resolve_hostname(Address *addr, const char *hostname, uint16_t port, #ifdef ENABLE_HTTP3 QUICKeyingMaterial::QUICKeyingMaterial(QUICKeyingMaterial &&other) noexcept : cid_encryption_ctx{std::exchange(other.cid_encryption_ctx, nullptr)}, + cid_decryption_ctx{std::exchange(other.cid_decryption_ctx, nullptr)}, reserved{other.reserved}, secret{other.secret}, salt{other.salt}, @@ -4710,11 +4745,16 @@ QUICKeyingMaterial::~QUICKeyingMaterial() noexcept { if (cid_encryption_ctx) { EVP_CIPHER_CTX_free(cid_encryption_ctx); } + + if (cid_decryption_ctx) { + EVP_CIPHER_CTX_free(cid_decryption_ctx); + } } QUICKeyingMaterial & QUICKeyingMaterial::operator=(QUICKeyingMaterial &&other) noexcept { cid_encryption_ctx = std::exchange(other.cid_encryption_ctx, nullptr); + cid_decryption_ctx = std::exchange(other.cid_decryption_ctx, nullptr); reserved = other.reserved; secret = other.secret; salt = other.salt; diff --git a/src/shrpx_config.h b/src/shrpx_config.h index 335b0f9..f264b6a 100644 --- a/src/shrpx_config.h +++ b/src/shrpx_config.h @@ -406,6 +406,12 @@ constexpr auto SHRPX_OPT_REQUIRE_HTTP_SCHEME = StringRef::from_lit("require-http-scheme"); constexpr auto SHRPX_OPT_TLS_KTLS = StringRef::from_lit("tls-ktls"); constexpr auto SHRPX_OPT_ALPN_LIST = StringRef::from_lit("alpn-list"); +constexpr auto SHRPX_OPT_FRONTEND_HEADER_TIMEOUT = + StringRef::from_lit("frontend-header-timeout"); +constexpr auto SHRPX_OPT_FRONTEND_HTTP2_IDLE_TIMEOUT = + StringRef::from_lit("frontend-http2-idle-timeout"); +constexpr auto SHRPX_OPT_FRONTEND_HTTP3_IDLE_TIMEOUT = + StringRef::from_lit("frontend-http3-idle-timeout"); constexpr size_t SHRPX_OBFUSCATED_NODE_LENGTH = 8; @@ -641,6 +647,7 @@ struct QUICKeyingMaterial { ~QUICKeyingMaterial() noexcept; QUICKeyingMaterial &operator=(QUICKeyingMaterial &&other) noexcept; EVP_CIPHER_CTX *cid_encryption_ctx; + EVP_CIPHER_CTX *cid_decryption_ctx; std::array<uint8_t, SHRPX_QUIC_SECRET_RESERVEDLEN> reserved; std::array<uint8_t, SHRPX_QUIC_SECRETLEN> secret; std::array<uint8_t, SHRPX_QUIC_SALTLEN> salt; @@ -815,7 +822,7 @@ struct QUICConfig { StringRef prog_file; bool disabled; } bpf; - std::array<uint8_t, SHRPX_QUIC_SERVER_IDLEN> server_id; + uint32_t server_id; }; struct Http3Config { @@ -864,6 +871,9 @@ struct HttpConfig { struct { bool strip_incoming; } early_data; + struct { + ev_tstamp header; + } timeout; std::vector<AltSvc> altsvcs; // altsvcs serialized in a wire format. StringRef altsvc_header_value; @@ -1048,11 +1058,10 @@ struct ConnectionConfig { struct { struct { - ev_tstamp http2_read; - ev_tstamp http3_read; - ev_tstamp read; + ev_tstamp http2_idle; + ev_tstamp http3_idle; ev_tstamp write; - ev_tstamp idle_read; + ev_tstamp idle; } timeout; struct { RateLimitConfig read; @@ -1249,12 +1258,14 @@ enum { SHRPX_OPTID_FORWARDED_FOR, SHRPX_OPTID_FRONTEND, SHRPX_OPTID_FRONTEND_FRAME_DEBUG, + SHRPX_OPTID_FRONTEND_HEADER_TIMEOUT, SHRPX_OPTID_FRONTEND_HTTP2_CONNECTION_WINDOW_BITS, SHRPX_OPTID_FRONTEND_HTTP2_CONNECTION_WINDOW_SIZE, SHRPX_OPTID_FRONTEND_HTTP2_DECODER_DYNAMIC_TABLE_SIZE, SHRPX_OPTID_FRONTEND_HTTP2_DUMP_REQUEST_HEADER, SHRPX_OPTID_FRONTEND_HTTP2_DUMP_RESPONSE_HEADER, SHRPX_OPTID_FRONTEND_HTTP2_ENCODER_DYNAMIC_TABLE_SIZE, + SHRPX_OPTID_FRONTEND_HTTP2_IDLE_TIMEOUT, SHRPX_OPTID_FRONTEND_HTTP2_MAX_CONCURRENT_STREAMS, SHRPX_OPTID_FRONTEND_HTTP2_OPTIMIZE_WINDOW_SIZE, SHRPX_OPTID_FRONTEND_HTTP2_OPTIMIZE_WRITE_BUFFER_SIZE, @@ -1263,6 +1274,7 @@ enum { SHRPX_OPTID_FRONTEND_HTTP2_WINDOW_BITS, SHRPX_OPTID_FRONTEND_HTTP2_WINDOW_SIZE, SHRPX_OPTID_FRONTEND_HTTP3_CONNECTION_WINDOW_SIZE, + SHRPX_OPTID_FRONTEND_HTTP3_IDLE_TIMEOUT, SHRPX_OPTID_FRONTEND_HTTP3_MAX_CONCURRENT_STREAMS, SHRPX_OPTID_FRONTEND_HTTP3_MAX_CONNECTION_WINDOW_SIZE, SHRPX_OPTID_FRONTEND_HTTP3_MAX_WINDOW_SIZE, diff --git a/src/shrpx_connection_handler.cc b/src/shrpx_connection_handler.cc index af4b8fc..b29ce9a 100644 --- a/src/shrpx_connection_handler.cc +++ b/src/shrpx_connection_handler.cc @@ -278,15 +278,14 @@ int ConnectionHandler::create_single_worker() { #endif // ENABLE_HTTP3 && HAVE_LIBBPF #ifdef ENABLE_HTTP3 - assert(cid_prefixes_.size() == 1); - const auto &cid_prefix = cid_prefixes_[0]; + assert(worker_ids_.size() == 1); + const auto &wid = worker_ids_[0]; #endif // ENABLE_HTTP3 single_worker_ = std::make_unique<Worker>( loop_, sv_ssl_ctx, cl_ssl_ctx, session_cache_ssl_ctx, cert_tree_.get(), #ifdef ENABLE_HTTP3 - quic_sv_ssl_ctx, quic_cert_tree_.get(), cid_prefix.data(), - cid_prefix.size(), + quic_sv_ssl_ctx, quic_cert_tree_.get(), wid, # ifdef HAVE_LIBBPF /* index = */ 0, # endif // HAVE_LIBBPF @@ -376,21 +375,20 @@ int ConnectionHandler::create_worker_thread(size_t num) { } # ifdef ENABLE_HTTP3 - assert(cid_prefixes_.size() == num); + assert(worker_ids_.size() == num); # endif // ENABLE_HTTP3 for (size_t i = 0; i < num; ++i) { auto loop = ev_loop_new(config->ev_loop_flags); # ifdef ENABLE_HTTP3 - const auto &cid_prefix = cid_prefixes_[i]; + const auto &wid = worker_ids_[i]; # endif // ENABLE_HTTP3 auto worker = std::make_unique<Worker>( loop, sv_ssl_ctx, cl_ssl_ctx, session_cache_ssl_ctx, cert_tree_.get(), # ifdef ENABLE_HTTP3 - quic_sv_ssl_ctx, quic_cert_tree_.get(), cid_prefix.data(), - cid_prefix.size(), + quic_sv_ssl_ctx, quic_cert_tree_.get(), wid, # ifdef HAVE_LIBBPF i, # endif // HAVE_LIBBPF @@ -1008,27 +1006,23 @@ void ConnectionHandler::set_enable_acceptor_on_ocsp_completion(bool f) { #ifdef ENABLE_HTTP3 int ConnectionHandler::forward_quic_packet( const UpstreamAddr *faddr, const Address &remote_addr, - const Address &local_addr, const ngtcp2_pkt_info &pi, - const uint8_t *cid_prefix, const uint8_t *data, size_t datalen) { + const Address &local_addr, const ngtcp2_pkt_info &pi, const WorkerID &wid, + const uint8_t *data, size_t datalen) { assert(!get_config()->single_thread); - for (auto &worker : workers_) { - if (!std::equal(cid_prefix, cid_prefix + SHRPX_QUIC_CID_PREFIXLEN, - worker->get_cid_prefix())) { - continue; - } - - WorkerEvent wev{}; - wev.type = WorkerEventType::QUIC_PKT_FORWARD; - wev.quic_pkt = std::make_unique<QUICPacket>(faddr->index, remote_addr, - local_addr, pi, data, datalen); + auto worker = find_worker(wid); + if (worker == nullptr) { + return -1; + } - worker->send(std::move(wev)); + WorkerEvent wev{}; + wev.type = WorkerEventType::QUIC_PKT_FORWARD; + wev.quic_pkt = std::make_unique<QUICPacket>(faddr->index, remote_addr, + local_addr, pi, data, datalen); - return 0; - } + worker->send(std::move(wev)); - return -1; + return 0; } void ConnectionHandler::set_quic_keying_materials( @@ -1041,22 +1035,40 @@ ConnectionHandler::get_quic_keying_materials() const { return quic_keying_materials_; } -void ConnectionHandler::set_cid_prefixes( - const std::vector<std::array<uint8_t, SHRPX_QUIC_CID_PREFIXLEN>> - &cid_prefixes) { - cid_prefixes_ = cid_prefixes; +void ConnectionHandler::set_worker_ids(std::vector<WorkerID> worker_ids) { + worker_ids_ = std::move(worker_ids); } -QUICLingeringWorkerProcess * -ConnectionHandler::match_quic_lingering_worker_process_cid_prefix( - const uint8_t *dcid, size_t dcidlen) { - assert(dcidlen >= SHRPX_QUIC_CID_PREFIXLEN); +namespace { +ssize_t find_worker_index(const std::vector<WorkerID> &worker_ids, + const WorkerID &wid) { + assert(!worker_ids.empty()); + if (wid.server != worker_ids[0].server || + wid.worker_process != worker_ids[0].worker_process || + wid.thread >= worker_ids.size()) { + return -1; + } + + return wid.thread; +} +} // namespace + +Worker *ConnectionHandler::find_worker(const WorkerID &wid) const { + auto idx = find_worker_index(worker_ids_, wid); + if (idx == -1) { + return nullptr; + } + + return workers_[idx].get(); +} + +QUICLingeringWorkerProcess * +ConnectionHandler::match_quic_lingering_worker_process_worker_id( + const WorkerID &wid) { for (auto &lwps : quic_lingering_worker_processes_) { - for (auto &cid_prefix : lwps.cid_prefixes) { - if (std::equal(std::begin(cid_prefix), std::end(cid_prefix), dcid)) { - return &lwps; - } + if (find_worker_index(lwps.worker_ids, wid) != -1) { + return &lwps; } } @@ -1275,33 +1287,29 @@ int ConnectionHandler::quic_ipc_read() { auto &qkm = quic_keying_materials_->keying_materials.front(); - std::array<uint8_t, SHRPX_QUIC_DECRYPTED_DCIDLEN> decrypted_dcid; + ConnectionID decrypted_dcid; - if (decrypt_quic_connection_id(decrypted_dcid.data(), - vc.dcid + SHRPX_QUIC_CID_PREFIX_OFFSET, - qkm.cid_encryption_ctx) != 0) { + if (decrypt_quic_connection_id(decrypted_dcid, + vc.dcid + SHRPX_QUIC_CID_WORKER_ID_OFFSET, + qkm.cid_decryption_ctx) != 0) { return -1; } - for (auto &worker : workers_) { - if (!std::equal(std::begin(decrypted_dcid), - std::begin(decrypted_dcid) + SHRPX_QUIC_CID_PREFIXLEN, - worker->get_cid_prefix())) { - continue; + auto worker = find_worker(decrypted_dcid.worker); + if (worker == nullptr) { + if (LOG_ENABLED(INFO)) { + LOG(INFO) << "No worker to match Worker ID"; } - WorkerEvent wev{ - .type = WorkerEventType::QUIC_PKT_FORWARD, - .quic_pkt = std::move(pkt), - }; - worker->send(std::move(wev)); - return 0; } - if (LOG_ENABLED(INFO)) { - LOG(INFO) << "No worker to match CID prefix"; - } + WorkerEvent wev{ + .type = WorkerEventType::QUIC_PKT_FORWARD, + .quic_pkt = std::move(pkt), + }; + + worker->send(std::move(wev)); return 0; } diff --git a/src/shrpx_connection_handler.h b/src/shrpx_connection_handler.h index f3748ab..47ec209 100644 --- a/src/shrpx_connection_handler.h +++ b/src/shrpx_connection_handler.h @@ -108,7 +108,7 @@ struct SerialEvent { struct BPFRef { bpf_object *obj; bpf_map *reuseport_array; - bpf_map *cid_prefix_map; + bpf_map *worker_id_map; }; # endif // HAVE_LIBBPF @@ -121,12 +121,10 @@ enum class QUICIPCType { // WorkerProcesses which are in graceful shutdown period. struct QUICLingeringWorkerProcess { - QUICLingeringWorkerProcess( - std::vector<std::array<uint8_t, SHRPX_QUIC_CID_PREFIXLEN>> cid_prefixes, - int quic_ipc_fd) - : cid_prefixes{std::move(cid_prefixes)}, quic_ipc_fd{quic_ipc_fd} {} + QUICLingeringWorkerProcess(std::vector<WorkerID> worker_ids, int quic_ipc_fd) + : worker_ids{std::move(worker_ids)}, quic_ipc_fd{quic_ipc_fd} {} - std::vector<std::array<uint8_t, SHRPX_QUIC_CID_PREFIXLEN>> cid_prefixes; + std::vector<WorkerID> worker_ids; // Socket to send QUIC IPC message to this worker process. int quic_ipc_fd; }; @@ -197,25 +195,23 @@ public: int forward_quic_packet(const UpstreamAddr *faddr, const Address &remote_addr, const Address &local_addr, const ngtcp2_pkt_info &pi, - const uint8_t *cid_prefix, const uint8_t *data, + const WorkerID &wid, const uint8_t *data, size_t datalen); void set_quic_keying_materials(std::shared_ptr<QUICKeyingMaterials> qkms); const std::shared_ptr<QUICKeyingMaterials> &get_quic_keying_materials() const; - void set_cid_prefixes( - const std::vector<std::array<uint8_t, SHRPX_QUIC_CID_PREFIXLEN>> - &cid_prefixes); + void set_worker_ids(std::vector<WorkerID> worker_ids); + Worker *find_worker(const WorkerID &wid) const; void set_quic_lingering_worker_processes( const std::vector<QUICLingeringWorkerProcess> &quic_lwps); - // Return matching QUICLingeringWorkerProcess which has a CID prefix + // Return matching QUICLingeringWorkerProcess which has a Worker ID // such that |dcid| starts with it. If no such // QUICLingeringWorkerProcess, it returns nullptr. QUICLingeringWorkerProcess * - match_quic_lingering_worker_process_cid_prefix(const uint8_t *dcid, - size_t dcidlen); + match_quic_lingering_worker_process_worker_id(const WorkerID &wid); int forward_quic_packet_to_lingering_worker_process( QUICLingeringWorkerProcess *quic_lwp, const Address &remote_addr, @@ -260,9 +256,8 @@ private: // and signature algorithm presented by client. std::vector<std::vector<SSL_CTX *>> indexed_ssl_ctx_; #ifdef ENABLE_HTTP3 - std::vector<std::array<uint8_t, SHRPX_QUIC_CID_PREFIXLEN>> cid_prefixes_; - std::vector<std::array<uint8_t, SHRPX_QUIC_CID_PREFIXLEN>> - lingering_cid_prefixes_; + std::vector<WorkerID> worker_ids_; + std::vector<WorkerID> lingering_worker_ids_; int quic_ipc_fd_; std::vector<QUICLingeringWorkerProcess> quic_lingering_worker_processes_; # ifdef HAVE_LIBBPF diff --git a/src/shrpx_dns_resolver.cc b/src/shrpx_dns_resolver.cc index f83ecb7..8253942 100644 --- a/src/shrpx_dns_resolver.cc +++ b/src/shrpx_dns_resolver.cc @@ -55,9 +55,11 @@ void sock_state_cb(void *data, int s, int read, int write) { } // namespace namespace { -void host_cb(void *arg, int status, int timeouts, hostent *hostent) { +void addrinfo_cb(void *arg, int status, int timeouts, ares_addrinfo *result) { auto resolv = static_cast<DNSResolver *>(arg); - resolv->on_result(status, hostent); + resolv->on_result(status, result); + + ares_freeaddrinfo(result); } } // namespace @@ -173,7 +175,10 @@ int DNSResolver::resolve(const StringRef &name, int family) { channel_ = chan; status_ = DNSResolverStatus::RUNNING; - ares_gethostbyname(channel_, name_.c_str(), family_, host_cb, this); + ares_addrinfo_hints hints{}; + hints.ai_family = family_; + + ares_getaddrinfo(channel_, name_.c_str(), nullptr, &hints, addrinfo_cb, this); reset_timeout(); return 0; @@ -285,7 +290,7 @@ void DNSResolver::start_wev(int fd) { void DNSResolver::stop_wev(int fd) { stop_ev(wevs_, loop_, fd, EV_WRITE); } -void DNSResolver::on_result(int status, hostent *hostent) { +void DNSResolver::on_result(int status, ares_addrinfo *ai) { stop_ev(loop_, revs_); stop_ev(loop_, wevs_); ev_timer_stop(loop_, &timer_); @@ -299,40 +304,44 @@ void DNSResolver::on_result(int status, hostent *hostent) { return; } - auto ap = *hostent->h_addr_list; + auto ap = ai->nodes; + + for (; ap; ap = ap->ai_next) { + switch (ap->ai_family) { + case AF_INET: + status_ = DNSResolverStatus::OK; + result_.len = sizeof(result_.su.in); + + assert(sizeof(result_.su.in) == ap->ai_addrlen); + + memcpy(&result_.su.in, ap->ai_addr, sizeof(result_.su.in)); + + break; + case AF_INET6: + status_ = DNSResolverStatus::OK; + result_.len = sizeof(result_.su.in6); + + assert(sizeof(result_.su.in6) == ap->ai_addrlen); + + memcpy(&result_.su.in6, ap->ai_addr, sizeof(result_.su.in6)); + + break; + default: + continue; + } + + break; + } + if (!ap) { if (LOG_ENABLED(INFO)) { - LOG(INFO) << "Name lookup for " << name_ << "failed: no address returned"; + LOG(INFO) << "Name lookup for " << name_ + << " failed: no address returned"; } status_ = DNSResolverStatus::ERROR; return; } - switch (hostent->h_addrtype) { - case AF_INET: - status_ = DNSResolverStatus::OK; - result_.len = sizeof(result_.su.in); - result_.su.in = {}; - result_.su.in.sin_family = AF_INET; -#ifdef HAVE_SOCKADDR_IN_SIN_LEN - result_.su.in.sin_len = sizeof(result_.su.in); -#endif // HAVE_SOCKADDR_IN_SIN_LEN - memcpy(&result_.su.in.sin_addr, ap, sizeof(result_.su.in.sin_addr)); - break; - case AF_INET6: - status_ = DNSResolverStatus::OK; - result_.len = sizeof(result_.su.in6); - result_.su.in6 = {}; - result_.su.in6.sin6_family = AF_INET6; -#ifdef HAVE_SOCKADDR_IN6_SIN6_LEN - result_.su.in6.sin6_len = sizeof(result_.su.in6); -#endif // HAVE_SOCKADDR_IN6_SIN6_LEN - memcpy(&result_.su.in6.sin6_addr, ap, sizeof(result_.su.in6.sin6_addr)); - break; - default: - assert(0); - } - if (status_ == DNSResolverStatus::OK) { if (LOG_ENABLED(INFO)) { LOG(INFO) << "Name lookup succeeded: " << name_ << " -> " diff --git a/src/shrpx_dns_resolver.h b/src/shrpx_dns_resolver.h index e622f99..4d68273 100644 --- a/src/shrpx_dns_resolver.h +++ b/src/shrpx_dns_resolver.h @@ -88,7 +88,7 @@ public: int on_write(int fd); int on_timeout(); // Calls this function when DNS query finished. - void on_result(int status, hostent *hostent); + void on_result(int status, ares_addrinfo *result); void reset_timeout(); void start_rev(int fd); diff --git a/src/shrpx_downstream.cc b/src/shrpx_downstream.cc index 9ea52b4..5fd717e 100644 --- a/src/shrpx_downstream.cc +++ b/src/shrpx_downstream.cc @@ -46,6 +46,23 @@ namespace shrpx { namespace { +void header_timeoutcb(struct ev_loop *loop, ev_timer *w, int revents) { + auto downstream = static_cast<Downstream *>(w->data); + auto upstream = downstream->get_upstream(); + + if (LOG_ENABLED(INFO)) { + DLOG(INFO, downstream) << "request header timeout stream_id=" + << downstream->get_stream_id(); + } + + downstream->disable_upstream_rtimer(); + downstream->disable_upstream_wtimer(); + + upstream->on_timeout(downstream); +} +} // namespace + +namespace { void upstream_timeoutcb(struct ev_loop *loop, ev_timer *w, int revents) { auto downstream = static_cast<Downstream *>(w->data); auto upstream = downstream->get_upstream(); @@ -148,7 +165,12 @@ Downstream::Downstream(Upstream *upstream, MemchunkPool *mcpool, expect_100_continue_(false), stop_reading_(false) { - auto &timeoutconf = get_config()->http2.timeout; + auto config = get_config(); + auto &httpconf = config->http; + + ev_timer_init(&header_timer_, header_timeoutcb, 0., httpconf.timeout.header); + + auto &timeoutconf = config->http2.timeout; ev_timer_init(&upstream_rtimer_, &upstream_rtimeoutcb, 0., timeoutconf.stream_read); @@ -159,6 +181,7 @@ Downstream::Downstream(Upstream *upstream, MemchunkPool *mcpool, ev_timer_init(&downstream_wtimer_, &downstream_wtimeoutcb, 0., timeoutconf.stream_write); + header_timer_.data = this; upstream_rtimer_.data = this; upstream_wtimer_.data = this; downstream_rtimer_.data = this; @@ -183,6 +206,7 @@ Downstream::~Downstream() { ev_timer_stop(loop, &upstream_wtimer_); ev_timer_stop(loop, &downstream_rtimer_); ev_timer_stop(loop, &downstream_wtimer_); + ev_timer_stop(loop, &header_timer_); #ifdef HAVE_MRUBY auto handler = upstream_->get_client_handler(); @@ -946,6 +970,18 @@ bool Downstream::expect_response_trailer() const { (resp_.http_major == 3 || resp_.http_major == 2); } +void Downstream::repeat_header_timer() { + auto loop = upstream_->get_client_handler()->get_loop(); + + ev_timer_again(loop, &header_timer_); +} + +void Downstream::stop_header_timer() { + auto loop = upstream_->get_client_handler()->get_loop(); + + ev_timer_stop(loop, &header_timer_); +} + namespace { void reset_timer(struct ev_loop *loop, ev_timer *w) { ev_timer_again(loop, w); } } // namespace diff --git a/src/shrpx_downstream.h b/src/shrpx_downstream.h index 146cae5..15f3a47 100644 --- a/src/shrpx_downstream.h +++ b/src/shrpx_downstream.h @@ -448,6 +448,9 @@ public: // connection. int on_read(); + void repeat_header_timer(); + void stop_header_timer(); + // Resets upstream read timer. If it is active, timeout value is // reset. If it is not active, timer will be started. void reset_upstream_rtimer(); @@ -562,6 +565,8 @@ private: // if frontend uses RFC 8441 WebSocket bootstrapping via HTTP/2. StringRef ws_key_; + ev_timer header_timer_; + ev_timer upstream_rtimer_; ev_timer upstream_wtimer_; diff --git a/src/shrpx_http2_upstream.cc b/src/shrpx_http2_upstream.cc index 2cb5436..7816f5f 100644 --- a/src/shrpx_http2_upstream.cc +++ b/src/shrpx_http2_upstream.cc @@ -285,7 +285,10 @@ void Http2Upstream::on_start_request(const nghttp2_frame *frame) { downstream->reset_upstream_rtimer(); - handler_->repeat_read_timer(); + auto config = get_config(); + auto &httpconf = config->http; + + handler_->reset_upstream_read_timeout(httpconf.timeout.header); auto &req = downstream->request(); @@ -298,8 +301,6 @@ void Http2Upstream::on_start_request(const nghttp2_frame *frame) { ++num_requests_; - auto config = get_config(); - auto &httpconf = config->http; if (httpconf.max_requests <= num_requests_) { start_graceful_shutdown(); } @@ -1132,7 +1133,7 @@ Http2Upstream::Http2Upstream(ClientHandler *handler) #endif // defined(TCP_INFO) && defined(TCP_NOTSENT_LOWAT) handler_->reset_upstream_read_timeout( - config->conn.upstream.timeout.http2_read); + config->conn.upstream.timeout.http2_idle); handler_->signal_write(); } @@ -1640,7 +1641,10 @@ void Http2Upstream::remove_downstream(Downstream *downstream) { if (downstream_queue_.get_downstreams() == nullptr) { // There is no downstream at the moment. Start idle timer now. - handler_->repeat_read_timer(); + auto config = get_config(); + auto &upstreamconf = config->conn.upstream; + + handler_->reset_upstream_read_timeout(upstreamconf.timeout.http2_idle); } } diff --git a/src/shrpx_http3_upstream.cc b/src/shrpx_http3_upstream.cc index b8667a3..d12d2da 100644 --- a/src/shrpx_http3_upstream.cc +++ b/src/shrpx_http3_upstream.cc @@ -118,7 +118,6 @@ Http3Upstream::Http3Upstream(ClientHandler *handler) httpconn_{nullptr}, downstream_queue_{downstream_queue_size(handler->get_worker()), !get_config()->http2_proxy}, - retry_close_{false}, tx_{ .data = std::unique_ptr<uint8_t[]>(new uint8_t[64_k]), } { @@ -212,8 +211,10 @@ int get_new_connection_id(ngtcp2_conn *conn, ngtcp2_cid *cid, uint8_t *token, auto &qkms = conn_handler->get_quic_keying_materials(); auto &qkm = qkms->keying_materials.front(); - if (generate_quic_connection_id(*cid, cidlen, worker->get_cid_prefix(), - qkm.id, qkm.cid_encryption_ctx) != 0) { + assert(SHRPX_QUIC_SCIDLEN == cidlen); + + if (generate_quic_connection_id(*cid, worker->get_worker_id(), qkm.id, + qkm.cid_encryption_ctx) != 0) { return NGTCP2_ERR_CALLBACK_FAILURE; } @@ -250,8 +251,9 @@ void Http3Upstream::http_begin_request_headers(int64_t stream_id) { nghttp3_conn_set_stream_user_data(httpconn_, stream_id, downstream.get()); downstream->reset_upstream_rtimer(); + downstream->repeat_header_timer(); - handler_->repeat_read_timer(); + handler_->stop_read_timer(); auto &req = downstream->request(); req.http_major = 3; @@ -609,8 +611,7 @@ int Http3Upstream::init(const UpstreamAddr *faddr, const Address &remote_addr, ngtcp2_cid scid; - if (generate_quic_connection_id(scid, SHRPX_QUIC_SCIDLEN, - worker->get_cid_prefix(), qkm.id, + if (generate_quic_connection_id(scid, worker->get_worker_id(), qkm.id, qkm.cid_encryption_ctx) != 0) { return -1; } @@ -997,7 +998,18 @@ int Http3Upstream::write_streams() { return 0; } -int Http3Upstream::on_timeout(Downstream *downstream) { return 0; } +int Http3Upstream::on_timeout(Downstream *downstream) { + if (LOG_ENABLED(INFO)) { + ULOG(INFO, this) << "Stream timeout stream_id=" + << downstream->get_stream_id(); + } + + shutdown_stream(downstream, NGHTTP3_H3_INTERNAL_ERROR); + + handler_->signal_write(); + + return 0; +} int Http3Upstream::on_downstream_abort_request(Downstream *downstream, unsigned int status_code) { @@ -1528,8 +1540,13 @@ void Http3Upstream::on_handler_delete() { quic_conn_handler->remove_connection_id(cid); } - if (retry_close_ || last_error_.type == NGTCP2_CCERR_TYPE_IDLE_CLOSE) { + switch (last_error_.type) { + case NGTCP2_CCERR_TYPE_IDLE_CLOSE: + case NGTCP2_CCERR_TYPE_DROP_CONN: + case NGTCP2_CCERR_TYPE_RETRY: return; + default: + break; } // If this is not idle close, send CONNECTION_CLOSE. @@ -1823,7 +1840,8 @@ int Http3Upstream::on_read(const UpstreamAddr *faddr, return -1; } - retry_close_ = true; + // Overwrite error if any is set + ngtcp2_ccerr_set_liberr(&last_error_, rv, nullptr, 0); quic_conn_handler->send_retry(handler_->get_upstream_addr(), vc.version, vc.dcid, vc.dcidlen, vc.scid, vc.scidlen, @@ -1838,6 +1856,9 @@ int Http3Upstream::on_read(const UpstreamAddr *faddr, } break; case NGTCP2_ERR_DROP_CONN: + // Overwrite error if any is set + ngtcp2_ccerr_set_liberr(&last_error_, rv, nullptr, 0); + return -1; default: if (!last_error_.error_code) { @@ -2149,6 +2170,11 @@ int Http3Upstream::http_recv_request_header(Downstream *downstream, // just ignore if this is a trailer part. if (trailer) { + if (shutdown_stream_read(downstream->get_stream_id(), + NGHTTP3_H3_NO_ERROR) != 0) { + return -1; + } + return 0; } @@ -2182,7 +2208,6 @@ namespace { int http_end_request_headers(nghttp3_conn *conn, int64_t stream_id, int fin, void *user_data, void *stream_user_data) { auto upstream = static_cast<Http3Upstream *>(user_data); - auto handler = upstream->get_client_handler(); auto downstream = static_cast<Downstream *>(stream_user_data); if (!downstream || downstream->get_stop_reading()) { @@ -2194,7 +2219,7 @@ int http_end_request_headers(nghttp3_conn *conn, int64_t stream_id, int fin, } downstream->reset_upstream_rtimer(); - handler->stop_read_timer(); + downstream->stop_header_timer(); return 0; } diff --git a/src/shrpx_http3_upstream.h b/src/shrpx_http3_upstream.h index 89dfc17..53c73ae 100644 --- a/src/shrpx_http3_upstream.h +++ b/src/shrpx_http3_upstream.h @@ -167,7 +167,6 @@ private: ngtcp2_ccerr last_error_; nghttp3_conn *httpconn_; DownstreamQueue downstream_queue_; - bool retry_close_; std::vector<uint8_t> conn_close_; struct { diff --git a/src/shrpx_https_upstream.cc b/src/shrpx_https_upstream.cc index 49d2088..0412384 100644 --- a/src/shrpx_https_upstream.cc +++ b/src/shrpx_https_upstream.cc @@ -115,12 +115,9 @@ void HttpsUpstream::on_start_request() { attach_downstream(std::move(downstream)); - auto conn = handler_->get_connection(); - auto &upstreamconf = get_config()->conn.upstream; - - conn->rt.repeat = upstreamconf.timeout.read; + auto &httpconf = get_config()->http; - handler_->repeat_read_timer(); + handler_->reset_upstream_read_timeout(httpconf.timeout.header); ++num_requests_; } @@ -795,12 +792,9 @@ int HttpsUpstream::on_write() { return 0; } - auto conn = handler_->get_connection(); auto &upstreamconf = get_config()->conn.upstream; - conn->rt.repeat = upstreamconf.timeout.idle_read; - - handler_->repeat_read_timer(); + handler_->reset_upstream_read_timeout(upstreamconf.timeout.idle); return resume_read(SHRPX_NO_BUFFER, nullptr, 0); } else { diff --git a/src/shrpx_quic.cc b/src/shrpx_quic.cc index a6d4dfa..c52eee4 100644 --- a/src/shrpx_quic.cc +++ b/src/shrpx_quic.cc @@ -173,42 +173,34 @@ int quic_send_packet(const UpstreamAddr *faddr, const sockaddr *remote_sa, return 0; } -int generate_quic_retry_connection_id(ngtcp2_cid &cid, size_t cidlen, - const uint8_t *server_id, uint8_t km_id, - EVP_CIPHER_CTX *ctx) { - assert(cidlen == SHRPX_QUIC_SCIDLEN); - - if (RAND_bytes(cid.data, cidlen) != 1) { +int generate_quic_retry_connection_id(ngtcp2_cid &cid, uint32_t server_id, + uint8_t km_id, EVP_CIPHER_CTX *ctx) { + if (RAND_bytes(cid.data, SHRPX_QUIC_SCIDLEN) != 1) { return -1; } - cid.datalen = cidlen; - - cid.data[0] = (cid.data[0] & 0x3f) | km_id; + cid.datalen = SHRPX_QUIC_SCIDLEN; + cid.data[0] = (cid.data[0] & (~SHRPX_QUIC_DCID_KM_ID_MASK)) | km_id; - auto p = cid.data + SHRPX_QUIC_CID_PREFIX_OFFSET; + auto p = cid.data + SHRPX_QUIC_CID_WORKER_ID_OFFSET; - std::copy_n(server_id, SHRPX_QUIC_SERVER_IDLEN, p); + std::copy_n(reinterpret_cast<uint8_t *>(&server_id), sizeof(server_id), p); return encrypt_quic_connection_id(p, p, ctx); } -int generate_quic_connection_id(ngtcp2_cid &cid, size_t cidlen, - const uint8_t *cid_prefix, uint8_t km_id, - EVP_CIPHER_CTX *ctx) { - assert(cidlen == SHRPX_QUIC_SCIDLEN); - - if (RAND_bytes(cid.data, cidlen) != 1) { +int generate_quic_connection_id(ngtcp2_cid &cid, const WorkerID &wid, + uint8_t km_id, EVP_CIPHER_CTX *ctx) { + if (RAND_bytes(cid.data, SHRPX_QUIC_SCIDLEN) != 1) { return -1; } - cid.datalen = cidlen; + cid.datalen = SHRPX_QUIC_SCIDLEN; + cid.data[0] = (cid.data[0] & (~SHRPX_QUIC_DCID_KM_ID_MASK)) | km_id; - cid.data[0] = (cid.data[0] & 0x3f) | km_id; + auto p = cid.data + SHRPX_QUIC_CID_WORKER_ID_OFFSET; - auto p = cid.data + SHRPX_QUIC_CID_PREFIX_OFFSET; - - std::copy_n(cid_prefix, SHRPX_QUIC_CID_PREFIXLEN, p); + std::copy_n(reinterpret_cast<const uint8_t *>(&wid), sizeof(wid), p); return encrypt_quic_connection_id(p, p, ctx); } @@ -225,12 +217,13 @@ int encrypt_quic_connection_id(uint8_t *dest, const uint8_t *src, return 0; } -int decrypt_quic_connection_id(uint8_t *dest, const uint8_t *src, +int decrypt_quic_connection_id(ConnectionID &dest, const uint8_t *src, EVP_CIPHER_CTX *ctx) { int len; + auto p = reinterpret_cast<uint8_t *>(&dest); - if (!EVP_EncryptUpdate(ctx, dest, &len, src, SHRPX_QUIC_DECRYPTED_DCIDLEN) || - !EVP_EncryptFinal_ex(ctx, dest + len, &len)) { + if (!EVP_DecryptUpdate(ctx, p, &len, src, SHRPX_QUIC_DECRYPTED_DCIDLEN) || + !EVP_DecryptFinal_ex(ctx, p + len, &len)) { return -1; } diff --git a/src/shrpx_quic.h b/src/shrpx_quic.h index 88388e9..dae6e31 100644 --- a/src/shrpx_quic.h +++ b/src/shrpx_quic.h @@ -65,19 +65,50 @@ struct UpstreamAddr; struct QUICKeyingMaterials; struct QUICKeyingMaterial; -constexpr size_t SHRPX_QUIC_SCIDLEN = 20; +constexpr size_t SHRPX_QUIC_CID_WORKER_ID_OFFSET = 1; constexpr size_t SHRPX_QUIC_SERVER_IDLEN = 4; -// SHRPX_QUIC_CID_PREFIXLEN includes SHRPX_QUIC_SERVER_IDLEN. -constexpr size_t SHRPX_QUIC_CID_PREFIXLEN = 8; -constexpr size_t SHRPX_QUIC_CID_PREFIX_OFFSET = 1; -constexpr size_t SHRPX_QUIC_DECRYPTED_DCIDLEN = 16; +constexpr size_t SHRPX_QUIC_SOCK_IDLEN = 4; +constexpr size_t SHRPX_QUIC_WORKER_IDLEN = + SHRPX_QUIC_SERVER_IDLEN + SHRPX_QUIC_SOCK_IDLEN; +constexpr size_t SHRPX_QUIC_CLIENT_IDLEN = 8; +constexpr size_t SHRPX_QUIC_DECRYPTED_DCIDLEN = + SHRPX_QUIC_WORKER_IDLEN + SHRPX_QUIC_CLIENT_IDLEN; +constexpr size_t SHRPX_QUIC_SCIDLEN = + SHRPX_QUIC_CID_WORKER_ID_OFFSET + SHRPX_QUIC_DECRYPTED_DCIDLEN; constexpr size_t SHRPX_QUIC_CID_ENCRYPTION_KEYLEN = 16; constexpr size_t SHRPX_QUIC_CONN_CLOSE_PKTLEN = 256; constexpr size_t SHRPX_QUIC_STATELESS_RESET_BURST = 100; constexpr size_t SHRPX_QUIC_SECRET_RESERVEDLEN = 4; constexpr size_t SHRPX_QUIC_SECRETLEN = 32; constexpr size_t SHRPX_QUIC_SALTLEN = 32; -constexpr uint8_t SHRPX_QUIC_DCID_KM_ID_MASK = 0xc0; +constexpr uint8_t SHRPX_QUIC_DCID_KM_ID_MASK = 0xe0; + +struct WorkerID { + union { + struct { + uint32_t server; + uint16_t worker_process; + uint16_t thread; + }; + uint64_t worker; + }; +}; + +static_assert(sizeof(WorkerID) == SHRPX_QUIC_WORKER_IDLEN, + "WorkerID length assertion failure"); + +inline bool operator==(const WorkerID &lhd, const WorkerID &rhd) { + return lhd.worker == rhd.worker; +} + +inline bool operator!=(const WorkerID &lhd, const WorkerID &rhd) { + return lhd.worker != rhd.worker; +} + +struct ConnectionID { + WorkerID worker; + uint64_t client; +}; ngtcp2_tstamp quic_timestamp(); @@ -86,18 +117,16 @@ int quic_send_packet(const UpstreamAddr *faddr, const sockaddr *remote_sa, size_t local_salen, const ngtcp2_pkt_info &pi, const uint8_t *data, size_t datalen, size_t gso_size); -int generate_quic_retry_connection_id(ngtcp2_cid &cid, size_t cidlen, - const uint8_t *server_id, uint8_t km_id, - EVP_CIPHER_CTX *ctx); +int generate_quic_retry_connection_id(ngtcp2_cid &cid, uint32_t server_id, + uint8_t km_id, EVP_CIPHER_CTX *ctx); -int generate_quic_connection_id(ngtcp2_cid &cid, size_t cidlen, - const uint8_t *cid_prefix, uint8_t km_id, - EVP_CIPHER_CTX *ctx); +int generate_quic_connection_id(ngtcp2_cid &cid, const WorkerID &wid, + uint8_t km_id, EVP_CIPHER_CTX *ctx); int encrypt_quic_connection_id(uint8_t *dest, const uint8_t *src, EVP_CIPHER_CTX *ctx); -int decrypt_quic_connection_id(uint8_t *dest, const uint8_t *src, +int decrypt_quic_connection_id(ConnectionID &dest, const uint8_t *src, EVP_CIPHER_CTX *ctx); int generate_quic_hashed_connection_id(ngtcp2_cid &dest, diff --git a/src/shrpx_quic_connection_handler.cc b/src/shrpx_quic_connection_handler.cc index 13f710b..b810aa6 100644 --- a/src/shrpx_quic_connection_handler.cc +++ b/src/shrpx_quic_connection_handler.cc @@ -123,7 +123,7 @@ int QUICConnectionHandler::handle_packet(const UpstreamAddr *faddr, } if (it == std::end(connections_)) { - std::array<uint8_t, SHRPX_QUIC_DECRYPTED_DCIDLEN> decrypted_dcid; + ConnectionID decrypted_dcid; auto &qkms = conn_handler->get_quic_keying_materials(); const QUICKeyingMaterial *qkm = nullptr; @@ -132,19 +132,17 @@ int QUICConnectionHandler::handle_packet(const UpstreamAddr *faddr, qkm = select_quic_keying_material( *qkms.get(), vc.dcid[0] & SHRPX_QUIC_DCID_KM_ID_MASK); - if (decrypt_quic_connection_id(decrypted_dcid.data(), - vc.dcid + SHRPX_QUIC_CID_PREFIX_OFFSET, - qkm->cid_encryption_ctx) != 0) { + if (decrypt_quic_connection_id(decrypted_dcid, + vc.dcid + SHRPX_QUIC_CID_WORKER_ID_OFFSET, + qkm->cid_decryption_ctx) != 0) { return 0; } if (qkm != &qkms->keying_materials.front() || - !std::equal(std::begin(decrypted_dcid), - std::begin(decrypted_dcid) + SHRPX_QUIC_CID_PREFIXLEN, - worker_->get_cid_prefix())) { + decrypted_dcid.worker != worker_->get_worker_id()) { auto quic_lwp = - conn_handler->match_quic_lingering_worker_process_cid_prefix( - decrypted_dcid.data(), decrypted_dcid.size()); + conn_handler->match_quic_lingering_worker_process_worker_id( + decrypted_dcid.worker); if (quic_lwp) { if (conn_handler->forward_quic_packet_to_lingering_worker_process( quic_lwp, remote_addr, local_addr, pi, data, datalen) == 0) { @@ -177,23 +175,21 @@ int QUICConnectionHandler::handle_packet(const UpstreamAddr *faddr, switch (ngtcp2_accept(&hd, data, datalen)) { case 0: { - // If we get Initial and it has the CID prefix of this worker, - // it is likely that client is intentionally use the prefix. - // Just drop it. + // If we get Initial and it has the Worker ID of this worker, it + // is likely that client is intentionally use the prefix. Just + // drop it. if (vc.dcidlen == SHRPX_QUIC_SCIDLEN) { if (qkm != &qkms->keying_materials.front()) { qkm = &qkms->keying_materials.front(); - if (decrypt_quic_connection_id(decrypted_dcid.data(), - vc.dcid + SHRPX_QUIC_CID_PREFIX_OFFSET, - qkm->cid_encryption_ctx) != 0) { + if (decrypt_quic_connection_id( + decrypted_dcid, vc.dcid + SHRPX_QUIC_CID_WORKER_ID_OFFSET, + qkm->cid_decryption_ctx) != 0) { return 0; } } - if (std::equal(std::begin(decrypted_dcid), - std::begin(decrypted_dcid) + SHRPX_QUIC_CID_PREFIXLEN, - worker_->get_cid_prefix())) { + if (decrypted_dcid.worker == worker_->get_worker_id()) { return 0; } } @@ -324,22 +320,19 @@ int QUICConnectionHandler::handle_packet(const UpstreamAddr *faddr, break; } default: - if (!config->single_thread && !(data[0] & 0x80) && - vc.dcidlen == SHRPX_QUIC_SCIDLEN && - !std::equal(std::begin(decrypted_dcid), - std::begin(decrypted_dcid) + SHRPX_QUIC_CID_PREFIXLEN, - worker_->get_cid_prefix())) { - if (conn_handler->forward_quic_packet(faddr, remote_addr, local_addr, - pi, decrypted_dcid.data(), data, + if (!(data[0] & 0x80) && vc.dcidlen == SHRPX_QUIC_SCIDLEN && + decrypted_dcid.worker != worker_->get_worker_id()) { + if (!config->single_thread && + conn_handler->forward_quic_packet(faddr, remote_addr, local_addr, + pi, decrypted_dcid.worker, data, datalen) == 0) { return 0; } - } - if (!(data[0] & 0x80)) { - // TODO Must be rate limited - send_stateless_reset(faddr, vc.dcid, vc.dcidlen, remote_addr, - local_addr); + if (datalen >= SHRPX_QUIC_SCIDLEN + 22) { + send_stateless_reset(faddr, datalen, vc.dcid, vc.dcidlen, remote_addr, + local_addr); + } } return 0; @@ -478,8 +471,7 @@ int QUICConnectionHandler::send_retry( ngtcp2_cid retry_scid; - if (generate_quic_retry_connection_id(retry_scid, SHRPX_QUIC_SCIDLEN, - quicconf.server_id.data(), qkm.id, + if (generate_quic_retry_connection_id(retry_scid, quicconf.server_id, qkm.id, qkm.cid_encryption_ctx) != 0) { return -1; } @@ -563,11 +555,9 @@ int QUICConnectionHandler::send_version_negotiation( buf.data(), nwrite, 0); } -int QUICConnectionHandler::send_stateless_reset(const UpstreamAddr *faddr, - const uint8_t *dcid, - size_t dcidlen, - const Address &remote_addr, - const Address &local_addr) { +int QUICConnectionHandler::send_stateless_reset( + const UpstreamAddr *faddr, size_t pktlen, const uint8_t *dcid, + size_t dcidlen, const Address &remote_addr, const Address &local_addr) { if (stateless_reset_bucket_ == 0) { if (LOG_ENABLED(INFO)) { LOG(INFO) << "Stateless Reset bucket has been depleted"; @@ -598,17 +588,30 @@ int QUICConnectionHandler::send_stateless_reset(const UpstreamAddr *faddr, return -1; } - std::array<uint8_t, NGTCP2_MIN_STATELESS_RESET_RANDLEN> rand_bytes; + // SCID + minimum expansion - NGTCP2_STATELESS_RESET_TOKENLEN + constexpr size_t max_rand_byteslen = + SHRPX_QUIC_SCIDLEN + 22 - NGTCP2_STATELESS_RESET_TOKENLEN; + + size_t rand_byteslen; + + if (pktlen <= 43) { + // As per + // https://datatracker.ietf.org/doc/html/rfc9000#section-10.3 + rand_byteslen = pktlen - NGTCP2_STATELESS_RESET_TOKENLEN - 1; + } else { + rand_byteslen = max_rand_byteslen; + } + + std::array<uint8_t, max_rand_byteslen> rand_bytes; - if (RAND_bytes(rand_bytes.data(), rand_bytes.size()) != 1) { + if (RAND_bytes(rand_bytes.data(), rand_byteslen) != 1) { return -1; } std::array<uint8_t, NGTCP2_MAX_UDP_PAYLOAD_SIZE> buf; - auto nwrite = - ngtcp2_pkt_write_stateless_reset(buf.data(), buf.size(), token.data(), - rand_bytes.data(), rand_bytes.size()); + auto nwrite = ngtcp2_pkt_write_stateless_reset( + buf.data(), buf.size(), token.data(), rand_bytes.data(), rand_byteslen); if (nwrite < 0) { LOG(ERROR) << "ngtcp2_pkt_write_stateless_reset: " << ngtcp2_strerror(nwrite); diff --git a/src/shrpx_quic_connection_handler.h b/src/shrpx_quic_connection_handler.h index 29e73a4..7f65370 100644 --- a/src/shrpx_quic_connection_handler.h +++ b/src/shrpx_quic_connection_handler.h @@ -103,8 +103,9 @@ public: const uint8_t *ini_scid, size_t ini_scidlen, const Address &remote_addr, const Address &local_addr); - int send_stateless_reset(const UpstreamAddr *faddr, const uint8_t *dcid, - size_t dcidlen, const Address &remote_addr, + int send_stateless_reset(const UpstreamAddr *faddr, size_t pktlen, + const uint8_t *dcid, size_t dcidlen, + const Address &remote_addr, const Address &local_addr); // Send Initial CONNECTION_CLOSE. |ini_dcid| is the destination // Connection ID which appeared in Client Initial packet. diff --git a/src/shrpx_quic_listener.cc b/src/shrpx_quic_listener.cc index 9b9f120..681f605 100644 --- a/src/shrpx_quic_listener.cc +++ b/src/shrpx_quic_listener.cc @@ -74,6 +74,19 @@ void QUICListener::on_read() { return; } + // Packets less than 22 bytes never be a valid QUIC packet. + if (nread < 22) { + ++pktcnt; + + continue; + } + + if (util::quic_prohibited_port(util::get_port(&su))) { + ++pktcnt; + + continue; + } + Address local_addr{}; if (util::msghdr_get_local_addr(local_addr, &msg, su.storage.ss_family) != 0) { @@ -108,7 +121,8 @@ void QUICListener::on_read() { << " bytes"; } - if (datalen == 0) { + // Packets less than 22 bytes never be a valid QUIC packet. + if (datalen < 22) { break; } diff --git a/src/shrpx_worker.cc b/src/shrpx_worker.cc index e7d6740..9f5911f 100644 --- a/src/shrpx_worker.cc +++ b/src/shrpx_worker.cc @@ -148,7 +148,7 @@ Worker::Worker(struct ev_loop *loop, SSL_CTX *sv_ssl_ctx, SSL_CTX *cl_ssl_ctx, tls::CertLookupTree *cert_tree, #ifdef ENABLE_HTTP3 SSL_CTX *quic_sv_ssl_ctx, tls::CertLookupTree *quic_cert_tree, - const uint8_t *cid_prefix, size_t cid_prefixlen, + WorkerID wid, # ifdef HAVE_LIBBPF size_t index, # endif // HAVE_LIBBPF @@ -164,6 +164,7 @@ Worker::Worker(struct ev_loop *loop, SSL_CTX *sv_ssl_ctx, SSL_CTX *cl_ssl_ctx, worker_stat_{}, dns_tracker_(loop, get_config()->conn.downstream->family), #ifdef ENABLE_HTTP3 + worker_id_{std::move(wid)}, quic_upstream_addrs_{get_config()->conn.quic_listener.addrs}, #endif // ENABLE_HTTP3 loop_(loop), @@ -180,10 +181,6 @@ Worker::Worker(struct ev_loop *loop, SSL_CTX *sv_ssl_ctx, SSL_CTX *cl_ssl_ctx, connect_blocker_( std::make_unique<ConnectBlocker>(randgen_, loop_, nullptr, nullptr)), graceful_shutdown_(false) { -#ifdef ENABLE_HTTP3 - std::copy_n(cid_prefix, cid_prefixlen, std::begin(cid_prefix_)); -#endif // ENABLE_HTTP3 - ev_async_init(&w_, eventcb); w_.data = this; ev_async_start(loop_, &w_); @@ -1071,10 +1068,10 @@ int Worker::create_quic_server_socket(UpstreamAddr &faddr) { return -1; } - ref.cid_prefix_map = bpf_object__find_map_by_name(obj, "cid_prefix_map"); - if (!ref.cid_prefix_map) { + ref.worker_id_map = bpf_object__find_map_by_name(obj, "worker_id_map"); + if (!ref.worker_id_map) { auto error = errno; - LOG(FATAL) << "Failed to get cid_prefix_map: " + LOG(FATAL) << "Failed to get worker_id_map: " << xsi_strerror(error, errbuf.data(), errbuf.size()); close(fd); return -1; @@ -1155,12 +1152,12 @@ int Worker::create_quic_server_socket(UpstreamAddr &faddr) { return -1; } - rv = bpf_map__update_elem(ref.cid_prefix_map, cid_prefix_.data(), - cid_prefix_.size(), &sk_index, sizeof(sk_index), + rv = bpf_map__update_elem(ref.worker_id_map, &worker_id_, + sizeof(worker_id_), &sk_index, sizeof(sk_index), BPF_NOEXIST); if (rv != 0) { auto error = errno; - LOG(FATAL) << "Failed to update cid_prefix_map: " + LOG(FATAL) << "Failed to update worker_id_map: " << xsi_strerror(error, errbuf.data(), errbuf.size()); close(fd); return -1; @@ -1187,7 +1184,7 @@ int Worker::create_quic_server_socket(UpstreamAddr &faddr) { return 0; } -const uint8_t *Worker::get_cid_prefix() const { return cid_prefix_.data(); } +const WorkerID &Worker::get_worker_id() const { return worker_id_; } const UpstreamAddr *Worker::find_quic_upstream_addr(const Address &local_addr) { std::array<char, NI_MAXHOST> host; @@ -1444,16 +1441,4 @@ void downstream_failure(DownstreamAddr *addr, const Address *raddr) { } } -#ifdef ENABLE_HTTP3 -int create_cid_prefix(uint8_t *cid_prefix, const uint8_t *server_id) { - auto p = std::copy_n(server_id, SHRPX_QUIC_SERVER_IDLEN, cid_prefix); - - if (RAND_bytes(p, SHRPX_QUIC_CID_PREFIXLEN - SHRPX_QUIC_SERVER_IDLEN) != 1) { - return -1; - } - - return 0; -} -#endif // ENABLE_HTTP3 - } // namespace shrpx diff --git a/src/shrpx_worker.h b/src/shrpx_worker.h index 3cc7b57..f8a2d84 100644 --- a/src/shrpx_worker.h +++ b/src/shrpx_worker.h @@ -312,7 +312,7 @@ public: tls::CertLookupTree *cert_tree, #ifdef ENABLE_HTTP3 SSL_CTX *quic_sv_ssl_ctx, tls::CertLookupTree *quic_cert_tree, - const uint8_t *cid_prefix, size_t cid_prefixlen, + WorkerID wid, # ifdef HAVE_LIBBPF size_t index, # endif // HAVE_LIBBPF @@ -377,7 +377,7 @@ public: int setup_quic_server_socket(); - const uint8_t *get_cid_prefix() const; + const WorkerID &get_worker_id() const; # ifdef HAVE_LIBBPF bool should_attach_bpf() const; @@ -414,7 +414,7 @@ private: DNSTracker dns_tracker_; #ifdef ENABLE_HTTP3 - std::array<uint8_t, SHRPX_QUIC_CID_PREFIXLEN> cid_prefix_; + WorkerID worker_id_; std::vector<UpstreamAddr> quic_upstream_addrs_; std::vector<std::unique_ptr<QUICListener>> quic_listeners_; #endif // ENABLE_HTTP3 @@ -468,13 +468,6 @@ size_t match_downstream_addr_group( // nullptr. This function may schedule live check. void downstream_failure(DownstreamAddr *addr, const Address *raddr); -#ifdef ENABLE_HTTP3 -// Creates unpredictable SHRPX_QUIC_CID_PREFIXLEN bytes sequence which -// is used as a prefix of QUIC Connection ID. This function returns -// -1 on failure. |server_id| must be 2 bytes long. -int create_cid_prefix(uint8_t *cid_prefix, const uint8_t *server_id); -#endif // ENABLE_HTTP3 - } // namespace shrpx #endif // SHRPX_WORKER_H diff --git a/src/shrpx_worker_process.cc b/src/shrpx_worker_process.cc index e3f7dae..6591e9b 100644 --- a/src/shrpx_worker_process.cc +++ b/src/shrpx_worker_process.cc @@ -593,11 +593,21 @@ int worker_process_event_loop(WorkerProcessConfig *wpconf) { } EVP_CIPHER_CTX_set_padding(qkm.cid_encryption_ctx, 0); + + qkm.cid_decryption_ctx = EVP_CIPHER_CTX_new(); + if (!EVP_DecryptInit_ex(qkm.cid_decryption_ctx, EVP_aes_128_ecb(), nullptr, + qkm.cid_encryption_key.data(), nullptr)) { + LOG(ERROR) + << "Failed to initialize QUIC Connection ID decryption context"; + return -1; + } + + EVP_CIPHER_CTX_set_padding(qkm.cid_decryption_ctx, 0); } conn_handler->set_quic_keying_materials(std::move(qkms)); - conn_handler->set_cid_prefixes(wpconf->cid_prefixes); + conn_handler->set_worker_ids(wpconf->worker_ids); conn_handler->set_quic_lingering_worker_processes( wpconf->quic_lingering_worker_processes); #endif // ENABLE_HTTP3 diff --git a/src/shrpx_worker_process.h b/src/shrpx_worker_process.h index f432503..155b565 100644 --- a/src/shrpx_worker_process.h +++ b/src/shrpx_worker_process.h @@ -49,8 +49,8 @@ struct WorkerProcessConfig { // IPv6 socket, or -1 if not used int server_fd6; #ifdef ENABLE_HTTP3 - // CID prefixes for the new worker process. - std::vector<std::array<uint8_t, SHRPX_QUIC_CID_PREFIXLEN>> cid_prefixes; + // Worker IDs for the new worker process. + std::vector<WorkerID> worker_ids; // IPC socket to read forwarded QUIC UDP datagram from the current // worker process. int quic_ipc_fd; @@ -25,9 +25,11 @@ #include "tls.h" #include <cassert> +#include <cstring> #include <vector> #include <mutex> #include <iostream> +#include <fstream> #include <openssl/crypto.h> #include <openssl/conf.h> @@ -176,6 +178,32 @@ int cert_decompress(SSL *ssl, CRYPTO_BUFFER **out, size_t uncompressed_len, } #endif // NGHTTP2_OPENSSL_IS_BORINGSSL && HAVE_LIBBROTLI +namespace { +std::ofstream keylog_file; + +void keylog_callback(const SSL *ssl, const char *line) { + keylog_file.write(line, strlen(line)); + keylog_file.put('\n'); + keylog_file.flush(); +} +} // namespace + +int setup_keylog_callback(SSL_CTX *ssl_ctx) { + auto keylog_filename = getenv("SSLKEYLOGFILE"); + if (!keylog_filename) { + return 0; + } + + keylog_file.open(keylog_filename, std::ios_base::app); + if (!keylog_file) { + return -1; + } + + SSL_CTX_set_keylog_callback(ssl_ctx, keylog_callback); + + return 0; +} + } // namespace tls } // namespace nghttp2 @@ -106,6 +106,9 @@ int cert_decompress(SSL *ssl, CRYPTO_BUFFER **out, size_t uncompressed_len, const uint8_t *in, size_t in_len); #endif // NGHTTP2_OPENSSL_IS_BORINGSSL && HAVE_LIBBROTLI +// Setup keylog callback. It returns 0 if it succeeds, or -1. +int setup_keylog_callback(SSL_CTX *ssl_ctx); + } // namespace tls } // namespace nghttp2 diff --git a/src/util.cc b/src/util.cc index 0996c0a..47151b2 100644 --- a/src/util.cc +++ b/src/util.cc @@ -802,6 +802,30 @@ void set_port(Address &addr, uint16_t port) { } } +uint16_t get_port(const sockaddr_union *su) { + switch (su->storage.ss_family) { + case AF_INET: + return ntohs(su->in.sin_port); + case AF_INET6: + return ntohs(su->in6.sin6_port); + default: + return 0; + } +} + +bool quic_prohibited_port(uint16_t port) { + switch (port) { + case 1900: + case 5353: + case 11211: + case 20800: + case 27015: + return true; + default: + return port < 1024; + } +} + std::string ascii_dump(const uint8_t *data, size_t len) { std::string res; @@ -1348,66 +1372,166 @@ StringRef make_hostport(BlockAllocator &balloc, const StringRef &host, } namespace { -void hexdump8(FILE *out, const uint8_t *first, const uint8_t *last) { - auto stop = std::min(first + 8, last); - for (auto k = first; k != stop; ++k) { - fprintf(out, "%02x ", *k); +uint8_t *hexdump_addr(uint8_t *dest, size_t addr) { + // Lower 32 bits are displayed. + for (size_t i = 0; i < 4; ++i) { + auto a = (addr >> (3 - i) * 8) & 0xff; + + *dest++ = LOWER_XDIGITS[a >> 4]; + *dest++ = LOWER_XDIGITS[a & 0xf]; + } + + return dest; +} +} // namespace + +namespace { +uint8_t *hexdump_ascii(uint8_t *dest, const uint8_t *data, size_t datalen) { + *dest++ = '|'; + + for (size_t i = 0; i < datalen; ++i) { + if (0x20 <= data[i] && data[i] <= 0x7e) { + *dest++ = data[i]; + } else { + *dest++ = '.'; + } + } + + *dest++ = '|'; + + return dest; +} +} // namespace + +namespace { +uint8_t *hexdump8(uint8_t *dest, const uint8_t *data, size_t datalen) { + size_t i; + + for (i = 0; i < datalen; ++i) { + *dest++ = LOWER_XDIGITS[data[i] >> 4]; + *dest++ = LOWER_XDIGITS[data[i] & 0xf]; + *dest++ = ' '; } - // each byte needs 3 spaces (2 hex value and space) - for (; stop != first + 8; ++stop) { - fputs(" ", out); + + for (; i < 8; ++i) { + *dest++ = ' '; + *dest++ = ' '; + *dest++ = ' '; } - // we have extra space after 8 bytes - fputc(' ', out); + + return dest; } } // namespace -void hexdump(FILE *out, const uint8_t *src, size_t len) { - if (len == 0) { - return; +namespace { +uint8_t *hexdump16(uint8_t *dest, const uint8_t *data, size_t datalen) { + if (datalen > 8) { + dest = hexdump8(dest, data, 8); + *dest++ = ' '; + dest = hexdump8(dest, data + 8, datalen - 8); + *dest++ = ' '; + } else { + dest = hexdump8(dest, data, datalen); + *dest++ = ' '; + dest = hexdump8(dest, nullptr, 0); + *dest++ = ' '; } - size_t buflen = 0; + + return dest; +} +} // namespace + +namespace { +uint8_t *hexdump_line(uint8_t *dest, const uint8_t *data, size_t datalen, + size_t addr) { + dest = hexdump_addr(dest, addr); + *dest++ = ' '; + *dest++ = ' '; + + dest = hexdump16(dest, data, datalen); + + return hexdump_ascii(dest, data, datalen); +} +} // namespace + +namespace { +int hexdump_write(int fd, const uint8_t *data, size_t datalen) { + ssize_t nwrite; + + for (; (nwrite = write(fd, data, datalen)) == -1 && errno == EINTR;) + ; + if (nwrite == -1) { + return -1; + } + + return 0; +} +} // namespace + +int hexdump(FILE *out, const void *data, size_t datalen) { + if (datalen == 0) { + return 0; + } + + // min_space is the additional minimum space that the buffer must + // accept, which is the size of a single full line output + one + // repeat line marker ("*\n"). If the remaining buffer size is less + // than that, flush the buffer and reset. + constexpr size_t min_space = 79 + 2; + + auto fd = fileno(out); + std::array<uint8_t, 4096> buf; + auto last = buf.data(); + auto in = reinterpret_cast<const uint8_t *>(data); auto repeated = false; - std::array<uint8_t, 16> buf{}; - auto end = src + len; - auto i = src; - for (;;) { - auto nextlen = - std::min(static_cast<size_t>(16), static_cast<size_t>(end - i)); - if (nextlen == buflen && - std::equal(std::begin(buf), std::begin(buf) + buflen, i)) { - // as long as adjacent 16 bytes block are the same, we just - // print single '*'. - if (!repeated) { - repeated = true; - fputs("*\n", out); + + for (size_t offset = 0; offset < datalen; offset += 16) { + auto n = datalen - offset; + auto s = in + offset; + + if (n >= 16) { + n = 16; + + if (offset > 0) { + if (std::equal(s - 16, s, s)) { + if (repeated) { + continue; + } + + repeated = true; + + *last++ = '*'; + *last++ = '\n'; + + continue; + } + + repeated = false; } - i += nextlen; - continue; } - repeated = false; - fprintf(out, "%08lx", static_cast<unsigned long>(i - src)); - if (i == end) { - fputc('\n', out); - break; - } - fputs(" ", out); - hexdump8(out, i, end); - hexdump8(out, i + 8, std::max(i + 8, end)); - fputc('|', out); - auto stop = std::min(i + 16, end); - buflen = stop - i; - auto p = buf.data(); - for (; i != stop; ++i) { - *p++ = *i; - if (0x20 <= *i && *i <= 0x7e) { - fputc(*i, out); - } else { - fputc('.', out); + + last = hexdump_line(last, s, n, offset); + *last++ = '\n'; + + auto len = static_cast<size_t>(last - buf.data()); + if (len + min_space > buf.size()) { + if (hexdump_write(fd, buf.data(), len) != 0) { + return -1; } + + last = buf.data(); } - fputs("|\n", out); } + + last = hexdump_addr(last, datalen); + *last++ = '\n'; + + auto len = static_cast<size_t>(last - buf.data()); + if (len) { + return hexdump_write(fd, buf.data(), len); + } + + return 0; } void put_uint16be(uint8_t *buf, uint16_t n) { @@ -567,6 +567,12 @@ std::string to_numeric_addr(const struct sockaddr *sa, socklen_t salen); // Sets |port| to |addr|. void set_port(Address &addr, uint16_t port); +// Get port from |su|. +uint16_t get_port(const sockaddr_union *su); + +// Returns true if |port| is prohibited as a QUIC client port. +bool quic_prohibited_port(uint16_t port); + // Returns ASCII dump of |data| of length |len|. Only ASCII printable // characters are preserved. Other characters are replaced with ".". std::string ascii_dump(const uint8_t *data, size_t len); @@ -840,8 +846,10 @@ StringRef make_http_hostport(OutputIt first, const StringRef &host, return StringRef{first, p}; } -// Dumps |src| of length |len| in the format similar to `hexdump -C`. -void hexdump(FILE *out, const uint8_t *src, size_t len); +// hexdump dumps |data| of length |datalen| in the format similar to +// hexdump(1) with -C option. This function returns 0 if it succeeds, +// or -1. +int hexdump(FILE *out, const void *data, size_t datalen); // Copies 2 byte unsigned integer |n| in host byte order to |buf| in // network byte order. diff --git a/tests/nghttp2_session_test.c b/tests/nghttp2_session_test.c index c155102..bb505c2 100644 --- a/tests/nghttp2_session_test.c +++ b/tests/nghttp2_session_test.c @@ -10536,6 +10536,7 @@ void test_nghttp2_session_open_idle_stream(void) { nghttp2_stream *opened_stream; nghttp2_priority_spec pri_spec; nghttp2_frame frame; + nghttp2_ext_priority_update priority_update; memset(&callbacks, 0, sizeof(nghttp2_session_callbacks)); @@ -10567,6 +10568,35 @@ void test_nghttp2_session_open_idle_stream(void) { nghttp2_frame_priority_free(&frame.priority); nghttp2_session_del(session); + + /* No RFC 7540 priorities */ + nghttp2_session_server_new(&session, &callbacks, NULL); + + session->pending_no_rfc7540_priorities = 1; + + frame.ext.payload = &priority_update; + + nghttp2_frame_priority_update_init(&frame.ext, 1, (uint8_t *)"u=3", + strlen("u=3")); + + assert_int(0, ==, + nghttp2_session_on_priority_update_received(session, &frame)); + + stream = nghttp2_session_get_stream_raw(session, 1); + + assert_enum(nghttp2_stream_state, NGHTTP2_STREAM_IDLE, ==, stream->state); + assert_null(stream->closed_next); + assert_size(1, ==, session->num_idle_streams); + + opened_stream = open_recv_stream2(session, 1, NGHTTP2_STREAM_OPENING); + + assert_ptr_equal(stream, opened_stream); + assert_enum(nghttp2_stream_state, NGHTTP2_STREAM_OPENING, ==, stream->state); + assert_size(0, ==, session->num_idle_streams); + + nghttp2_frame_priority_free(&frame.priority); + + nghttp2_session_del(session); } void test_nghttp2_session_cancel_reserved_remote(void) { @@ -10903,6 +10933,21 @@ void test_nghttp2_session_detach_item_from_closed_stream(void) { assert_int(0, ==, nghttp2_session_send(session)); nghttp2_session_del(session); + + /* No RFC 7540 priorities */ + nghttp2_session_server_new(&session, &callbacks, NULL); + + session->pending_no_rfc7540_priorities = 1; + + open_recv_stream(session, 1); + open_recv_stream(session, 3); + + nghttp2_session_close_stream(session, 1, NGHTTP2_NO_ERROR); + nghttp2_session_close_stream(session, 3, NGHTTP2_NO_ERROR); + + assert_int(0, ==, nghttp2_session_send(session)); + + nghttp2_session_del(session); } void test_nghttp2_session_flooding(void) { |