From c18514225db2835dfe22843100307c4bc8a59576 Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Mon, 5 Aug 2024 12:00:11 +0200 Subject: Merging upstream version 2.4.62. Signed-off-by: Daniel Baumann --- CHANGES | 34 ++++- docs/manual/misc/security_tips.html.fr.utf8 | 8 +- docs/manual/mod/core.html.de | 2 +- docs/manual/mod/core.html.en | 20 ++- docs/manual/mod/core.html.es | 2 +- docs/manual/mod/core.html.fr.utf8 | 36 +++-- docs/manual/mod/core.html.ja.utf8 | 2 +- docs/manual/mod/core.html.tr.utf8 | 2 +- docs/manual/mod/mod_rewrite.html.en | 6 + docs/manual/mod/mod_rewrite.html.fr.utf8 | 15 +++ docs/manual/mod/mod_ssl.html.en | 13 +- docs/manual/mod/mod_ssl.html.fr.utf8 | 15 ++- docs/manual/mod/quickreference.html.de | 2 +- docs/manual/mod/quickreference.html.en | 2 +- docs/manual/mod/quickreference.html.es | 2 +- docs/manual/mod/quickreference.html.fr.utf8 | 2 +- docs/manual/mod/quickreference.html.ja.utf8 | 2 +- docs/manual/mod/quickreference.html.ko.euc-kr | 2 +- docs/manual/mod/quickreference.html.tr.utf8 | 2 +- docs/manual/mod/quickreference.html.zh-cn.utf8 | 2 +- docs/manual/rewrite/flags.html.en | 7 + docs/manual/rewrite/flags.html.fr.utf8 | 20 ++- docs/manual/style/version.ent | 2 +- httpd.spec | 2 +- include/ap_release.h | 2 +- modules/http/http_request.c | 2 +- modules/mappers/mod_rewrite.c | 14 ++ modules/proxy/balancers/mod_lbmethod_bytraffic.c | 2 - modules/proxy/mod_proxy.c | 111 ++------------- modules/proxy/mod_proxy.h | 1 + modules/proxy/mod_proxy_balancer.c | 98 +++++--------- modules/proxy/proxy_util.c | 154 +++++++++++++++++---- modules/proxy/proxy_util.h | 17 +++ modules/ssl/ssl_engine_init.c | 13 +- modules/ssl/ssl_engine_io.c | 82 ++++++++--- modules/ssl/ssl_engine_kernel.c | 4 +- modules/ssl/ssl_engine_pphrase.c | 159 ++++++++++++++++++++-- modules/ssl/ssl_private.h | 14 +- modules/ssl/ssl_util.c | 2 +- server/mpm/event/event.c | 29 ++-- server/mpm/worker/worker.c | 55 +++++--- test/modules/core/conftest.py | 22 +-- test/modules/core/env.py | 25 ++++ test/modules/core/test_001_encoding.py | 41 +++--- test/modules/core/test_002_restarts.py | 150 +++++++++++++++++++++ test/modules/http1/__init__.py | 0 test/modules/http1/conftest.py | 36 +++++ test/modules/http1/env.py | 81 +++++++++++ test/modules/http1/htdocs/cgi/files/empty.txt | 0 test/modules/http1/htdocs/cgi/hello.py | 15 +++ test/modules/http1/htdocs/cgi/requestparser.py | 57 ++++++++ test/modules/http1/htdocs/cgi/upload.py | 55 ++++++++ test/modules/http1/mod_h1test/mod_h1test.c | 129 ++++++++++++++++++ test/modules/http1/mod_h1test/mod_h1test.slo | 0 test/modules/http1/test_001_alive.py | 20 +++ test/modules/http1/test_003_get.py | 27 ++++ test/modules/http1/test_004_post.py | 53 ++++++++ test/modules/http1/test_005_trailers.py | 42 ++++++ test/modules/http1/test_006_unsafe.py | 134 ++++++++++++++++++ test/modules/http1/test_007_strict.py | 126 +++++++++++++++++ test/modules/http2/conftest.py | 11 +- test/modules/http2/env.py | 36 +---- test/modules/http2/test_007_ssi.py | 1 - test/modules/http2/test_008_ranges.py | 20 +-- test/modules/http2/test_100_conn_reuse.py | 12 ++ test/modules/http2/test_101_ssl_reneg.py | 46 +++++++ test/modules/http2/test_102_require.py | 6 + test/modules/http2/test_103_upgrade.py | 3 + test/modules/http2/test_105_timeout.py | 14 ++ test/modules/http2/test_106_shutdown.py | 8 +- test/modules/http2/test_200_header_invalid.py | 22 +++ test/modules/http2/test_203_rfc9113.py | 24 ++-- test/modules/http2/test_500_proxy.py | 12 ++ test/modules/http2/test_600_h2proxy.py | 7 +- test/modules/http2/test_700_load_get.py | 34 +++++ test/modules/http2/test_712_buffering.py | 14 +- test/modules/http2/test_800_websockets.py | 5 +- test/modules/md/conftest.py | 45 +------ test/modules/md/test_300_conf_validate.py | 165 ++++++++++++++++++++++- test/modules/md/test_702_auto.py | 54 ++++++++ test/modules/md/test_720_wildcard.py | 28 ++++ test/modules/md/test_730_static.py | 7 + test/modules/md/test_740_acme_errors.py | 18 +++ test/modules/md/test_741_setup_errors.py | 10 ++ test/modules/md/test_750_eab.py | 105 ++++++++++++++- test/modules/md/test_780_tailscale.py | 12 ++ test/modules/md/test_790_failover.py | 18 +++ test/modules/md/test_900_notify.py | 14 ++ test/modules/md/test_901_message.py | 32 ++++- test/modules/md/test_920_status.py | 6 + test/modules/proxy/conftest.py | 20 --- test/modules/proxy/env.py | 1 - test/modules/proxy/test_02_unix.py | 6 + test/modules/tls/conf.py | 11 +- test/modules/tls/conftest.py | 6 - test/modules/tls/env.py | 5 +- test/modules/tls/test_02_conf.py | 12 +- test/modules/tls/test_03_sni.py | 18 +++ test/modules/tls/test_06_ciphers.py | 21 +-- test/modules/tls/test_08_vars.py | 21 ++- test/modules/tls/test_14_proxy_ssl.py | 49 ++++++- test/modules/tls/test_15_proxy_tls.py | 10 +- test/modules/tls/test_16_proxy_mixed.py | 3 + test/modules/tls/test_17_proxy_machine_cert.py | 3 +- test/pyhttpd/conf.py | 87 +++++++++++- test/pyhttpd/curl.py | 2 - test/pyhttpd/env.py | 29 +++- test/pyhttpd/log.py | 165 +++++++++++------------ 108 files changed, 2629 insertions(+), 603 deletions(-) create mode 100644 test/modules/core/env.py create mode 100644 test/modules/core/test_002_restarts.py create mode 100644 test/modules/http1/__init__.py create mode 100644 test/modules/http1/conftest.py create mode 100644 test/modules/http1/env.py create mode 100644 test/modules/http1/htdocs/cgi/files/empty.txt create mode 100755 test/modules/http1/htdocs/cgi/hello.py create mode 100644 test/modules/http1/htdocs/cgi/requestparser.py create mode 100755 test/modules/http1/htdocs/cgi/upload.py create mode 100644 test/modules/http1/mod_h1test/mod_h1test.c create mode 100644 test/modules/http1/mod_h1test/mod_h1test.slo create mode 100644 test/modules/http1/test_001_alive.py create mode 100644 test/modules/http1/test_003_get.py create mode 100644 test/modules/http1/test_004_post.py create mode 100644 test/modules/http1/test_005_trailers.py create mode 100644 test/modules/http1/test_006_unsafe.py create mode 100644 test/modules/http1/test_007_strict.py diff --git a/CHANGES b/CHANGES index eea1e55..cd86fe7 100644 --- a/CHANGES +++ b/CHANGES @@ -1,6 +1,38 @@ -*- coding: utf-8 -*- +Changes with Apache 2.4.62 + + *) mod_proxy: Fix canonicalisation and FCGI env (PATH_INFO, SCRIPT_NAME) for + "balancer:" URLs set via SetHandler, also allowing for "unix:" sockets + with BalancerMember(s). PR 69168. [Yann Ylavic] + + *) mod_proxy: Avoid AH01059 parsing error for SetHandler "unix:" URLs. + PR 69160 [Yann Ylavic] + + *) mod_ssl: Fix crashes in PKCS#11 ENGINE support with OpenSSL 3.2. + [Joe Orton] + + *) mod_ssl: Add support for loading certs/keys from pkcs11: URIs + via OpenSSL 3.x providers. [Ingo Franzki ] + + *) mod_ssl: Restore SSL dumping on trace7 loglevel with OpenSSL >= 3.0. + [Ruediger Pluem, Yann Ylavic] + + *) mpm_worker: Fix possible warning (AH00045) about children processes not + terminating timely. [Yann Ylavic] + Changes with Apache 2.4.61 + *) SECURITY: CVE-2024-39884: Apache HTTP Server: source code + disclosure with handlers configured via AddType (cve.mitre.org) + A regression in the core of Apache HTTP Server 2.4.60 ignores + some use of the legacy content-type based configuration of + handlers. "AddType" and similar configuration, under some + circumstances where files are requested indirectly, result in + source code disclosure of local content. For example, PHP + scripts may be served instead of interpreted. + Users are recommended to upgrade to version 2.4.61, which fixes + this issue. + Changes with Apache 2.4.60 *) SECURITY: CVE-2024-39573: Apache HTTP Server: mod_rewrite proxy @@ -67,7 +99,7 @@ Changes with Apache 2.4.60 crafted requests. Credits: Orange Tsai (@orange_8361) from DEVCORE - *) SECURITY: CVE-2024-38472: Apache HTTP Server on WIndows UNC SSRF + *) SECURITY: CVE-2024-38472: Apache HTTP Server on Windows UNC SSRF (cve.mitre.org) SSRF in Apache HTTP Server on Windows allows to potentially leak NTML hashes to a malicious server via SSRF and malicious diff --git a/docs/manual/misc/security_tips.html.fr.utf8 b/docs/manual/misc/security_tips.html.fr.utf8 index 741a0e7..3efa7d7 100644 --- a/docs/manual/misc/security_tips.html.fr.utf8 +++ b/docs/manual/misc/security_tips.html.fr.utf8 @@ -28,8 +28,6 @@  ko  |  tr 

-
Cette traduction peut être périmée. Vérifiez la version - anglaise pour les changements récents.

Ce document propose quelques conseils et astuces concernant les problèmes de sécurité liés @@ -145,11 +143,7 @@ vous permet de traiter d'avantage de connexions simultanées, ce qui minimise l'effet des attaques DoS. Dans le futur, le module mpm event utilisera un traitement asynchrone afin de ne pas - dédier un thread à chaque connexion. De par la - nature de la bibliothèque OpenSSL, le module mpm event est actuellement incompatible - avec le module mod_ssl ainsi que d'autres filtres - en entrée. Dans ces cas, son comportement se ramène à celui - du module mpm worker. + dédier un thread à chaque connexion.

  • Il existe de nombreux modules tiers qui peuvent restreindre les comportements de certains clients et ainsi minimiser les problèmes de diff --git a/docs/manual/mod/core.html.de b/docs/manual/mod/core.html.de index a025992..06368d2 100644 --- a/docs/manual/mod/core.html.de +++ b/docs/manual/mod/core.html.de @@ -3624,7 +3624,7 @@ bevor er die Anfrage abbricht - + diff --git a/docs/manual/mod/core.html.en b/docs/manual/mod/core.html.en index ba049f3..f5efca3 100644 --- a/docs/manual/mod/core.html.en +++ b/docs/manual/mod/core.html.en @@ -5000,7 +5000,7 @@ certain events before failing a request
    Beschreibung:Controls what UNC host names can be accessed by the server
    Syntax:UNCListhostname ...
    Syntax:UNCList hostname [hostname...]
    Voreinstellung:unset
    Kontext:Serverkonfiguration
    Status:Core
    - + @@ -5012,10 +5012,20 @@ certain events before failing a request has been specified by this directive. The intent is to limit access to paths derived from untrusted inputs.

    -

    Security

    -

    UNC paths accessed outside of request processing, such as during startup, -are not checked against the hosts configured with this directive.

    -
    +

    + UNCList example.com other.example.com +

    + +

    Security

    +

    UNC paths accessed outside of request processing, such as during startup, + are not necessarily checked against the hosts configured with this directive.

    +
    + +

    Directive Ordering

    +

    This directive should be placed before UNC paths used in httpd.conf. + Multiple occurrences of the directive reset the list.

    +
    +
    top
    diff --git a/docs/manual/mod/core.html.es b/docs/manual/mod/core.html.es index a563573..5efc11d 100644 --- a/docs/manual/mod/core.html.es +++ b/docs/manual/mod/core.html.es @@ -4323,7 +4323,7 @@ certain events before failing a request
    Description:Controls what UNC host names can be accessed by the server
    Syntax:UNCListhostname ...
    Syntax:UNCList hostname [hostname...]
    Default:unset
    Context:server config
    Status:Core
    - + diff --git a/docs/manual/mod/core.html.fr.utf8 b/docs/manual/mod/core.html.fr.utf8 index 4ca5ec6..cc7a612 100644 --- a/docs/manual/mod/core.html.fr.utf8 +++ b/docs/manual/mod/core.html.fr.utf8 @@ -33,8 +33,6 @@  ja  |  tr 

    -
    Cette traduction peut être périmée. Vérifiez la version - anglaise pour les changements récents.
    Descripción:Controls what UNC host names can be accessed by the server
    Sintaxis:UNCListhostname ...
    Sintaxis:UNCList hostname [hostname...]
    Valor por defecto:unset
    Contexto:server config
    Estado:Core
    Description:Fonctionnalités de base du serveur HTTP Apache toujours disponibles
    Statut:Noyau httpd
    @@ -5358,17 +5356,39 @@ dernière.
    top

    Directive UNCList

    - - + - -
    Description:Controls what UNC host names can be accessed by the server +
    Description:Définit quels sont les noms d’hôte UNC auxquels le serveur peut accéder
    Syntaxe:UNCListhostname ...
    Syntaxe:UNCList hostname [hostname...]
    Défaut:unset
    Contexte:configuration globale
    Statut:Noyau httpd
    Module:core
    Compatibilité:Added in 2.4.60, Windows only.

    La documentation de cette directive - n'a pas encore t traduite. Veuillez vous reporter la version - en langue anglaise.

    +Compatibilité:Windows seulement. Disponible à partir de la version 2.4.60 du +serveur HTTP Apache. + +

    Au cours de leur traitement, les requêtes pour accéder à un chemin du + système de fichiers qui aboutissent à un chemin UNC échoueront si le nom + d’hôte dans le chemin UNC n’a pas été spécifié par cette directive. Le but + est de limiter l’accès aux chemins dérivés d’entrées non fiables.

    + +

    + UNCList example.com other.example.com +

    + +

    Sécurité

    +

    Les chemins UNC accédés en dehors du traitement d’une requête, par + exemple au cours du démarrage, ne font pas nécessairement l’objet d’une + vérification par rapport aux noms d’hôte configurés avec cette directive.

    +
    + +

    Ordre des directives

    +

    Cette directive doit être placée avant les chemins UNC utilisés dans le + fichier httpd.conf. Plusieurs occurences de la directive redéfinissent la + liste.

    +
    + + +
    top

    Directive UnDefine

    diff --git a/docs/manual/mod/core.html.ja.utf8 b/docs/manual/mod/core.html.ja.utf8 index e807744..96d4454 100644 --- a/docs/manual/mod/core.html.ja.utf8 +++ b/docs/manual/mod/core.html.ja.utf8 @@ -3552,7 +3552,7 @@ of a request or the last 63, assuming the request itself is greater than
    - + diff --git a/docs/manual/mod/core.html.tr.utf8 b/docs/manual/mod/core.html.tr.utf8 index c3743b8..cbe05ac 100644 --- a/docs/manual/mod/core.html.tr.utf8 +++ b/docs/manual/mod/core.html.tr.utf8 @@ -4971,7 +4971,7 @@ gerçekleşmesi için sunucunun geçmesini bekleyeceği süre.
    説明:Controls what UNC host names can be accessed by the server
    構文:UNCListhostname ...
    構文:UNCList hostname [hostname...]
    デフォルト:unset
    コンテキスト:サーバ設定ファイル
    ステータス:Core
    - + diff --git a/docs/manual/mod/mod_rewrite.html.en b/docs/manual/mod/mod_rewrite.html.en index 83bce1b..5728d8f 100644 --- a/docs/manual/mod/mod_rewrite.html.en +++ b/docs/manual/mod/mod_rewrite.html.en @@ -1463,6 +1463,12 @@ cannot use $N in the substitution string! details ... + + + +
    Açıklama:Controls what UNC host names can be accessed by the server
    Sözdizimi:UNCListhostname ...
    Sözdizimi:UNCList hostname [hostname...]
    Öntanımlı:unset
    Bağlam:sunucu geneli
    Durum:Çekirdek
    UNCPrevents the merging of multiple leading slashes, as used by Windows UNC paths. + details ... +

    Home directory expansion

    diff --git a/docs/manual/mod/mod_rewrite.html.fr.utf8 b/docs/manual/mod/mod_rewrite.html.fr.utf8 index 621d369..41867ce 100644 --- a/docs/manual/mod/mod_rewrite.html.fr.utf8 +++ b/docs/manual/mod/mod_rewrite.html.fr.utf8 @@ -1564,6 +1564,21 @@ substitution ! Force l'attribution du Type-MIME spécifié au fichier cible. détails ... + + UnsafeAllow3F + Autorise les substitutions à partir d’URL potentiellement non + fiables. + détails ... + + + + UnsafePrefixStat + Autorise les substitutions potentiellement non fiables à partir + d’une variable de tête ou d’une référence arrière vers un chemin du + système de fichiers. + détails ... + +

    Développement du répertoire home

    diff --git a/docs/manual/mod/mod_ssl.html.en b/docs/manual/mod/mod_ssl.html.en index ee92ffb..3fc8a48 100644 --- a/docs/manual/mod/mod_ssl.html.en +++ b/docs/manual/mod/mod_ssl.html.en @@ -661,7 +661,7 @@ key is encrypted, the pass phrase dialog is forced at startup time. files, a certificate identifier can be used to identify a certificate stored in a token. Currently, only PKCS#11 URIs are recognized as certificate identifiers, and can be used in conjunction -with the OpenSSL pkcs11 engine. If SSLCertificateKeyFile is omitted, the +with the OpenSSL pkcs11 engine or provider. If SSLCertificateKeyFile is omitted, the certificate and private key can be loaded through the single identifier specified with SSLCertificateFile.

    @@ -749,7 +749,7 @@ key file.

    identifier can be used to identify a private key stored in a token. Currently, only PKCS#11 URIs are recognized as private key identifiers, and can be used in conjunction with the OpenSSL -pkcs11 engine.

    +pkcs11 engine or provider.

    Example

    # To use a private key from a PEM-encoded file:
 SSLCertificateKeyFile "/usr/local/apache2/conf/ssl.key/server.key"
@@ -983,6 +983,15 @@ separate "-engine" releases of OpenSSL 0.9.6 must be used.
    
 SSLCryptoDevice ubsec
    +

    +With OpenSSL 3.0 or later, if no engine is specified but the key or certificate +is specified using a PKCS#11 URIs +then it is tried to load the key and certificate from an OpenSSL provider. +The OpenSSL provider to use must be defined and configured in the OpenSSL config file, +and it must support the STORE method +for PKCS#11 URIs. +

    +
    top

    SSLEngine Directive

    diff --git a/docs/manual/mod/mod_ssl.html.fr.utf8 b/docs/manual/mod/mod_ssl.html.fr.utf8 index 8f3a9b6..3889746 100644 --- a/docs/manual/mod/mod_ssl.html.fr.utf8 +++ b/docs/manual/mod/mod_ssl.html.fr.utf8 @@ -749,7 +749,7 @@ passe de la clé s'ouvre au démarrage du serveur. on peut utiliser un identificateur de certificat pour identifier un certificat stocké dans un jeton. Actuellement, seuls les URIs PKCS#11 sont reconnus comme identificateurs de certificats et peuvent être utilisés en conjonction avec le -moteur OpenSSL pkcs11. Si la directive SSLCertificateKeyFile est absente, le certificat et +moteur ou le fournisseur OpenSSL pkcs11. Si la directive SSLCertificateKeyFile est absente, le certificat et la clé privée peuvent être chargés avec l'identificateur spécifié via la directive SSLCertificateFile.

    @@ -844,7 +844,8 @@ certificats qui utilisent un fichier de clé séparé.

    d'identifier une clé privée via un identifiant stocké dans un jeton. Actuellement, seuls les PKCS#11 URIs sont reconnus comme identifiants de clés privées et peuvent être -utilisés en conjonction avec le moteur OpenSSL pkcs11.

    +utilisés en conjonction avec le moteur ou le fournisseur OpenSSL +pkcs11.

    Exemple

    # Pour utiliser une clé privée stockée dans fichier encodé PEM :
 SSLCertificateKeyFile "/usr/local/apache2/conf/ssl.key/server.key"
@@ -1125,6 +1126,16 @@ qu'avec la version 0.9.6, il faut utiliser les distributions séparées
 SSLCryptoDevice ubsec
    +

    +À partir de la version 3.0 d'OpenSSL, si aucun moteur n'est spécifié alors +que la clé ou le certificat sont spécifiés à l'aide d'URIs PKCS#11, le chargement de la +clé et du certificat est tenté à partir d'un fournisseur OpenSSL. Le fournisseur +OpenSSL à utiliser doit être défini et configuré dans le fichier de +configuration d'OpenSSL et il doit prendre en charge la méthode +STORE pour les URIs PKCS#11. +

    + +
    top

    Directive SSLEngine

    diff --git a/docs/manual/mod/quickreference.html.de b/docs/manual/mod/quickreference.html.de index eb417c0..fbd90a9 100644 --- a/docs/manual/mod/quickreference.html.de +++ b/docs/manual/mod/quickreference.html.de @@ -1197,7 +1197,7 @@ bevor er die Anfrage abbricht TraceEnable [on|off|extended] on sCLegt das Verhalten von TRACE-Anfragen fest TransferLog file|pipesvBSpecify location of a log file TypesConfig file-path conf/mime.types sBThe location of the mime.types file -UNCListhostname ...sCControls what UNC host names can be accessed by the server +UNCList hostname [hostname...]sCControls what UNC host names can be accessed by the server UnDefine parameter-namesCUndefine the existence of a variable UndefMacro namesvdBUndefine a macro diff --git a/docs/manual/mod/quickreference.html.en b/docs/manual/mod/quickreference.html.en index 9da6b32..0c81760 100644 --- a/docs/manual/mod/quickreference.html.en +++ b/docs/manual/mod/quickreference.html.en @@ -1183,7 +1183,7 @@ certain events before failing a request TraceEnable [on|off|extended] on svCDetermines the behavior on TRACE requests TransferLog file|pipesvBSpecify location of a log file TypesConfig file-path conf/mime.types sBThe location of the mime.types file -UNCListhostname ...sCControls what UNC host names can be accessed by the server +UNCList hostname [hostname...]sCControls what UNC host names can be accessed by the server UnDefine parameter-namesCUndefine the existence of a variable UndefMacro namesvdBUndefine a macro diff --git a/docs/manual/mod/quickreference.html.es b/docs/manual/mod/quickreference.html.es index 84d952f..4773df5 100644 --- a/docs/manual/mod/quickreference.html.es +++ b/docs/manual/mod/quickreference.html.es @@ -1186,7 +1186,7 @@ certain events before failing a request TraceEnable [on|off|extended] on sCDetermines the behaviour on TRACE requests TransferLog file|pipesvBSpecify location of a log file TypesConfig file-path conf/mime.types sBThe location of the mime.types file -UNCListhostname ...sCControls what UNC host names can be accessed by the server +UNCList hostname [hostname...]sCControls what UNC host names can be accessed by the server UnDefine parameter-namesCUndefine the existence of a variable UndefMacro namesvdBUndefine a macro diff --git a/docs/manual/mod/quickreference.html.fr.utf8 b/docs/manual/mod/quickreference.html.fr.utf8 index 4b795fa..ba0870f 100644 --- a/docs/manual/mod/quickreference.html.fr.utf8 +++ b/docs/manual/mod/quickreference.html.fr.utf8 @@ -1516,7 +1516,7 @@ traitent les connexions clients TRACE TransferLog fichier|pipesvBSpécifie l'emplacement d'un fichier journal TypesConfig chemin-fichier conf/mime.types sBLe chemin du fichier mime.types -UNCListhostname ...sCControls what UNC host names can be accessed by the server +UNCList hostname [hostname...]sCDéfinit quels sont les noms d’hôte UNC auxquels le serveur peut accéder UnDefine nom-variablesCInvalide la définition d'une variable UndefMacro nomsvdBSupprime une macro diff --git a/docs/manual/mod/quickreference.html.ja.utf8 b/docs/manual/mod/quickreference.html.ja.utf8 index fa80388..74e84a6 100644 --- a/docs/manual/mod/quickreference.html.ja.utf8 +++ b/docs/manual/mod/quickreference.html.ja.utf8 @@ -1114,7 +1114,7 @@ Certificate verification TransferLog file|pipesvBログファイルの位置を指定 TypesConfig file-path conf/mime.types smime.types ファイルの位置 -UNCListhostname ...sCControls what UNC host names can be accessed by the server +UNCList hostname [hostname...]sCControls what UNC host names can be accessed by the server UnDefine parameter-namesCUndefine the existence of a variable UndefMacro namesvdBUndefine a macro diff --git a/docs/manual/mod/quickreference.html.ko.euc-kr b/docs/manual/mod/quickreference.html.ko.euc-kr index 60a7ae2..e409637 100644 --- a/docs/manual/mod/quickreference.html.ko.euc-kr +++ b/docs/manual/mod/quickreference.html.ko.euc-kr @@ -1142,7 +1142,7 @@ certain events before failing a request TraceEnable [on|off|extended] on svCDetermines the behavior on TRACE requests TransferLog file|pipesvBα ġ Ѵ TypesConfig file-path conf/mime.types sBThe location of the mime.types file -UNCListhostname ...sCControls what UNC host names can be accessed by the server +UNCList hostname [hostname...]sCControls what UNC host names can be accessed by the server UnDefine parameter-namesCUndefine the existence of a variable UndefMacro namesvdBUndefine a macro diff --git a/docs/manual/mod/quickreference.html.tr.utf8 b/docs/manual/mod/quickreference.html.tr.utf8 index 2c5261e..9dc9099 100644 --- a/docs/manual/mod/quickreference.html.tr.utf8 +++ b/docs/manual/mod/quickreference.html.tr.utf8 @@ -1181,7 +1181,7 @@ gerçekleşmesi için sunucunun geçmesini bekleyeceği süre. TransferLog dosya|borulu-süreç [takma-ad]skTBir günlük dosyasının yerini belirtir. TypesConfig file-path conf/mime.types sTThe location of the mime.types file -UNCListhostname ...sÇControls what UNC host names can be accessed by the server +UNCList hostname [hostname...]sÇControls what UNC host names can be accessed by the server UnDefine değişken-ismisÇBir değişkeni tanımsız yapar UndefMacro nameskdTUndefine a macro diff --git a/docs/manual/mod/quickreference.html.zh-cn.utf8 b/docs/manual/mod/quickreference.html.zh-cn.utf8 index a358a1a..22748a6 100644 --- a/docs/manual/mod/quickreference.html.zh-cn.utf8 +++ b/docs/manual/mod/quickreference.html.zh-cn.utf8 @@ -1178,7 +1178,7 @@ certain events before failing a request TraceEnable [on|off|extended] on svCDetermines the behavior on TRACE requests TransferLog file|pipesvBSpecify location of a log file TypesConfig file-path conf/mime.types sBThe location of the mime.types file -UNCListhostname ...sCControls what UNC host names can be accessed by the server +UNCList hostname [hostname...]sCControls what UNC host names can be accessed by the server UnDefine parameter-namesCUndefine the existence of a variable UndefMacro namesvdBUndefine a macro diff --git a/docs/manual/rewrite/flags.html.en b/docs/manual/rewrite/flags.html.en index 604e278..fa4aa93 100644 --- a/docs/manual/rewrite/flags.html.en +++ b/docs/manual/rewrite/flags.html.en @@ -59,6 +59,7 @@ providing detailed explanations and examples.

  • T|type
  • UnsafeAllow3F
  • UnsafePrefixStat
    • +
  • UNC

    • See also

    top
    @@ -838,6 +839,12 @@ The L flag can be useful in this context to end the These substitutions are not prefixed with the document root. This protects from a malicious URL causing the expanded substitution to map to an unexpected filesystem location.

    +
    top
    +
    +

    UNC

    +

    Setting this flag prevents the merging of multiple leading slashes, + as used in Windows UNC paths. The flag is not necessary when the rules + substitution starts with multiple literal slashes.

    Available Languages:  en  | diff --git a/docs/manual/rewrite/flags.html.fr.utf8 b/docs/manual/rewrite/flags.html.fr.utf8 index 1e07037..35d76cc 100644 --- a/docs/manual/rewrite/flags.html.fr.utf8 +++ b/docs/manual/rewrite/flags.html.fr.utf8 @@ -58,6 +58,8 @@ l'espace en +)

  • R|redirect
  • S|skip
  • T|type
    • +
  • UnsafeAllow3F
    • +
  • UnsafePrefixStat

    • Voir aussi

    top
    @@ -882,7 +884,23 @@ réécriture suivantes de mod_rewrite). Dans ce contexte, vous pouvez utiliser le drapeau L pour terminer la séquence courante de réécriture de mod_rewrite.

    - +
    top
    +
    +

    UnsafeAllow3F

    +

    Il est nécessaire de définir ce drapeau pour permettre à une réécriture + de continuer si la requête HTTP en cours d’écriture possède un point d'interrogation encodé, « %3f », et si le résultat réécrit contient un « ? » dans + la substitution. Cela protège d’une URL malveillante tirant avantage d’une + capture et d’une resubstitution du point d'interrogation encodé.

    +
    top
    +
    +

    UnsafePrefixStat

    +

    La définition de ce drapeau est requise dans les substitutions à + l'échelle du serveur qui commencent par une variable ou une référence + arrière et se résolvent en un chemin du système de fichiers. Ces + substitutions ne sont pas préfixées par la racine des documents. Cela protège + d’une URL malveillante faisant correspondre la substitution expansée à un + emplacement non souhaité du système de fichiers.

    +

    Langues Disponibles:  en  |  fr 

    diff --git a/docs/manual/style/version.ent b/docs/manual/style/version.ent index 9e62acc..e3df8e2 100644 --- a/docs/manual/style/version.ent +++ b/docs/manual/style/version.ent @@ -19,6 +19,6 @@ - + diff --git a/httpd.spec b/httpd.spec index 9719769..e903574 100644 --- a/httpd.spec +++ b/httpd.spec @@ -4,7 +4,7 @@ Summary: Apache HTTP Server Name: httpd -Version: 2.4.61 +Version: 2.4.62 Release: 1 URL: http://httpd.apache.org/ Vendor: Apache Software Foundation diff --git a/include/ap_release.h b/include/ap_release.h index 3be7bae..8d362c4 100644 --- a/include/ap_release.h +++ b/include/ap_release.h @@ -43,7 +43,7 @@ #define AP_SERVER_MAJORVERSION_NUMBER 2 #define AP_SERVER_MINORVERSION_NUMBER 4 -#define AP_SERVER_PATCHLEVEL_NUMBER 61 +#define AP_SERVER_PATCHLEVEL_NUMBER 62 #define AP_SERVER_DEVBUILD_BOOLEAN 0 /* Synchronize the above with docs/manual/style/version.ent */ diff --git a/modules/http/http_request.c b/modules/http/http_request.c index 71ecc2b..7e9477b 100644 --- a/modules/http/http_request.c +++ b/modules/http/http_request.c @@ -708,7 +708,7 @@ AP_DECLARE(void) ap_internal_fast_redirect(request_rec *rr, request_rec *r) r->args = rr->args; r->finfo = rr->finfo; r->handler = rr->handler; - ap_set_content_type_ex(r, rr->content_type, AP_REQUEST_IS_TRUSTED_CT(r)); + ap_set_content_type_ex(r, rr->content_type, AP_REQUEST_IS_TRUSTED_CT(rr)); r->content_encoding = rr->content_encoding; r->content_languages = rr->content_languages; r->per_dir_config = rr->per_dir_config; diff --git a/modules/mappers/mod_rewrite.c b/modules/mappers/mod_rewrite.c index 3fc2baf..f1c22e3 100644 --- a/modules/mappers/mod_rewrite.c +++ b/modules/mappers/mod_rewrite.c @@ -179,6 +179,7 @@ static const char* really_last_key = "rewrite_really_last"; #define RULEFLAG_ESCAPECTLS (1<<21) #define RULEFLAG_UNSAFE_PREFIX_STAT (1<<22) #define RULEFLAG_UNSAFE_ALLOW3F (1<<23) +#define RULEFLAG_UNC (1<<24) /* return code of the rewrite rule * the result may be escaped - or not @@ -3843,6 +3844,9 @@ static const char *cmd_rewriterule_setflag(apr_pool_t *p, void *_cfg, else if(!strcasecmp(key, "nsafeAllow3F")) { cfg->flags |= RULEFLAG_UNSAFE_ALLOW3F; } + else if(!strcasecmp(key, "NC")) { + cfg->flags |= RULEFLAG_UNC; + } else { ++error; } @@ -4462,6 +4466,16 @@ static rule_return_type apply_rewrite_rule(rewriterule_entry *p, return RULE_RC_MATCH; } + if (!(p->flags & RULEFLAG_UNC)) { + /* merge leading slashes, unless they were literals in the sub */ + if (!AP_IS_SLASH(p->output[0]) || !AP_IS_SLASH(p->output[1])) { + while (AP_IS_SLASH(r->filename[0]) && + AP_IS_SLASH(r->filename[1])) { + r->filename++; + } + } + } + /* Finally remember the forced mime-type */ force_type_handler(p, ctx); diff --git a/modules/proxy/balancers/mod_lbmethod_bytraffic.c b/modules/proxy/balancers/mod_lbmethod_bytraffic.c index 6cfab94..724b028 100644 --- a/modules/proxy/balancers/mod_lbmethod_bytraffic.c +++ b/modules/proxy/balancers/mod_lbmethod_bytraffic.c @@ -73,8 +73,6 @@ static apr_status_t reset(proxy_balancer *balancer, server_rec *s) proxy_worker **worker; worker = (proxy_worker **)balancer->workers->elts; for (i = 0; i < balancer->workers->nelts; i++, worker++) { - (*worker)->s->lbstatus = 0; - (*worker)->s->busy = 0; (*worker)->s->transferred = 0; (*worker)->s->read = 0; } diff --git a/modules/proxy/mod_proxy.c b/modules/proxy/mod_proxy.c index ad0c031..8f13e68 100644 --- a/modules/proxy/mod_proxy.c +++ b/modules/proxy/mod_proxy.c @@ -822,60 +822,6 @@ static int proxy_detect(request_rec *r) return DECLINED; } -static const char *proxy_interpolate(request_rec *r, const char *str) -{ - /* Interpolate an env str in a configuration string - * Syntax ${var} --> value_of(var) - * Method: replace one var, and recurse on remainder of string - * Nothing clever here, and crap like nested vars may do silly things - * but we'll at least avoid sending the unwary into a loop - */ - const char *start; - const char *end; - const char *var; - const char *val; - const char *firstpart; - - start = ap_strstr_c(str, "${"); - if (start == NULL) { - return str; - } - end = ap_strchr_c(start+2, '}'); - if (end == NULL) { - return str; - } - /* OK, this is syntax we want to interpolate. Is there such a var ? */ - var = apr_pstrmemdup(r->pool, start+2, end-(start+2)); - val = apr_table_get(r->subprocess_env, var); - firstpart = apr_pstrmemdup(r->pool, str, (start-str)); - - if (val == NULL) { - return apr_pstrcat(r->pool, firstpart, - proxy_interpolate(r, end+1), NULL); - } - else { - return apr_pstrcat(r->pool, firstpart, val, - proxy_interpolate(r, end+1), NULL); - } -} -static apr_array_header_t *proxy_vars(request_rec *r, - apr_array_header_t *hdr) -{ - int i; - apr_array_header_t *ret = apr_array_make(r->pool, hdr->nelts, - sizeof (struct proxy_alias)); - struct proxy_alias *old = (struct proxy_alias *) hdr->elts; - - for (i = 0; i < hdr->nelts; ++i) { - struct proxy_alias *newcopy = apr_array_push(ret); - newcopy->fake = (old[i].flags & PROXYPASS_INTERPOLATE) - ? proxy_interpolate(r, old[i].fake) : old[i].fake; - newcopy->real = (old[i].flags & PROXYPASS_INTERPOLATE) - ? proxy_interpolate(r, old[i].real) : old[i].real; - } - return ret; -} - PROXY_DECLARE(int) ap_proxy_trans_match(request_rec *r, struct proxy_alias *ent, proxy_dir_conf *dconf) { @@ -891,8 +837,8 @@ PROXY_DECLARE(int) ap_proxy_trans_match(request_rec *r, struct proxy_alias *ent, const char *servlet_uri = NULL; if (dconf && (dconf->interpolate_env == 1) && (ent->flags & PROXYPASS_INTERPOLATE)) { - fake = proxy_interpolate(r, ent->fake); - real = proxy_interpolate(r, ent->real); + fake = ap_proxy_interpolate(r, ent->fake); + real = ap_proxy_interpolate(r, ent->real); } else { fake = ent->fake; @@ -1212,38 +1158,12 @@ static int proxy_map_location(request_rec *r) */ static int proxy_fixup(request_rec *r) { - char *url, *p; - int access_status; - proxy_dir_conf *dconf = ap_get_module_config(r->per_dir_config, - &proxy_module); - if (!r->proxyreq || !r->filename || strncmp(r->filename, "proxy:", 6) != 0) return DECLINED; /* XXX: Shouldn't we try this before we run the proxy_walk? */ - url = &r->filename[6]; - if ((dconf->interpolate_env == 1) && (r->proxyreq == PROXYREQ_REVERSE)) { - /* create per-request copy of reverse proxy conf, - * and interpolate vars in it - */ - proxy_req_conf *rconf = apr_palloc(r->pool, sizeof(proxy_req_conf)); - ap_set_module_config(r->request_config, &proxy_module, rconf); - rconf->raliases = proxy_vars(r, dconf->raliases); - rconf->cookie_paths = proxy_vars(r, dconf->cookie_paths); - rconf->cookie_domains = proxy_vars(r, dconf->cookie_domains); - } - - /* canonicalise each specific scheme */ - if ((access_status = proxy_run_canon_handler(r, url))) { - return access_status; - } - - p = strchr(url, ':'); - if (p == NULL || p == url) - return HTTP_BAD_REQUEST; - - return OK; /* otherwise; we've done the best we can */ + return ap_proxy_canon_url(r); } /* Send a redirection if the request contains a hostname which is not */ @@ -1321,11 +1241,8 @@ static int proxy_handler(request_rec *r) r->proxyreq = PROXYREQ_REVERSE; r->filename = apr_pstrcat(r->pool, r->handler, r->filename, NULL); - /* Still need to fixup/canonicalize r->filename */ - rc = ap_proxy_fixup_uds_filename(r); - if (rc <= OK) { - rc = proxy_fixup(r); - } + /* Still need to canonicalize r->filename */ + rc = ap_proxy_canon_url(r); if (rc != OK) { r->filename = old_filename; r->proxyreq = 0; @@ -1338,6 +1255,15 @@ static int proxy_handler(request_rec *r) return rc; } + uri = r->filename + 6; + p = strchr(uri, ':'); + if (p == NULL) { + ap_log_rerror(APLOG_MARK, APLOG_DEBUG, 0, r, APLOGNO(01141) + "proxy_handler no URL in %s", r->filename); + return HTTP_BAD_REQUEST; + } + scheme = apr_pstrmemdup(r->pool, uri, p - uri); + /* handle max-forwards / OPTIONS / TRACE */ if ((str = apr_table_get(r->headers_in, "Max-Forwards"))) { char *end; @@ -1417,14 +1343,6 @@ static int proxy_handler(request_rec *r) } } - uri = r->filename + 6; - p = strchr(uri, ':'); - if (p == NULL) { - ap_log_rerror(APLOG_MARK, APLOG_DEBUG, 0, r, APLOGNO(01141) - "proxy_handler no URL in %s", r->filename); - return HTTP_BAD_REQUEST; - } - /* If the host doesn't have a domain name, add one and redirect. */ if (conf->domain != NULL) { rc = proxy_needsdomain(r, uri, conf->domain); @@ -1432,7 +1350,6 @@ static int proxy_handler(request_rec *r) return HTTP_MOVED_PERMANENTLY; } - scheme = apr_pstrmemdup(r->pool, uri, p - uri); /* Check URI's destination host against NoProxy hosts */ /* Bypass ProxyRemote server lookup if configured as NoProxy */ for (direct_connect = i = 0; i < conf->dirconn->nelts && diff --git a/modules/proxy/mod_proxy.h b/modules/proxy/mod_proxy.h index 59572bf..cd38889 100644 --- a/modules/proxy/mod_proxy.h +++ b/modules/proxy/mod_proxy.h @@ -1008,6 +1008,7 @@ PROXY_DECLARE(proxy_balancer_shared *) ap_proxy_find_balancershm(ap_slotmem_prov * r->notes ("uds_path") * @param r current request * @return OK if fixed up, DECLINED if not UDS, or an HTTP_XXX error + * @remark Deprecated (for internal use only) */ PROXY_DECLARE(int) ap_proxy_fixup_uds_filename(request_rec *r); diff --git a/modules/proxy/mod_proxy_balancer.c b/modules/proxy/mod_proxy_balancer.c index 3c0f5a8..79cb8bb 100644 --- a/modules/proxy/mod_proxy_balancer.c +++ b/modules/proxy/mod_proxy_balancer.c @@ -17,6 +17,7 @@ /* Load balancer module for Apache proxy */ #include "mod_proxy.h" +#include "proxy_util.h" #include "scoreboard.h" #include "ap_mpm.h" #include "apr_version.h" @@ -69,23 +70,21 @@ extern void proxy_update_members(proxy_balancer **balancer, request_rec *r, static int proxy_balancer_canon(request_rec *r, char *url) { - char *host, *path; - char *search = NULL; - const char *err; + char *host; apr_port_t port = 0; + const char *err; /* TODO: offset of BALANCER_PREFIX ?? */ if (ap_cstr_casecmpn(url, "balancer:", 9) == 0) { + ap_log_rerror(APLOG_MARK, APLOG_TRACE1, 0, r, "canonicalising URL %s", url); url += 9; } else { return DECLINED; } - ap_log_rerror(APLOG_MARK, APLOG_TRACE1, 0, r, "canonicalising URL %s", url); - /* do syntatic check. - * We break the URL into host, port, path, search + * We break the URL into host, port, path */ err = ap_proxy_canon_netloc(r->pool, &url, NULL, NULL, &host, &port); if (err) { @@ -94,50 +93,12 @@ static int proxy_balancer_canon(request_rec *r, char *url) url, err); return HTTP_BAD_REQUEST; } - /* - * now parse path/search args, according to rfc1738: - * process the path. With proxy-noncanon set (by - * mod_proxy) we use the raw, unparsed uri - */ - if (apr_table_get(r->notes, "proxy-nocanon")) { - path = url; /* this is the raw path */ - } - else if (apr_table_get(r->notes, "proxy-noencode")) { - path = url; /* this is the encoded path already */ - search = r->args; - } - else { - core_dir_config *d = ap_get_core_module_config(r->per_dir_config); - int flags = d->allow_encoded_slashes && !d->decode_encoded_slashes ? PROXY_CANONENC_NOENCODEDSLASHENCODING : 0; - path = ap_proxy_canonenc_ex(r->pool, url, strlen(url), enc_path, flags, - r->proxyreq); - if (!path) { - return HTTP_BAD_REQUEST; - } - search = r->args; - } - /* - * If we have a raw control character or a ' ' in nocanon path or - * r->args, correct encoding was missed. + /* The canon_handler hooks are run per the BalancerMember in + * balancer_fixup(), keep the original/raw path for now. */ - if (path == url && *ap_scan_vchar_obstext(path)) { - ap_log_rerror(APLOG_MARK, APLOG_ERR, 0, r, APLOGNO(10416) - "To be forwarded path contains control " - "characters or spaces"); - return HTTP_FORBIDDEN; - } - if (search && *ap_scan_vchar_obstext(search)) { - ap_log_rerror(APLOG_MARK, APLOG_ERR, 0, r, APLOGNO(10407) - "To be forwarded query string contains control " - "characters or spaces"); - return HTTP_FORBIDDEN; - } - - r->filename = apr_pstrcat(r->pool, "proxy:" BALANCER_PREFIX, host, - "/", path, (search) ? "?" : "", (search) ? search : "", NULL); - - r->path_info = apr_pstrcat(r->pool, "/", path, NULL); + r->filename = apr_pstrcat(r->pool, "proxy:" BALANCER_PREFIX, + host, "/", url, NULL); return OK; } @@ -429,25 +390,25 @@ static proxy_worker *find_best_worker(proxy_balancer *balancer, } -static int rewrite_url(request_rec *r, proxy_worker *worker, - char **url) +static int balancer_fixup(request_rec *r, proxy_worker *worker, char **url) { - const char *scheme = strstr(*url, "://"); - const char *path = NULL; + const char *path; + int rc; - if (scheme) - path = ap_strchr_c(scheme + 3, '/'); - - /* we break the URL into host, port, uri */ - if (!worker) { - return ap_proxyerror(r, HTTP_BAD_REQUEST, apr_pstrcat(r->pool, - "missing worker. URI cannot be parsed: ", *url, - NULL)); + /* Build the proxy URL from the worker URL and the actual path */ + path = strstr(*url, "://"); + if (path) { + path = ap_strchr_c(path + 3, '/'); } + r->filename = apr_pstrcat(r->pool, "proxy:", worker->s->name_ex, path, NULL); - *url = apr_pstrcat(r->pool, worker->s->name_ex, path, NULL); - - return OK; + /* Canonicalize r->filename per the worker scheme's canon_handler hook */ + rc = ap_proxy_canon_url(r); + if (rc == OK) { + AP_DEBUG_ASSERT(strncmp(r->filename, "proxy:", 6) == 0); + *url = apr_pstrdup(r->pool, r->filename + 6); + } + return rc; } static void force_recovery(proxy_balancer *balancer, server_rec *s) @@ -515,7 +476,8 @@ static int proxy_balancer_pre_request(proxy_worker **worker, * for balancer, because this is failover attempt. */ if (!*balancer && - !(*balancer = ap_proxy_get_balancer(r->pool, conf, *url, 1))) + (ap_cstr_casecmpn(*url, BALANCER_PREFIX, sizeof(BALANCER_PREFIX) - 1) + || !(*balancer = ap_proxy_get_balancer(r->pool, conf, *url, 1)))) return DECLINED; /* Step 2: Lock the LoadBalancer @@ -649,10 +611,12 @@ static int proxy_balancer_pre_request(proxy_worker **worker, /* Rewrite the url from 'balancer://url' * to the 'worker_scheme://worker_hostname[:worker_port]/url' - * This replaces the balancers fictional name with the - * real hostname of the elected worker. + * This replaces the balancers fictional name with the real + * hostname of the elected worker and canonicalizes according + * to the worker scheme (calls canon_handler hooks). */ - access_status = rewrite_url(r, *worker, url); + access_status = balancer_fixup(r, *worker, url); + /* Add the session route to request notes if present */ if (route) { apr_table_setn(r->notes, "session-sticky", sticky); diff --git a/modules/proxy/proxy_util.c b/modules/proxy/proxy_util.c index e71cbd8..7c0d315 100644 --- a/modules/proxy/proxy_util.c +++ b/modules/proxy/proxy_util.c @@ -1358,8 +1358,6 @@ PROXY_DECLARE(apr_status_t) ap_proxy_initialize_balancer(proxy_balancer *balance ap_log_error(APLOG_MARK, APLOG_CRIT, 0, s, APLOGNO(00921) "slotmem_attach failed"); return APR_EGENERAL; } - if (balancer->lbmethod && balancer->lbmethod->reset) - balancer->lbmethod->reset(balancer, s); #if APR_HAS_THREADS if (balancer->tmutex == NULL) { @@ -2429,14 +2427,14 @@ static int ap_proxy_retry_worker(const char *proxy_function, proxy_worker *worke * were passed a UDS url (eg: from mod_proxy) and adjust uds_path * as required. */ -PROXY_DECLARE(int) ap_proxy_fixup_uds_filename(request_rec *r) +static int fixup_uds_filename(request_rec *r) { char *uds_url = r->filename + 6, *origin_url; if (!strncmp(r->filename, "proxy:", 6) && !ap_cstr_casecmpn(uds_url, "unix:", 5) && (origin_url = ap_strchr(uds_url + 5, '|'))) { - char *uds_path = NULL; + char *uds_path = NULL, *end; apr_uri_t urisock; apr_status_t rv; @@ -2448,9 +2446,10 @@ PROXY_DECLARE(int) ap_proxy_fixup_uds_filename(request_rec *r) || !urisock.hostname[0])) { uds_path = ap_runtime_dir_relative(r->pool, urisock.path); } - if (!uds_path) { + if (!uds_path || !(end = ap_strchr(origin_url, ':'))) { ap_log_rerror(APLOG_MARK, APLOG_ERR, 0, r, APLOGNO(10292) "Invalid proxy UDS filename (%s)", r->filename); + apr_table_unset(r->notes, "uds_path"); return HTTP_BAD_REQUEST; } apr_table_setn(r->notes, "uds_path", uds_path); @@ -2459,14 +2458,136 @@ PROXY_DECLARE(int) ap_proxy_fixup_uds_filename(request_rec *r) "*: fixup UDS from %s: %s (%s)", r->filename, origin_url, uds_path); - /* Overwrite the UDS part in place */ - memmove(uds_url, origin_url, strlen(origin_url) + 1); + /* The hostname part of the URL is not mandated for UDS though + * the canon_handler hooks will require it, so add "localhost" + * if it's missing (won't be used anyway for an AF_UNIX socket). + */ + if (!end[1]) { + r->filename = apr_pstrcat(r->pool, "proxy:", + origin_url, "//localhost", + NULL); + } + else if (end[1] == '/' && end[2] == '/' && !end[3]) { + r->filename = apr_pstrcat(r->pool, "proxy:", + origin_url, "localhost", + NULL); + } + else { + /* Overwrite the UDS part of r->filename in place */ + memmove(uds_url, origin_url, strlen(origin_url) + 1); + } return OK; } + apr_table_unset(r->notes, "uds_path"); return DECLINED; } +/* Deprecated (unused upstream) */ +PROXY_DECLARE(int) ap_proxy_fixup_uds_filename(request_rec *r) +{ + return fixup_uds_filename(r); +} + +PROXY_DECLARE(const char *) ap_proxy_interpolate(request_rec *r, + const char *str) +{ + /* Interpolate an env str in a configuration string + * Syntax ${var} --> value_of(var) + * Method: replace one var, and recurse on remainder of string + * Nothing clever here, and crap like nested vars may do silly things + * but we'll at least avoid sending the unwary into a loop + */ + const char *start; + const char *end; + const char *var; + const char *val; + const char *firstpart; + + start = ap_strstr_c(str, "${"); + if (start == NULL) { + return str; + } + end = ap_strchr_c(start+2, '}'); + if (end == NULL) { + return str; + } + /* OK, this is syntax we want to interpolate. Is there such a var ? */ + var = apr_pstrmemdup(r->pool, start+2, end-(start+2)); + val = apr_table_get(r->subprocess_env, var); + firstpart = apr_pstrmemdup(r->pool, str, (start-str)); + + if (val == NULL) { + return apr_pstrcat(r->pool, firstpart, + ap_proxy_interpolate(r, end+1), NULL); + } + else { + return apr_pstrcat(r->pool, firstpart, val, + ap_proxy_interpolate(r, end+1), NULL); + } +} + +static apr_array_header_t *proxy_vars(request_rec *r, apr_array_header_t *hdr) +{ + int i; + apr_array_header_t *ret = apr_array_make(r->pool, hdr->nelts, + sizeof (struct proxy_alias)); + struct proxy_alias *old = (struct proxy_alias *) hdr->elts; + + for (i = 0; i < hdr->nelts; ++i) { + struct proxy_alias *newcopy = apr_array_push(ret); + newcopy->fake = (old[i].flags & PROXYPASS_INTERPOLATE) + ? ap_proxy_interpolate(r, old[i].fake) : old[i].fake; + newcopy->real = (old[i].flags & PROXYPASS_INTERPOLATE) + ? ap_proxy_interpolate(r, old[i].real) : old[i].real; + } + return ret; +} + +PROXY_DECLARE(int) ap_proxy_canon_url(request_rec *r) +{ + char *url, *p; + int access_status; + proxy_dir_conf *dconf = ap_get_module_config(r->per_dir_config, + &proxy_module); + + if (!r->proxyreq || !r->filename || strncmp(r->filename, "proxy:", 6) != 0) + return DECLINED; + + /* Put the UDS path appart if any (and not already stripped) */ + if (r->proxyreq == PROXYREQ_REVERSE) { + access_status = fixup_uds_filename(r); + if (ap_is_HTTP_ERROR(access_status)) { + return access_status; + } + } + + /* Keep this after fixup_uds_filename() */ + url = apr_pstrdup(r->pool, r->filename + 6); + + if ((dconf->interpolate_env == 1) && (r->proxyreq == PROXYREQ_REVERSE)) { + /* create per-request copy of reverse proxy conf, + * and interpolate vars in it + */ + proxy_req_conf *rconf = apr_palloc(r->pool, sizeof(proxy_req_conf)); + ap_set_module_config(r->request_config, &proxy_module, rconf); + rconf->raliases = proxy_vars(r, dconf->raliases); + rconf->cookie_paths = proxy_vars(r, dconf->cookie_paths); + rconf->cookie_domains = proxy_vars(r, dconf->cookie_domains); + } + + /* canonicalise each specific scheme */ + if ((access_status = proxy_run_canon_handler(r, url))) { + return access_status; + } + + p = strchr(url, ':'); + if (p == NULL || p == url) + return HTTP_BAD_REQUEST; + + return OK; /* otherwise; we've done the best we can */ +} + PROXY_DECLARE(int) ap_proxy_pre_request(proxy_worker **worker, proxy_balancer **balancer, request_rec *r, @@ -2476,16 +2597,16 @@ PROXY_DECLARE(int) ap_proxy_pre_request(proxy_worker **worker, access_status = proxy_run_pre_request(worker, balancer, r, conf, url); if (access_status == DECLINED && *balancer == NULL) { - const int forward = (r->proxyreq == PROXYREQ_PROXY); + /* UDS path stripped from *url by proxy_fixup() already */ *worker = ap_proxy_get_worker_ex(r->pool, NULL, conf, *url, - forward ? AP_PROXY_WORKER_NO_UDS : 0); + AP_PROXY_WORKER_NO_UDS); if (*worker) { ap_log_rerror(APLOG_MARK, APLOG_TRACE2, 0, r, "%s: found worker %s for %s", (*worker)->s->scheme, (*worker)->s->name_ex, *url); access_status = OK; } - else if (forward) { + else if (r->proxyreq == PROXYREQ_PROXY) { if (conf->forward) { ap_log_rerror(APLOG_MARK, APLOG_TRACE2, 0, r, "*: found forward proxy worker for %s", *url); @@ -2522,19 +2643,6 @@ PROXY_DECLARE(int) ap_proxy_pre_request(proxy_worker **worker, access_status = HTTP_SERVICE_UNAVAILABLE; } - if (access_status == OK && r->proxyreq == PROXYREQ_REVERSE) { - int rc = ap_proxy_fixup_uds_filename(r); - if (ap_is_HTTP_ERROR(rc)) { - return rc; - } - /* If the URL has changed in r->filename, take everything after - * the "proxy:" prefix. - */ - if (rc == OK) { - *url = apr_pstrdup(r->pool, r->filename + 6); - } - } - return access_status; } diff --git a/modules/proxy/proxy_util.h b/modules/proxy/proxy_util.h index bc131da..9bae20b 100644 --- a/modules/proxy/proxy_util.h +++ b/modules/proxy/proxy_util.h @@ -40,6 +40,23 @@ extern PROXY_DECLARE_DATA const apr_strmatch_pattern *ap_proxy_strmatch_domain; */ void proxy_util_register_hooks(apr_pool_t *p); +/* + * interpolate an env str in a configuration string + * + * @param r current request + * @param str the string to interpolcate + * @return the interpolated string + */ +PROXY_DECLARE(const char *) ap_proxy_interpolate(request_rec *r, + const char *str); + +/* + * Canonicalize the URL in r->filename + * @param r current request + * @return OK or an HTTP_XXX error + */ +PROXY_DECLARE(int) ap_proxy_canon_url(request_rec *r); + /** @} */ #endif /* PROXY_UTIL_H_ */ diff --git a/modules/ssl/ssl_engine_init.c b/modules/ssl/ssl_engine_init.c index 443eac4..598e89f 100644 --- a/modules/ssl/ssl_engine_init.c +++ b/modules/ssl/ssl_engine_init.c @@ -1424,7 +1424,7 @@ static apr_status_t ssl_init_server_certs(server_rec *s, if (modssl_is_engine_id(keyfile)) { apr_status_t rv; - if ((rv = modssl_load_engine_keypair(s, ptemp, vhost_id, + if ((rv = modssl_load_engine_keypair(s, p, ptemp, vhost_id, engine_certfile, keyfile, &cert, &pkey))) { return rv; @@ -1433,8 +1433,10 @@ static apr_status_t ssl_init_server_certs(server_rec *s, if (cert) { if (SSL_CTX_use_certificate(mctx->ssl_ctx, cert) < 1) { ap_log_error(APLOG_MARK, APLOG_EMERG, 0, s, APLOGNO(10137) - "Failed to configure engine certificate %s, check %s", - key_id, certfile); + "Failed to configure certificate %s from %s, check %s", + key_id, mc->szCryptoDevice ? + mc->szCryptoDevice : "provider", + certfile); ssl_log_ssl_error(SSLLOG_MARK, APLOG_EMERG, s); return APR_EGENERAL; } @@ -1445,8 +1447,9 @@ static apr_status_t ssl_init_server_certs(server_rec *s, if (SSL_CTX_use_PrivateKey(mctx->ssl_ctx, pkey) < 1) { ap_log_error(APLOG_MARK, APLOG_EMERG, 0, s, APLOGNO(10130) - "Failed to configure private key %s from engine", - keyfile); + "Failed to configure private key %s from %s", + keyfile, mc->szCryptoDevice ? + mc->szCryptoDevice : "provider"); ssl_log_ssl_error(SSLLOG_MARK, APLOG_EMERG, s); return APR_EGENERAL; } diff --git a/modules/ssl/ssl_engine_io.c b/modules/ssl/ssl_engine_io.c index 9c7d216..0be5318 100644 --- a/modules/ssl/ssl_engine_io.c +++ b/modules/ssl/ssl_engine_io.c @@ -2285,9 +2285,7 @@ void ssl_io_filter_init(conn_rec *c, request_rec *r, SSL *ssl) apr_pool_cleanup_register(c->pool, (void*)filter_ctx, ssl_io_filter_cleanup, apr_pool_cleanup_null); - if (APLOG_CS_IS_LEVEL(c, mySrvFromConn(c), APLOG_TRACE4)) { - modssl_set_io_callbacks(ssl); - } + modssl_set_io_callbacks(ssl, c, mySrvFromConn(c)); return; } @@ -2312,7 +2310,7 @@ void ssl_io_filter_register(apr_pool_t *p) #define DUMP_WIDTH 16 static void ssl_io_data_dump(conn_rec *c, server_rec *s, - const char *b, long len) + const char *b, int len) { char buf[256]; int i, j, rows, trunc, pos; @@ -2365,11 +2363,13 @@ static void ssl_io_data_dump(conn_rec *c, server_rec *s, } if (trunc > 0) ap_log_cserror(APLOG_MARK, APLOG_TRACE7, 0, c, s, - "| %04ld - ", len + trunc); + "| %04d - ", len + trunc); ap_log_cserror(APLOG_MARK, APLOG_TRACE7, 0, c, s, "+-------------------------------------------------------------------------+"); } +#define MODSSL_IO_DUMP_MAX APR_UINT16_MAX + #if OPENSSL_VERSION_NUMBER >= 0x30000000L static long modssl_io_cb(BIO *bio, int cmd, const char *argp, size_t len, int argi, long argl, int rc, @@ -2382,10 +2382,12 @@ static long modssl_io_cb(BIO *bio, int cmd, const char *argp, SSL *ssl; conn_rec *c; server_rec *s; + + /* unused */ #if OPENSSL_VERSION_NUMBER >= 0x30000000L - (void)len; - (void)processed; + (void)argi; #endif + (void)argl; if ((ssl = (SSL *)BIO_get_callback_arg(bio)) == NULL) return rc; @@ -2395,28 +2397,59 @@ static long modssl_io_cb(BIO *bio, int cmd, const char *argp, if ( cmd == (BIO_CB_WRITE|BIO_CB_RETURN) || cmd == (BIO_CB_READ |BIO_CB_RETURN) ) { - if (rc >= 0) { +#if OPENSSL_VERSION_NUMBER >= 0x30000000L + apr_size_t requested_len = len; + /* + * On OpenSSL >= 3 rc uses the meaning of the BIO_read_ex and + * BIO_write_ex functions return value and not the one of + * BIO_read and BIO_write. Hence 0 indicates an error. + */ + int ok = (rc > 0); +#else + apr_size_t requested_len = (apr_size_t)argi; + int ok = (rc >= 0); +#endif + if (ok) { +#if OPENSSL_VERSION_NUMBER >= 0x30000000L + apr_size_t actual_len = *processed; +#else + apr_size_t actual_len = (apr_size_t)rc; +#endif const char *dump = ""; if (APLOG_CS_IS_LEVEL(c, s, APLOG_TRACE7)) { - if (argp != NULL) - dump = "(BIO dump follows)"; - else + if (argp == NULL) dump = "(Oops, no memory buffer?)"; + else if (actual_len > MODSSL_IO_DUMP_MAX) + dump = "(BIO dump follows, truncated to " + APR_STRINGIFY(MODSSL_IO_DUMP_MAX) ")"; + else + dump = "(BIO dump follows)"; } ap_log_cserror(APLOG_MARK, APLOG_TRACE4, 0, c, s, - "%s: %s %ld/%d bytes %s BIO#%pp [mem: %pp] %s", + "%s: %s %" APR_SIZE_T_FMT "/%" APR_SIZE_T_FMT + " bytes %s BIO#%pp [mem: %pp] %s", MODSSL_LIBRARY_NAME, - (cmd == (BIO_CB_WRITE|BIO_CB_RETURN) ? "write" : "read"), - (long)rc, argi, (cmd == (BIO_CB_WRITE|BIO_CB_RETURN) ? "to" : "from"), + (cmd & BIO_CB_WRITE) ? "write" : "read", + actual_len, requested_len, + (cmd & BIO_CB_WRITE) ? "to" : "from", bio, argp, dump); - if (*dump != '\0' && argp != NULL) - ssl_io_data_dump(c, s, argp, rc); + /* + * *dump will only be != '\0' if + * APLOG_CS_IS_LEVEL(c, s, APLOG_TRACE7) + */ + if (*dump != '\0' && argp != NULL) { + int dump_len = (actual_len >= MODSSL_IO_DUMP_MAX + ? MODSSL_IO_DUMP_MAX + : actual_len); + ssl_io_data_dump(c, s, argp, dump_len); + } } else { ap_log_cserror(APLOG_MARK, APLOG_TRACE4, 0, c, s, - "%s: I/O error, %d bytes expected to %s on BIO#%pp [mem: %pp]", - MODSSL_LIBRARY_NAME, argi, - (cmd == (BIO_CB_WRITE|BIO_CB_RETURN) ? "write" : "read"), + "%s: I/O error, %" APR_SIZE_T_FMT + " bytes expected to %s on BIO#%pp [mem: %pp]", + MODSSL_LIBRARY_NAME, requested_len, + (cmd & BIO_CB_WRITE) ? "write" : "read", bio, argp); } } @@ -2433,10 +2466,15 @@ static APR_INLINE void set_bio_callback(BIO *bio, void *arg) BIO_set_callback_arg(bio, arg); } -void modssl_set_io_callbacks(SSL *ssl) +void modssl_set_io_callbacks(SSL *ssl, conn_rec *c, server_rec *s) { - BIO *rbio = SSL_get_rbio(ssl), - *wbio = SSL_get_wbio(ssl); + BIO *rbio, *wbio; + + if (!APLOG_CS_IS_LEVEL(c, s, APLOG_TRACE4)) + return; + + rbio = SSL_get_rbio(ssl); + wbio = SSL_get_wbio(ssl); if (rbio) { set_bio_callback(rbio, ssl); } diff --git a/modules/ssl/ssl_engine_kernel.c b/modules/ssl/ssl_engine_kernel.c index fa1b3a8..9c51021 100644 --- a/modules/ssl/ssl_engine_kernel.c +++ b/modules/ssl/ssl_engine_kernel.c @@ -2585,9 +2585,7 @@ static int ssl_find_vhost(void *servername, conn_rec *c, server_rec *s) * (and the first vhost doesn't use APLOG_TRACE4), then * we need to set that callback here. */ - if (APLOGtrace4(s)) { - modssl_set_io_callbacks(ssl); - } + modssl_set_io_callbacks(ssl, c, s); return 1; } diff --git a/modules/ssl/ssl_engine_pphrase.c b/modules/ssl/ssl_engine_pphrase.c index 699019f..8a08ede 100644 --- a/modules/ssl/ssl_engine_pphrase.c +++ b/modules/ssl/ssl_engine_pphrase.c @@ -31,6 +31,9 @@ #include "ssl_private.h" #include +#if MODSSL_HAVE_OPENSSL_STORE +#include +#endif typedef struct { server_rec *s; @@ -608,7 +611,7 @@ int ssl_pphrase_Handle_CB(char *buf, int bufsize, int verify, void *srv) return (len); } -#if MODSSL_HAVE_ENGINE_API +#if MODSSL_HAVE_ENGINE_API || MODSSL_HAVE_OPENSSL_STORE /* OpenSSL UI implementation for passphrase entry; largely duplicated * from ssl_pphrase_Handle_CB but adjusted for UI API. TODO: Might be @@ -826,21 +829,32 @@ static UI_METHOD *get_passphrase_ui(apr_pool_t *p) } #endif +#if MODSSL_HAVE_ENGINE_API +static apr_status_t modssl_engine_cleanup(void *engine) +{ + ENGINE *e = engine; -apr_status_t modssl_load_engine_keypair(server_rec *s, apr_pool_t *p, - const char *vhostid, - const char *certid, const char *keyid, - X509 **pubkey, EVP_PKEY **privkey) + ENGINE_finish(e); + + return APR_SUCCESS; +} + +static apr_status_t modssl_load_keypair_engine(server_rec *s, apr_pool_t *pconf, + apr_pool_t *ptemp, + const char *vhostid, + const char *certid, + const char *keyid, + X509 **pubkey, + EVP_PKEY **privkey) { -#if MODSSL_HAVE_ENGINE_API const char *c, *scheme; ENGINE *e; - UI_METHOD *ui_method = get_passphrase_ui(p); + UI_METHOD *ui_method = get_passphrase_ui(ptemp); pphrase_cb_arg_t ppcb; memset(&ppcb, 0, sizeof ppcb); ppcb.s = s; - ppcb.p = p; + ppcb.p = ptemp; ppcb.bPassPhraseDialogOnce = TRUE; ppcb.key_id = vhostid; ppcb.pkey_file = keyid; @@ -853,7 +867,7 @@ apr_status_t modssl_load_engine_keypair(server_rec *s, apr_pool_t *p, return ssl_die(s); } - scheme = apr_pstrmemdup(p, keyid, c - keyid); + scheme = apr_pstrmemdup(ptemp, keyid, c - keyid); if (!(e = ENGINE_by_id(scheme))) { ap_log_error(APLOG_MARK, APLOG_EMERG, 0, s, APLOGNO(10132) "Init: Failed to load engine for private key %s", @@ -902,11 +916,136 @@ apr_status_t modssl_load_engine_keypair(server_rec *s, apr_pool_t *p, return ssl_die(s); } - ENGINE_finish(e); + /* Release the functional reference obtained by ENGINE_init() only + * when after the ENGINE is no longer used. */ + apr_pool_cleanup_register(pconf, e, modssl_engine_cleanup, modssl_engine_cleanup); + + /* Release the structural reference obtained by ENGINE_by_id() + * immediately. */ ENGINE_free(e); return APR_SUCCESS; +} +#endif + +#if MODSSL_HAVE_OPENSSL_STORE +static OSSL_STORE_INFO *modssl_load_store_uri(server_rec *s, apr_pool_t *p, + const char *vhostid, + const char *uri, int info_type) +{ + OSSL_STORE_CTX *sctx; + UI_METHOD *ui_method = get_passphrase_ui(p); + pphrase_cb_arg_t ppcb; + OSSL_STORE_INFO *info = NULL; + + memset(&ppcb, 0, sizeof ppcb); + ppcb.s = s; + ppcb.p = p; + ppcb.bPassPhraseDialogOnce = TRUE; + ppcb.key_id = vhostid; + ppcb.pkey_file = uri; + + sctx = OSSL_STORE_open(uri, ui_method, &ppcb, NULL, NULL); + if (!sctx) { + ap_log_error(APLOG_MARK, APLOG_ERR, 0, s, APLOGNO(10491) + "Init: OSSL_STORE_open failed for PKCS#11 URI `%s'", + uri); + return NULL; + } + + while (!OSSL_STORE_eof(sctx)) { + info = OSSL_STORE_load(sctx); + if (!info) + break; + + if (OSSL_STORE_INFO_get_type(info) == info_type) + break; + + OSSL_STORE_INFO_free(info); + info = NULL; + } + + OSSL_STORE_close(sctx); + + return info; +} + +static apr_status_t modssl_load_keypair_store(server_rec *s, apr_pool_t *p, + const char *vhostid, + const char *certid, + const char *keyid, + X509 **pubkey, + EVP_PKEY **privkey) +{ + OSSL_STORE_INFO *info = NULL; + + *privkey = NULL; + *pubkey = NULL; + + info = modssl_load_store_uri(s, p, vhostid, keyid, OSSL_STORE_INFO_PKEY); + if (!info) { + ap_log_error(APLOG_MARK, APLOG_EMERG, 0, s, APLOGNO(10492) + "Init: OSSL_STORE_INFO_PKEY lookup failed for private key identifier `%s'", + keyid); + return ssl_die(s); + } + + *privkey = OSSL_STORE_INFO_get1_PKEY(info); + OSSL_STORE_INFO_free(info); + if (!*privkey) { + ap_log_error(APLOG_MARK, APLOG_EMERG, 0, s, APLOGNO(10493) + "Init: OSSL_STORE_INFO_PKEY lookup failed for private key identifier `%s'", + keyid); + return ssl_die(s); + } + + if (certid) { + info = modssl_load_store_uri(s, p, vhostid, certid, OSSL_STORE_INFO_CERT); + if (!info) { + ap_log_error(APLOG_MARK, APLOG_EMERG, 0, s, APLOGNO(10494) + "Init: OSSL_STORE_INFO_CERT lookup failed for certificate identifier `%s'", + keyid); + return ssl_die(s); + } + + *pubkey = OSSL_STORE_INFO_get1_CERT(info); + OSSL_STORE_INFO_free(info); + if (!*pubkey) { + ap_log_error(APLOG_MARK, APLOG_EMERG, 0, s, APLOGNO(10495) + "Init: OSSL_STORE_INFO_CERT lookup failed for certificate identifier `%s'", + certid); + return ssl_die(s); + } + } + + return APR_SUCCESS; +} +#endif + +apr_status_t modssl_load_engine_keypair(server_rec *s, + apr_pool_t *pconf, apr_pool_t *ptemp, + const char *vhostid, + const char *certid, const char *keyid, + X509 **pubkey, EVP_PKEY **privkey) +{ +#if MODSSL_HAVE_ENGINE_API + SSLModConfigRec *mc = myModConfig(s); + + /* For OpenSSL 3.x, use the STORE-based API if either ENGINE + * support was not present compile-time, or if it's built but + * SSLCryptoDevice is not configured. */ + if (mc->szCryptoDevice) + return modssl_load_keypair_engine(s, pconf, ptemp, + vhostid, certid, keyid, + pubkey, privkey); +#endif +#if MODSSL_HAVE_OPENSSL_STORE + return modssl_load_keypair_store(s, ptemp, vhostid, certid, keyid, + pubkey, privkey); #else + ap_log_error(APLOG_MARK, APLOG_EMERG, 0, s, APLOGNO(10496) + "Init: no method for loading keypair for %s (%s | %s)", + vhostid, certid ? certid : "no cert", keyid); return APR_ENOTIMPL; #endif } diff --git a/modules/ssl/ssl_private.h b/modules/ssl/ssl_private.h index 25d79ce..c517a7b 100644 --- a/modules/ssl/ssl_private.h +++ b/modules/ssl/ssl_private.h @@ -118,6 +118,15 @@ #define MODSSL_HAVE_ENGINE_API 0 #endif +/* Use OpenSSL 3.x STORE for loading URI keys and certificates starting with + * OpenSSL 3.0 + */ +#if OPENSSL_VERSION_NUMBER >= 0x30000000 +#define MODSSL_HAVE_OPENSSL_STORE 1 +#else +#define MODSSL_HAVE_OPENSSL_STORE 0 +#endif + #if (OPENSSL_VERSION_NUMBER < 0x0090801f) #error mod_ssl requires OpenSSL 0.9.8a or later #endif @@ -1049,7 +1058,7 @@ void modssl_callback_keylog(const SSL *ssl, const char *line); /** I/O */ void ssl_io_filter_init(conn_rec *, request_rec *r, SSL *); void ssl_io_filter_register(apr_pool_t *); -void modssl_set_io_callbacks(SSL *ssl); +void modssl_set_io_callbacks(SSL *ssl, conn_rec *c, server_rec *s); /* ssl_io_buffer_fill fills the setaside buffering of the HTTP request * to allow an SSL renegotiation to take place. */ @@ -1081,7 +1090,8 @@ apr_status_t ssl_load_encrypted_pkey(server_rec *, apr_pool_t *, int, /* Load public and/or private key from the configured ENGINE. Private * key returned as *pkey. certid can be NULL, in which case *pubkey * is not altered. Errors logged on failure. */ -apr_status_t modssl_load_engine_keypair(server_rec *s, apr_pool_t *p, +apr_status_t modssl_load_engine_keypair(server_rec *s, + apr_pool_t *pconf, apr_pool_t *ptemp, const char *vhostid, const char *certid, const char *keyid, X509 **pubkey, EVP_PKEY **privkey); diff --git a/modules/ssl/ssl_util.c b/modules/ssl/ssl_util.c index 87ddfa7..7473edb 100644 --- a/modules/ssl/ssl_util.c +++ b/modules/ssl/ssl_util.c @@ -476,7 +476,7 @@ void ssl_util_thread_id_setup(apr_pool_t *p) int modssl_is_engine_id(const char *name) { -#if MODSSL_HAVE_ENGINE_API +#if MODSSL_HAVE_ENGINE_API || MODSSL_HAVE_OPENSSL_STORE /* ### Can handle any other special ENGINE key names here? */ return strncmp(name, "pkcs11:", 7) == 0; #else diff --git a/server/mpm/event/event.c b/server/mpm/event/event.c index 3672f44..7e7a7e9 100644 --- a/server/mpm/event/event.c +++ b/server/mpm/event/event.c @@ -2319,31 +2319,32 @@ static void setup_threads_runtime(void) clean_child_exit(APEXIT_CHILDFATAL); } - /* Create the main pollset */ + /* Create the main pollset. When APR_POLLSET_WAKEABLE is asked we account + * for the wakeup pipe explicitely with pollset_size+1 because some pollset + * implementations don't do it implicitely in APR. + */ pollset_flags = APR_POLLSET_THREADSAFE | APR_POLLSET_NOCOPY | - APR_POLLSET_NODEFAULT | APR_POLLSET_WAKEABLE; + APR_POLLSET_WAKEABLE | APR_POLLSET_NODEFAULT; for (i = 0; i < sizeof(good_methods) / sizeof(good_methods[0]); i++) { - rv = apr_pollset_create_ex(&event_pollset, pollset_size, pruntime, + rv = apr_pollset_create_ex(&event_pollset, pollset_size + 1, pruntime, pollset_flags, good_methods[i]); if (rv == APR_SUCCESS) { listener_is_wakeable = 1; break; } } - if (rv != APR_SUCCESS) { - pollset_flags &= ~APR_POLLSET_WAKEABLE; - for (i = 0; i < sizeof(good_methods) / sizeof(good_methods[0]); i++) { - rv = apr_pollset_create_ex(&event_pollset, pollset_size, pruntime, - pollset_flags, good_methods[i]); - if (rv == APR_SUCCESS) { - break; - } - } - } if (rv != APR_SUCCESS) { pollset_flags &= ~APR_POLLSET_NODEFAULT; - rv = apr_pollset_create(&event_pollset, pollset_size, pruntime, + rv = apr_pollset_create(&event_pollset, pollset_size + 1, pruntime, pollset_flags); + if (rv == APR_SUCCESS) { + listener_is_wakeable = 1; + } + else { + pollset_flags &= ~APR_POLLSET_WAKEABLE; + rv = apr_pollset_create(&event_pollset, pollset_size, pruntime, + pollset_flags); + } } if (rv != APR_SUCCESS) { ap_log_error(APLOG_MARK, APLOG_ERR, rv, ap_server_conf, APLOGNO(03103) diff --git a/server/mpm/worker/worker.c b/server/mpm/worker/worker.c index 7b572bd..315371d 100644 --- a/server/mpm/worker/worker.c +++ b/server/mpm/worker/worker.c @@ -125,10 +125,11 @@ static int max_workers = 0; static int server_limit = 0; static int thread_limit = 0; static int had_healthy_child = 0; -static int dying = 0; +static volatile int dying = 0; static int workers_may_exit = 0; static int start_thread_may_exit = 0; static int listener_may_exit = 0; +static int listener_is_wakeable = 0; /* Pollset supports APR_POLLSET_WAKEABLE */ static int requests_this_child; static int num_listensocks = 0; static int resource_shortage = 0; @@ -272,6 +273,15 @@ static void close_worker_sockets(void) static void wakeup_listener(void) { listener_may_exit = 1; + + /* Unblock the listener if it's poll()ing */ + if (worker_pollset && listener_is_wakeable) { + apr_pollset_wakeup(worker_pollset); + } + + /* unblock the listener if it's waiting for a worker */ + ap_queue_info_term(worker_queue_info); + if (!listener_os_thread) { /* XXX there is an obscure path that this doesn't handle perfectly: * right after listener thread is created but before @@ -280,10 +290,6 @@ static void wakeup_listener(void) */ return; } - - /* unblock the listener if it's waiting for a worker */ - ap_queue_info_term(worker_queue_info); - /* * we should just be able to "kill(ap_my_pid, LISTENER_SIGNAL)" on all * platforms and wake up the listener thread since it is the only thread @@ -861,6 +867,7 @@ static void create_listener_thread(thread_starter *ts) static void setup_threads_runtime(void) { ap_listen_rec *lr; + int pollset_flags; apr_status_t rv; /* All threads (listener, workers) and synchronization objects (queues, @@ -893,9 +900,21 @@ static void setup_threads_runtime(void) clean_child_exit(APEXIT_CHILDFATAL); } - /* Create the main pollset */ - rv = apr_pollset_create(&worker_pollset, num_listensocks, pruntime, - APR_POLLSET_NOCOPY); + /* Create the main pollset. When APR_POLLSET_WAKEABLE is asked we account + * for the wakeup pipe explicitely with num_listensocks+1 because some + * pollset implementations don't do it implicitely in APR. + */ + pollset_flags = APR_POLLSET_NOCOPY | APR_POLLSET_WAKEABLE; + rv = apr_pollset_create(&worker_pollset, num_listensocks + 1, pruntime, + pollset_flags); + if (rv == APR_SUCCESS) { + listener_is_wakeable = 1; + } + else { + pollset_flags &= ~APR_POLLSET_WAKEABLE; + rv = apr_pollset_create(&worker_pollset, num_listensocks, pruntime, + pollset_flags); + } if (rv != APR_SUCCESS) { ap_log_error(APLOG_MARK, APLOG_EMERG, rv, ap_server_conf, APLOGNO(03285) "Couldn't create pollset in thread;" @@ -1031,19 +1050,17 @@ static void join_workers(apr_thread_t *listener, apr_thread_t **threads, */ iter = 0; - while (iter < 10 && -#ifdef HAVE_PTHREAD_KILL - pthread_kill(*listener_os_thread, 0) -#else - kill(ap_my_pid, 0) -#endif - == 0) { - /* listener not dead yet */ - apr_sleep(apr_time_make(0, 500000)); + while (!dying) { + apr_sleep(apr_time_from_msec(500)); + if (dying || ++iter > 10) { + break; + } + /* listener has not stopped accepting yet */ + ap_log_error(APLOG_MARK, APLOG_TRACE1, 0, ap_server_conf, + "listener has not stopped accepting yet (%d iter)", iter); wakeup_listener(); - ++iter; } - if (iter >= 10) { + if (iter > 10) { ap_log_error(APLOG_MARK, APLOG_DEBUG, 0, ap_server_conf, APLOGNO(00276) "the listener thread didn't exit"); } diff --git a/test/modules/core/conftest.py b/test/modules/core/conftest.py index 439cd22..22906ef 100644 --- a/test/modules/core/conftest.py +++ b/test/modules/core/conftest.py @@ -4,41 +4,27 @@ import os import pytest import sys +from .env import CoreTestEnv from pyhttpd.env import HttpdTestEnv sys.path.append(os.path.join(os.path.dirname(__file__), '../..')) def pytest_report_header(config, startdir): - env = HttpdTestEnv() + env = CoreTestEnv() return f"core [apache: {env.get_httpd_version()}, mpm: {env.mpm_module}, {env.prefix}]" @pytest.fixture(scope="package") -def env(pytestconfig) -> HttpdTestEnv: +def env(pytestconfig) -> CoreTestEnv: level = logging.INFO console = logging.StreamHandler() console.setLevel(level) console.setFormatter(logging.Formatter('%(levelname)s: %(message)s')) logging.getLogger('').addHandler(console) logging.getLogger('').setLevel(level=level) - env = HttpdTestEnv(pytestconfig=pytestconfig) + env = CoreTestEnv(pytestconfig=pytestconfig) env.setup_httpd() env.apache_access_log_clear() env.httpd_error_log.clear_log() return env - - -@pytest.fixture(autouse=True, scope="package") -def _session_scope(env): - env.httpd_error_log.set_ignored_lognos([ - 'AH10244', # core: invalid URI path - 'AH01264', # mod_cgid script not found - ]) - yield - assert env.apache_stop() == 0 - errors, warnings = env.httpd_error_log.get_missed() - assert (len(errors), len(warnings)) == (0, 0),\ - f"apache logged {len(errors)} errors and {len(warnings)} warnings: \n"\ - "{0}\n{1}\n".format("\n".join(errors), "\n".join(warnings)) - diff --git a/test/modules/core/env.py b/test/modules/core/env.py new file mode 100644 index 0000000..9c63380 --- /dev/null +++ b/test/modules/core/env.py @@ -0,0 +1,25 @@ +import inspect +import logging +import os + +from pyhttpd.env import HttpdTestEnv, HttpdTestSetup + +log = logging.getLogger(__name__) + + +class CoreTestSetup(HttpdTestSetup): + + def __init__(self, env: 'HttpdTestEnv'): + super().__init__(env=env) + self.add_source_dir(os.path.dirname(inspect.getfile(CoreTestSetup))) + self.add_modules(["cgid"]) + + +class CoreTestEnv(HttpdTestEnv): + + def __init__(self, pytestconfig=None): + super().__init__(pytestconfig=pytestconfig) + self.add_httpd_log_modules(["http", "core"]) + + def setup_httpd(self, setup: HttpdTestSetup = None): + super().setup_httpd(setup=CoreTestSetup(env=self)) diff --git a/test/modules/core/test_001_encoding.py b/test/modules/core/test_001_encoding.py index b7ffbaa..a3b24d0 100644 --- a/test/modules/core/test_001_encoding.py +++ b/test/modules/core/test_001_encoding.py @@ -1,12 +1,11 @@ import pytest +from typing import List, Optional from pyhttpd.conf import HttpdConf class TestEncoding: - EXP_AH10244_ERRS = 0 - @pytest.fixture(autouse=True, scope='class') def _class_scope(self, env): conf = HttpdConf(env, extras={ @@ -57,29 +56,29 @@ class TestEncoding: assert r.response["status"] == 200 # check path traversals - @pytest.mark.parametrize(["path", "status"], [ - ["/../echo.py", 400], - ["/nothing/../../echo.py", 400], - ["/cgi-bin/../../echo.py", 400], - ["/nothing/%2e%2e/%2e%2e/echo.py", 400], - ["/cgi-bin/%2e%2e/%2e%2e/echo.py", 400], - ["/nothing/%%32%65%%32%65/echo.py", 400], - ["/cgi-bin/%%32%65%%32%65/echo.py", 400], - ["/nothing/%%32%65%%32%65/%%32%65%%32%65/h2_env.py", 400], - ["/cgi-bin/%%32%65%%32%65/%%32%65%%32%65/h2_env.py", 400], - ["/nothing/%25%32%65%25%32%65/echo.py", 404], - ["/cgi-bin/%25%32%65%25%32%65/echo.py", 404], - ["/nothing/%25%32%65%25%32%65/%25%32%65%25%32%65/h2_env.py", 404], - ["/cgi-bin/%25%32%65%25%32%65/%25%32%65%25%32%65/h2_env.py", 404], + @pytest.mark.parametrize(["path", "status", "lognos"], [ + ["/../echo.py", 400, ["AH10244"]], + ["/nothing/../../echo.py", 400, ["AH10244"]], + ["/cgi-bin/../../echo.py", 400, ["AH10244"]], + ["/nothing/%2e%2e/%2e%2e/echo.py", 400, ["AH10244"]], + ["/cgi-bin/%2e%2e/%2e%2e/echo.py", 400, ["AH10244"]], + ["/nothing/%%32%65%%32%65/echo.py", 400, ["AH10244"]], + ["/cgi-bin/%%32%65%%32%65/echo.py", 400, ["AH10244"]], + ["/nothing/%%32%65%%32%65/%%32%65%%32%65/h2_env.py", 400, ["AH10244"]], + ["/cgi-bin/%%32%65%%32%65/%%32%65%%32%65/h2_env.py", 400, ["AH10244"]], + ["/nothing/%25%32%65%25%32%65/echo.py", 404, ["AH01264"]], + ["/cgi-bin/%25%32%65%25%32%65/echo.py", 404, ["AH01264"]], + ["/nothing/%25%32%65%25%32%65/%25%32%65%25%32%65/h2_env.py", 404, ["AH01264"]], + ["/cgi-bin/%25%32%65%25%32%65/%25%32%65%25%32%65/h2_env.py", 404, ["AH01264"]], ]) - def test_core_001_04(self, env, path, status): + def test_core_001_04(self, env, path, status, lognos: Optional[List[str]]): url = env.mkurl("https", "test1", path) r = env.curl_get(url) assert r.response["status"] == status - if status == 400: - TestEncoding.EXP_AH10244_ERRS += 1 - # the log will have a core:err about invalid URI path - + # + if lognos is not None: + env.httpd_error_log.ignore_recent(lognos = lognos) + # check handling of %2f url encodings that are not decoded by default @pytest.mark.parametrize(["host", "path", "status"], [ ["test1", "/006%2f006.css", 404], diff --git a/test/modules/core/test_002_restarts.py b/test/modules/core/test_002_restarts.py new file mode 100644 index 0000000..cf203bc --- /dev/null +++ b/test/modules/core/test_002_restarts.py @@ -0,0 +1,150 @@ +import os +import re +import time +from datetime import datetime, timedelta +from threading import Thread + +import pytest + +from .env import CoreTestEnv +from pyhttpd.conf import HttpdConf + + +class Loader: + + def __init__(self, env, url: str, clients: int, req_per_client: int = 10): + self.env = env + self.url = url + self.clients = clients + self.req_per_client = req_per_client + self.result = None + self.total_request = 0 + self._thread = None + + def run(self): + self.total_requests = self.clients * self.req_per_client + conn_per_client = 5 + args = [self.env.h2load, f"--connect-to=localhost:{self.env.https_port}", + "--h1", # use only http/1.1 + "-n", str(self.total_requests), # total # of requests to make + "-c", str(conn_per_client * self.clients), # total # of connections to make + "-r", str(self.clients), # connections at a time + "--rate-period", "2", # create conns every 2 sec + self.url, + ] + self.result = self.env.run(args) + + def start(self): + self._thread = Thread(target=self.run) + self._thread.start() + + def join(self): + self._thread.join() + + +class ChildDynamics: + + RE_DATE_TIME = re.compile(r'\[(?P[^\]]+)\] .*') + RE_TIME_FRAC = re.compile(r'(?P
    .* \d\d:\d\d:\d\d)(?P.(?P.\d+)) (?P\d+)') + RE_CHILD_CHANGE = re.compile(r'\[(?P[^\]]+)\] ' + r'\[mpm_event:\w+\]' + r' \[pid (?P\d+):tid \w+\] ' + r'.* Child (?P\d+) (?P\w+): ' + r'pid (?P\d+), gen (?P\d+), .*') + + def __init__(self, env: CoreTestEnv): + self.env = env + self.changes = list() + self._start = None + for l in open(env.httpd_error_log.path): + m = self.RE_CHILD_CHANGE.match(l) + if m: + self.changes.append({ + 'pid': int(m.group('pid')), + 'child_no': int(m.group('child_no')), + 'gen': int(m.group('generation')), + 'action': m.group('action'), + 'rtime' : self._rtime(m.group('date_time')) + }) + continue + if self._start is None: + m = self.RE_DATE_TIME.match(l) + if m: + self._rtime(m.group('date_time')) + + def _rtime(self, s: str) -> timedelta: + micros = 0 + m = self.RE_TIME_FRAC.match(s) + if m: + micros = int(m.group('micros')) + s = f"{m.group('dt')} {m.group('year')}" + d = datetime.strptime(s, '%a %b %d %H:%M:%S %Y') + timedelta(microseconds=micros) + if self._start is None: + self._start = d + delta = d - self._start + return f"{delta.seconds:+02d}.{delta.microseconds:06d}" + + + +@pytest.mark.skipif(condition='STRESS_TEST' not in os.environ, + reason="STRESS_TEST not set in env") +@pytest.mark.skipif(condition=not CoreTestEnv().h2load_is_at_least('1.41.0'), + reason="h2load unavailable or misses --connect-to option") +class TestRestarts: + + def test_core_002_01(self, env): + # Lets make a tight config that triggers dynamic child behaviour + conf = HttpdConf(env, extras={ + 'base': f""" + StartServers 1 + ServerLimit 3 + ThreadLimit 4 + ThreadsPerChild 4 + MinSpareThreads 4 + MaxSpareThreads 6 + MaxRequestWorkers 12 + MaxConnectionsPerChild 0 + + LogLevel mpm_event:trace6 + """, + }) + conf.add_vhost_cgi() + conf.install() + + # clear logs and start server, start load + env.httpd_error_log.clear_log() + assert env.apache_restart() == 0 + # we should see a single child started + cd = ChildDynamics(env) + assert len(cd.changes) == 1, f"{cd.changes}" + assert cd.changes[0]['action'] == 'started' + # This loader simulates 6 clients, each making 10 requests. + # delay.py sleeps for 1sec, so this should run for about 10 seconds + loader = Loader(env=env, url=env.mkurl("https", "cgi", "/delay.py"), + clients=6, req_per_client=10) + loader.start() + # Expect 2 more children to have been started after half time + time.sleep(5) + cd = ChildDynamics(env) + assert len(cd.changes) == 3, f"{cd.changes}" + assert len([x for x in cd.changes if x['action'] == 'started']) == 3, f"{cd.changes}" + + # Trigger a server reload + assert env.apache_reload() == 0 + # a graceful reload lets ongoing requests continue, but + # after a while all gen 0 children should have stopped + time.sleep(3) # FIXME: this pbly depends on the runtime a lot, do we have expectations? + cd = ChildDynamics(env) + gen0 = [x for x in cd.changes if x['gen'] == 0] + assert len([x for x in gen0 if x['action'] == 'stopped']) == 3 + + # wait for the loader to finish and stop the server + loader.join() + env.apache_stop() + + # Similar to before the reload, we expect 3 children to have + # been started and stopped again on server stop + cd = ChildDynamics(env) + gen1 = [x for x in cd.changes if x['gen'] == 1] + assert len([x for x in gen1 if x['action'] == 'started']) == 3 + assert len([x for x in gen1 if x['action'] == 'stopped']) == 3 diff --git a/test/modules/http1/__init__.py b/test/modules/http1/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/test/modules/http1/conftest.py b/test/modules/http1/conftest.py new file mode 100644 index 0000000..33a16a1 --- /dev/null +++ b/test/modules/http1/conftest.py @@ -0,0 +1,36 @@ +import logging +import os + +import pytest +import sys + +sys.path.append(os.path.join(os.path.dirname(__file__), '../..')) + +from .env import H1TestEnv + + +def pytest_report_header(config, startdir): + env = H1TestEnv() + return f"mod_http [apache: {env.get_httpd_version()}, mpm: {env.mpm_module}, {env.prefix}]" + + +def pytest_generate_tests(metafunc): + if "repeat" in metafunc.fixturenames: + count = int(metafunc.config.getoption("repeat")) + metafunc.fixturenames.append('tmp_ct') + metafunc.parametrize('repeat', range(count)) + + +@pytest.fixture(scope="package") +def env(pytestconfig) -> H1TestEnv: + level = logging.INFO + console = logging.StreamHandler() + console.setLevel(level) + console.setFormatter(logging.Formatter('%(levelname)s: %(message)s')) + logging.getLogger('').addHandler(console) + logging.getLogger('').setLevel(level=level) + env = H1TestEnv(pytestconfig=pytestconfig) + env.setup_httpd() + env.apache_access_log_clear() + env.httpd_error_log.clear_log() + return env diff --git a/test/modules/http1/env.py b/test/modules/http1/env.py new file mode 100644 index 0000000..e2df1a5 --- /dev/null +++ b/test/modules/http1/env.py @@ -0,0 +1,81 @@ +import inspect +import logging +import os +import subprocess +from typing import Dict, Any + +from pyhttpd.certs import CertificateSpec +from pyhttpd.conf import HttpdConf +from pyhttpd.env import HttpdTestEnv, HttpdTestSetup + +log = logging.getLogger(__name__) + + +class H1TestSetup(HttpdTestSetup): + + def __init__(self, env: 'HttpdTestEnv'): + super().__init__(env=env) + self.add_source_dir(os.path.dirname(inspect.getfile(H1TestSetup))) + self.add_modules(["cgid", "autoindex", "ssl"]) + + def make(self): + super().make() + self._add_h1test() + self._setup_data_1k_1m() + + def _add_h1test(self): + local_dir = os.path.dirname(inspect.getfile(H1TestSetup)) + p = subprocess.run([self.env.apxs, '-c', 'mod_h1test.c'], + capture_output=True, + cwd=os.path.join(local_dir, 'mod_h1test')) + rv = p.returncode + if rv != 0: + log.error(f"compiling md_h1test failed: {p.stderr}") + raise Exception(f"compiling md_h1test failed: {p.stderr}") + + modules_conf = os.path.join(self.env.server_dir, 'conf/modules.conf') + with open(modules_conf, 'a') as fd: + # load our test module which is not installed + fd.write(f"LoadModule h1test_module \"{local_dir}/mod_h1test/.libs/mod_h1test.so\"\n") + + def _setup_data_1k_1m(self): + s90 = "01234567890123456789012345678901234567890123456789012345678901234567890123456789012345678\n" + with open(os.path.join(self.env.gen_dir, "data-1k"), 'w') as f: + for i in range(10): + f.write(f"{i:09d}-{s90}") + with open(os.path.join(self.env.gen_dir, "data-10k"), 'w') as f: + for i in range(100): + f.write(f"{i:09d}-{s90}") + with open(os.path.join(self.env.gen_dir, "data-100k"), 'w') as f: + for i in range(1000): + f.write(f"{i:09d}-{s90}") + with open(os.path.join(self.env.gen_dir, "data-1m"), 'w') as f: + for i in range(10000): + f.write(f"{i:09d}-{s90}") + + +class H1TestEnv(HttpdTestEnv): + + def __init__(self, pytestconfig=None): + super().__init__(pytestconfig=pytestconfig) + self.add_httpd_log_modules(["http", "core"]) + + def setup_httpd(self, setup: HttpdTestSetup = None): + super().setup_httpd(setup=H1TestSetup(env=self)) + + +class H1Conf(HttpdConf): + + def __init__(self, env: HttpdTestEnv, extras: Dict[str, Any] = None): + super().__init__(env=env, extras=HttpdConf.merge_extras(extras, { + "base": [ + "LogLevel http:trace4", + ], + f"cgi.{env.http_tld}": [ + "SSLOptions +StdEnvVars", + "AddHandler cgi-script .py", + "", + " SetHandler h1test-echo", + "", + ] + })) diff --git a/test/modules/http1/htdocs/cgi/files/empty.txt b/test/modules/http1/htdocs/cgi/files/empty.txt new file mode 100644 index 0000000..e69de29 diff --git a/test/modules/http1/htdocs/cgi/hello.py b/test/modules/http1/htdocs/cgi/hello.py new file mode 100755 index 0000000..191acb2 --- /dev/null +++ b/test/modules/http1/htdocs/cgi/hello.py @@ -0,0 +1,15 @@ +#!/usr/bin/env python3 + +import os + +print("Content-Type: application/json") +print() +print("{") +print(" \"https\" : \"%s\"," % (os.getenv('HTTPS', ''))) +print(" \"host\" : \"%s\"," % (os.getenv('SERVER_NAME', ''))) +print(" \"protocol\" : \"%s\"," % (os.getenv('SERVER_PROTOCOL', ''))) +print(" \"ssl_protocol\" : \"%s\"," % (os.getenv('SSL_PROTOCOL', ''))) +print(" \"h2\" : \"%s\"," % (os.getenv('HTTP2', ''))) +print(" \"h2push\" : \"%s\"" % (os.getenv('H2PUSH', ''))) +print("}") + diff --git a/test/modules/http1/htdocs/cgi/requestparser.py b/test/modules/http1/htdocs/cgi/requestparser.py new file mode 100644 index 0000000..c7e0648 --- /dev/null +++ b/test/modules/http1/htdocs/cgi/requestparser.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python3 +import os +import sys +from urllib import parse +import multipart # https://github.com/andrew-d/python-multipart (`apt install python3-multipart`) +import shutil + + +try: # Windows needs stdio set for binary mode. + import msvcrt + + msvcrt.setmode(0, os.O_BINARY) # stdin = 0 + msvcrt.setmode(1, os.O_BINARY) # stdout = 1 +except ImportError: + pass + + +class FileItem: + + def __init__(self, mparse_item): + self.item = mparse_item + + @property + def file_name(self): + return os.path.basename(self.item.file_name.decode()) + + def save_to(self, destpath: str): + fsrc = self.item.file_object + fsrc.seek(0) + with open(destpath, 'wb') as fd: + shutil.copyfileobj(fsrc, fd) + + +def get_request_params(): + oforms = {} + ofiles = {} + if "REQUEST_URI" in os.environ: + qforms = parse.parse_qs(parse.urlsplit(os.environ["REQUEST_URI"]).query) + for name, values in qforms.items(): + oforms[name] = values[0] + if "CONTENT_TYPE" in os.environ: + ctype = os.environ["CONTENT_TYPE"] + if ctype == "application/x-www-form-urlencoded": + s = sys.stdin.read() + qforms = parse.parse_qs(s) + for name, values in qforms.items(): + oforms[name] = values[0] + elif ctype.startswith("multipart/"): + def on_field(field): + oforms[field.field_name.decode()] = field.value.decode() + def on_file(file): + ofiles[file.field_name.decode()] = FileItem(file) + multipart.parse_form(headers={"Content-Type": ctype}, + input_stream=sys.stdin.buffer, + on_field=on_field, on_file=on_file) + return oforms, ofiles + diff --git a/test/modules/http1/htdocs/cgi/upload.py b/test/modules/http1/htdocs/cgi/upload.py new file mode 100755 index 0000000..632b7e9 --- /dev/null +++ b/test/modules/http1/htdocs/cgi/upload.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python3 +import os +import sys +from requestparser import get_request_params + + +forms, files = get_request_params() + +status = '200 Ok' + +# Test if the file was uploaded +if 'file' in files: + fitem = files['file'] + # strip leading path from file name to avoid directory traversal attacks + fname = fitem.file_name + fpath = f'{os.environ["DOCUMENT_ROOT"]}/files/{fname}' + fitem.save_to(fpath) + message = "The file %s was uploaded successfully" % (fname) + print("Status: 201 Created") + print("Content-Type: text/html") + print("Location: %s://%s/files/%s" % (os.environ["REQUEST_SCHEME"], os.environ["HTTP_HOST"], fname)) + print("") + print("

    %s

    " % (message)) + +elif 'remove' in forms: + remove = forms['remove'] + try: + fname = os.path.basename(remove) + os.remove('./files/' + fname) + message = 'The file "' + fname + '" was removed successfully' + except OSError as e: + message = 'Error removing ' + fname + ': ' + e.strerror + status = '404 File Not Found' + print("Status: %s" % (status)) + print(""" +Content-Type: text/html + + +

    %s

    +""" % (message)) + +else: + message = '''\ + Upload File
    + +
    + ''' + print("Status: %s" % (status)) + print("""\ +Content-Type: text/html + + +

    %s

    +""" % (message)) + diff --git a/test/modules/http1/mod_h1test/mod_h1test.c b/test/modules/http1/mod_h1test/mod_h1test.c new file mode 100644 index 0000000..cbd87b5 --- /dev/null +++ b/test/modules/http1/mod_h1test/mod_h1test.c @@ -0,0 +1,129 @@ +/* Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +static void h1test_hooks(apr_pool_t *pool); + +AP_DECLARE_MODULE(h1test) = { + STANDARD20_MODULE_STUFF, + NULL, /* func to create per dir config */ + NULL, /* func to merge per dir config */ + NULL, /* func to create per server config */ + NULL, /* func to merge per server config */ + NULL, /* command handlers */ + h1test_hooks, +#if defined(AP_MODULE_FLAG_NONE) + AP_MODULE_FLAG_ALWAYS_MERGE +#endif +}; + + +static int h1test_echo_handler(request_rec *r) +{ + conn_rec *c = r->connection; + apr_bucket_brigade *bb; + apr_bucket *b; + apr_status_t rv; + char buffer[8192]; + const char *ct; + long l; + + if (strcmp(r->handler, "h1test-echo")) { + return DECLINED; + } + if (r->method_number != M_GET && r->method_number != M_POST) { + return DECLINED; + } + + ap_log_rerror(APLOG_MARK, APLOG_TRACE1, 0, r, "echo_handler: processing request"); + r->status = 200; + r->clength = -1; + r->chunked = 1; + ct = apr_table_get(r->headers_in, "content-type"); + ap_set_content_type(r, ct? ct : "application/octet-stream"); + + bb = apr_brigade_create(r->pool, c->bucket_alloc); + /* copy any request body into the response */ + if ((rv = ap_setup_client_block(r, REQUEST_CHUNKED_DECHUNK))) goto cleanup; + if (ap_should_client_block(r)) { + while (0 < (l = ap_get_client_block(r, &buffer[0], sizeof(buffer)))) { + ap_log_rerror(APLOG_MARK, APLOG_TRACE1, 0, r, + "echo_handler: copying %ld bytes from request body", l); + rv = apr_brigade_write(bb, NULL, NULL, buffer, l); + if (APR_SUCCESS != rv) goto cleanup; + rv = ap_pass_brigade(r->output_filters, bb); + if (APR_SUCCESS != rv) goto cleanup; + ap_log_rerror(APLOG_MARK, APLOG_TRACE1, 0, r, + "echo_handler: passed %ld bytes from request body", l); + } + } + /* we are done */ + b = apr_bucket_eos_create(c->bucket_alloc); + APR_BRIGADE_INSERT_TAIL(bb, b); + ap_log_rerror(APLOG_MARK, APLOG_TRACE1, 0, r, "echo_handler: request read"); + + if (r->trailers_in && !apr_is_empty_table(r->trailers_in)) { + ap_log_rerror(APLOG_MARK, APLOG_TRACE2, 0, r, + "echo_handler: seeing incoming trailers"); + apr_table_setn(r->trailers_out, "h1test-trailers-in", + apr_itoa(r->pool, 1)); + } + if (apr_table_get(r->headers_in, "Add-Trailer")) { + ap_log_rerror(APLOG_MARK, APLOG_TRACE2, 0, r, + "echo_handler: seeing incoming Add-Trailer header"); + apr_table_setn(r->trailers_out, "h1test-add-trailer", + apr_table_get(r->headers_in, "Add-Trailer")); + } + + rv = ap_pass_brigade(r->output_filters, bb); + +cleanup: + if (rv == APR_SUCCESS + || r->status != HTTP_OK + || c->aborted) { + ap_log_rerror(APLOG_MARK, APLOG_TRACE1, rv, r, "echo_handler: request handled"); + return OK; + } + else { + /* no way to know what type of error occurred */ + ap_log_rerror(APLOG_MARK, APLOG_TRACE1, rv, r, "h1test_echo_handler failed"); + return AP_FILTER_ERROR; + } + return DECLINED; +} + + +/* Install this module into the apache2 infrastructure. + */ +static void h1test_hooks(apr_pool_t *pool) +{ + ap_log_perror(APLOG_MARK, APLOG_TRACE1, 0, pool, "installing hooks and handlers"); + + /* test h1 handlers */ + ap_hook_handler(h1test_echo_handler, NULL, NULL, APR_HOOK_MIDDLE); +} + diff --git a/test/modules/http1/mod_h1test/mod_h1test.slo b/test/modules/http1/mod_h1test/mod_h1test.slo new file mode 100644 index 0000000..e69de29 diff --git a/test/modules/http1/test_001_alive.py b/test/modules/http1/test_001_alive.py new file mode 100644 index 0000000..0a1de1d --- /dev/null +++ b/test/modules/http1/test_001_alive.py @@ -0,0 +1,20 @@ +import pytest + +from .env import H1Conf + + +class TestBasicAlive: + + @pytest.fixture(autouse=True, scope='class') + def _class_scope(self, env): + H1Conf(env).add_vhost_test1().install() + assert env.apache_restart() == 0 + + # we expect to see the document from the generic server + def test_h1_001_01(self, env): + url = env.mkurl("https", "test1", "/alive.json") + r = env.curl_get(url, 5) + assert r.exit_code == 0, r.stderr + r.stdout + assert r.response["json"] + assert r.response["json"]["alive"] is True + assert r.response["json"]["host"] == "test1" diff --git a/test/modules/http1/test_003_get.py b/test/modules/http1/test_003_get.py new file mode 100644 index 0000000..1cd5917 --- /dev/null +++ b/test/modules/http1/test_003_get.py @@ -0,0 +1,27 @@ +import socket + +import pytest + +from .env import H1Conf + + +class TestGet: + + @pytest.fixture(autouse=True, scope='class') + def _class_scope(self, env): + H1Conf(env).add_vhost_cgi( + proxy_self=True + ).add_vhost_test1( + proxy_self=True + ).install() + assert env.apache_restart() == 0 + + # check SSL environment variables from CGI script + def test_h1_003_01(self, env): + url = env.mkurl("https", "cgi", "/hello.py") + r = env.curl_get(url) + assert r.response["status"] == 200 + assert r.response["json"]["protocol"] == "HTTP/1.1" + assert r.response["json"]["https"] == "on" + tls_version = r.response["json"]["ssl_protocol"] + assert tls_version in ["TLSv1.2", "TLSv1.3"] diff --git a/test/modules/http1/test_004_post.py b/test/modules/http1/test_004_post.py new file mode 100644 index 0000000..005a8c2 --- /dev/null +++ b/test/modules/http1/test_004_post.py @@ -0,0 +1,53 @@ +import difflib +import email.parser +import inspect +import json +import os +import sys + +import pytest + +from .env import H1Conf + + +class TestPost: + + @pytest.fixture(autouse=True, scope='class') + def _class_scope(self, env): + TestPost._local_dir = os.path.dirname(inspect.getfile(TestPost)) + H1Conf(env).add_vhost_cgi().install() + assert env.apache_restart() == 0 + + def local_src(self, fname): + return os.path.join(TestPost._local_dir, fname) + + # upload and GET again using curl, compare to original content + def curl_upload_and_verify(self, env, fname, options=None): + url = env.mkurl("https", "cgi", "/upload.py") + fpath = os.path.join(env.gen_dir, fname) + r = env.curl_upload(url, fpath, options=options) + assert r.exit_code == 0, f"{r}" + assert 200 <= r.response["status"] < 300 + + r2 = env.curl_get(r.response["header"]["location"]) + assert r2.exit_code == 0 + assert r2.response["status"] == 200 + with open(self.local_src(fpath), mode='rb') as file: + src = file.read() + assert src == r2.response["body"] + return r + + def test_h1_004_01(self, env): + self.curl_upload_and_verify(env, "data-1k", ["-vvv"]) + + def test_h1_004_02(self, env): + self.curl_upload_and_verify(env, "data-10k", []) + + def test_h1_004_03(self, env): + self.curl_upload_and_verify(env, "data-100k", []) + + def test_h1_004_04(self, env): + self.curl_upload_and_verify(env, "data-1m", []) + + def test_h1_004_05(self, env): + r = self.curl_upload_and_verify(env, "data-1k", ["-vvv", "-H", "Expect: 100-continue"]) diff --git a/test/modules/http1/test_005_trailers.py b/test/modules/http1/test_005_trailers.py new file mode 100644 index 0000000..ca717a0 --- /dev/null +++ b/test/modules/http1/test_005_trailers.py @@ -0,0 +1,42 @@ +import os +import pytest + +from .env import H1Conf + + +# The trailer tests depend on "nghttp" as no other client seems to be able to send those +# rare things. +class TestTrailers: + + @pytest.fixture(autouse=True, scope='class') + def _class_scope(self, env): + H1Conf(env).add_vhost_cgi(proxy_self=True).install() + assert env.apache_restart() == 0 + + # check that we get a trailer out when telling the handler to add one + def test_h1_005_01(self, env): + if not env.httpd_is_at_least("2.5.0"): + pytest.skip(f'need at least httpd 2.5.0 for this') + url = env.mkurl("https", "cgi", "/h1test/echo") + host = f"cgi.{env.http_tld}" + fpath = os.path.join(env.gen_dir, "data-1k") + r = env.curl_upload(url, fpath, options=["--header", "Add-Trailer: 005_01"]) + assert r.exit_code == 0, f"{r}" + assert 200 <= r.response["status"] < 300 + assert r.response["trailer"], f"no trailers received: {r}" + assert "h1test-add-trailer" in r.response["trailer"] + assert r.response["trailer"]["h1test-add-trailer"] == "005_01" + + # check that we get out trailers through the proxy + def test_h1_005_02(self, env): + if not env.httpd_is_at_least("2.5.0"): + pytest.skip(f'need at least httpd 2.5.0 for this') + url = env.mkurl("https", "cgi", "/proxy/h1test/echo") + host = f"cgi.{env.http_tld}" + fpath = os.path.join(env.gen_dir, "data-1k") + r = env.curl_upload(url, fpath, options=["--header", "Add-Trailer: 005_01"]) + assert r.exit_code == 0, f"{r}" + assert 200 <= r.response["status"] < 300 + assert r.response["trailer"], f"no trailers received: {r}" + assert "h1test-add-trailer" in r.response["trailer"] + assert r.response["trailer"]["h1test-add-trailer"] == "005_01" diff --git a/test/modules/http1/test_006_unsafe.py b/test/modules/http1/test_006_unsafe.py new file mode 100644 index 0000000..eb83217 --- /dev/null +++ b/test/modules/http1/test_006_unsafe.py @@ -0,0 +1,134 @@ +import re +import socket +from typing import List, Optional + +import pytest + +from .env import H1Conf + +class TestRequestUnsafe: + + @pytest.fixture(autouse=True, scope='class') + def _class_scope(self, env): + conf = H1Conf(env) + conf.add([ + "HttpProtocolOptions Unsafe", + ]) + conf.install() + assert env.apache_restart() == 0 + + # unsafe tests from t/apache/http_strict.t + # possible expected results: + # 0: any HTTP error + # 1: any HTTP success + # 200-500: specific HTTP status code + # None: HTTPD should drop connection without error message + @pytest.mark.parametrize(["intext", "status", "lognos"], [ + ["GET / HTTP/1.0\r\n\r\n", 1, None], + ["GET / HTTP/1.0\n\n", 1, None], + ["get / HTTP/1.0\r\n\r\n", 501, ["AH00135"]], + ["G ET / HTTP/1.0\r\n\r\n", 400, None], + ["G\0ET / HTTP/1.0\r\n\r\n", 400, None], + ["G/T / HTTP/1.0\r\n\r\n", 501, ["AH00135"]], + ["GET /\0 HTTP/1.0\r\n\r\n", 400, None], + ["GET / HTTP/1.0\0\r\n\r\n", 400, None], + ["GET\f/ HTTP/1.0\r\n\r\n", 400, None], + ["GET\r/ HTTP/1.0\r\n\r\n", 400, None], + ["GET\t/ HTTP/1.0\r\n\r\n", 400, None], + ["GET / HTT/1.0\r\n\r\n", 0, None], + ["GET / HTTP/1.0\r\nHost: localhost\r\n\r\n", 1, None], + ["GET / HTTP/2.0\r\nHost: localhost\r\n\r\n", 1, None], + ["GET / HTTP/1.2\r\nHost: localhost\r\n\r\n", 1, None], + ["GET / HTTP/1.11\r\nHost: localhost\r\n\r\n", 400, None], + ["GET / HTTP/10.0\r\nHost: localhost\r\n\r\n", 400, None], + ["GET / HTTP/1.0 \r\nHost: localhost\r\n\r\n", 200, None], + ["GET / HTTP/1.0 x\r\nHost: localhost\r\n\r\n", 400, None], + ["GET / HTTP/\r\nHost: localhost\r\n\r\n", 0, None], + ["GET / HTTP/0.9\r\n\r\n", 0, None], + ["GET / HTTP/0.8\r\n\r\n", 0, None], + ["GET /\x01 HTTP/1.0\r\n\r\n", 400, None], + ["GET / HTTP/1.0\r\nFoo: bar\r\n\r\n", 200, None], + ["GET / HTTP/1.0\r\nFoo:bar\r\n\r\n", 200, None], + ["GET / HTTP/1.0\r\nFoo: b\0ar\r\n\r\n", 400, None], + ["GET / HTTP/1.0\r\nFoo: b\x01ar\r\n\r\n", 200, None], + ["GET / HTTP/1.0\r\nFoo\r\n\r\n", 400, None], + ["GET / HTTP/1.0\r\nFoo bar\r\n\r\n", 400, None], + ["GET / HTTP/1.0\r\n: bar\r\n\r\n", 400, None], + ["GET / HTTP/1.0\r\nX: bar\r\n\r\n", 200, None], + ["GET / HTTP/1.0\r\nFoo bar:bash\r\n\r\n", 400, None], + ["GET / HTTP/1.0\r\nFoo :bar\r\n\r\n", 400, None], + ["GET / HTTP/1.0\r\n Foo:bar\r\n\r\n", 400, None], + ["GET / HTTP/1.0\r\nF\x01o: bar\r\n\r\n", 200, None], + ["GET / HTTP/1.0\r\nF\ro: bar\r\n\r\n", 400, None], + ["GET / HTTP/1.0\r\nF\to: bar\r\n\r\n", 400, None], + ["GET / HTTP/1.0\r\nFo: b\tar\r\n\r\n", 200, None], + ["GET / HTTP/1.0\r\nFo: bar\r\r\n\r\n", 400, None], + ["GET / HTTP/1.0\r\r", None, None], + ["GET /\r\n", 0, None], + ["GET /#frag HTTP/1.0\r\n", 400, None], + ["GET / HTTP/1.0\r\nHost: localhost\r\nHost: localhost\r\n\r\n", 200, None], + ["GET http://017700000001/ HTTP/1.0\r\n\r\n", 200, None], + ["GET http://0x7f.1/ HTTP/1.0\r\n\r\n", 200, None], + ["GET http://127.0.0.1/ HTTP/1.0\r\n\r\n", 200, None], + ["GET http://127.01.0.1/ HTTP/1.0\r\n\r\n", 200, None], + ["GET http://%3127.0.0.1/ HTTP/1.0\r\n\r\n", 200, None], + ["GET / HTTP/1.0\r\nHost: localhost:80\r\nHost: localhost:80\r\n\r\n", 200, None], + ["GET / HTTP/1.0\r\nHost: localhost:80 x\r\n\r", 400, None], + ["GET http://localhost:80/ HTTP/1.0\r\n\r\n", 200, None], + ["GET http://localhost:80x/ HTTP/1.0\r\n\r\n", 400, None], + ["GET http://localhost:80:80/ HTTP/1.0\r\n\r\n", 400, None], + ["GET http://localhost::80/ HTTP/1.0\r\n\r\n", 400, None], + ["GET http://foo@localhost:80/ HTTP/1.0\r\n\r\n", 200, None], + ["GET http://[::1]/ HTTP/1.0\r\n\r\n", 1, None], + ["GET http://[::1:2]/ HTTP/1.0\r\n\r\n", 1, None], + ["GET http://[4712::abcd]/ HTTP/1.0\r\n\r\n", 1, None], + ["GET http://[4712::abcd:1]/ HTTP/1.0\r\n\r\n", 1, None], + ["GET http://[4712::abcd::]/ HTTP/1.0\r\n\r\n", 400, None], + ["GET http://[4712:abcd::,]/ HTTP/1.0\r\n\r\n", 1, None], + ["GET http://[4712::abcd]:8000/ HTTP/1.0\r\n\r\n", 1, None], + ["GET http://4713::abcd:8001/ HTTP/1.0\r\n\r\n", 400, None], + ["GET / HTTP/1.0\r\nHost: [::1]\r\n\r\n", 1, None], + ["GET / HTTP/1.0\r\nHost: [::1:2]\r\n\r\n", 1, None], + ["GET / HTTP/1.0\r\nHost: [4711::abcd]\r\n\r\n", 1, None], + ["GET / HTTP/1.0\r\nHost: [4711::abcd:1]\r\n\r\n", 1, None], + ["GET / HTTP/1.0\r\nHost: [4711:abcd::]\r\n\r\n", 1, None], + ["GET / HTTP/1.0\r\nHost: [4711::abcd]:8000\r\n\r\n", 1, None], + ["GET / HTTP/1.0\r\nHost: 4714::abcd:8001\r\n\r\n", 200, None], + ["GET / HTTP/1.0\r\nHost: abc\xa0\r\n\r\n", 200, None], + ["GET / HTTP/1.0\r\nHost: abc\\foo\r\n\r\n", 400, None], + ["GET http://foo/ HTTP/1.0\r\nHost: bar\r\n\r\n", 200, None], + ["GET http://foo:81/ HTTP/1.0\r\nHost: bar\r\n\r\n", 200, None], + ["GET http://[::1]:81/ HTTP/1.0\r\nHost: bar\r\n\r\n", 200, None], + ["GET http://10.0.0.1:81/ HTTP/1.0\r\nHost: bar\r\n\r\n", 200, None], + ["GET / HTTP/1.0\r\nHost: foo-bar.example.com\r\n\r\n", 200, None], + ["GET / HTTP/1.0\r\nHost: foo_bar.example.com\r\n\r\n", 200, None], + ["GET http://foo_bar/ HTTP/1.0\r\n\r\n", 200, None], + ]) + def test_h1_006_01(self, env, intext, status: Optional[int], lognos: Optional[List[str]]): + with socket.create_connection(('localhost', int(env.http_port))) as sock: + # on some OS, the server does not see our connection until there is + # something incoming + sock.sendall(intext.encode()) + sock.shutdown(socket.SHUT_WR) + buff = sock.recv(1024) + msg = buff.decode() + if status is None: + assert len(msg) == 0, f"unexpected answer: {msg}" + else: + assert len(msg) > 0, "no answer from server" + rlines = msg.splitlines() + response = rlines[0] + m = re.match(r'^HTTP/1.1 (\d+)\s+(\S+)', response) + assert m or status == 0, f"unrecognized response: {rlines}" + if status == 1: + assert int(m.group(1)) >= 200 + elif status == 0: + # headerless 0.9 response, yuk + assert len(rlines) >= 1, f"{rlines}" + elif status > 0: + assert int(m.group(1)) == status, f"{rlines}" + else: + assert int(m.group(1)) >= 400, f"{rlines}" + # + if lognos is not None: + env.httpd_error_log.ignore_recent(lognos = lognos) diff --git a/test/modules/http1/test_007_strict.py b/test/modules/http1/test_007_strict.py new file mode 100644 index 0000000..7c52f68 --- /dev/null +++ b/test/modules/http1/test_007_strict.py @@ -0,0 +1,126 @@ +import re +import socket +from typing import List, Optional + +import pytest + +from .env import H1Conf + + +class TestRequestStrict: + + @pytest.fixture(autouse=True, scope='class') + def _class_scope(self, env): + conf = H1Conf(env) + conf.add([ + "HttpProtocolOptions Strict", + ]) + conf.install() + assert env.apache_restart() == 0 + + # strict tests from t/apache/http_strict.t + # possible expected results: + # 0: any HTTP error + # 1: any HTTP success + # 200-500: specific HTTP status code + # undef: HTTPD should drop connection without error message + @pytest.mark.parametrize(["intext", "status"], [ + ["GET / HTTP/1.0\n\n", 400], + ["G/T / HTTP/1.0\r\n\r\n", 400], + ["GET / HTTP/1.0 \r\nHost: localhost\r\n\r\n", 400], + ["GET / HTTP/1.0\r\nFoo: b\x01ar\r\n\r\n", 400], + ["GET / HTTP/1.0\r\nF\x01o: bar\r\n\r\n", 400], + ["GET / HTTP/1.0\r\r", None], + ["GET / HTTP/1.0\r\nHost: localhost\r\nHost: localhost\r\n\r\n", 400], + ["GET http://017700000001/ HTTP/1.0\r\n\r\n", 400], + ["GET http://0x7f.1/ HTTP/1.0\r\n\r\n", 400], + ["GET http://127.01.0.1/ HTTP/1.0\r\n\r\n", 400], + ["GET http://%3127.0.0.1/ HTTP/1.0\r\n\r\n", 400], + ["GET / HTTP/1.0\r\nHost: localhost:80\r\nHost: localhost:80\r\n\r\n", 400], + ["GET http://foo@localhost:80/ HTTP/1.0\r\n\r\n", 400], + ["GET / HTTP/1.0\r\nHost: 4714::abcd:8001\r\n\r\n", 400], + ["GET / HTTP/1.0\r\nHost: abc\xa0\r\n\r\n", 400], + ["GET / HTTP/1.0\r\nHost: foo_bar.example.com\r\n\r\n", 200], + ["GET http://foo_bar/ HTTP/1.0\r\n\r\n", 200], + ]) + def test_h1_007_01(self, env, intext, status: Optional[int]): + with socket.create_connection(('localhost', int(env.http_port))) as sock: + # on some OS, the server does not see our connection until there is + # something incoming + sock.sendall(intext.encode()) + sock.shutdown(socket.SHUT_WR) + buff = sock.recv(1024) + msg = buff.decode() + if status is None: + assert len(msg) == 0, f"unexpected answer: {msg}" + else: + assert len(msg) > 0, "no answer from server" + rlines = msg.splitlines() + response = rlines[0] + m = re.match(r'^HTTP/1.1 (\d+)\s+(\S+)', response) + assert m, f"unrecognized response: {rlines}" + if status == 1: + assert int(m.group(1)) >= 200 + elif status == 90: + assert len(rlines) >= 1, f"{rlines}" + elif status > 0: + assert int(m.group(1)) == status, f"{rlines}" + else: + assert int(m.group(1)) >= 400, f"{rlines}" + + @pytest.mark.parametrize(["hvalue", "expvalue", "status", "lognos"], [ + ['"123"', '123', 200, None], + ['"123 "', '123 ', 200, None], # trailing space stays + ['"123\t"', '123\t', 200, None], # trailing tab stays + ['" 123"', '123', 200, None], # leading space is stripped + ['" 123"', '123', 200, None], # leading spaces are stripped + ['"\t123"', '123', 200, None], # leading tab is stripped + ['"expr=%{unescape:123%0A 123}"', '', 500, ["AH02430"]], # illegal char + ['" \t "', '', 200, None], # just ws + ]) + def test_h1_007_02(self, env, hvalue, expvalue, status, lognos: Optional[List[str]]): + hname = 'ap-test-007' + conf = H1Conf(env, extras={ + f'test1.{env.http_tld}': [ + '', + f'Header add {hname} {hvalue}', + '', + ] + }) + conf.add_vhost_test1(proxy_self=True) + conf.install() + assert env.apache_restart() == 0 + url = env.mkurl("https", "test1", "/index.html") + r = env.curl_get(url, options=['--http1.1']) + assert r.response["status"] == status + if int(status) < 400: + assert r.response["header"][hname] == expvalue + # + if lognos is not None: + env.httpd_error_log.ignore_recent(lognos = lognos) + + @pytest.mark.parametrize(["hvalue", "expvalue"], [ + ['123', '123'], + ['123 ', '123'], # trailing space is stripped + ['123\t', '123'], # trailing tab is stripped + [' 123', '123'], # leading space is stripped + [' 123', '123'], # leading spaces are stripped + ['\t123', '123'], # leading tab is stripped + ]) + def test_h1_007_03(self, env, hvalue, expvalue): + # same as 007_02, but http1 proxied + hname = 'ap-test-007' + conf = H1Conf(env, extras={ + f'test1.{env.http_tld}': [ + '', + f'Header add {hname} "{hvalue}"', + '', + ] + }) + conf.add_vhost_test1(proxy_self=True) + conf.install() + assert env.apache_restart() == 0 + url = env.mkurl("https", "test1", "/proxy/index.html") + r = env.curl_get(url, options=['--http1.1']) + assert r.response["status"] == 200 + assert r.response["header"][hname] == expvalue diff --git a/test/modules/http2/conftest.py b/test/modules/http2/conftest.py index 55d0c3a..118cef1 100644 --- a/test/modules/http2/conftest.py +++ b/test/modules/http2/conftest.py @@ -30,11 +30,10 @@ def env(pytestconfig) -> H2TestEnv: @pytest.fixture(autouse=True, scope="package") -def _session_scope(env): +def _h2_package_scope(env): + env.httpd_error_log.add_ignored_lognos([ + 'AH10400', # warning that 'enablereuse' has not effect in certain configs + 'AH00045', # child did not exit in time, SIGTERM was sent + ]) yield assert env.apache_stop() == 0 - errors, warnings = env.httpd_error_log.get_missed() - assert (len(errors), len(warnings)) == (0, 0),\ - f"apache logged {len(errors)} errors and {len(warnings)} warnings: \n"\ - "{0}\n{1}\n".format("\n".join(errors), "\n".join(warnings)) - diff --git a/test/modules/http2/env.py b/test/modules/http2/env.py index 34d196d..b2443e0 100644 --- a/test/modules/http2/env.py +++ b/test/modules/http2/env.py @@ -1,8 +1,8 @@ import inspect import logging import os -import re import subprocess +from shutil import copyfile from typing import Dict, Any from pyhttpd.certs import CertificateSpec @@ -53,6 +53,12 @@ class H2TestSetup(HttpdTestSetup): with open(os.path.join(self.env.gen_dir, "data-1m"), 'w') as f: for i in range(10000): f.write(f"{i:09d}-{s90}") + test1_docs = os.path.join(self.env.server_docs_dir, 'test1') + self.env.mkpath(test1_docs) + for fname in ["data-1k", "data-10k", "data-100k", "data-1m"]: + src = os.path.join(self.env.gen_dir, fname) + dest = os.path.join(test1_docs, fname) + copyfile(src, dest) class H2TestEnv(HttpdTestEnv): @@ -85,34 +91,6 @@ class H2TestEnv(HttpdTestEnv): CertificateSpec(domains=[f"noh2.{self.http_tld}"], key_type='rsa2048'), ]) - self.httpd_error_log.set_ignored_lognos([ - 'AH02032', - 'AH01276', - 'AH01630', - 'AH00135', - 'AH02261', # Re-negotiation handshake failed (our test_101) - 'AH03490', # scoreboard full, happens on limit tests - 'AH02429', # invalid chars in response header names, see test_h2_200 - 'AH02430', # invalid chars in response header values, see test_h2_200 - 'AH10373', # SSL errors on uncompleted handshakes, see test_h2_105 - 'AH01247', # mod_cgid sometimes freaks out on load tests - 'AH01110', # error by proxy reading response - 'AH10400', # warning that 'enablereuse' has not effect in certain configs test_h2_600 - 'AH00045', # child did not exit in time, SIGTERM was sent - ]) - self.httpd_error_log.add_ignored_patterns([ - re.compile(r'.*malformed header from script \'hecho.py\': Bad header: x.*'), - re.compile(r'.*:tls_post_process_client_hello:.*'), - # OSSL 3 dropped the function name from the error description. Use the code instead: - # 0A0000C1 = no shared cipher -- Too restrictive SSLCipherSuite or using DSA server certificate? - re.compile(r'.*SSL Library Error: error:0A0000C1:.*'), - re.compile(r'.*:tls_process_client_certificate:.*'), - # OSSL 3 dropped the function name from the error description. Use the code instead: - # 0A0000C7 = peer did not return a certificate -- No CAs known to server for verification? - re.compile(r'.*SSL Library Error: error:0A0000C7:.*'), - re.compile(r'.*have incompatible TLS configurations.'), - ]) - def setup_httpd(self, setup: HttpdTestSetup = None): super().setup_httpd(setup=H2TestSetup(env=self)) diff --git a/test/modules/http2/test_007_ssi.py b/test/modules/http2/test_007_ssi.py index 97e38df..f5411bc 100644 --- a/test/modules/http2/test_007_ssi.py +++ b/test/modules/http2/test_007_ssi.py @@ -1,4 +1,3 @@ -import re import pytest from .env import H2Conf, H2TestEnv diff --git a/test/modules/http2/test_008_ranges.py b/test/modules/http2/test_008_ranges.py index 4dcdcc8..dd695bb 100644 --- a/test/modules/http2/test_008_ranges.py +++ b/test/modules/http2/test_008_ranges.py @@ -1,13 +1,16 @@ import inspect import json +import logging import os import re -import time import pytest from .env import H2Conf, H2TestEnv +log = logging.getLogger(__name__) + + @pytest.mark.skipif(condition=H2TestEnv.is_unsupported, reason="mod_http2 not supported here") class TestRanges: @@ -123,13 +126,17 @@ class TestRanges: '--limit-rate', '2k', '-m', '2' ]) assert r.exit_code != 0, f'{r}' + # Restart for logs to be flushed out + assert env.apache_restart() == 0 found = False for line in open(TestRanges.LOGFILE).readlines(): e = json.loads(line) + log.info(f'inspecting logged request: {e["request"]}') if e['request'] == f'GET {path}?03broken HTTP/2.0': assert e['bytes_rx_I'] > 0 assert e['bytes_resp_B'] == 100*1024*1024 assert e['bytes_tx_O'] > 1024 + assert e['bytes_tx_O'] < 100*1024*1024 # curl buffers, but not that much found = True break assert found, f'request not found in {self.LOGFILE}' @@ -141,18 +148,13 @@ class TestRanges: assert env.apache_restart() == 0 stats = self.get_server_status(env) # we see the server uptime check request here - assert 1 == int(stats['Total Accesses']), f'{stats}' - assert 1 == int(stats['Total kBytes']), f'{stats}' + assert 1 == int(stats['Total Accesses']) + assert 1 == int(stats['Total kBytes']) count = 10 url = env.mkurl("https", "test1", f'/data-100m?[0-{count-1}]') r = env.curl_get(url, 5, options=['--http2', '-H', f'Range: bytes=0-{4096}']) assert r.exit_code == 0, f'{r}' - for _ in range(10): - # slow cpu might not success on first read - stats = self.get_server_status(env) - if (4*count)+1 <= int(stats['Total kBytes']): - break - time.sleep(0.1) + stats = self.get_server_status(env) # amount reported is larger than (count *4k), the net payload # but does not exceed an additional 4k assert (4*count)+1 <= int(stats['Total kBytes']) diff --git a/test/modules/http2/test_100_conn_reuse.py b/test/modules/http2/test_100_conn_reuse.py index 3ebac24..103166f 100644 --- a/test/modules/http2/test_100_conn_reuse.py +++ b/test/modules/http2/test_100_conn_reuse.py @@ -48,6 +48,12 @@ class TestConnReuse: hostname = ("noh2.%s" % env.http_tld) r = env.curl_get(url, 5, options=[ "-H", "Host:%s" % hostname ]) assert 421 == r.response["status"] + # + env.httpd_error_log.ignore_recent( + lognos = [ + "AH02032" # Hostname provided via SNI and hostname provided via HTTP have no compatible SSL setup + ] + ) # access an unknown vhost, after using ServerName in SNI def test_h2_100_05(self, env): @@ -55,3 +61,9 @@ class TestConnReuse: hostname = ("unknown.%s" % env.http_tld) r = env.curl_get(url, 5, options=[ "-H", "Host:%s" % hostname ]) assert 421 == r.response["status"] + # + env.httpd_error_log.ignore_recent( + lognos = [ + "AH02032" # Hostname provided via SNI and hostname provided via HTTP have no compatible SSL setup + ] + ) diff --git a/test/modules/http2/test_101_ssl_reneg.py b/test/modules/http2/test_101_ssl_reneg.py index 528002f..d278af2 100644 --- a/test/modules/http2/test_101_ssl_reneg.py +++ b/test/modules/http2/test_101_ssl_reneg.py @@ -56,6 +56,12 @@ class TestSslRenegotiation: assert 0 == r.exit_code, f"{r}" assert r.response assert 403 == r.response["status"] + # + env.httpd_error_log.ignore_recent( + lognos = [ + "AH01276" # No matching DirectoryIndex found + ] + ) # try to renegotiate the cipher, should fail with correct code def test_h2_101_02(self, env): @@ -68,6 +74,16 @@ class TestSslRenegotiation: assert 0 != r.exit_code assert not r.response assert re.search(r'HTTP_1_1_REQUIRED \(err 13\)', r.stderr) + # + env.httpd_error_log.ignore_recent( + lognos = [ + "AH02261" # Re-negotiation handshake failed + ], + matches = [ + r'.*:tls_post_process_client_hello:.*', + r'.*SSL Library Error:.*:SSL routines::no shared cipher.*' + ] + ) # try to renegotiate a client certificate from Location # needs to fail with correct code @@ -79,6 +95,16 @@ class TestSslRenegotiation: assert 0 != r.exit_code assert not r.response assert re.search(r'HTTP_1_1_REQUIRED \(err 13\)', r.stderr) + # + env.httpd_error_log.ignore_recent( + lognos = [ + "AH02261" # Re-negotiation handshake failed + ], + matches = [ + r'.*:tls_process_client_certificate:.*', + r'.*SSL Library Error:.*:SSL routines::peer did not return a certificate.*' + ] + ) # try to renegotiate a client certificate from Directory # needs to fail with correct code @@ -90,6 +116,16 @@ class TestSslRenegotiation: assert 0 != r.exit_code, f"{r}" assert not r.response assert re.search(r'HTTP_1_1_REQUIRED \(err 13\)', r.stderr) + # + env.httpd_error_log.ignore_recent( + lognos = [ + "AH02261" # Re-negotiation handshake failed + ], + matches = [ + r'.*:tls_process_client_certificate:.*', + r'.*SSL Library Error:.*:SSL routines::peer did not return a certificate.*' + ] + ) # make 10 requests on the same connection, none should produce a status code # reported by erki@example.ee @@ -136,3 +172,13 @@ class TestSslRenegotiation: assert 0 != r.exit_code assert not r.response assert re.search(r'HTTP_1_1_REQUIRED \(err 13\)', r.stderr) + # + env.httpd_error_log.ignore_recent( + lognos = [ + "AH02261" # Re-negotiation handshake failed + ], + matches = [ + r'.*:tls_post_process_client_hello:.*', + r'.*SSL Library Error:.*:SSL routines::no shared cipher.*' + ] + ) diff --git a/test/modules/http2/test_102_require.py b/test/modules/http2/test_102_require.py index b7e4eae..4b0cad5 100644 --- a/test/modules/http2/test_102_require.py +++ b/test/modules/http2/test_102_require.py @@ -39,3 +39,9 @@ class TestRequire: assert 0 == r.exit_code assert r.response assert 403 == r.response["status"] + # + env.httpd_error_log.ignore_recent( + lognos = [ + "AH01630" # client denied by server configuration + ] + ) diff --git a/test/modules/http2/test_103_upgrade.py b/test/modules/http2/test_103_upgrade.py index 2fa7d1d..1542450 100644 --- a/test/modules/http2/test_103_upgrade.py +++ b/test/modules/http2/test_103_upgrade.py @@ -90,6 +90,9 @@ class TestUpgrade: url = env.mkurl("http", "test1", "/index.html") r = env.nghttp().get(url, options=["-u"]) assert r.response["status"] == 200 + # check issue #272 + assert 'date' in r.response["header"], f'{r.response}' + assert r.response["header"]["date"] != 'Sun, 00 Jan 1900 00:00:00 GMT', f'{r.response}' # upgrade to h2c for a request where http/1.1 is preferred, but the clients upgrade # wish is honored nevertheless diff --git a/test/modules/http2/test_105_timeout.py b/test/modules/http2/test_105_timeout.py index f7d3859..22160b4 100644 --- a/test/modules/http2/test_105_timeout.py +++ b/test/modules/http2/test_105_timeout.py @@ -42,6 +42,13 @@ class TestTimeout: except Exception as ex: print(f"as expected: {ex}") sock.close() + # + time.sleep(1) # let the log flush + env.httpd_error_log.ignore_recent( + lognos = [ + "AH10373" # SSL handshake was not completed + ] + ) # Check that mod_reqtimeout handshake setting takes effect def test_h2_105_02(self, env): @@ -77,6 +84,13 @@ class TestTimeout: except Exception as ex: print(f"as expected: {ex}") sock.close() + # + time.sleep(1) # let the log flush + env.httpd_error_log.ignore_recent( + lognos = [ + "AH10373" # SSL handshake was not completed + ] + ) # Check that mod_reqtimeout handshake setting do no longer apply to handshaked # connections. See . diff --git a/test/modules/http2/test_106_shutdown.py b/test/modules/http2/test_106_shutdown.py index 83e143c..fab881b 100644 --- a/test/modules/http2/test_106_shutdown.py +++ b/test/modules/http2/test_106_shutdown.py @@ -72,4 +72,10 @@ class TestShutdown: else: assert r.exit_code == 0, f"failed on {i}. request: {r.stdout} {r.stderr}" assert r.response["status"] == 200 - assert "HTTP/2" == r.response["protocol"] \ No newline at end of file + assert "HTTP/2" == r.response["protocol"] + # + env.httpd_error_log.ignore_recent( + lognos = [ + "AH03490" # scoreboard is full, not at MaxRequestWorkers + ] + ) \ No newline at end of file diff --git a/test/modules/http2/test_200_header_invalid.py b/test/modules/http2/test_200_header_invalid.py index 5b3aafd..04c022c 100644 --- a/test/modules/http2/test_200_header_invalid.py +++ b/test/modules/http2/test_200_header_invalid.py @@ -28,6 +28,15 @@ class TestInvalidHeaders: assert 500 == r.response["status"], f'unexpected status for char 0x{x:02}' else: assert 0 != r.exit_code, f'unexpected exit code for char 0x{x:02}' + # + env.httpd_error_log.ignore_recent( + lognos = [ + "AH02429" # Response header name contains invalid characters + ], + matches = [ + r'.*malformed header from script \'hecho.py\': Bad header: x.*' + ] + ) # let the hecho.py CGI echo chars < 0x20 in field value # for almost all such characters, the stream returns a 500 @@ -46,6 +55,12 @@ class TestInvalidHeaders: assert 500 == r.response["status"], f'unexpected status for char 0x{x:02}' else: assert 0 != r.exit_code, "unexpected exit code for char 0x%02x" % x + # + env.httpd_error_log.ignore_recent( + lognos = [ + "AH02430" # Response header value contains invalid characters + ] + ) # let the hecho.py CGI echo 0x10 and 0x7f in field name and value def test_h2_200_03(self, env): @@ -63,6 +78,13 @@ class TestInvalidHeaders: assert 500 == r.response["status"], f"unexpected exit code for char 0x{h:02}" else: assert 0 != r.exit_code + # + env.httpd_error_log.ignore_recent( + lognos = [ + "AH02429", # Response header name contains invalid characters + "AH02430" # Response header value contains invalid characters + ] + ) # test header field lengths check, LimitRequestLine def test_h2_200_10(self, env): diff --git a/test/modules/http2/test_203_rfc9113.py b/test/modules/http2/test_203_rfc9113.py index 9fc8f3b..1fe3e13 100644 --- a/test/modules/http2/test_203_rfc9113.py +++ b/test/modules/http2/test_203_rfc9113.py @@ -1,4 +1,5 @@ import pytest +from typing import List, Optional from pyhttpd.env import HttpdTestEnv from .env import H2Conf @@ -22,17 +23,17 @@ class TestRfc9113: assert r.response["status"] == 200, f'curl output: {r.stdout}' # response header are also handled, but we strip ws before sending - @pytest.mark.parametrize(["hvalue", "expvalue", "status"], [ - ['"123"', '123', 200], - ['"123 "', '123', 200], # trailing space stripped - ['"123\t"', '123', 200], # trailing tab stripped - ['" 123"', '123', 200], # leading space is stripped - ['" 123"', '123', 200], # leading spaces are stripped - ['"\t123"', '123', 200], # leading tab is stripped - ['"expr=%{unescape:123%0A 123}"', '', 500], # illegal char - ['" \t "', '', 200], # just ws + @pytest.mark.parametrize(["hvalue", "expvalue", "status", "lognos"], [ + ['"123"', '123', 200, None], + ['"123 "', '123', 200, None], # trailing space stripped + ['"123\t"', '123', 200, None], # trailing tab stripped + ['" 123"', '123', 200, None], # leading space is stripped + ['" 123"', '123', 200, None], # leading spaces are stripped + ['"\t123"', '123', 200, None], # leading tab is stripped + ['"expr=%{unescape:123%0A 123}"', '', 500, ["AH02430"]], # illegal char + ['" \t "', '', 200, None], # just ws ]) - def test_h2_203_02(self, env, hvalue, expvalue, status): + def test_h2_203_02(self, env, hvalue, expvalue, status, lognos: Optional[List[str]]): hname = 'ap-test-007' conf = H2Conf(env, extras={ f'test1.{env.http_tld}': [ @@ -53,4 +54,7 @@ class TestRfc9113: assert r.response["status"] == status if int(status) < 400: assert r.response["header"][hname] == expvalue + # + if lognos is not None: + env.httpd_error_log.ignore_recent(lognos = lognos) diff --git a/test/modules/http2/test_500_proxy.py b/test/modules/http2/test_500_proxy.py index 88a8ece..87e523c 100644 --- a/test/modules/http2/test_500_proxy.py +++ b/test/modules/http2/test_500_proxy.py @@ -149,9 +149,21 @@ class TestProxy: url = env.mkurl("https", "cgi", "/proxy/h2test/error?body_error=timeout") r = env.curl_get(url) assert r.exit_code != 0, r + # + env.httpd_error_log.ignore_recent( + lognos = [ + "AH01110" # Network error reading response + ] + ) # produce an error, fail to generate an error bucket def test_h2_500_32(self, env, repeat): url = env.mkurl("https", "cgi", "/proxy/h2test/error?body_error=timeout&error_bucket=0") r = env.curl_get(url) assert r.exit_code != 0, r + # + env.httpd_error_log.ignore_recent( + lognos = [ + "AH01110" # Network error reading response + ] + ) diff --git a/test/modules/http2/test_600_h2proxy.py b/test/modules/http2/test_600_h2proxy.py index 18d5d1d..18a528e 100644 --- a/test/modules/http2/test_600_h2proxy.py +++ b/test/modules/http2/test_600_h2proxy.py @@ -78,8 +78,8 @@ class TestH2Proxy: conf.install() assert env.apache_restart() == 0 url = env.mkurl("https", "cgi", f"/h2proxy/{env.http_port}/hello.py") - # httpd 2.4.59 disables reuse, not matter the config - if enable_reuse == "on" and not env.httpd_is_at_least("2.4.59"): + # httpd 2.5.0 disables reuse, not matter the config + if enable_reuse == "on" and not env.httpd_is_at_least("2.4.60"): # reuse is not guaranteed for each request, but we expect some # to do it and run on a h2 stream id > 1 reused = False @@ -132,7 +132,7 @@ class TestH2Proxy: assert int(r.json[0]["port"]) == env.http_port assert r.response["status"] == 200 exp_port = env.http_port if enable_reuse == "on" \ - and not env.httpd_is_at_least("2.4.59")\ + and not env.httpd_is_at_least("2.4.60")\ else env.http_port2 assert int(r.json[1]["port"]) == exp_port @@ -188,7 +188,6 @@ class TestH2Proxy: # produce an error, fail to generate an error bucket def test_h2_600_32(self, env, repeat): - pytest.skip('only works reliable with r1911964 from trunk') conf = H2Conf(env) conf.add_vhost_cgi(h2proxy_self=True) conf.install() diff --git a/test/modules/http2/test_700_load_get.py b/test/modules/http2/test_700_load_get.py index 78760fb..138e74c 100644 --- a/test/modules/http2/test_700_load_get.py +++ b/test/modules/http2/test_700_load_get.py @@ -61,3 +61,37 @@ class TestLoadGet: args.append(env.mkurl("https", "cgi", ("/mnot164.py?count=%d&text=%s" % (start+(n*chunk)+i, text)))) r = env.run(args) self.check_h2load_ok(env, r, chunk) + + # test window sizes, connection and stream + @pytest.mark.parametrize("connbits,streambits", [ + [10, 16], # 1k connection window, 64k stream windows + [10, 30], # 1k connection window, huge stream windows + [30, 8], # huge conn window, 256 bytes stream windows + ]) + @pytest.mark.skip('awaiting mpm_event improvements') + def test_h2_700_20(self, env, connbits, streambits): + if not env.httpd_is_at_least("2.5.0"): + pytest.skip(f'need at least httpd 2.5.0 for this') + conf = H2Conf(env, extras={ + 'base': [ + 'StartServers 1', + ] + }) + conf.add_vhost_cgi().add_vhost_test1().install() + assert env.apache_restart() == 0 + assert env.is_live() + n = 2000 + conns = 50 + parallel = 10 + args = [ + env.h2load, + '-n', f'{n}', '-t', '1', + '-c', f'{conns}', '-m', f'{parallel}', + '-W', f'{connbits}', # connection window bits + '-w', f'{streambits}', # stream window bits + f'--connect-to=localhost:{env.https_port}', + f'--base-uri={env.mkurl("https", "test1", "/")}', + "/data-100k" + ] + r = env.run(args) + self.check_h2load_ok(env, r, n) \ No newline at end of file diff --git a/test/modules/http2/test_712_buffering.py b/test/modules/http2/test_712_buffering.py index 6658441..0a6978b 100644 --- a/test/modules/http2/test_712_buffering.py +++ b/test/modules/http2/test_712_buffering.py @@ -33,7 +33,7 @@ class TestBuffering: url = env.mkurl("https", "cgi", "/h2test/echo") base_chunk = "0123456789" chunks = ["chunk-{0:03d}-{1}\n".format(i, base_chunk) for i in range(5)] - stutter = timedelta(seconds=0.2) # this is short, but works on my machine (tm) + stutter = timedelta(seconds=0.2) piper = CurlPiper(env=env, url=url) piper.stutter_check(chunks, stutter) @@ -43,6 +43,16 @@ class TestBuffering: url = env.mkurl("https", "cgi", "/h2proxy/h2test/echo") base_chunk = "0123456789" chunks = ["chunk-{0:03d}-{1}\n".format(i, base_chunk) for i in range(3)] - stutter = timedelta(seconds=1) # need a bit more delay since we have the extra connection + stutter = timedelta(seconds=0.4) # need a bit more delay since we have the extra connection + piper = CurlPiper(env=env, url=url) + piper.stutter_check(chunks, stutter) + + def test_h2_712_03(self, env): + # same as 712_02 but with smaller chunks + # + url = env.mkurl("https", "cgi", "/h2proxy/h2test/echo") + base_chunk = "0" + chunks = ["ck{0}-{1}\n".format(i, base_chunk) for i in range(3)] + stutter = timedelta(seconds=0.4) # need a bit more delay since we have the extra connection piper = CurlPiper(env=env, url=url) piper.stutter_check(chunks, stutter) diff --git a/test/modules/http2/test_800_websockets.py b/test/modules/http2/test_800_websockets.py index 52af1a3..c0fc0c2 100644 --- a/test/modules/http2/test_800_websockets.py +++ b/test/modules/http2/test_800_websockets.py @@ -84,8 +84,8 @@ def ws_run(env: H2TestEnv, path, authority=None, do_input=None, inbytes=None, @pytest.mark.skipif(condition=H2TestEnv.is_unsupported, reason="mod_http2 not supported here") -@pytest.mark.skipif(condition=not H2TestEnv().httpd_is_at_least("2.4.58"), - reason=f'need at least httpd 2.4.58 for this') +@pytest.mark.skipif(condition=not H2TestEnv().httpd_is_at_least("2.4.60"), + reason=f'need at least httpd 2.4.60 for this') @pytest.mark.skipif(condition=ws_version < ws_version_min, reason=f'websockets is {ws_version}, need at least {ws_version_min}') class TestWebSockets: @@ -154,7 +154,6 @@ class TestWebSockets: r, infos, frames = ws_run(env, path='/ws/echo/', scenario='fail-proto') assert r.exit_code == 0, f'{r}' assert infos == ['[1] :status: 501', '[1] EOF'], f'{r}' - env.httpd_error_log.ignore_recent() # a correct CONNECT, send CLOSE, expect CLOSE, basic success def test_h2_800_02_ws_empty(self, env: H2TestEnv, ws_server): diff --git a/test/modules/md/conftest.py b/test/modules/md/conftest.py index 04165a2..0f9e4a9 100755 --- a/test/modules/md/conftest.py +++ b/test/modules/md/conftest.py @@ -1,6 +1,5 @@ import logging import os -import re import sys import pytest @@ -33,48 +32,18 @@ def env(pytestconfig) -> MDTestEnv: env.setup_httpd() env.apache_access_log_clear() env.httpd_error_log.clear_log() - return env + yield env + env.apache_stop() @pytest.fixture(autouse=True, scope="package") -def _session_scope(env): - # we'd like to check the httpd error logs after the test suite has - # run to catch anything unusual. For this, we setup the ignore list - # of errors and warnings that we do expect. - env.httpd_error_log.set_ignored_lognos([ - 'AH10040', # mod_md, setup complain - 'AH10045', # mod_md complains that there is no vhost for an MDomain - 'AH10056', # mod_md, invalid params - 'AH10105', # mod_md does not find a vhost with SSL enabled for an MDomain - 'AH10085', # mod_ssl complains about fallback certificates - 'AH01909', # mod_ssl, cert alt name complains - 'AH10170', # mod_md, wrong config, tested - 'AH10171', # mod_md, wrong config, tested - 'AH10373', # SSL errors on uncompleted handshakes - 'AH10398', # test on global store lock +def _md_package_scope(env): + env.httpd_error_log.add_ignored_lognos([ + "AH10085", # There are no SSL certificates configured and no other module contributed any + "AH10045", # No VirtualHost matches Managed Domain + "AH10105", # MDomain does not match any VirtualHost with 'SSLEngine on' ]) - env.httpd_error_log.add_ignored_patterns([ - re.compile(r'.*urn:ietf:params:acme:error:.*'), - re.compile(r'.*None of the ACME challenge methods configured for this domain are suitable.*'), - re.compile(r'.*problem\[(challenge-mismatch|challenge-setup-failure|apache:eab-hmac-invalid)].*'), - re.compile(r'.*CA considers answer to challenge invalid.].*'), - re.compile(r'.*problem\[urn:org:apache:httpd:log:AH\d+:].*'), - re.compile(r'.*Unsuccessful in contacting ACME server at :*'), - re.compile(r'.*test-md-720-002-\S+.org: dns-01 setup command failed .*'), - re.compile(r'.*AH\d*: unable to obtain global registry lock, .*'), - ]) - if env.lacks_ocsp(): - env.httpd_error_log.add_ignored_patterns([ - re.compile(r'.*certificate with serial \S+ has no OCSP responder URL.*'), - ]) - yield - assert env.apache_stop() == 0 - errors, warnings = env.httpd_error_log.get_missed() - assert (len(errors), len(warnings)) == (0, 0),\ - f"apache logged {len(errors)} errors and {len(warnings)} warnings: \n"\ - "{0}\n{1}\n".format("\n".join(errors), "\n".join(warnings)) - @pytest.fixture(scope="package") def acme(env): diff --git a/test/modules/md/test_300_conf_validate.py b/test/modules/md/test_300_conf_validate.py index 85371ba..88df168 100644 --- a/test/modules/md/test_300_conf_validate.py +++ b/test/modules/md/test_300_conf_validate.py @@ -15,7 +15,8 @@ from .md_env import MDTestEnv class TestConf: @pytest.fixture(autouse=True, scope='class') - def _class_scope(self, env): + def _class_scope(self, env, acme): + acme.start(config='default') env.clear_store() # test case: just one MDomain definition @@ -24,6 +25,12 @@ class TestConf: MDomain not-forbidden.org www.not-forbidden.org mail.not-forbidden.org """).install() assert env.apache_restart() == 0 + # + env.httpd_error_log.ignore_recent( + lognos = [ + "AH10045" # No VirtualHost matches Managed Domain + ] + ) # test case: two MDomain definitions, non-overlapping def test_md_300_002(self, env): @@ -32,6 +39,12 @@ class TestConf: MDomain example2.org www.example2.org mail.example2.org """).install() assert env.apache_restart() == 0 + # + env.httpd_error_log.ignore_recent( + lognos = [ + "AH10045" # No VirtualHost matches Managed Domain + ] + ) # test case: two MDomain definitions, exactly the same def test_md_300_003(self, env): @@ -41,6 +54,12 @@ class TestConf: MDomain not-forbidden.org www.not-forbidden.org mail.not-forbidden.org test3.not-forbidden.org """).install() assert env.apache_fail() == 0 + # + env.httpd_error_log.ignore_recent( + lognos = [ + "AH10038" # two Managed Domains have an overlap in domain + ] + ) # test case: two MDomain definitions, overlapping def test_md_300_004(self, env): @@ -50,6 +69,12 @@ class TestConf: MDomain example2.org test3.not-forbidden.org www.example2.org mail.example2.org """).install() assert env.apache_fail() == 0 + # + env.httpd_error_log.ignore_recent( + lognos = [ + "AH10038" # two Managed Domains have an overlap in domain + ] + ) # test case: two MDomains, one inside a virtual host def test_md_300_005(self, env): @@ -60,6 +85,12 @@ class TestConf: """).install() assert env.apache_restart() == 0 + # + env.httpd_error_log.ignore_recent( + lognos = [ + "AH10045" # No VirtualHost matches Managed Domain + ] + ) # test case: two MDomains, one correct vhost name def test_md_300_006(self, env): @@ -71,6 +102,12 @@ class TestConf: """).install() assert env.apache_restart() == 0 + # + env.httpd_error_log.ignore_recent( + lognos = [ + "AH10045" # No VirtualHost matches Managed Domain + ] + ) # test case: two MDomains, two correct vhost names def test_md_300_007(self, env): @@ -85,6 +122,12 @@ class TestConf: """).install() assert env.apache_restart() == 0 + # + env.httpd_error_log.ignore_recent( + lognos = [ + "AH10045" # No VirtualHost matches Managed Domain + ] + ) # test case: two MDomains, overlapping vhosts def test_md_300_008(self, env): @@ -102,6 +145,12 @@ class TestConf: """).install() assert env.apache_restart() == 0 + # + env.httpd_error_log.ignore_recent( + lognos = [ + "AH10045" # No VirtualHost matches Managed Domain + ] + ) # test case: vhosts with overlapping MDs def test_md_300_009(self, env): @@ -118,7 +167,12 @@ class TestConf: conf.install() assert env.apache_fail() == 0 env.apache_stop() - env.httpd_error_log.ignore_recent() + # + env.httpd_error_log.ignore_recent( + lognos = [ + "AH10238" # 2 MDs match Virtualhost + ] + ) # test case: MDomain, vhost with matching ServerAlias def test_md_300_010(self, env): @@ -146,6 +200,9 @@ class TestConf: conf.install() assert env.apache_fail() == 0 env.apache_stop() + env.httpd_error_log.ignore_recent([ + "AH10040" # A requested MD certificate will not match ServerName + ]) # test case: MDomain, misses one ServerAlias, but auto add enabled def test_md_300_011b(self, env): @@ -171,6 +228,12 @@ class TestConf: """).install() assert env.apache_restart() == 0 + # + env.httpd_error_log.ignore_recent( + lognos = [ + "AH10045" # No VirtualHost matches Managed Domain + ] + ) # test case: one md covers two vhosts def test_md_300_013(self, env): @@ -261,7 +324,6 @@ class TestConf: MDConf(env, text=line).install() assert env.apache_fail() == 0, "Server accepted test config {}".format(line) assert exp_err_msg in env.apachectl_stderr - env.httpd_error_log.ignore_recent() # test case: alt-names incomplete detection, github isse #68 def test_md_300_021(self, env): @@ -294,6 +356,12 @@ class TestConf: """).install() assert env.apache_restart() == 0 + # + env.httpd_error_log.ignore_recent( + lognos = [ + "AH10105" # MD secret.com does not match any VirtualHost with 'SSLEngine on' + ] + ) # test case: use MDRequireHttps not in + ServerName {domaina} + ServerAlias {dalias} + SSLEngine on + + + ServerName {domainb} + ServerAlias {dalias} + SSLEngine on + + """) + conf.install() + # This does not work as we have both MDs match domain's vhost + assert env.apache_fail() == 0 + env.httpd_error_log.ignore_recent( + lognos=[ + "AH10238", # 2 MDs match the same vhost + ] + ) + # It works, if we only match on ServerNames + conf.add("MDMatchNames servernames") + conf.install() + assert env.apache_restart() == 0 + env.httpd_error_log.ignore_recent( + lognos=[ + "AH10040", # ServerAlias not covered + ] + ) + + # wildcard and specfic MD overlaps + def test_md_300_029(self, env): + assert env.apache_stop() == 0 + conf = MDConf(env) + domain = f"t300-029.{env.http_tld}" + subdomain = f"sub.{domain}" + conf.add_vhost(port=env.http_port, domains=[domain, subdomain], with_ssl=False) + conf.add(f""" + MDMembers manual + MDomain {domain} *.{domain} + MDomain {subdomain} + """) + conf.add(f""" + + ServerName {domain} + SSLEngine on + + + ServerName another.{domain} + SSLEngine on + + + ServerName {subdomain} + SSLEngine on + + """) + conf.install() + # This does not work as we have overlapping names in MDs + assert env.apache_fail() == 0 + env.httpd_error_log.ignore_recent( + lognos = [ + "AH10038" # 2 MDs overlap + ] + ) + # It works, if we only match on ServerNames + conf.add("MDMatchNames servernames") + conf.install() + assert env.apache_restart() == 0 + time.sleep(2) + assert env.apache_stop() == 0 + # we need dns-01 challenge for the wildcard, which is not configured + env.httpd_error_log.ignore_recent(matches=[ + r'.*None of offered challenge types.*are supported.*' + ]) + diff --git a/test/modules/md/test_702_auto.py b/test/modules/md/test_702_auto.py index 8e8f5f1..04a9c75 100644 --- a/test/modules/md/test_702_auto.py +++ b/test/modules/md/test_702_auto.py @@ -64,6 +64,12 @@ class TestAutov2: # file system needs to have correct permissions env.check_dir_empty(env.store_challenges()) env.check_file_permissions(domain) + # + env.httpd_error_log.ignore_recent( + lognos = [ + "AH10045" # No VirtualHost matches Managed Domain test-md-702-001-1688648129.org + ] + ) # test case: same as test_702_001, but with two parallel managed domains def test_md_702_002(self, env): @@ -234,6 +240,15 @@ class TestAutov2: cert = env.get_cert(name_a) assert name_a in cert.get_san_list() assert env.get_http_status(name_a, "/name.txt") == 503 + # + env.httpd_error_log.ignore_recent( + lognos = [ + "AH10056" # None of offered challenge types + ], + matches = [ + r'.*problem\[challenge-mismatch\].*' + ] + ) # Specify a non-working http proxy def test_md_702_008(self, env): @@ -254,6 +269,15 @@ class TestAutov2: assert md['renewal']['errors'] > 0 assert md['renewal']['last']['status-description'] == 'Connection refused' assert 'account' not in md['ca'] + # + env.httpd_error_log.ignore_recent( + lognos = [ + "AH10056" # Unsuccessful in contacting ACME server + ], + matches = [ + r'.*Unsuccessful in contacting ACME server at .*' + ] + ) # Specify a valid http proxy def test_md_702_008a(self, env): @@ -335,6 +359,16 @@ class TestAutov2: assert env.apache_restart() == 0 env.check_md(domains) assert env.await_completion([domain]) + # + env.httpd_error_log.ignore_recent( + lognos = [ + "AH10173", # None of the ACME challenge methods configured for this domain are suitable + "AH10056" # None of the ACME challenge methods configured for this domain are suitable + ], + matches = [ + r'.*None of the ACME challenge methods configured for this domain are suitable.*' + ] + ) def test_md_702_011(self, env): domain = self.test_domain @@ -364,6 +398,16 @@ class TestAutov2: assert env.apache_restart() == 0 env.check_md(domains) assert env.await_completion([domain]) + # + env.httpd_error_log.ignore_recent( + lognos = [ + "AH10173", # None of the ACME challenge methods configured for this domain are suitable + "AH10056" # None of the ACME challenge methods configured for this domain are suitable + ], + matches = [ + r'.*None of the ACME challenge methods configured for this domain are suitable.*' + ] + ) # test case: one MD with several dns names. sign up. remove the *first* name # in the MD. restart. should find and keep the existing MD. @@ -648,6 +692,16 @@ class TestAutov2: conf.install() assert env.apache_restart() == 0 assert env.await_error(domain) + # + env.httpd_error_log.ignore_recent( + lognos = [ + "AH10173", # None of the ACME challenge methods configured for this domain are suitable + "AH10056" # None of the ACME challenge methods configured for this domain are suitable + ], + matches = [ + r'.*None of the ACME challenge methods configured for this domain are suitable.*' + ] + ) # Make a setup using the base server without http:, but with acme-tls/1, should work. def test_md_702_052(self, env): diff --git a/test/modules/md/test_720_wildcard.py b/test/modules/md/test_720_wildcard.py index 23b311c..916c47a 100644 --- a/test/modules/md/test_720_wildcard.py +++ b/test/modules/md/test_720_wildcard.py @@ -44,6 +44,15 @@ class TestWildcard: assert md assert md['renewal']['errors'] > 0 assert md['renewal']['last']['problem'] == 'challenge-mismatch' + # + env.httpd_error_log.ignore_recent( + lognos = [ + "AH10056" # None of offered challenge types + ], + matches = [ + r'.*problem\[challenge-mismatch\].*' + ] + ) # test case: a wildcard certificate with ACMEv2, only dns-01 configured, invalid command path def test_md_720_002(self, env): @@ -67,6 +76,16 @@ class TestWildcard: assert md assert md['renewal']['errors'] > 0 assert md['renewal']['last']['problem'] == 'challenge-setup-failure' + # + env.httpd_error_log.ignore_recent( + lognos = [ + "AH10056" # None of offered challenge types + ], + matches = [ + r'.*problem\[challenge-setup-failure\].*', + r'.*setup command failed to execute.*' + ] + ) # variation, invalid cmd path, other challenges still get certificate for non-wildcard def test_md_720_002b(self, env): @@ -113,6 +132,15 @@ class TestWildcard: assert md assert md['renewal']['errors'] > 0 assert md['renewal']['last']['problem'] == 'challenge-setup-failure' + # + env.httpd_error_log.ignore_recent( + lognos = [ + "AH10056" # None of offered challenge types + ], + matches = [ + r'.*problem\[challenge-setup-failure\].*' + ] + ) # test case: a wildcard name certificate with ACMEv2, only dns-01 configured def test_md_720_004(self, env): diff --git a/test/modules/md/test_730_static.py b/test/modules/md/test_730_static.py index f7f7b4b..891ae62 100644 --- a/test/modules/md/test_730_static.py +++ b/test/modules/md/test_730_static.py @@ -115,3 +115,10 @@ class TestStatic: conf.add_vhost(domain) conf.install() assert env.apache_fail() == 0 + # + env.httpd_error_log.ignore_recent( + lognos = [ + "AH10170", # Managed Domain needs one MDCertificateKeyFile for each MDCertificateFile + "AH10171" # Managed Domain has MDCertificateKeyFile(s) but no MDCertificateFile + ] + ) diff --git a/test/modules/md/test_740_acme_errors.py b/test/modules/md/test_740_acme_errors.py index 670c9ab..364aaca 100644 --- a/test/modules/md/test_740_acme_errors.py +++ b/test/modules/md/test_740_acme_errors.py @@ -46,6 +46,15 @@ class TestAcmeErrors: assert md['renewal']['last']['detail'] == ( "Error creating new order :: Cannot issue for " "\"%s\": Domain name contains an invalid character" % domains[1]) + # + env.httpd_error_log.ignore_recent( + lognos = [ + "AH10056" # Order included DNS identifier with a value containing an illegal character + ], + matches = [ + r'.*urn:ietf:params:acme:error:malformed.*' + ] + ) # test case: MD with 3 names, 2 invalid # @@ -70,3 +79,12 @@ class TestAcmeErrors: "Error creating new order :: Cannot issue for") assert md['renewal']['last']['subproblems'] assert len(md['renewal']['last']['subproblems']) == 2 + # + env.httpd_error_log.ignore_recent( + lognos = [ + "AH10056" # Order included DNS identifier with a value containing an illegal character + ], + matches = [ + r'.*urn:ietf:params:acme:error:malformed.*' + ] + ) diff --git a/test/modules/md/test_741_setup_errors.py b/test/modules/md/test_741_setup_errors.py index 49b4e78..9ad79f0 100644 --- a/test/modules/md/test_741_setup_errors.py +++ b/test/modules/md/test_741_setup_errors.py @@ -46,3 +46,13 @@ class TestSetupErrors: md = env.await_error(domain, errors=2, timeout=10) assert md assert md['renewal']['errors'] > 0 + # + env.httpd_error_log.ignore_recent( + lognos = [ + "AH10056" # CA considers answer to challenge invalid + ], + matches = [ + r'.*The key authorization file from the server did not match this challenge.*', + r'.*CA considers answer to challenge invalid.*' + ] + ) diff --git a/test/modules/md/test_750_eab.py b/test/modules/md/test_750_eab.py index af1be95..aec7e89 100644 --- a/test/modules/md/test_750_eab.py +++ b/test/modules/md/test_750_eab.py @@ -37,6 +37,15 @@ class TestEab: md = env.await_error(domain) assert md['renewal']['errors'] > 0 assert md['renewal']['last']['problem'] == 'urn:ietf:params:acme:error:externalAccountRequired' + # + env.httpd_error_log.ignore_recent( + lognos = [ + "AH10056" # ACME server policy requires newAccount requests must include a value for the 'externalAccountBinding' field + ], + matches = [ + r'.*urn:ietf:params:acme:error:externalAccountRequired.*' + ] + ) def test_md_750_002(self, env): # md with known EAB KID and non base64 hmac key configured @@ -51,6 +60,15 @@ class TestEab: md = env.await_error(domain) assert md['renewal']['errors'] > 0 assert md['renewal']['last']['problem'] == 'apache:eab-hmac-invalid' + # + env.httpd_error_log.ignore_recent( + lognos = [ + "AH10056" # external account binding HMAC value is not valid base64 + ], + matches = [ + r'.*problem\[apache:eab-hmac-invalid\].*' + ] + ) def test_md_750_003(self, env): # md with empty EAB KID configured @@ -64,7 +82,19 @@ class TestEab: assert env.apache_restart() == 0 md = env.await_error(domain) assert md['renewal']['errors'] > 0 - assert md['renewal']['last']['problem'] == 'urn:ietf:params:acme:error:unauthorized' + assert md['renewal']['last']['problem'] in [ + 'urn:ietf:params:acme:error:unauthorized', + 'urn:ietf:params:acme:error:malformed', + ] + # + env.httpd_error_log.ignore_recent( + lognos = [ + "AH10056" # the field 'kid' references a key that is not known to the ACME server + ], + matches = [ + r'.*urn:ietf:params:acme:error:(unauthorized|malformed).*' + ] + ) def test_md_750_004(self, env): # md with unknown EAB KID configured @@ -78,7 +108,19 @@ class TestEab: assert env.apache_restart() == 0 md = env.await_error(domain) assert md['renewal']['errors'] > 0 - assert md['renewal']['last']['problem'] == 'urn:ietf:params:acme:error:unauthorized' + assert md['renewal']['last']['problem'] in [ + 'urn:ietf:params:acme:error:unauthorized', + 'urn:ietf:params:acme:error:malformed', + ] + # + env.httpd_error_log.ignore_recent( + lognos = [ + "AH10056" # the field 'kid' references a key that is not known to the ACME server + ], + matches = [ + r'.*urn:ietf:params:acme:error:(unauthorized|malformed).*' + ] + ) def test_md_750_005(self, env): # md with known EAB KID but wrong HMAC configured @@ -92,7 +134,19 @@ class TestEab: assert env.apache_restart() == 0 md = env.await_error(domain) assert md['renewal']['errors'] > 0 - assert md['renewal']['last']['problem'] == 'urn:ietf:params:acme:error:unauthorized' + assert md['renewal']['last']['problem'] in [ + 'urn:ietf:params:acme:error:unauthorized', + 'urn:ietf:params:acme:error:malformed', + ] + # + env.httpd_error_log.ignore_recent( + lognos = [ + "AH10056" # external account binding JWS verification error: square/go-jose: error in cryptographic primitive + ], + matches = [ + r'.*urn:ietf:params:acme:error:(unauthorized|malformed).*' + ] + ) def test_md_750_010(self, env): # md with correct EAB configured @@ -125,6 +179,15 @@ class TestEab: md = env.await_error(domain_b) assert md['renewal']['errors'] > 0 assert md['renewal']['last']['problem'] == 'urn:ietf:params:acme:error:externalAccountRequired' + # + env.httpd_error_log.ignore_recent( + lognos = [ + "AH10056" # ACME server policy requires newAccount requests must include a value for the 'externalAccountBinding' field + ], + matches = [ + r'.*urn:ietf:params:acme:error:externalAccountRequired.*' + ] + ) def test_md_750_012(self, env): # first one md without EAB, then one with @@ -144,6 +207,15 @@ class TestEab: md = env.await_error(domain_a) assert md['renewal']['errors'] > 0 assert md['renewal']['last']['problem'] == 'urn:ietf:params:acme:error:externalAccountRequired' + # + env.httpd_error_log.ignore_recent( + lognos = [ + "AH10056" # ACME server policy requires newAccount requests must include a value for the 'externalAccountBinding' field + ], + matches = [ + r'.*urn:ietf:params:acme:error:externalAccountRequired.*' + ] + ) def test_md_750_013(self, env): # 2 mds with the same EAB, should one create a single account @@ -215,6 +287,15 @@ class TestEab: md = env.await_error(domain) assert md['renewal']['errors'] > 0 assert md['renewal']['last']['problem'] == 'urn:ietf:params:acme:error:externalAccountRequired' + # + env.httpd_error_log.ignore_recent( + lognos = [ + "AH10056" # ACME server policy requires newAccount requests must include a value for the 'externalAccountBinding' field + ], + matches = [ + r'.*urn:ietf:params:acme:error:externalAccountRequired.*' + ] + ) def test_md_750_016(self, env): # md with correct EAB, get cert, change to invalid EAB @@ -241,6 +322,15 @@ class TestEab: md = env.await_error(domain) assert md['renewal']['errors'] > 0 assert md['renewal']['last']['problem'] == 'urn:ietf:params:acme:error:unauthorized' + # + env.httpd_error_log.ignore_recent( + lognos = [ + "AH10056" # the field 'kid' references a key that is not known to the ACME server + ], + matches = [ + r'.*urn:ietf:params:acme:error:unauthorized.*' + ] + ) def test_md_750_017(self, env): # md without EAB explicitly set to none @@ -257,6 +347,15 @@ class TestEab: md = env.await_error(domain) assert md['renewal']['errors'] > 0 assert md['renewal']['last']['problem'] == 'urn:ietf:params:acme:error:externalAccountRequired' + # + env.httpd_error_log.ignore_recent( + lognos = [ + "AH10056" # ACME server policy requires newAccount requests must include a value for the 'externalAccountBinding' field + ], + matches = [ + r'.*urn:ietf:params:acme:error:externalAccountRequired.*' + ] + ) def test_md_750_018(self, env): # md with EAB file that does not exist diff --git a/test/modules/md/test_780_tailscale.py b/test/modules/md/test_780_tailscale.py index 84a266b..27a2df4 100644 --- a/test/modules/md/test_780_tailscale.py +++ b/test/modules/md/test_780_tailscale.py @@ -140,6 +140,12 @@ class TestTailscale: assert md['renewal']['last']['status-description'] == 'No such file or directory' assert md['renewal']['last']['detail'] == \ f"tailscale socket not available, may not be up: {socket_path}" + # + env.httpd_error_log.ignore_recent( + lognos = [ + "AH10056" # retrieving certificate from tailscale + ] + ) # create a MD using `tailscale` as protocol, path to faker, should succeed def test_md_780_002(self, env): @@ -184,3 +190,9 @@ class TestTailscale: assert md['renewal']['errors'] > 0 assert md['renewal']['last']['status-description'] == 'No such file or directory' assert md['renewal']['last']['detail'] == "retrieving certificate from tailscale" + # + env.httpd_error_log.ignore_recent( + lognos = [ + "AH10056" # retrieving certificate from tailscale + ] + ) diff --git a/test/modules/md/test_790_failover.py b/test/modules/md/test_790_failover.py index a939912..696161f 100644 --- a/test/modules/md/test_790_failover.py +++ b/test/modules/md/test_790_failover.py @@ -63,6 +63,15 @@ class TestFailover: assert env.apache_restart() == 0 assert env.await_completion([domain]) env.check_md_complete(domain) + # + env.httpd_error_log.ignore_recent( + lognos = [ + "AH10056" # Unsuccessful in contacting ACME server + ], + matches = [ + r'.*Unsuccessful in contacting ACME server at .*' + ] + ) # set 3 ACME certificata authority, invalid + invalid + valid def test_md_790_003(self, env): @@ -85,3 +94,12 @@ class TestFailover: assert env.apache_restart() == 0 assert env.await_completion([domain]) env.check_md_complete(domain) + # + env.httpd_error_log.ignore_recent( + lognos = [ + "AH10056" # Unsuccessful in contacting ACME server + ], + matches = [ + r'.*Unsuccessful in contacting ACME server at .*' + ] + ) diff --git a/test/modules/md/test_900_notify.py b/test/modules/md/test_900_notify.py index 30e0742..9d18da5 100644 --- a/test/modules/md/test_900_notify.py +++ b/test/modules/md/test_900_notify.py @@ -49,6 +49,12 @@ class TestNotify: assert env.await_error(self.domain) stat = env.get_md_status(self.domain) assert stat["renewal"]["last"]["problem"] == "urn:org:apache:httpd:log:AH10108:" + # + env.httpd_error_log.ignore_recent( + matches = [ + r'.*urn:org:apache:httpd:log:AH10108:.*' + ] + ) # test: valid notify cmd that fails, check error def test_md_900_002(self, env): @@ -61,6 +67,14 @@ class TestNotify: assert env.await_error(self.domain) stat = env.get_md_status(self.domain) assert stat["renewal"]["last"]["problem"] == "urn:org:apache:httpd:log:AH10108:" + # + env.httpd_error_log.ignore_recent( + matches = [ + r'.*urn:org:apache:httpd:log:AH10108:.*', + r'.*urn:org:apache:httpd:log:AH10109:.*' + r'.*problem\[challenge-setup-failure\].*', + ] + ) # test: valid notify that logs to file def test_md_900_010(self, env): diff --git a/test/modules/md/test_901_message.py b/test/modules/md/test_901_message.py index 8d03bfd..b18cfd3 100644 --- a/test/modules/md/test_901_message.py +++ b/test/modules/md/test_901_message.py @@ -46,6 +46,16 @@ class TestMessage: stat = env.get_md_status(domain) # this command should have failed and logged an error assert stat["renewal"]["last"]["problem"] == "urn:org:apache:httpd:log:AH10109:" + # + env.httpd_error_log.ignore_recent( + lognos = [ + "AH10056" # None of the offered challenge types + ], + matches = [ + r'.*urn:org:apache:httpd:log:AH10109:.*', + r'.*problem\[challenge-setup-failure\].*' + ] + ) # test: signup with configured message cmd that is valid but returns != 0 def test_md_901_002(self, env): @@ -63,6 +73,16 @@ class TestMessage: stat = env.get_md_status(domain) # this command should have failed and logged an error assert stat["renewal"]["last"]["problem"] == "urn:org:apache:httpd:log:AH10109:" + # + env.httpd_error_log.ignore_recent( + lognos = [ + "AH10056" # None of the offered challenge types + ], + matches = [ + r'.*urn:org:apache:httpd:log:AH10109:.*', + r'.*problem\[challenge-setup-failure\].*' + ] + ) # test: signup with working message cmd and see that it logs the right things def test_md_901_003(self, env): @@ -247,7 +267,6 @@ class TestMessage: assert job["last"]["problem"] == "urn:org:apache:httpd:log:AH10109:" break time.sleep(0.1) - env.httpd_error_log.ignore_recent() # reconfigure to a working notification command and restart conf = MDConf(env) @@ -294,4 +313,13 @@ class TestMessage: stat = env.get_md_status(domain) # this command should have failed and logged an error assert stat["renewal"]["last"]["problem"] == "challenge-setup-failure" - + # + env.httpd_error_log.ignore_recent( + lognos = [ + "AH10056" # None of the offered challenge types + ], + matches = [ + r'.*urn:org:apache:httpd:log:AH10109:.*', + r'.*problem\[challenge-setup-failure\].*' + ] + ) diff --git a/test/modules/md/test_920_status.py b/test/modules/md/test_920_status.py index c89ce6d..6ad7087 100644 --- a/test/modules/md/test_920_status.py +++ b/test/modules/md/test_920_status.py @@ -243,3 +243,9 @@ Protocols h2 http/1.1 acme-tls/1 assert ktype in stat['cert'] if env.acme_server == 'boulder': assert 'ocsp' in stat['cert'][ktype] + # + env.httpd_error_log.ignore_recent( + matches = [ + r'.*certificate with serial \w+ has no OCSP responder URL.*' + ] + ) diff --git a/test/modules/proxy/conftest.py b/test/modules/proxy/conftest.py index 23c5f14..7e6f4e7 100644 --- a/test/modules/proxy/conftest.py +++ b/test/modules/proxy/conftest.py @@ -29,23 +29,3 @@ def env(pytestconfig) -> ProxyTestEnv: env.apache_access_log_clear() env.httpd_error_log.clear_log() return env - - -@pytest.fixture(autouse=True, scope="package") -def _session_scope(env): - # we'd like to check the httpd error logs after the test suite has - # run to catch anything unusual. For this, we setup the ignore list - # of errors and warnings that we do expect. - env.httpd_error_log.set_ignored_lognos([ - 'AH01144', # No protocol handler was valid for the URL - ]) - - env.httpd_error_log.add_ignored_patterns([ - #re.compile(r'.*urn:ietf:params:acme:error:.*'), - ]) - yield - assert env.apache_stop() == 0 - errors, warnings = env.httpd_error_log.get_missed() - assert (len(errors), len(warnings)) == (0, 0),\ - f"apache logged {len(errors)} errors and {len(warnings)} warnings: \n"\ - "{0}\n{1}\n".format("\n".join(errors), "\n".join(warnings)) diff --git a/test/modules/proxy/env.py b/test/modules/proxy/env.py index 9ed635c..098d4d4 100644 --- a/test/modules/proxy/env.py +++ b/test/modules/proxy/env.py @@ -1,7 +1,6 @@ import inspect import logging import os -import re import subprocess from typing import Dict, Any diff --git a/test/modules/proxy/test_02_unix.py b/test/modules/proxy/test_02_unix.py index 7f3d4d5..0c39bc9 100644 --- a/test/modules/proxy/test_02_unix.py +++ b/test/modules/proxy/test_02_unix.py @@ -153,6 +153,12 @@ Host: {domain} r2 = self.parse_response(rlines) assert r2.response assert r2.response['status'] == exp_status + # + env.httpd_error_log.ignore_recent( + lognos = [ + "AH01144" # No protocol handler was valid for the URL + ] + ) def parse_response(self, lines) -> ExecResult: exp_body = False diff --git a/test/modules/tls/conf.py b/test/modules/tls/conf.py index ddeb91f..b34f746 100644 --- a/test/modules/tls/conf.py +++ b/test/modules/tls/conf.py @@ -13,7 +13,10 @@ class TlsTestConf(HttpdConf): def start_tls_vhost(self, domains: List[str], port=None, ssl_module=None): if ssl_module is None: - ssl_module = 'mod_tls' + if not self.env.has_shared_module("tls"): + ssl_module = "mod_ssl" + else: + ssl_module = 'mod_tls' super().start_vhost(domains=domains, port=port, doc_root=f"htdocs/{domains[0]}", ssl_module=ssl_module) def end_tls_vhost(self): @@ -39,8 +42,12 @@ class TlsTestConf(HttpdConf): f" MDCertificateKeyFile {pkey_file}", ]) self.add("") + if self.env.has_shared_module("tls"): + ssl_module= "mod_tls" + else: + ssl_module= "mod_ssl" super().add_vhost(domains=[domain], port=port, doc_root=f"htdocs/{domain}", - with_ssl=True, with_certificates=False, ssl_module='mod_tls') + with_ssl=True, with_certificates=False, ssl_module=ssl_module) def add_md_base(self, domain: str): self.add([ diff --git a/test/modules/tls/conftest.py b/test/modules/tls/conftest.py index cde4be6..c7cb858 100644 --- a/test/modules/tls/conftest.py +++ b/test/modules/tls/conftest.py @@ -31,9 +31,3 @@ def env(pytestconfig) -> TlsTestEnv: env.apache_access_log_clear() env.httpd_error_log.clear_log() return env - - -@pytest.fixture(autouse=True, scope="package") -def _session_scope(env): - yield - assert env.apache_stop() == 0 diff --git a/test/modules/tls/env.py b/test/modules/tls/env.py index 0e457bf..6afc472 100644 --- a/test/modules/tls/env.py +++ b/test/modules/tls/env.py @@ -129,7 +129,10 @@ class TlsTestEnv(HttpdTestEnv): ]), CertificateSpec(name="user1", client=True, single_file=True), ]) - self.add_httpd_log_modules(['tls']) + if not HttpdTestEnv.has_shared_module("tls"): + self.add_httpd_log_modules(['ssl']) + else: + self.add_httpd_log_modules(['tls']) def setup_httpd(self, setup: TlsTestSetup = None): diff --git a/test/modules/tls/test_02_conf.py b/test/modules/tls/test_02_conf.py index 4d6aa60..88be80c 100644 --- a/test/modules/tls/test_02_conf.py +++ b/test/modules/tls/test_02_conf.py @@ -64,9 +64,15 @@ class TestConf: ]) def test_tls_02_conf_cert_listen_valid(self, env, listen: str): conf = TlsTestConf(env=env) - conf.add("TLSEngine {listen}".format(listen=listen)) - conf.install() - assert env.apache_restart() == 0 + if not env.has_shared_module("tls"): + # Without cert/key openssl will complain + conf.add("SSLEngine on"); + conf.install() + assert env.apache_restart() == 1 + else: + conf.add("TLSEngine {listen}".format(listen=listen)) + conf.install() + assert env.apache_restart() == 0 def test_tls_02_conf_cert_listen_cert(self, env): domain = env.domain_a diff --git a/test/modules/tls/test_03_sni.py b/test/modules/tls/test_03_sni.py index cf421c0..cbd142a 100644 --- a/test/modules/tls/test_03_sni.py +++ b/test/modules/tls/test_03_sni.py @@ -34,6 +34,12 @@ class TestSni: domain_unknown = "unknown.test" r = env.tls_get(domain_unknown, "/index.json") assert r.exit_code != 0 + # + env.httpd_error_log.ignore_recent( + lognos = [ + "AH10353" # cannot decrypt peer's message + ] + ) def test_tls_03_sni_request_other_same_config(self, env): # do we see the first vhost response for another domain with different certs? @@ -44,6 +50,12 @@ class TestSni: assert r.exit_code == 0 assert r.json is None assert r.response['status'] == 421 + # + env.httpd_error_log.ignore_recent( + lognos = [ + "AH10345" # Connection host selected via SNI and request have incompatible TLS configurations + ] + ) def test_tls_03_sni_request_other_other_honor(self, env): # do we see the first vhost response for an unknown domain? @@ -60,6 +72,12 @@ class TestSni: # request denied assert r.exit_code == 0 assert r.json is None + # + env.httpd_error_log.ignore_recent( + lognos = [ + "AH10345" # Connection host selected via SNI and request have incompatible TLS configurations + ] + ) @pytest.mark.skip('openssl behaviour changed on ventura, unreliable') def test_tls_03_sni_bad_hostname(self, env): diff --git a/test/modules/tls/test_06_ciphers.py b/test/modules/tls/test_06_ciphers.py index 2e60bdd..4bedd69 100644 --- a/test/modules/tls/test_06_ciphers.py +++ b/test/modules/tls/test_06_ciphers.py @@ -176,16 +176,21 @@ class TestCiphers: def test_tls_06_ciphers_pref_unsupported(self, env): # a warning on preferring a known, but not supported cipher - env.httpd_error_log.ignore_recent() conf = TlsTestConf(env=env, extras={ env.domain_b: "TLSCiphersPrefer TLS_NULL_WITH_NULL_NULL" }) conf.add_tls_vhosts(domains=[env.domain_a, env.domain_b]) conf.install() - assert env.apache_restart() == 0 - (errors, warnings) = env.httpd_error_log.get_recent_count() - assert errors == 0 - assert warnings == 2 # once on dry run, once on start + if not conf.env.has_shared_module("tls"): + assert env.apache_restart() != 0 + else: + assert env.apache_restart() == 0 + # + env.httpd_error_log.ignore_recent( + lognos = [ + "AH10319" # Server has TLSCiphersPrefer configured that are not supported by rustls + ] + ) def test_tls_06_ciphers_supp_unknown(self, env): conf = TlsTestConf(env=env, extras={ @@ -197,13 +202,11 @@ class TestCiphers: def test_tls_06_ciphers_supp_unsupported(self, env): # no warnings on suppressing known, but not supported ciphers - env.httpd_error_log.ignore_recent() conf = TlsTestConf(env=env, extras={ env.domain_b: "TLSCiphersSuppress TLS_NULL_WITH_NULL_NULL" }) conf.add_tls_vhosts(domains=[env.domain_a, env.domain_b]) conf.install() + if not conf.env.has_shared_module("tls"): + return assert env.apache_restart() == 0 - (errors, warnings) = env.httpd_error_log.get_recent_count() - assert errors == 0 - assert warnings == 0 diff --git a/test/modules/tls/test_08_vars.py b/test/modules/tls/test_08_vars.py index a8df99a..0e3ee74 100644 --- a/test/modules/tls/test_08_vars.py +++ b/test/modules/tls/test_08_vars.py @@ -23,7 +23,10 @@ class TestVars: def test_tls_08_vars_root(self, env): # in domain_b root, the StdEnvVars is switch on exp_proto = "TLSv1.2" - exp_cipher = "TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384" + if env.has_shared_module("tls"): + exp_cipher = "TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384" + else: + exp_cipher = "ECDHE-ECDSA-AES256-GCM-SHA384" options = [ '--tls-max', '1.2'] r = env.tls_get(env.domain_b, "/vars.py", options=options) assert r.exit_code == 0, r.stderr @@ -47,7 +50,12 @@ class TestVars: def test_tls_08_vars_const(self, env, name: str, value: str): r = env.tls_get(env.domain_b, f"/vars.py?name={name}") assert r.exit_code == 0, r.stderr - assert r.json == {name: value}, r.stdout + if env.has_shared_module("tls"): + assert r.json == {name: value}, r.stdout + else: + if name == "SSL_SECURE_RENEG": + value = "true" + assert r.json == {name: value}, r.stdout @pytest.mark.parametrize("name, pattern", [ ("SSL_VERSION_INTERFACE", r'mod_tls/\d+\.\d+\.\d+'), @@ -57,4 +65,11 @@ class TestVars: r = env.tls_get(env.domain_b, f"/vars.py?name={name}") assert r.exit_code == 0, r.stderr assert name in r.json - assert re.match(pattern, r.json[name]), r.json + if env.has_shared_module("tls"): + assert re.match(pattern, r.json[name]), r.json + else: + if name == "SSL_VERSION_INTERFACE": + pattern = r'mod_ssl/\d+\.\d+\.\d+' + else: + pattern = r'OpenSSL/\d+\.\d+\.\d+' + assert re.match(pattern, r.json[name]), r.json diff --git a/test/modules/tls/test_14_proxy_ssl.py b/test/modules/tls/test_14_proxy_ssl.py index cefcbf6..87e04c2 100644 --- a/test/modules/tls/test_14_proxy_ssl.py +++ b/test/modules/tls/test_14_proxy_ssl.py @@ -2,6 +2,7 @@ import re import pytest from .conf import TlsTestConf +from pyhttpd.env import HttpdTestEnv class TestProxySSL: @@ -9,6 +10,12 @@ class TestProxySSL: @pytest.fixture(autouse=True, scope='class') def _class_scope(self, env): # add vhosts a+b and a ssl proxy from a to b + if not HttpdTestEnv.has_shared_module("tls"): + myoptions="SSLOptions +StdEnvVars" + myssl="mod_ssl" + else: + myoptions="TLSOptions +StdEnvVars" + myssl="mod_tls" conf = TlsTestConf(env=env, extras={ 'base': [ "LogLevel proxy:trace1 proxy_http:trace1 ssl:trace1 proxy_http2:trace1", @@ -33,10 +40,10 @@ class TestProxySSL: f'ProxyPass /proxy-ssl/ https://127.0.0.1:{env.https_port}/', f'ProxyPass /proxy-local/ https://localhost:{env.https_port}/', f'ProxyPass /proxy-h2-ssl/ h2://127.0.0.1:{env.https_port}/', - "TLSOptions +StdEnvVars", + myoptions, ], }) - conf.add_tls_vhosts(domains=[env.domain_a, env.domain_b]) + conf.add_tls_vhosts(domains=[env.domain_a, env.domain_b], ssl_module=myssl) conf.install() assert env.apache_restart() == 0 @@ -48,6 +55,13 @@ class TestProxySSL: # does not work, since SSLProxy* not configured data = env.tls_get_json(env.domain_b, "/proxy-local/index.json") assert data is None + # + env.httpd_error_log.ignore_recent( + lognos = [ + "AH01961", # failed to enable ssl support [Hint: if using mod_ssl, see SSLProxyEngine] + "AH00961" # failed to enable ssl support (mod_proxy) + ] + ) def test_tls_14_proxy_ssl_h2_get(self, env): r = env.tls_get(env.domain_b, "/proxy-h2-ssl/index.json") @@ -62,7 +76,24 @@ class TestProxySSL: ("SSL_CIPHER_EXPORT", "false"), ("SSL_CLIENT_VERIFY", "NONE"), ]) + def test_tls_14_proxy_tsl_vars_const(self, env, name: str, value: str): + if not HttpdTestEnv.has_shared_module("tls"): + return + r = env.tls_get(env.domain_b, f"/proxy-ssl/vars.py?name={name}") + assert r.exit_code == 0, r.stderr + assert r.json == {name: value}, r.stdout + + @pytest.mark.parametrize("name, value", [ + ("SERVER_NAME", "b.mod-tls.test"), + ("SSL_SESSION_RESUMED", "Initial"), + ("SSL_SECURE_RENEG", "true"), + ("SSL_COMPRESS_METHOD", "NULL"), + ("SSL_CIPHER_EXPORT", "false"), + ("SSL_CLIENT_VERIFY", "NONE"), + ]) def test_tls_14_proxy_ssl_vars_const(self, env, name: str, value: str): + if HttpdTestEnv.has_shared_module("tls"): + return r = env.tls_get(env.domain_b, f"/proxy-ssl/vars.py?name={name}") assert r.exit_code == 0, r.stderr assert r.json == {name: value}, r.stdout @@ -71,7 +102,21 @@ class TestProxySSL: ("SSL_VERSION_INTERFACE", r'mod_tls/\d+\.\d+\.\d+'), ("SSL_VERSION_LIBRARY", r'rustls-ffi/\d+\.\d+\.\d+/rustls/\d+\.\d+(\.\d+)?'), ]) + def test_tls_14_proxy_tsl_vars_match(self, env, name: str, pattern: str): + if not HttpdTestEnv.has_shared_module("tls"): + return + r = env.tls_get(env.domain_b, f"/proxy-ssl/vars.py?name={name}") + assert r.exit_code == 0, r.stderr + assert name in r.json + assert re.match(pattern, r.json[name]), r.json + + @pytest.mark.parametrize("name, pattern", [ + ("SSL_VERSION_INTERFACE", r'mod_ssl/\d+\.\d+\.\d+'), + ("SSL_VERSION_LIBRARY", r'OpenSSL/\d+\.\d+\.\d+'), + ]) def test_tls_14_proxy_ssl_vars_match(self, env, name: str, pattern: str): + if HttpdTestEnv.has_shared_module("tls"): + return r = env.tls_get(env.domain_b, f"/proxy-ssl/vars.py?name={name}") assert r.exit_code == 0, r.stderr assert name in r.json diff --git a/test/modules/tls/test_15_proxy_tls.py b/test/modules/tls/test_15_proxy_tls.py index f2f670d..e7eb103 100644 --- a/test/modules/tls/test_15_proxy_tls.py +++ b/test/modules/tls/test_15_proxy_tls.py @@ -1,10 +1,11 @@ -import re from datetime import timedelta import pytest from .conf import TlsTestConf +from pyhttpd.env import HttpdTestEnv +@pytest.mark.skipif(condition=not HttpdTestEnv.has_shared_module("tls"), reason="no mod_tls available") class TestProxyTLS: @@ -53,6 +54,13 @@ class TestProxyTLS: # does not work, since SSLProxy* not configured data = env.tls_get_json(env.domain_b, "/proxy-local/index.json") assert data is None + # + env.httpd_error_log.ignore_recent( + lognos = [ + "AH01961", # failed to enable ssl support [Hint: if using mod_ssl, see SSLProxyEngine] + "AH00961" # failed to enable ssl support (mod_proxy) + ] + ) def test_tls_15_proxy_tls_h2_get(self, env): r = env.tls_get(env.domain_b, "/proxy-h2-tls/index.json") diff --git a/test/modules/tls/test_16_proxy_mixed.py b/test/modules/tls/test_16_proxy_mixed.py index ca08236..88b351f 100644 --- a/test/modules/tls/test_16_proxy_mixed.py +++ b/test/modules/tls/test_16_proxy_mixed.py @@ -3,6 +3,9 @@ import time import pytest from .conf import TlsTestConf +from pyhttpd.env import HttpdTestEnv + +@pytest.mark.skipif(condition=not HttpdTestEnv.has_shared_module("tls"), reason="no mod_tls available") class TestProxyMixed: diff --git a/test/modules/tls/test_17_proxy_machine_cert.py b/test/modules/tls/test_17_proxy_machine_cert.py index 7b5ef44..a5410d6 100644 --- a/test/modules/tls/test_17_proxy_machine_cert.py +++ b/test/modules/tls/test_17_proxy_machine_cert.py @@ -3,8 +3,9 @@ import os import pytest from .conf import TlsTestConf +from pyhttpd.env import HttpdTestEnv - +@pytest.mark.skipif(condition=not HttpdTestEnv.has_shared_module("tls"), reason="no mod_tls available") class TestProxyMachineCert: @pytest.fixture(autouse=True, scope='class') diff --git a/test/pyhttpd/conf.py b/test/pyhttpd/conf.py index cd3363f..e1c6bf5 100644 --- a/test/pyhttpd/conf.py +++ b/test/pyhttpd/conf.py @@ -26,15 +26,96 @@ class HttpdConf(object): def install(self): self.env.install_test_conf(self._lines) + def replacetlsstr(self, line): + l = line.replace("TLS_", "") + l = l.replace("\n", " ") + l = l.replace("\\", " ") + l = " ".join(l.split()) + l = l.replace(" ", ":") + l = l.replace("_", "-") + l = l.replace("-WITH", "") + l = l.replace("AES-", "AES") + l = l.replace("POLY1305-SHA256", "POLY1305") + return l + + def replaceinstr(self, line): + if line.startswith("TLSCiphersPrefer"): + # the "TLS_" are changed into "". + l = self.replacetlsstr(line) + l = l.replace("TLSCiphersPrefer:", "SSLCipherSuite ") + elif line.startswith("TLSCiphersSuppress"): + # like SSLCipherSuite but with :! + l = self.replacetlsstr(line) + l = l.replace("TLSCiphersSuppress:", "SSLCipherSuite !") + l = l.replace(":", ":!") + elif line.startswith("TLSCertificate"): + l = line.replace("TLSCertificate", "SSLCertificateFile") + elif line.startswith("TLSProtocol"): + # mod_ssl is different (+ no supported and 0x code have to be translated) + l = line.replace("TLSProtocol", "SSLProtocol") + l = l.replace("+", "") + l = l.replace("default", "all") + l = l.replace("0x0303", "1.2") # need to check 1.3 and 1.1 + elif line.startswith("SSLProtocol"): + l = line # we have that in test/modules/tls/test_05_proto.py + elif line.startswith("TLSHonorClientOrder"): + # mod_ssl has SSLHonorCipherOrder on = use server off = use client. + l = line.lower() + if "on" in l: + l = "SSLHonorCipherOrder off" + else: + l = "SSLHonorCipherOrder on" + elif line.startswith("TLSEngine"): + # In fact it should go in the corresponding VirtualHost... Not sure how to do that. + l = "SSLEngine On" + else: + if line != "": + l = line.replace("TLS", "SSL") + else: + l = line + return l + def add(self, line: Any): + # make we transform the TLS to SSL if we are using mod_ssl if isinstance(line, str): + if not HttpdTestEnv.has_shared_module("tls"): + line = self.replaceinstr(line) if self._indents > 0: line = f"{' ' * self._indents}{line}" self._lines.append(line) else: - if self._indents > 0: - line = [f"{' ' * self._indents}{l}" for l in line] - self._lines.extend(line) + if not HttpdTestEnv.has_shared_module("tls"): + new = [] + previous = "" + for l in line: + if previous.startswith("SSLCipherSuite"): + if l.startswith("TLSCiphersPrefer") or l.startswith("TLSCiphersSuppress"): + # we need to merge it + l = self.replaceinstr(l) + l = l.replace("SSLCipherSuite ", ":") + previous = previous + l + continue + else: + if self._indents > 0: + previous = f"{' ' * self._indents}{previous}" + new.append(previous) + previous = "" + l = self.replaceinstr(l) + if l.startswith("SSLCipherSuite"): + previous = l + continue + if self._indents > 0: + l = f"{' ' * self._indents}{l}" + new.append(l) + if previous != "": + if self._indents > 0: + previous = f"{' ' * self._indents}{previous}" + new.append(previous) + self._lines.extend(new) + else: + if self._indents > 0: + line = [f"{' ' * self._indents}{l}" for l in line] + self._lines.extend(line) return self def add_certificate(self, cert_file, key_file, ssl_module=None): diff --git a/test/pyhttpd/curl.py b/test/pyhttpd/curl.py index 3d7993f..7dcc25b 100644 --- a/test/pyhttpd/curl.py +++ b/test/pyhttpd/curl.py @@ -131,8 +131,6 @@ class CurlPiper: recv_deltas.append(datetime.timedelta(microseconds=delta_mics)) last_mics = mics stutter_td = datetime.timedelta(seconds=stutter.total_seconds() * 0.75) # 25% leeway - # TODO: the first two chunks are often close together, it seems - # there still is a little buffering delay going on for idx, td in enumerate(recv_deltas[1:]): assert stutter_td < td, \ f"chunk {idx} arrived too early \n{recv_deltas}\nafter {td}\n{recv_err}" diff --git a/test/pyhttpd/env.py b/test/pyhttpd/env.py index 1d4e8b1..8a20d92 100644 --- a/test/pyhttpd/env.py +++ b/test/pyhttpd/env.py @@ -93,6 +93,7 @@ class HttpdTestSetup: self._make_modules_conf() self._make_htdocs() self._add_aptest() + self._build_clients() self.env.clear_curl_headerfiles() def _make_dirs(self): @@ -196,6 +197,16 @@ class HttpdTestSetup: # load our test module which is not installed fd.write(f"LoadModule aptest_module \"{local_dir}/mod_aptest/.libs/mod_aptest.so\"\n") + def _build_clients(self): + clients_dir = os.path.join( + os.path.dirname(os.path.dirname(inspect.getfile(HttpdTestSetup))), + 'clients') + p = subprocess.run(['make'], capture_output=True, cwd=clients_dir) + rv = p.returncode + if rv != 0: + log.error(f"compiling test clients failed: {p.stderr}") + raise Exception(f"compiling test clients failed: {p.stderr}") + class HttpdTestEnv: @@ -324,6 +335,12 @@ class HttpdTestEnv: for name in self._httpd_log_modules: self._log_interesting += f" {name}:{log_level}" + def check_error_log(self): + errors, warnings = self._error_log.get_missed() + assert (len(errors), len(warnings)) == (0, 0),\ + f"apache logged {len(errors)} errors and {len(warnings)} warnings: \n"\ + "{0}\n{1}\n".format("\n".join(errors), "\n".join(warnings)) + @property def curl(self) -> str: return self._curl @@ -572,16 +589,22 @@ class HttpdTestEnv: return f"{scheme}://{hostname}.{self.http_tld}:{port}{path}" def install_test_conf(self, lines: List[str]): + self.apache_stop() with open(self._test_conf, 'w') as fd: fd.write('\n'.join(self._httpd_base_conf)) fd.write('\n') fd.write(f"CoreDumpDirectory {self._server_dir}\n") - if self._verbosity >= 2: - fd.write(f"LogLevel core:trace5 {self.mpm_module}:trace5 http:trace5\n") + fd.write('\n') if self._verbosity >= 3: - fd.write(f"LogLevel dumpio:trace7\n") + fd.write(f"LogLevel trace7 ssl:trace6\n") fd.write(f"DumpIoOutput on\n") fd.write(f"DumpIoInput on\n") + elif self._verbosity >= 2: + fd.write(f"LogLevel debug core:trace5 {self.mpm_module}:trace5 ssl:trace5 http:trace5\n") + elif self._verbosity >= 1: + fd.write(f"LogLevel info\n") + else: + fd.write(f"LogLevel warn\n") if self._log_interesting: fd.write(self._log_interesting) fd.write('\n\n') diff --git a/test/pyhttpd/log.py b/test/pyhttpd/log.py index dff7623..17b0502 100644 --- a/test/pyhttpd/log.py +++ b/test/pyhttpd/log.py @@ -8,33 +8,32 @@ from typing import List, Tuple, Any class HttpdErrorLog: """Checking the httpd error log for errors and warnings, including - limiting checks from a last known position forward. + limiting checks from a recent known position forward. """ - RE_ERRLOG_ERROR = re.compile(r'.*\[(?P[^:]+):error].*') - RE_ERRLOG_WARN = re.compile(r'.*\[(?P[^:]+):warn].*') - RE_APLOGNO = re.compile(r'.*\[(?P[^:]+):(error|warn)].* (?PAH\d+): .+') - RE_SSL_LIB_ERR = re.compile(r'.*\[ssl:error].* SSL Library Error: error:(?P\S+):.+') + RE_ERRLOG_WARN = re.compile(r'.*\[[^:]+:warn].*') + RE_ERRLOG_ERROR = re.compile(r'.*\[[^:]+:error].*') + RE_APLOGNO = re.compile(r'.*\[[^:]+:(error|warn)].* (?PAH\d+): .+') def __init__(self, path: str): self._path = path - self._ignored_modules = [] + self._ignored_matches = [] self._ignored_lognos = set() - self._ignored_patterns = [] # remember the file position we started with self._start_pos = 0 if os.path.isfile(self._path): with open(self._path) as fd: self._start_pos = fd.seek(0, SEEK_END) - self._last_pos = self._start_pos - self._last_errors = [] - self._last_warnings = [] - self._observed_erros = set() - self._observed_warnings = set() + self._recent_pos = self._start_pos + self._recent_errors = [] + self._recent_warnings = [] + self._caught_errors = set() + self._caught_warnings = set() + self._caught_matches = set() def __repr__(self): - return f"HttpdErrorLog[{self._path}, errors: {' '.join(self._last_errors)}, " \ - f"warnings: {' '.join(self._last_warnings)}]" + return f"HttpdErrorLog[{self._path}, errors: {' '.join(self._recent_errors)}, " \ + f"warnings: {' '.join(self._recent_warnings)}]" @property def path(self) -> str: @@ -42,118 +41,108 @@ class HttpdErrorLog: def clear_log(self): if os.path.isfile(self.path): - os.remove(self.path) - self._start_pos = 0 - self._last_pos = self._start_pos - self._last_errors = [] - self._last_warnings = [] - self._observed_erros = set() - self._observed_warnings = set() + os.truncate(self.path, 0) + self._start_pos = self._recent_pos = 0 + self._recent_errors = [] + self._recent_warnings = [] + self._caught_errors = set() + self._caught_warnings = set() + self._caught_matches = set() + + def _lookup_matches(self, line: str, matches: List[str]) -> bool: + for m in matches: + if re.match(m, line): + return True + return False + + def _lookup_lognos(self, line: str, lognos: set) -> bool: + if len(lognos) > 0: + m = self.RE_APLOGNO.match(line) + if m and m.group('aplogno') in lognos: + return True + return False - def set_ignored_modules(self, modules: List[str]): - self._ignored_modules = modules.copy() if modules else [] + def clear_ignored_matches(self): + self._ignored_matches = [] - def set_ignored_lognos(self, lognos: List[str]): - if lognos: - for l in lognos: - self._ignored_lognos.add(l) + def add_ignored_matches(self, matches: List[str]): + for m in matches: + self._ignored_matches.append(re.compile(m)) - def add_ignored_patterns(self, patterns: List[Any]): - self._ignored_patterns.extend(patterns) + def clear_ignored_lognos(self): + self._ignored_lognos = set() + + def add_ignored_lognos(self, lognos: List[str]): + for l in lognos: + self._ignored_lognos.add(l) def _is_ignored(self, line: str) -> bool: - for p in self._ignored_patterns: - if p.match(line): - return True - m = self.RE_APLOGNO.match(line) - if m and m.group('aplogno') in self._ignored_lognos: + if self._lookup_matches(line, self._ignored_matches): + return True + if self._lookup_lognos(line, self._ignored_lognos): return True return False - def get_recent(self, advance=True) -> Tuple[List[str], List[str]]: - """Collect error and warning from the log since the last remembered position - :param advance: advance the position to the end of the log afterwards - :return: list of error and list of warnings as tuple - """ - self._last_errors = [] - self._last_warnings = [] + def ignore_recent(self, lognos: List[str] = [], matches: List[str] = []): + """After a test case triggered errors/warnings on purpose, add + those to our 'caught' list so the do not get reported as 'missed'. + """ + self._recent_errors = [] + self._recent_warnings = [] if os.path.isfile(self._path): with open(self._path) as fd: - fd.seek(self._last_pos, os.SEEK_SET) + fd.seek(self._recent_pos, os.SEEK_SET) + lognos_set = set(lognos) for line in fd: if self._is_ignored(line): continue - m = self.RE_ERRLOG_ERROR.match(line) - if m and m.group('module') not in self._ignored_modules: - self._last_errors.append(line) + if self._lookup_matches(line, matches): + self._caught_matches.add(line) continue m = self.RE_ERRLOG_WARN.match(line) - if m: - if m and m.group('module') not in self._ignored_modules: - self._last_warnings.append(line) - continue - if advance: - self._last_pos = fd.tell() - self._observed_erros.update(set(self._last_errors)) - self._observed_warnings.update(set(self._last_warnings)) - return self._last_errors, self._last_warnings - - def get_recent_count(self, advance=True): - errors, warnings = self.get_recent(advance=advance) - return len(errors), len(warnings) - - def ignore_recent(self): - """After a test case triggered errors/warnings on purpose, add - those to our 'observed' list so the do not get reported as 'missed'. - """ - self._last_errors = [] - self._last_warnings = [] - if os.path.isfile(self._path): - with open(self._path) as fd: - fd.seek(self._last_pos, os.SEEK_SET) - for line in fd: - if self._is_ignored(line): + if m and self._lookup_lognos(line, lognos_set): + self._caught_warnings.add(line) continue m = self.RE_ERRLOG_ERROR.match(line) - if m and m.group('module') not in self._ignored_modules: - self._observed_erros.add(line) + if m and self._lookup_lognos(line, lognos_set): + self._caught_errors.add(line) continue - m = self.RE_ERRLOG_WARN.match(line) - if m: - if m and m.group('module') not in self._ignored_modules: - self._observed_warnings.add(line) - continue - self._last_pos = fd.tell() + self._recent_pos = fd.tell() def get_missed(self) -> Tuple[List[str], List[str]]: errors = [] warnings = [] + self._recent_errors = [] + self._recent_warnings = [] if os.path.isfile(self._path): with open(self._path) as fd: fd.seek(self._start_pos, os.SEEK_SET) for line in fd: if self._is_ignored(line): continue + if line in self._caught_matches: + continue + m = self.RE_ERRLOG_WARN.match(line) + if m and line not in self._caught_warnings: + warnings.append(line) + continue m = self.RE_ERRLOG_ERROR.match(line) - if m and m.group('module') not in self._ignored_modules \ - and line not in self._observed_erros: + if m and line not in self._caught_errors: errors.append(line) continue - m = self.RE_ERRLOG_WARN.match(line) - if m: - if m and m.group('module') not in self._ignored_modules \ - and line not in self._observed_warnings: - warnings.append(line) - continue + self._start_pos = self._recent_pos = fd.tell() + self._caught_errors = set() + self._caught_warnings = set() + self._caught_matches = set() return errors, warnings - def scan_recent(self, pattern: re, timeout=10): + def scan_recent(self, pattern: re.Pattern, timeout=10): if not os.path.isfile(self.path): return False with open(self.path) as fd: end = datetime.now() + timedelta(seconds=timeout) while True: - fd.seek(self._last_pos, os.SEEK_SET) + fd.seek(self._recent_pos, os.SEEK_SET) for line in fd: if pattern.match(line): return True -- cgit v1.2.3