Merging upstream version 2.4.62.

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

4 files changed, 199 insertions, 39 deletions

diff --git a/test/modules/core/conftest.py b/test/modules/core/conftest.py
index 439cd22..22906ef 100644
--- a/test/modules/core/conftest.py
+++ b/test/modules/core/conftest.py
@@ -4,41 +4,27 @@ import os
 import pytest
 import sys
 
+from .env import CoreTestEnv
 from pyhttpd.env import HttpdTestEnv
 
 sys.path.append(os.path.join(os.path.dirname(__file__), '../..'))
 
 
 def pytest_report_header(config, startdir):
-    env = HttpdTestEnv()
+    env = CoreTestEnv()
     return f"core [apache: {env.get_httpd_version()}, mpm: {env.mpm_module}, {env.prefix}]"
 
 
 @pytest.fixture(scope="package")
-def env(pytestconfig) -> HttpdTestEnv:
+def env(pytestconfig) -> CoreTestEnv:
     level = logging.INFO
     console = logging.StreamHandler()
     console.setLevel(level)
     console.setFormatter(logging.Formatter('%(levelname)s: %(message)s'))
     logging.getLogger('').addHandler(console)
     logging.getLogger('').setLevel(level=level)
-    env = HttpdTestEnv(pytestconfig=pytestconfig)
+    env = CoreTestEnv(pytestconfig=pytestconfig)
     env.setup_httpd()
     env.apache_access_log_clear()
     env.httpd_error_log.clear_log()
     return env
-
-
-@pytest.fixture(autouse=True, scope="package")
-def _session_scope(env):
-    env.httpd_error_log.set_ignored_lognos([
-        'AH10244',  # core: invalid URI path
-        'AH01264',  # mod_cgid script not found
-    ])
-    yield
-    assert env.apache_stop() == 0
-    errors, warnings = env.httpd_error_log.get_missed()
-    assert (len(errors), len(warnings)) == (0, 0),\
-            f"apache logged {len(errors)} errors and {len(warnings)} warnings: \n"\
-            "{0}\n{1}\n".format("\n".join(errors), "\n".join(warnings))
-
diff --git a/test/modules/core/env.py b/test/modules/core/env.py
new file mode 100644
index 0000000..9c63380
--- /dev/null
+++ b/test/modules/core/env.py
@@ -0,0 +1,25 @@
+import inspect
+import logging
+import os
+
+from pyhttpd.env import HttpdTestEnv, HttpdTestSetup
+
+log = logging.getLogger(__name__)
+
+
+class CoreTestSetup(HttpdTestSetup):
+
+    def __init__(self, env: 'HttpdTestEnv'):
+        super().__init__(env=env)
+        self.add_source_dir(os.path.dirname(inspect.getfile(CoreTestSetup)))
+        self.add_modules(["cgid"])
+
+
+class CoreTestEnv(HttpdTestEnv):
+
+    def __init__(self, pytestconfig=None):
+        super().__init__(pytestconfig=pytestconfig)
+        self.add_httpd_log_modules(["http", "core"])
+
+    def setup_httpd(self, setup: HttpdTestSetup = None):
+        super().setup_httpd(setup=CoreTestSetup(env=self))
diff --git a/test/modules/core/test_001_encoding.py b/test/modules/core/test_001_encoding.py
index b7ffbaa..a3b24d0 100644
--- a/test/modules/core/test_001_encoding.py
+++ b/test/modules/core/test_001_encoding.py
@@ -1,12 +1,11 @@
 import pytest
+from typing import List, Optional
 
 from pyhttpd.conf import HttpdConf
 
 
 class TestEncoding:
 
-    EXP_AH10244_ERRS = 0
-
     @pytest.fixture(autouse=True, scope='class')
     def _class_scope(self, env):
         conf = HttpdConf(env, extras={
@@ -57,29 +56,29 @@ class TestEncoding:
         assert r.response["status"] == 200
 
     # check path traversals
-    @pytest.mark.parametrize(["path", "status"], [
-        ["/../echo.py", 400],
-        ["/nothing/../../echo.py", 400],
-        ["/cgi-bin/../../echo.py", 400],
-        ["/nothing/%2e%2e/%2e%2e/echo.py", 400],
-        ["/cgi-bin/%2e%2e/%2e%2e/echo.py", 400],
-        ["/nothing/%%32%65%%32%65/echo.py", 400],
-        ["/cgi-bin/%%32%65%%32%65/echo.py", 400],
-        ["/nothing/%%32%65%%32%65/%%32%65%%32%65/h2_env.py", 400],
-        ["/cgi-bin/%%32%65%%32%65/%%32%65%%32%65/h2_env.py", 400],
-        ["/nothing/%25%32%65%25%32%65/echo.py", 404],
-        ["/cgi-bin/%25%32%65%25%32%65/echo.py", 404],
-        ["/nothing/%25%32%65%25%32%65/%25%32%65%25%32%65/h2_env.py", 404],
-        ["/cgi-bin/%25%32%65%25%32%65/%25%32%65%25%32%65/h2_env.py", 404],
+    @pytest.mark.parametrize(["path", "status", "lognos"], [
+        ["/../echo.py", 400, ["AH10244"]],
+        ["/nothing/../../echo.py", 400, ["AH10244"]],
+        ["/cgi-bin/../../echo.py", 400, ["AH10244"]],
+        ["/nothing/%2e%2e/%2e%2e/echo.py", 400, ["AH10244"]],
+        ["/cgi-bin/%2e%2e/%2e%2e/echo.py", 400, ["AH10244"]],
+        ["/nothing/%%32%65%%32%65/echo.py", 400, ["AH10244"]],
+        ["/cgi-bin/%%32%65%%32%65/echo.py", 400, ["AH10244"]],
+        ["/nothing/%%32%65%%32%65/%%32%65%%32%65/h2_env.py", 400, ["AH10244"]],
+        ["/cgi-bin/%%32%65%%32%65/%%32%65%%32%65/h2_env.py", 400, ["AH10244"]],
+        ["/nothing/%25%32%65%25%32%65/echo.py", 404, ["AH01264"]],
+        ["/cgi-bin/%25%32%65%25%32%65/echo.py", 404, ["AH01264"]],
+        ["/nothing/%25%32%65%25%32%65/%25%32%65%25%32%65/h2_env.py", 404, ["AH01264"]],
+        ["/cgi-bin/%25%32%65%25%32%65/%25%32%65%25%32%65/h2_env.py", 404, ["AH01264"]],
     ])
-    def test_core_001_04(self, env, path, status):
+    def test_core_001_04(self, env, path, status, lognos: Optional[List[str]]):
         url = env.mkurl("https", "test1", path)
         r = env.curl_get(url)
         assert r.response["status"] == status
-        if status == 400:
-            TestEncoding.EXP_AH10244_ERRS += 1
-            # the log will have a core:err about invalid URI path
-
+        #
+        if lognos is not None:
+            env.httpd_error_log.ignore_recent(lognos = lognos)
+ 
     # check handling of %2f url encodings that are not decoded by default
     @pytest.mark.parametrize(["host", "path", "status"], [
         ["test1", "/006%2f006.css", 404],
diff --git a/test/modules/core/test_002_restarts.py b/test/modules/core/test_002_restarts.py
new file mode 100644
index 0000000..cf203bc
--- /dev/null
+++ b/test/modules/core/test_002_restarts.py
@@ -0,0 +1,150 @@
+import os
+import re
+import time
+from datetime import datetime, timedelta
+from threading import Thread
+
+import pytest
+
+from .env import CoreTestEnv
+from pyhttpd.conf import HttpdConf
+
+
+class Loader:
+
+    def __init__(self, env, url: str, clients: int, req_per_client: int = 10):
+        self.env = env
+        self.url = url
+        self.clients = clients
+        self.req_per_client = req_per_client
+        self.result = None
+        self.total_request = 0
+        self._thread = None
+
+    def run(self):
+        self.total_requests = self.clients * self.req_per_client
+        conn_per_client = 5
+        args = [self.env.h2load, f"--connect-to=localhost:{self.env.https_port}",
+                "--h1",                                 # use only http/1.1
+                "-n", str(self.total_requests),              # total # of requests to make
+                "-c", str(conn_per_client * self.clients),   # total # of connections to make
+                "-r", str(self.clients),                     # connections at a time
+                "--rate-period", "2",                   # create conns every 2 sec
+                self.url,
+                ]
+        self.result = self.env.run(args)
+
+    def start(self):
+        self._thread = Thread(target=self.run)
+        self._thread.start()
+
+    def join(self):
+        self._thread.join()
+
+
+class ChildDynamics:
+
+    RE_DATE_TIME = re.compile(r'\[(?P<date_time>[^\]]+)\] .*')
+    RE_TIME_FRAC = re.compile(r'(?P<dt>.* \d\d:\d\d:\d\d)(?P<frac>.(?P<micros>.\d+)) (?P<year>\d+)')
+    RE_CHILD_CHANGE = re.compile(r'\[(?P<date_time>[^\]]+)\] '
+                                 r'\[mpm_event:\w+\]'
+                                 r' \[pid (?P<main_pid>\d+):tid \w+\] '
+                                 r'.* Child (?P<child_no>\d+) (?P<action>\w+): '
+                                 r'pid (?P<pid>\d+), gen (?P<generation>\d+), .*')
+
+    def __init__(self, env: CoreTestEnv):
+        self.env = env
+        self.changes = list()
+        self._start = None
+        for l in open(env.httpd_error_log.path):
+            m = self.RE_CHILD_CHANGE.match(l)
+            if m:
+                self.changes.append({
+                    'pid': int(m.group('pid')),
+                    'child_no': int(m.group('child_no')),
+                    'gen': int(m.group('generation')),
+                    'action': m.group('action'),
+                    'rtime' : self._rtime(m.group('date_time'))
+                })
+                continue
+            if self._start is None:
+                m = self.RE_DATE_TIME.match(l)
+                if m:
+                    self._rtime(m.group('date_time'))
+
+    def _rtime(self, s: str) -> timedelta:
+        micros = 0
+        m = self.RE_TIME_FRAC.match(s)
+        if m:
+            micros = int(m.group('micros'))
+            s = f"{m.group('dt')} {m.group('year')}"
+        d = datetime.strptime(s, '%a %b %d %H:%M:%S %Y') + timedelta(microseconds=micros)
+        if self._start is None:
+            self._start = d
+        delta = d - self._start
+        return f"{delta.seconds:+02d}.{delta.microseconds:06d}"
+
+
+
+@pytest.mark.skipif(condition='STRESS_TEST' not in os.environ,
+                    reason="STRESS_TEST not set in env")
+@pytest.mark.skipif(condition=not CoreTestEnv().h2load_is_at_least('1.41.0'),
+                    reason="h2load unavailable or misses --connect-to option")
+class TestRestarts:
+
+    def test_core_002_01(self, env):
+        # Lets make a tight config that triggers dynamic child behaviour
+        conf = HttpdConf(env, extras={
+            'base': f"""
+        StartServers            1
+        ServerLimit             3
+        ThreadLimit             4
+        ThreadsPerChild         4
+        MinSpareThreads         4
+        MaxSpareThreads         6
+        MaxRequestWorkers       12
+        MaxConnectionsPerChild  0
+
+        LogLevel mpm_event:trace6
+                """,
+        })
+        conf.add_vhost_cgi()
+        conf.install()
+
+        # clear logs and start server, start load
+        env.httpd_error_log.clear_log()
+        assert env.apache_restart() == 0
+        # we should see a single child started
+        cd = ChildDynamics(env)
+        assert len(cd.changes) == 1, f"{cd.changes}"
+        assert cd.changes[0]['action'] == 'started'
+        # This loader simulates 6 clients, each making 10 requests.
+        # delay.py sleeps for 1sec, so this should run for about 10 seconds
+        loader = Loader(env=env, url=env.mkurl("https", "cgi", "/delay.py"),
+                        clients=6, req_per_client=10)
+        loader.start()
+        # Expect 2 more children to have been started after half time
+        time.sleep(5)
+        cd = ChildDynamics(env)
+        assert len(cd.changes) == 3, f"{cd.changes}"
+        assert len([x for x in cd.changes if x['action'] == 'started']) == 3, f"{cd.changes}"
+
+        # Trigger a server reload
+        assert env.apache_reload() == 0
+        # a graceful reload lets ongoing requests continue, but
+        # after a while all gen 0 children should have stopped
+        time.sleep(3)  # FIXME: this pbly depends on the runtime a lot, do we have expectations?
+        cd = ChildDynamics(env)
+        gen0 = [x for x in cd.changes if x['gen'] == 0]
+        assert len([x for x in gen0 if x['action'] == 'stopped']) == 3
+
+        # wait for the loader to finish and stop the server
+        loader.join()
+        env.apache_stop()
+
+        # Similar to before the reload, we expect 3 children to have
+        # been started and stopped again on server stop
+        cd = ChildDynamics(env)
+        gen1 = [x for x in cd.changes if x['gen'] == 1]
+        assert len([x for x in gen1 if x['action'] == 'started']) == 3
+        assert len([x for x in gen1 if x['action'] == 'stopped']) == 3