Adding upstream version 2.4.59.upstream/2.4.59upstream

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

8 files changed, 278 insertions, 48 deletions

diff --git a/test/pyhttpd/conf/httpd.conf.template b/test/pyhttpd/conf/httpd.conf.template
index f44935e..255b88a 100644
--- a/test/pyhttpd/conf/httpd.conf.template
+++ b/test/pyhttpd/conf/httpd.conf.template
@@ -6,7 +6,7 @@ Include "conf/modules.conf"
 DocumentRoot "${server_dir}/htdocs"
 
 <IfModule log_config_module>
-    LogFormat "%h %l %u %t \"%r\" %>s %O \"%{Referer}i\" \"%{User-Agent}i\" %k" combined
+    LogFormat "{ \"request\": \"%r\", \"status\": %>s, \"bytes_resp_B\": %B, \"bytes_tx_O\": %O, \"bytes_rx_I\": %I, \"bytes_rx_tx_S\": %S, \"time_taken\": %D }" combined
     LogFormat "%h %l %u %t \"%r\" %>s %b" common
     CustomLog "logs/access_log" combined
 
diff --git a/test/pyhttpd/conf/mime.types b/test/pyhttpd/conf/mime.types
index b90b165..2db5c09 100644
--- a/test/pyhttpd/conf/mime.types
+++ b/test/pyhttpd/conf/mime.types
@@ -1,6 +1,6 @@
 # This file maps Internet media types to unique file extension(s).
 # Although created for httpd, this file is used by many software systems
-# and has been placed in the public domain for unlimited redisribution.
+# and has been placed in the public domain for unlimited redistribution.
 #
 # The table below contains both registered and (common) unregistered types.
 # A type that has no unique extension can be ignored -- they are listed
diff --git a/test/pyhttpd/config.ini.in b/test/pyhttpd/config.ini.in
index e1ae070..3f42248 100644
--- a/test/pyhttpd/config.ini.in
+++ b/test/pyhttpd/config.ini.in
@@ -26,6 +26,7 @@ http_port = 5002
 https_port = 5001
 proxy_port = 5003
 http_port2 = 5004
+ws_port = 5100
 http_tld = tests.httpd.apache.org
 test_dir = @abs_srcdir@
 test_src_dir = @abs_srcdir@
diff --git a/test/pyhttpd/curl.py b/test/pyhttpd/curl.py
index 2b6840b..5a215cd 100644
--- a/test/pyhttpd/curl.py
+++ b/test/pyhttpd/curl.py
@@ -31,9 +31,14 @@ class CurlPiper:
     def response(self):
         return self._r.response if self._r else None
 
+    def __repr__(self):
+        return f'CurlPiper[exitcode={self._exitcode}, stderr={self._stderr}, stdout={self._stdout}]'
+
     def start(self):
         self.args, self.headerfile = self.env.curl_complete_args([self.url], timeout=5, options=[
-            "-T", "-", "-X", "POST", "--trace-ascii", "%", "--trace-time"])
+            "-T", "-", "-X", "POST", "--trace-ascii", "%", "--trace-time"
+        ])
+        self.args.append(self.url)
         sys.stderr.write("starting: {0}\n".format(self.args))
         self.proc = subprocess.Popen(self.args, stdin=subprocess.PIPE,
                                      stdout=subprocess.PIPE,
@@ -125,7 +130,7 @@ class CurlPiper:
                 delta_mics += datetime.time(23, 59, 59, 999999)
             recv_deltas.append(datetime.timedelta(microseconds=delta_mics))
             last_mics = mics
-        stutter_td = datetime.timedelta(seconds=stutter.total_seconds() * 0.9)  # 10% leeway
+        stutter_td = datetime.timedelta(seconds=stutter.total_seconds() * 0.75)  # 25% leeway
         # TODO: the first two chunks are often close together, it seems
         # there still is a little buffering delay going on
         for idx, td in enumerate(recv_deltas[1:]):
diff --git a/test/pyhttpd/env.py b/test/pyhttpd/env.py
index 2c91859..1d4e8b1 100644
--- a/test/pyhttpd/env.py
+++ b/test/pyhttpd/env.py
@@ -96,9 +96,8 @@ class HttpdTestSetup:
         self.env.clear_curl_headerfiles()
 
     def _make_dirs(self):
-        if os.path.exists(self.env.gen_dir):
-            shutil.rmtree(self.env.gen_dir)
-        os.makedirs(self.env.gen_dir)
+        if not os.path.exists(self.env.gen_dir):
+            os.makedirs(self.env.gen_dir)
         if not os.path.exists(self.env.server_logs_dir):
             os.makedirs(self.env.server_logs_dir)
 
@@ -238,6 +237,8 @@ class HttpdTestEnv:
         if HttpdTestEnv.LIBEXEC_DIR is None:
             HttpdTestEnv.LIBEXEC_DIR = self._libexec_dir = self.get_apxs_var('LIBEXECDIR')
         self._curl = self.config.get('global', 'curl_bin')
+        if 'CURL' in os.environ:
+            self._curl = os.environ['CURL']
         self._nghttp = self.config.get('global', 'nghttp')
         if self._nghttp is None:
             self._nghttp = 'nghttp'
@@ -249,8 +250,10 @@ class HttpdTestEnv:
         self._http_port2 = int(self.config.get('test', 'http_port2'))
         self._https_port = int(self.config.get('test', 'https_port'))
         self._proxy_port = int(self.config.get('test', 'proxy_port'))
+        self._ws_port = int(self.config.get('test', 'ws_port'))
         self._http_tld = self.config.get('test', 'http_tld')
         self._test_dir = self.config.get('test', 'test_dir')
+        self._clients_dir = os.path.join(os.path.dirname(self._test_dir), 'clients')
         self._gen_dir = self.config.get('test', 'gen_dir')
         self._server_dir = os.path.join(self._gen_dir, 'apache')
         self._server_conf_dir = os.path.join(self._server_dir, "conf")
@@ -288,6 +291,7 @@ class HttpdTestEnv:
 
         self._verify_certs = False
         self._curl_headerfiles_n = 0
+        self._curl_version = None
         self._h2load_version = None
         self._current_test = None
 
@@ -321,6 +325,10 @@ class HttpdTestEnv:
                 self._log_interesting += f" {name}:{log_level}"
 
     @property
+    def curl(self) -> str:
+        return self._curl
+
+    @property
     def apxs(self) -> str:
         return self._apxs
 
@@ -361,6 +369,10 @@ class HttpdTestEnv:
         return self._proxy_port
 
     @property
+    def ws_port(self) -> int:
+        return self._ws_port
+
+    @property
     def http_tld(self) -> str:
         return self._http_tld
 
@@ -385,6 +397,10 @@ class HttpdTestEnv:
         return self._test_dir
 
     @property
+    def clients_dir(self) -> str:
+        return self._clients_dir
+
+    @property
     def server_dir(self) -> str:
         return self._server_dir
 
@@ -473,6 +489,34 @@ class HttpdTestEnv:
             return self._h2load_version >= self._versiontuple(minv)
         return False
 
+    def curl_is_at_least(self, minv):
+        if self._curl_version is None:
+            p = subprocess.run([self._curl, '-V'], capture_output=True, text=True)
+            if p.returncode != 0:
+                return False
+            for l in p.stdout.splitlines():
+                m = re.match(r'curl ([0-9.]+)[- ].*', l)
+                if m:
+                    self._curl_version = self._versiontuple(m.group(1))
+                    break
+        if self._curl_version is not None:
+            return self._curl_version >= self._versiontuple(minv)
+        return False
+
+    def curl_is_less_than(self, version):
+        if self._curl_version is None:
+            p = subprocess.run([self._curl, '-V'], capture_output=True, text=True)
+            if p.returncode != 0:
+                return False
+            for l in p.stdout.splitlines():
+                m = re.match(r'curl ([0-9.]+)[- ].*', l)
+                if m:
+                    self._curl_version = self._versiontuple(m.group(1))
+                    break
+        if self._curl_version is not None:
+            return self._curl_version < self._versiontuple(version)
+        return False
+
     def has_nghttp(self):
         return self._nghttp != ""
 
@@ -499,12 +543,14 @@ class HttpdTestEnv:
         if not os.path.exists(path):
             return os.makedirs(path)
 
-    def run(self, args, stdout_list=False, intext=None, debug_log=True):
+    def run(self, args, stdout_list=False, intext=None, inbytes=None, debug_log=True):
         if debug_log:
             log.debug(f"run: {args}")
         start = datetime.now()
+        if intext is not None:
+            inbytes = intext.encode()
         p = subprocess.run(args, stderr=subprocess.PIPE, stdout=subprocess.PIPE,
-                           input=intext.encode() if intext else None)
+                           input=inbytes)
         stdout_as_list = None
         if stdout_list:
             try:
@@ -529,8 +575,13 @@ class HttpdTestEnv:
         with open(self._test_conf, 'w') as fd:
             fd.write('\n'.join(self._httpd_base_conf))
             fd.write('\n')
+            fd.write(f"CoreDumpDirectory {self._server_dir}\n")
             if self._verbosity >= 2:
-                fd.write(f"LogLevel core:trace5 {self.mpm_module}:trace5\n")
+                fd.write(f"LogLevel core:trace5 {self.mpm_module}:trace5 http:trace5\n")
+            if self._verbosity >= 3:
+                fd.write(f"LogLevel dumpio:trace7\n")
+                fd.write(f"DumpIoOutput on\n")
+                fd.write(f"DumpIoInput on\n")
             if self._log_interesting:
                 fd.write(self._log_interesting)
             fd.write('\n\n')
@@ -651,19 +702,11 @@ class HttpdTestEnv:
                 os.remove(os.path.join(self.gen_dir, fname))
         self._curl_headerfiles_n = 0
 
-    def curl_complete_args(self, urls, stdout_list=False,
-                           timeout=None, options=None,
-                           insecure=False, force_resolve=True):
-        u = urlparse(urls[0])
-        #assert u.hostname, f"hostname not in url: {urls[0]}"
-        headerfile = f"{self.gen_dir}/curl.headers.{self._curl_headerfiles_n}"
-        self._curl_headerfiles_n += 1
+    def curl_resolve_args(self, url, insecure=False, force_resolve=True, options=None):
+        u = urlparse(url)
 
         args = [
-            self._curl, "-s", "--path-as-is", "-D", headerfile,
         ]
-        if stdout_list:
-            args.extend(['-w', '%{stdout}' + HttpdTestSetup.CURL_STDOUT_SEPARATOR])
         if u.scheme == 'http':
             pass
         elif insecure:
@@ -675,19 +718,33 @@ class HttpdTestEnv:
             if ca_pem:
                 args.extend(["--cacert", ca_pem])
 
-        if self._current_test is not None:
-            args.extend(["-H", f'AP-Test-Name: {self._current_test}'])
-
         if force_resolve and u.hostname and u.hostname != 'localhost' \
                 and u.hostname != self._httpd_addr \
                 and not re.match(r'^(\d+|\[|:).*', u.hostname):
-            assert u.port, f"port not in url: {urls[0]}"
+            assert u.port, f"port not in url: {url}"
             args.extend(["--resolve", f"{u.hostname}:{u.port}:{self._httpd_addr}"])
+        return args
+
+    def curl_complete_args(self, urls, stdout_list=False,
+                           timeout=None, options=None,
+                           insecure=False, force_resolve=True):
+        headerfile = f"{self.gen_dir}/curl.headers.{self._curl_headerfiles_n}"
+        self._curl_headerfiles_n += 1
+
+        args = [
+            self._curl, "-s", "--path-as-is", "-D", headerfile,
+        ]
+        args.extend(self.curl_resolve_args(urls[0], insecure=insecure,
+                                           force_resolve=force_resolve,
+                                           options=options))
+        if stdout_list:
+            args.extend(['-w', '%{stdout}' + HttpdTestSetup.CURL_STDOUT_SEPARATOR])
+        if self._current_test is not None:
+            args.extend(["-H", f'AP-Test-Name: {self._current_test}'])
         if timeout is not None and int(timeout) > 0:
             args.extend(["--connect-timeout", str(int(timeout))])
         if options:
             args.extend(options)
-        args += urls
         return args, headerfile
 
     def curl_parse_headerfile(self, headerfile: str, r: ExecResult = None) -> ExecResult:
@@ -745,22 +802,24 @@ class HttpdTestEnv:
         return r
 
     def curl_raw(self, urls, timeout=10, options=None, insecure=False,
-                 force_resolve=True):
+                 force_resolve=True, no_stdout_list=False):
         if not isinstance(urls, list):
             urls = [urls]
         stdout_list = False
-        if len(urls) > 1:
+        if len(urls) > 1 and not no_stdout_list:
             stdout_list = True
         args, headerfile = self.curl_complete_args(
             urls=urls, stdout_list=stdout_list,
             timeout=timeout, options=options, insecure=insecure,
             force_resolve=force_resolve)
+        args += urls
         r = self.run(args, stdout_list=stdout_list)
         if r.exit_code == 0:
             self.curl_parse_headerfile(headerfile, r=r)
             if r.json:
                 r.response["json"] = r.json
-        os.remove(headerfile)
+        if os.path.isfile(headerfile):
+            os.remove(headerfile)
         return r
 
     def curl_get(self, url, insecure=False, options=None):
@@ -822,3 +881,18 @@ class HttpdTestEnv:
                 }
             run.add_results({"h2load": stats})
         return run
+
+    def make_data_file(self, indir: str, fname: str, fsize: int) -> str:
+        fpath = os.path.join(indir, fname)
+        s10 = "0123456789"
+        s = (101 * s10) + s10[0:3]
+        with open(fpath, 'w') as fd:
+            for i in range(int(fsize / 1024)):
+                fd.write(f"{i:09d}-{s}\n")
+            remain = int(fsize % 1024)
+            if remain != 0:
+                i = int(fsize / 1024) + 1
+                s = f"{i:09d}-{s}\n"
+                fd.write(s[0:remain])
+        return fpath
+
diff --git a/test/pyhttpd/nghttp.py b/test/pyhttpd/nghttp.py
index f27e40d..43721f5 100644
--- a/test/pyhttpd/nghttp.py
+++ b/test/pyhttpd/nghttp.py
@@ -37,6 +37,7 @@ class Nghttp:
                         "id": sid,
                         "body": b''
                     },
+                    "data_lengths": [],
                     "paddings": [],
                     "promises": []
             }
@@ -131,12 +132,13 @@ class Nghttp:
                 s = self.get_stream(streams, m.group(3))
                 blen = int(m.group(2))
                 if s:
-                    print("stream %d: %d DATA bytes added" % (s["id"], blen))
+                    print(f'stream {s["id"]}: {blen} DATA bytes added via "{l}"')
                     padlen = 0
                     if len(lines) > lidx + 2:
                         mpad = re.match(r' +\(padlen=(\d+)\)', lines[lidx+2])
                         if mpad: 
                             padlen = int(mpad.group(1))
+                    s["data_lengths"].append(blen)
                     s["paddings"].append(padlen)
                     blen -= padlen
                     s["response"]["body"] += body[-blen:].encode()
@@ -196,6 +198,7 @@ class Nghttp:
         if main_stream in streams:
             output["response"] = streams[main_stream]["response"]
             output["paddings"] = streams[main_stream]["paddings"]
+            output["data_lengths"] = streams[main_stream]["data_lengths"]
         return output
 
     def _raw(self, url, timeout, options):
@@ -244,11 +247,11 @@ class Nghttp:
     def post_name(self, url, name, timeout=5, options=None):
         reqbody = ("%s/nghttp.req.body" % self.TMP_DIR)
         with open(reqbody, 'w') as f:
-            f.write("--DSAJKcd9876\n")
-            f.write("Content-Disposition: form-data; name=\"value\"; filename=\"xxxxx\"\n")
-            f.write("Content-Type: text/plain\n")
-            f.write("\n%s\n" % name)
-            f.write("--DSAJKcd9876\n")
+            f.write("--DSAJKcd9876\r\n")
+            f.write("Content-Disposition: form-data; name=\"value\"; filename=\"xxxxx\"\r\n")
+            f.write("Content-Type: text/plain\r\n")
+            f.write(f"\r\n{name}")
+            f.write("\r\n--DSAJKcd9876\r\n")
         if not options:
             options = []
         options.extend([ 
@@ -267,20 +270,23 @@ class Nghttp:
         reqbody = ("%s/nghttp.req.body" % self.TMP_DIR)
         with open(fpath, 'rb') as fin:
             with open(reqbody, 'wb') as f:
-                f.write(("""--DSAJKcd9876
-Content-Disposition: form-data; name="xxx"; filename="xxxxx"
-Content-Type: text/plain
-
-testing mod_h2
---DSAJKcd9876
-Content-Disposition: form-data; name="file"; filename="%s"
-Content-Type: application/octet-stream
-Content-Transfer-Encoding: binary
-
-""" % fname).encode('utf-8'))
+                preamble = [
+                    '--DSAJKcd9876',
+                    'Content-Disposition: form-data; name="xxx"; filename="xxxxx"',
+                    'Content-Type: text/plain',
+                    '',
+                    'testing mod_h2',
+                    '\r\n--DSAJKcd9876',
+                    f'Content-Disposition: form-data; name="file"; filename="{fname}"',
+                    'Content-Type: application/octet-stream',
+                    'Content-Transfer-Encoding: binary',
+                    '', ''
+                ]
+                f.write('\r\n'.join(preamble).encode('utf-8'))
                 f.write(fin.read())
-                f.write("""
---DSAJKcd9876""".encode('utf-8'))
+                f.write('\r\n'.join([
+                    '\r\n--DSAJKcd9876', ''
+                ]).encode('utf-8'))
         if not options:
             options = []
         options.extend([ 
diff --git a/test/pyhttpd/result.py b/test/pyhttpd/result.py
index 3789461..4bf9ff2 100644
--- a/test/pyhttpd/result.py
+++ b/test/pyhttpd/result.py
@@ -28,7 +28,14 @@ class ExecResult:
             self._json_out = None
 
     def __repr__(self):
-        return f"ExecResult[code={self.exit_code}, args={self._args}, stdout={self._stdout}, stderr={self._stderr}]"
+        out = [
+            f"ExecResult[code={self.exit_code}, args={self._args}\n",
+            "----stdout---------------------------------------\n",
+            self._stdout.decode(),
+            "----stderr---------------------------------------\n",
+            self._stderr.decode()
+        ]
+        return ''.join(out)
 
     @property
     def exit_code(self) -> int:
diff --git a/test/pyhttpd/ws_util.py b/test/pyhttpd/ws_util.py
new file mode 100644
index 0000000..38a3cf7
--- /dev/null
+++ b/test/pyhttpd/ws_util.py
@@ -0,0 +1,137 @@
+import logging
+import struct
+
+
+log = logging.getLogger(__name__)
+
+
+class WsFrame:
+
+    CONT = 0
+    TEXT = 1
+    BINARY = 2
+    RSVD3 = 3
+    RSVD4 = 4
+    RSVD5 = 5
+    RSVD6 = 6
+    RSVD7 = 7
+    CLOSE = 8
+    PING = 9
+    PONG = 10
+    RSVD11 = 11
+    RSVD12 = 12
+    RSVD13 = 13
+    RSVD14 = 14
+    RSVD15 = 15
+
+    OP_NAMES = [
+        "CONT",
+        "TEXT",
+        "BINARY",
+        "RSVD3",
+        "RSVD4",
+        "RSVD5",
+        "RSVD6",
+        "RSVD7",
+        "CLOSE",
+        "PING",
+        "PONG",
+        "RSVD11",
+        "RSVD12",
+        "RSVD13",
+        "RSVD14",
+        "RSVD15",
+    ]
+
+    def __init__(self, opcode: int, fin: bool, mask: bytes, data: bytes):
+        self.opcode = opcode
+        self.fin = fin
+        self.mask = mask
+        self.data = data
+        self.length = len(data)
+
+    def __repr__(self):
+        return f'WsFrame[{self.OP_NAMES[self.opcode]} fin={self.fin}, mask={self.mask}, len={len(self.data)}]'
+
+    @property
+    def data_len(self) -> int:
+        return len(self.data) if self.data else 0
+
+    def to_network(self) -> bytes:
+        nd = bytearray()
+        h1 = self.opcode
+        if self.fin:
+            h1 |= 0x80
+        nd.extend(struct.pack("!B", h1))
+        mask_bit = 0x80 if self.mask is not None else 0x0
+        h2 = self.data_len
+        if h2 > 65535:
+            nd.extend(struct.pack("!BQ", 127|mask_bit, h2))
+        elif h2 > 126:
+            nd.extend(struct.pack("!BH", 126|mask_bit, h2))
+        else:
+            nd.extend(struct.pack("!B", h2|mask_bit))
+        if self.mask is not None:
+            nd.extend(self.mask)
+        if self.data is not None:
+            nd.extend(self.data)
+        return nd
+
+    @classmethod
+    def client_ping(cls, data: bytes, mask: bytes = None) -> 'WsFrame':
+        if mask is None:
+            mask = bytes.fromhex('00 00 00 00')
+        return WsFrame(opcode=WsFrame.PING, fin=True, mask=mask, data=data)
+
+    @classmethod
+    def client_close(cls, code: int, reason: str = None,
+                     mask: bytes = None) -> 'WsFrame':
+        data = bytearray(struct.pack("!H", code))
+        if reason is not None:
+            data.extend(reason.encode())
+        if mask is None:
+            mask = bytes.fromhex('00 00 00 00')
+        return WsFrame(opcode=WsFrame.CLOSE, fin=True, mask=mask, data=data)
+
+
+class WsFrameReader:
+
+    def __init__(self, data: bytes):
+        self.data = data
+
+    def _read(self, n: int):
+        if len(self.data) < n:
+            raise EOFError(f'have {len(self.data)} bytes left, but {n} requested')
+        elif n == 0:
+            return b''
+        chunk = self.data[:n]
+        del self.data[:n]
+        return chunk
+
+    def next_frame(self):
+        data = self._read(2)
+        h1, h2 = struct.unpack("!BB", data)
+        log.debug(f'parsed h1={h1} h2={h2} from {data}')
+        fin = True if h1 & 0x80 else False
+        opcode = h1 & 0xf
+        has_mask = True if h2 & 0x80 else False
+        mask = None
+        dlen = h2 & 0x7f
+        if dlen == 126:
+            (dlen,) = struct.unpack("!H", self._read(2))
+        elif dlen == 127:
+            (dlen,) = struct.unpack("!Q", self._read(8))
+        if has_mask:
+            mask = self._read(4)
+        return WsFrame(opcode=opcode, fin=fin, mask=mask, data=self._read(dlen))
+
+    def eof(self):
+        return len(self.data) == 0
+
+    @classmethod
+    def parse(cls, data: bytes):
+        frames = []
+        reader = WsFrameReader(data=data)
+        while not reader.eof():
+            frames.append(reader.next_frame())
+        return frames