summaryrefslogtreecommitdiffstats
path: root/test/pyhttpd
diff options
context:
space:
mode:
authorDaniel Baumann <daniel.baumann@progress-linux.org>2024-05-04 10:03:19 +0000
committerDaniel Baumann <daniel.baumann@progress-linux.org>2024-05-04 10:03:19 +0000
commitac45e40b4d3cdc2497d5b6f5efe7d8a927beca21 (patch)
tree96057c4c489cab1e8c112d689befe2b2b9fa34e7 /test/pyhttpd
parentReleasing progress-linux version 2.4.57-2~progress6.99u1. (diff)
downloadapache2-ac45e40b4d3cdc2497d5b6f5efe7d8a927beca21.tar.xz
apache2-ac45e40b4d3cdc2497d5b6f5efe7d8a927beca21.zip
Merging upstream version 2.4.59.
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'test/pyhttpd')
-rw-r--r--test/pyhttpd/conf/httpd.conf.template2
-rw-r--r--test/pyhttpd/conf/mime.types2
-rw-r--r--test/pyhttpd/config.ini.in1
-rw-r--r--test/pyhttpd/curl.py9
-rw-r--r--test/pyhttpd/env.py122
-rw-r--r--test/pyhttpd/nghttp.py44
-rw-r--r--test/pyhttpd/result.py9
-rw-r--r--test/pyhttpd/ws_util.py137
8 files changed, 278 insertions, 48 deletions
diff --git a/test/pyhttpd/conf/httpd.conf.template b/test/pyhttpd/conf/httpd.conf.template
index f44935e..255b88a 100644
--- a/test/pyhttpd/conf/httpd.conf.template
+++ b/test/pyhttpd/conf/httpd.conf.template
@@ -6,7 +6,7 @@ Include "conf/modules.conf"
DocumentRoot "${server_dir}/htdocs"
<IfModule log_config_module>
- LogFormat "%h %l %u %t \"%r\" %>s %O \"%{Referer}i\" \"%{User-Agent}i\" %k" combined
+ LogFormat "{ \"request\": \"%r\", \"status\": %>s, \"bytes_resp_B\": %B, \"bytes_tx_O\": %O, \"bytes_rx_I\": %I, \"bytes_rx_tx_S\": %S, \"time_taken\": %D }" combined
LogFormat "%h %l %u %t \"%r\" %>s %b" common
CustomLog "logs/access_log" combined
diff --git a/test/pyhttpd/conf/mime.types b/test/pyhttpd/conf/mime.types
index b90b165..2db5c09 100644
--- a/test/pyhttpd/conf/mime.types
+++ b/test/pyhttpd/conf/mime.types
@@ -1,6 +1,6 @@
# This file maps Internet media types to unique file extension(s).
# Although created for httpd, this file is used by many software systems
-# and has been placed in the public domain for unlimited redisribution.
+# and has been placed in the public domain for unlimited redistribution.
#
# The table below contains both registered and (common) unregistered types.
# A type that has no unique extension can be ignored -- they are listed
diff --git a/test/pyhttpd/config.ini.in b/test/pyhttpd/config.ini.in
index e1ae070..3f42248 100644
--- a/test/pyhttpd/config.ini.in
+++ b/test/pyhttpd/config.ini.in
@@ -26,6 +26,7 @@ http_port = 5002
https_port = 5001
proxy_port = 5003
http_port2 = 5004
+ws_port = 5100
http_tld = tests.httpd.apache.org
test_dir = @abs_srcdir@
test_src_dir = @abs_srcdir@
diff --git a/test/pyhttpd/curl.py b/test/pyhttpd/curl.py
index 2b6840b..5a215cd 100644
--- a/test/pyhttpd/curl.py
+++ b/test/pyhttpd/curl.py
@@ -31,9 +31,14 @@ class CurlPiper:
def response(self):
return self._r.response if self._r else None
+ def __repr__(self):
+ return f'CurlPiper[exitcode={self._exitcode}, stderr={self._stderr}, stdout={self._stdout}]'
+
def start(self):
self.args, self.headerfile = self.env.curl_complete_args([self.url], timeout=5, options=[
- "-T", "-", "-X", "POST", "--trace-ascii", "%", "--trace-time"])
+ "-T", "-", "-X", "POST", "--trace-ascii", "%", "--trace-time"
+ ])
+ self.args.append(self.url)
sys.stderr.write("starting: {0}\n".format(self.args))
self.proc = subprocess.Popen(self.args, stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
@@ -125,7 +130,7 @@ class CurlPiper:
delta_mics += datetime.time(23, 59, 59, 999999)
recv_deltas.append(datetime.timedelta(microseconds=delta_mics))
last_mics = mics
- stutter_td = datetime.timedelta(seconds=stutter.total_seconds() * 0.9) # 10% leeway
+ stutter_td = datetime.timedelta(seconds=stutter.total_seconds() * 0.75) # 25% leeway
# TODO: the first two chunks are often close together, it seems
# there still is a little buffering delay going on
for idx, td in enumerate(recv_deltas[1:]):
diff --git a/test/pyhttpd/env.py b/test/pyhttpd/env.py
index 2c91859..1d4e8b1 100644
--- a/test/pyhttpd/env.py
+++ b/test/pyhttpd/env.py
@@ -96,9 +96,8 @@ class HttpdTestSetup:
self.env.clear_curl_headerfiles()
def _make_dirs(self):
- if os.path.exists(self.env.gen_dir):
- shutil.rmtree(self.env.gen_dir)
- os.makedirs(self.env.gen_dir)
+ if not os.path.exists(self.env.gen_dir):
+ os.makedirs(self.env.gen_dir)
if not os.path.exists(self.env.server_logs_dir):
os.makedirs(self.env.server_logs_dir)
@@ -238,6 +237,8 @@ class HttpdTestEnv:
if HttpdTestEnv.LIBEXEC_DIR is None:
HttpdTestEnv.LIBEXEC_DIR = self._libexec_dir = self.get_apxs_var('LIBEXECDIR')
self._curl = self.config.get('global', 'curl_bin')
+ if 'CURL' in os.environ:
+ self._curl = os.environ['CURL']
self._nghttp = self.config.get('global', 'nghttp')
if self._nghttp is None:
self._nghttp = 'nghttp'
@@ -249,8 +250,10 @@ class HttpdTestEnv:
self._http_port2 = int(self.config.get('test', 'http_port2'))
self._https_port = int(self.config.get('test', 'https_port'))
self._proxy_port = int(self.config.get('test', 'proxy_port'))
+ self._ws_port = int(self.config.get('test', 'ws_port'))
self._http_tld = self.config.get('test', 'http_tld')
self._test_dir = self.config.get('test', 'test_dir')
+ self._clients_dir = os.path.join(os.path.dirname(self._test_dir), 'clients')
self._gen_dir = self.config.get('test', 'gen_dir')
self._server_dir = os.path.join(self._gen_dir, 'apache')
self._server_conf_dir = os.path.join(self._server_dir, "conf")
@@ -288,6 +291,7 @@ class HttpdTestEnv:
self._verify_certs = False
self._curl_headerfiles_n = 0
+ self._curl_version = None
self._h2load_version = None
self._current_test = None
@@ -321,6 +325,10 @@ class HttpdTestEnv:
self._log_interesting += f" {name}:{log_level}"
@property
+ def curl(self) -> str:
+ return self._curl
+
+ @property
def apxs(self) -> str:
return self._apxs
@@ -361,6 +369,10 @@ class HttpdTestEnv:
return self._proxy_port
@property
+ def ws_port(self) -> int:
+ return self._ws_port
+
+ @property
def http_tld(self) -> str:
return self._http_tld
@@ -385,6 +397,10 @@ class HttpdTestEnv:
return self._test_dir
@property
+ def clients_dir(self) -> str:
+ return self._clients_dir
+
+ @property
def server_dir(self) -> str:
return self._server_dir
@@ -473,6 +489,34 @@ class HttpdTestEnv:
return self._h2load_version >= self._versiontuple(minv)
return False
+ def curl_is_at_least(self, minv):
+ if self._curl_version is None:
+ p = subprocess.run([self._curl, '-V'], capture_output=True, text=True)
+ if p.returncode != 0:
+ return False
+ for l in p.stdout.splitlines():
+ m = re.match(r'curl ([0-9.]+)[- ].*', l)
+ if m:
+ self._curl_version = self._versiontuple(m.group(1))
+ break
+ if self._curl_version is not None:
+ return self._curl_version >= self._versiontuple(minv)
+ return False
+
+ def curl_is_less_than(self, version):
+ if self._curl_version is None:
+ p = subprocess.run([self._curl, '-V'], capture_output=True, text=True)
+ if p.returncode != 0:
+ return False
+ for l in p.stdout.splitlines():
+ m = re.match(r'curl ([0-9.]+)[- ].*', l)
+ if m:
+ self._curl_version = self._versiontuple(m.group(1))
+ break
+ if self._curl_version is not None:
+ return self._curl_version < self._versiontuple(version)
+ return False
+
def has_nghttp(self):
return self._nghttp != ""
@@ -499,12 +543,14 @@ class HttpdTestEnv:
if not os.path.exists(path):
return os.makedirs(path)
- def run(self, args, stdout_list=False, intext=None, debug_log=True):
+ def run(self, args, stdout_list=False, intext=None, inbytes=None, debug_log=True):
if debug_log:
log.debug(f"run: {args}")
start = datetime.now()
+ if intext is not None:
+ inbytes = intext.encode()
p = subprocess.run(args, stderr=subprocess.PIPE, stdout=subprocess.PIPE,
- input=intext.encode() if intext else None)
+ input=inbytes)
stdout_as_list = None
if stdout_list:
try:
@@ -529,8 +575,13 @@ class HttpdTestEnv:
with open(self._test_conf, 'w') as fd:
fd.write('\n'.join(self._httpd_base_conf))
fd.write('\n')
+ fd.write(f"CoreDumpDirectory {self._server_dir}\n")
if self._verbosity >= 2:
- fd.write(f"LogLevel core:trace5 {self.mpm_module}:trace5\n")
+ fd.write(f"LogLevel core:trace5 {self.mpm_module}:trace5 http:trace5\n")
+ if self._verbosity >= 3:
+ fd.write(f"LogLevel dumpio:trace7\n")
+ fd.write(f"DumpIoOutput on\n")
+ fd.write(f"DumpIoInput on\n")
if self._log_interesting:
fd.write(self._log_interesting)
fd.write('\n\n')
@@ -651,19 +702,11 @@ class HttpdTestEnv:
os.remove(os.path.join(self.gen_dir, fname))
self._curl_headerfiles_n = 0
- def curl_complete_args(self, urls, stdout_list=False,
- timeout=None, options=None,
- insecure=False, force_resolve=True):
- u = urlparse(urls[0])
- #assert u.hostname, f"hostname not in url: {urls[0]}"
- headerfile = f"{self.gen_dir}/curl.headers.{self._curl_headerfiles_n}"
- self._curl_headerfiles_n += 1
+ def curl_resolve_args(self, url, insecure=False, force_resolve=True, options=None):
+ u = urlparse(url)
args = [
- self._curl, "-s", "--path-as-is", "-D", headerfile,
]
- if stdout_list:
- args.extend(['-w', '%{stdout}' + HttpdTestSetup.CURL_STDOUT_SEPARATOR])
if u.scheme == 'http':
pass
elif insecure:
@@ -675,19 +718,33 @@ class HttpdTestEnv:
if ca_pem:
args.extend(["--cacert", ca_pem])
- if self._current_test is not None:
- args.extend(["-H", f'AP-Test-Name: {self._current_test}'])
-
if force_resolve and u.hostname and u.hostname != 'localhost' \
and u.hostname != self._httpd_addr \
and not re.match(r'^(\d+|\[|:).*', u.hostname):
- assert u.port, f"port not in url: {urls[0]}"
+ assert u.port, f"port not in url: {url}"
args.extend(["--resolve", f"{u.hostname}:{u.port}:{self._httpd_addr}"])
+ return args
+
+ def curl_complete_args(self, urls, stdout_list=False,
+ timeout=None, options=None,
+ insecure=False, force_resolve=True):
+ headerfile = f"{self.gen_dir}/curl.headers.{self._curl_headerfiles_n}"
+ self._curl_headerfiles_n += 1
+
+ args = [
+ self._curl, "-s", "--path-as-is", "-D", headerfile,
+ ]
+ args.extend(self.curl_resolve_args(urls[0], insecure=insecure,
+ force_resolve=force_resolve,
+ options=options))
+ if stdout_list:
+ args.extend(['-w', '%{stdout}' + HttpdTestSetup.CURL_STDOUT_SEPARATOR])
+ if self._current_test is not None:
+ args.extend(["-H", f'AP-Test-Name: {self._current_test}'])
if timeout is not None and int(timeout) > 0:
args.extend(["--connect-timeout", str(int(timeout))])
if options:
args.extend(options)
- args += urls
return args, headerfile
def curl_parse_headerfile(self, headerfile: str, r: ExecResult = None) -> ExecResult:
@@ -745,22 +802,24 @@ class HttpdTestEnv:
return r
def curl_raw(self, urls, timeout=10, options=None, insecure=False,
- force_resolve=True):
+ force_resolve=True, no_stdout_list=False):
if not isinstance(urls, list):
urls = [urls]
stdout_list = False
- if len(urls) > 1:
+ if len(urls) > 1 and not no_stdout_list:
stdout_list = True
args, headerfile = self.curl_complete_args(
urls=urls, stdout_list=stdout_list,
timeout=timeout, options=options, insecure=insecure,
force_resolve=force_resolve)
+ args += urls
r = self.run(args, stdout_list=stdout_list)
if r.exit_code == 0:
self.curl_parse_headerfile(headerfile, r=r)
if r.json:
r.response["json"] = r.json
- os.remove(headerfile)
+ if os.path.isfile(headerfile):
+ os.remove(headerfile)
return r
def curl_get(self, url, insecure=False, options=None):
@@ -822,3 +881,18 @@ class HttpdTestEnv:
}
run.add_results({"h2load": stats})
return run
+
+ def make_data_file(self, indir: str, fname: str, fsize: int) -> str:
+ fpath = os.path.join(indir, fname)
+ s10 = "0123456789"
+ s = (101 * s10) + s10[0:3]
+ with open(fpath, 'w') as fd:
+ for i in range(int(fsize / 1024)):
+ fd.write(f"{i:09d}-{s}\n")
+ remain = int(fsize % 1024)
+ if remain != 0:
+ i = int(fsize / 1024) + 1
+ s = f"{i:09d}-{s}\n"
+ fd.write(s[0:remain])
+ return fpath
+
diff --git a/test/pyhttpd/nghttp.py b/test/pyhttpd/nghttp.py
index f27e40d..43721f5 100644
--- a/test/pyhttpd/nghttp.py
+++ b/test/pyhttpd/nghttp.py
@@ -37,6 +37,7 @@ class Nghttp:
"id": sid,
"body": b''
},
+ "data_lengths": [],
"paddings": [],
"promises": []
}
@@ -131,12 +132,13 @@ class Nghttp:
s = self.get_stream(streams, m.group(3))
blen = int(m.group(2))
if s:
- print("stream %d: %d DATA bytes added" % (s["id"], blen))
+ print(f'stream {s["id"]}: {blen} DATA bytes added via "{l}"')
padlen = 0
if len(lines) > lidx + 2:
mpad = re.match(r' +\(padlen=(\d+)\)', lines[lidx+2])
if mpad:
padlen = int(mpad.group(1))
+ s["data_lengths"].append(blen)
s["paddings"].append(padlen)
blen -= padlen
s["response"]["body"] += body[-blen:].encode()
@@ -196,6 +198,7 @@ class Nghttp:
if main_stream in streams:
output["response"] = streams[main_stream]["response"]
output["paddings"] = streams[main_stream]["paddings"]
+ output["data_lengths"] = streams[main_stream]["data_lengths"]
return output
def _raw(self, url, timeout, options):
@@ -244,11 +247,11 @@ class Nghttp:
def post_name(self, url, name, timeout=5, options=None):
reqbody = ("%s/nghttp.req.body" % self.TMP_DIR)
with open(reqbody, 'w') as f:
- f.write("--DSAJKcd9876\n")
- f.write("Content-Disposition: form-data; name=\"value\"; filename=\"xxxxx\"\n")
- f.write("Content-Type: text/plain\n")
- f.write("\n%s\n" % name)
- f.write("--DSAJKcd9876\n")
+ f.write("--DSAJKcd9876\r\n")
+ f.write("Content-Disposition: form-data; name=\"value\"; filename=\"xxxxx\"\r\n")
+ f.write("Content-Type: text/plain\r\n")
+ f.write(f"\r\n{name}")
+ f.write("\r\n--DSAJKcd9876\r\n")
if not options:
options = []
options.extend([
@@ -267,20 +270,23 @@ class Nghttp:
reqbody = ("%s/nghttp.req.body" % self.TMP_DIR)
with open(fpath, 'rb') as fin:
with open(reqbody, 'wb') as f:
- f.write(("""--DSAJKcd9876
-Content-Disposition: form-data; name="xxx"; filename="xxxxx"
-Content-Type: text/plain
-
-testing mod_h2
---DSAJKcd9876
-Content-Disposition: form-data; name="file"; filename="%s"
-Content-Type: application/octet-stream
-Content-Transfer-Encoding: binary
-
-""" % fname).encode('utf-8'))
+ preamble = [
+ '--DSAJKcd9876',
+ 'Content-Disposition: form-data; name="xxx"; filename="xxxxx"',
+ 'Content-Type: text/plain',
+ '',
+ 'testing mod_h2',
+ '\r\n--DSAJKcd9876',
+ f'Content-Disposition: form-data; name="file"; filename="{fname}"',
+ 'Content-Type: application/octet-stream',
+ 'Content-Transfer-Encoding: binary',
+ '', ''
+ ]
+ f.write('\r\n'.join(preamble).encode('utf-8'))
f.write(fin.read())
- f.write("""
---DSAJKcd9876""".encode('utf-8'))
+ f.write('\r\n'.join([
+ '\r\n--DSAJKcd9876', ''
+ ]).encode('utf-8'))
if not options:
options = []
options.extend([
diff --git a/test/pyhttpd/result.py b/test/pyhttpd/result.py
index 3789461..4bf9ff2 100644
--- a/test/pyhttpd/result.py
+++ b/test/pyhttpd/result.py
@@ -28,7 +28,14 @@ class ExecResult:
self._json_out = None
def __repr__(self):
- return f"ExecResult[code={self.exit_code}, args={self._args}, stdout={self._stdout}, stderr={self._stderr}]"
+ out = [
+ f"ExecResult[code={self.exit_code}, args={self._args}\n",
+ "----stdout---------------------------------------\n",
+ self._stdout.decode(),
+ "----stderr---------------------------------------\n",
+ self._stderr.decode()
+ ]
+ return ''.join(out)
@property
def exit_code(self) -> int:
diff --git a/test/pyhttpd/ws_util.py b/test/pyhttpd/ws_util.py
new file mode 100644
index 0000000..38a3cf7
--- /dev/null
+++ b/test/pyhttpd/ws_util.py
@@ -0,0 +1,137 @@
+import logging
+import struct
+
+
+log = logging.getLogger(__name__)
+
+
+class WsFrame:
+
+ CONT = 0
+ TEXT = 1
+ BINARY = 2
+ RSVD3 = 3
+ RSVD4 = 4
+ RSVD5 = 5
+ RSVD6 = 6
+ RSVD7 = 7
+ CLOSE = 8
+ PING = 9
+ PONG = 10
+ RSVD11 = 11
+ RSVD12 = 12
+ RSVD13 = 13
+ RSVD14 = 14
+ RSVD15 = 15
+
+ OP_NAMES = [
+ "CONT",
+ "TEXT",
+ "BINARY",
+ "RSVD3",
+ "RSVD4",
+ "RSVD5",
+ "RSVD6",
+ "RSVD7",
+ "CLOSE",
+ "PING",
+ "PONG",
+ "RSVD11",
+ "RSVD12",
+ "RSVD13",
+ "RSVD14",
+ "RSVD15",
+ ]
+
+ def __init__(self, opcode: int, fin: bool, mask: bytes, data: bytes):
+ self.opcode = opcode
+ self.fin = fin
+ self.mask = mask
+ self.data = data
+ self.length = len(data)
+
+ def __repr__(self):
+ return f'WsFrame[{self.OP_NAMES[self.opcode]} fin={self.fin}, mask={self.mask}, len={len(self.data)}]'
+
+ @property
+ def data_len(self) -> int:
+ return len(self.data) if self.data else 0
+
+ def to_network(self) -> bytes:
+ nd = bytearray()
+ h1 = self.opcode
+ if self.fin:
+ h1 |= 0x80
+ nd.extend(struct.pack("!B", h1))
+ mask_bit = 0x80 if self.mask is not None else 0x0
+ h2 = self.data_len
+ if h2 > 65535:
+ nd.extend(struct.pack("!BQ", 127|mask_bit, h2))
+ elif h2 > 126:
+ nd.extend(struct.pack("!BH", 126|mask_bit, h2))
+ else:
+ nd.extend(struct.pack("!B", h2|mask_bit))
+ if self.mask is not None:
+ nd.extend(self.mask)
+ if self.data is not None:
+ nd.extend(self.data)
+ return nd
+
+ @classmethod
+ def client_ping(cls, data: bytes, mask: bytes = None) -> 'WsFrame':
+ if mask is None:
+ mask = bytes.fromhex('00 00 00 00')
+ return WsFrame(opcode=WsFrame.PING, fin=True, mask=mask, data=data)
+
+ @classmethod
+ def client_close(cls, code: int, reason: str = None,
+ mask: bytes = None) -> 'WsFrame':
+ data = bytearray(struct.pack("!H", code))
+ if reason is not None:
+ data.extend(reason.encode())
+ if mask is None:
+ mask = bytes.fromhex('00 00 00 00')
+ return WsFrame(opcode=WsFrame.CLOSE, fin=True, mask=mask, data=data)
+
+
+class WsFrameReader:
+
+ def __init__(self, data: bytes):
+ self.data = data
+
+ def _read(self, n: int):
+ if len(self.data) < n:
+ raise EOFError(f'have {len(self.data)} bytes left, but {n} requested')
+ elif n == 0:
+ return b''
+ chunk = self.data[:n]
+ del self.data[:n]
+ return chunk
+
+ def next_frame(self):
+ data = self._read(2)
+ h1, h2 = struct.unpack("!BB", data)
+ log.debug(f'parsed h1={h1} h2={h2} from {data}')
+ fin = True if h1 & 0x80 else False
+ opcode = h1 & 0xf
+ has_mask = True if h2 & 0x80 else False
+ mask = None
+ dlen = h2 & 0x7f
+ if dlen == 126:
+ (dlen,) = struct.unpack("!H", self._read(2))
+ elif dlen == 127:
+ (dlen,) = struct.unpack("!Q", self._read(8))
+ if has_mask:
+ mask = self._read(4)
+ return WsFrame(opcode=opcode, fin=fin, mask=mask, data=self._read(dlen))
+
+ def eof(self):
+ return len(self.data) == 0
+
+ @classmethod
+ def parse(cls, data: bytes):
+ frames = []
+ reader = WsFrameReader(data=data)
+ while not reader.eof():
+ frames.append(reader.next_frame())
+ return frames