diff options
Diffstat (limited to 'test/pyhttpd')
| -rw-r--r-- | test/pyhttpd/conf/httpd.conf.template | 2 | ||||
| -rw-r--r-- | test/pyhttpd/conf/mime.types | 2 | ||||
| -rw-r--r-- | test/pyhttpd/config.ini.in | 1 | ||||
| -rw-r--r-- | test/pyhttpd/curl.py | 11 | ||||
| -rw-r--r-- | test/pyhttpd/env.py | 143 | ||||
| -rw-r--r-- | test/pyhttpd/nghttp.py | 48 | ||||
| -rw-r--r-- | test/pyhttpd/result.py | 18 | ||||
| -rw-r--r-- | test/pyhttpd/ws_util.py | 137 |
8 files changed, 310 insertions, 52 deletions
diff --git a/test/pyhttpd/conf/httpd.conf.template b/test/pyhttpd/conf/httpd.conf.template index f44935e..255b88a 100644 --- a/test/pyhttpd/conf/httpd.conf.template +++ b/test/pyhttpd/conf/httpd.conf.template @@ -6,7 +6,7 @@ Include "conf/modules.conf" DocumentRoot "${server_dir}/htdocs" <IfModule log_config_module> - LogFormat "%h %l %u %t \"%r\" %>s %O \"%{Referer}i\" \"%{User-Agent}i\" %k" combined + LogFormat "{ \"request\": \"%r\", \"status\": %>s, \"bytes_resp_B\": %B, \"bytes_tx_O\": %O, \"bytes_rx_I\": %I, \"bytes_rx_tx_S\": %S, \"time_taken\": %D }" combined LogFormat "%h %l %u %t \"%r\" %>s %b" common CustomLog "logs/access_log" combined diff --git a/test/pyhttpd/conf/mime.types b/test/pyhttpd/conf/mime.types index b90b165..2db5c09 100644 --- a/test/pyhttpd/conf/mime.types +++ b/test/pyhttpd/conf/mime.types @@ -1,6 +1,6 @@ # This file maps Internet media types to unique file extension(s). # Although created for httpd, this file is used by many software systems -# and has been placed in the public domain for unlimited redisribution. +# and has been placed in the public domain for unlimited redistribution. # # The table below contains both registered and (common) unregistered types. # A type that has no unique extension can be ignored -- they are listed diff --git a/test/pyhttpd/config.ini.in b/test/pyhttpd/config.ini.in index e1ae070..3f42248 100644 --- a/test/pyhttpd/config.ini.in +++ b/test/pyhttpd/config.ini.in @@ -26,6 +26,7 @@ http_port = 5002 https_port = 5001 proxy_port = 5003 http_port2 = 5004 +ws_port = 5100 http_tld = tests.httpd.apache.org test_dir = @abs_srcdir@ test_src_dir = @abs_srcdir@ diff --git a/test/pyhttpd/curl.py b/test/pyhttpd/curl.py index 338e82c..5a215cd 100644 --- a/test/pyhttpd/curl.py +++ b/test/pyhttpd/curl.py @@ -31,9 +31,14 @@ class CurlPiper: def response(self): return self._r.response if self._r else None + def __repr__(self): + return f'CurlPiper[exitcode={self._exitcode}, stderr={self._stderr}, stdout={self._stdout}]' + def start(self): - self.args, self.headerfile = self.env.curl_complete_args(self.url, timeout=5, options=[ - "-T", "-", "-X", "POST", "--trace-ascii", "%", "--trace-time"]) + self.args, self.headerfile = self.env.curl_complete_args([self.url], timeout=5, options=[ + "-T", "-", "-X", "POST", "--trace-ascii", "%", "--trace-time" + ]) + self.args.append(self.url) sys.stderr.write("starting: {0}\n".format(self.args)) self.proc = subprocess.Popen(self.args, stdin=subprocess.PIPE, stdout=subprocess.PIPE, @@ -125,7 +130,7 @@ class CurlPiper: delta_mics += datetime.time(23, 59, 59, 999999) recv_deltas.append(datetime.timedelta(microseconds=delta_mics)) last_mics = mics - stutter_td = datetime.timedelta(seconds=stutter.total_seconds() * 0.9) # 10% leeway + stutter_td = datetime.timedelta(seconds=stutter.total_seconds() * 0.75) # 25% leeway # TODO: the first two chunks are often close together, it seems # there still is a little buffering delay going on for idx, td in enumerate(recv_deltas[1:]): diff --git a/test/pyhttpd/env.py b/test/pyhttpd/env.py index af856ef..1d4e8b1 100644 --- a/test/pyhttpd/env.py +++ b/test/pyhttpd/env.py @@ -65,6 +65,8 @@ class HttpdTestSetup: "proxy_http", ] + CURL_STDOUT_SEPARATOR = "===CURL_STDOUT_SEPARATOR===" + def __init__(self, env: 'HttpdTestEnv'): self.env = env self._source_dirs = [os.path.dirname(inspect.getfile(HttpdTestSetup))] @@ -94,9 +96,8 @@ class HttpdTestSetup: self.env.clear_curl_headerfiles() def _make_dirs(self): - if os.path.exists(self.env.gen_dir): - shutil.rmtree(self.env.gen_dir) - os.makedirs(self.env.gen_dir) + if not os.path.exists(self.env.gen_dir): + os.makedirs(self.env.gen_dir) if not os.path.exists(self.env.server_logs_dir): os.makedirs(self.env.server_logs_dir) @@ -236,6 +237,8 @@ class HttpdTestEnv: if HttpdTestEnv.LIBEXEC_DIR is None: HttpdTestEnv.LIBEXEC_DIR = self._libexec_dir = self.get_apxs_var('LIBEXECDIR') self._curl = self.config.get('global', 'curl_bin') + if 'CURL' in os.environ: + self._curl = os.environ['CURL'] self._nghttp = self.config.get('global', 'nghttp') if self._nghttp is None: self._nghttp = 'nghttp' @@ -247,8 +250,10 @@ class HttpdTestEnv: self._http_port2 = int(self.config.get('test', 'http_port2')) self._https_port = int(self.config.get('test', 'https_port')) self._proxy_port = int(self.config.get('test', 'proxy_port')) + self._ws_port = int(self.config.get('test', 'ws_port')) self._http_tld = self.config.get('test', 'http_tld') self._test_dir = self.config.get('test', 'test_dir') + self._clients_dir = os.path.join(os.path.dirname(self._test_dir), 'clients') self._gen_dir = self.config.get('test', 'gen_dir') self._server_dir = os.path.join(self._gen_dir, 'apache') self._server_conf_dir = os.path.join(self._server_dir, "conf") @@ -286,6 +291,7 @@ class HttpdTestEnv: self._verify_certs = False self._curl_headerfiles_n = 0 + self._curl_version = None self._h2load_version = None self._current_test = None @@ -319,6 +325,10 @@ class HttpdTestEnv: self._log_interesting += f" {name}:{log_level}" @property + def curl(self) -> str: + return self._curl + + @property def apxs(self) -> str: return self._apxs @@ -359,6 +369,10 @@ class HttpdTestEnv: return self._proxy_port @property + def ws_port(self) -> int: + return self._ws_port + + @property def http_tld(self) -> str: return self._http_tld @@ -383,6 +397,10 @@ class HttpdTestEnv: return self._test_dir @property + def clients_dir(self) -> str: + return self._clients_dir + + @property def server_dir(self) -> str: return self._server_dir @@ -471,6 +489,34 @@ class HttpdTestEnv: return self._h2load_version >= self._versiontuple(minv) return False + def curl_is_at_least(self, minv): + if self._curl_version is None: + p = subprocess.run([self._curl, '-V'], capture_output=True, text=True) + if p.returncode != 0: + return False + for l in p.stdout.splitlines(): + m = re.match(r'curl ([0-9.]+)[- ].*', l) + if m: + self._curl_version = self._versiontuple(m.group(1)) + break + if self._curl_version is not None: + return self._curl_version >= self._versiontuple(minv) + return False + + def curl_is_less_than(self, version): + if self._curl_version is None: + p = subprocess.run([self._curl, '-V'], capture_output=True, text=True) + if p.returncode != 0: + return False + for l in p.stdout.splitlines(): + m = re.match(r'curl ([0-9.]+)[- ].*', l) + if m: + self._curl_version = self._versiontuple(m.group(1)) + break + if self._curl_version is not None: + return self._curl_version < self._versiontuple(version) + return False + def has_nghttp(self): return self._nghttp != "" @@ -497,14 +543,28 @@ class HttpdTestEnv: if not os.path.exists(path): return os.makedirs(path) - def run(self, args, intext=None, debug_log=True): + def run(self, args, stdout_list=False, intext=None, inbytes=None, debug_log=True): if debug_log: log.debug(f"run: {args}") start = datetime.now() + if intext is not None: + inbytes = intext.encode() p = subprocess.run(args, stderr=subprocess.PIPE, stdout=subprocess.PIPE, - input=intext.encode() if intext else None) + input=inbytes) + stdout_as_list = None + if stdout_list: + try: + out = p.stdout.decode() + if HttpdTestSetup.CURL_STDOUT_SEPARATOR in out: + stdout_as_list = out.split(HttpdTestSetup.CURL_STDOUT_SEPARATOR) + if not stdout_as_list[len(stdout_as_list) - 1]: + stdout_as_list.pop() + p.stdout.replace(HttpdTestSetup.CURL_STDOUT_SEPARATOR.encode(), b'') + except: + pass return ExecResult(args=args, exit_code=p.returncode, stdout=p.stdout, stderr=p.stderr, + stdout_as_list=stdout_as_list, duration=datetime.now() - start) def mkurl(self, scheme, hostname, path='/'): @@ -515,8 +575,13 @@ class HttpdTestEnv: with open(self._test_conf, 'w') as fd: fd.write('\n'.join(self._httpd_base_conf)) fd.write('\n') + fd.write(f"CoreDumpDirectory {self._server_dir}\n") if self._verbosity >= 2: - fd.write(f"LogLevel core:trace5 {self.mpm_module}:trace5\n") + fd.write(f"LogLevel core:trace5 {self.mpm_module}:trace5 http:trace5\n") + if self._verbosity >= 3: + fd.write(f"LogLevel dumpio:trace7\n") + fd.write(f"DumpIoOutput on\n") + fd.write(f"DumpIoInput on\n") if self._log_interesting: fd.write(self._log_interesting) fd.write('\n\n') @@ -637,17 +702,10 @@ class HttpdTestEnv: os.remove(os.path.join(self.gen_dir, fname)) self._curl_headerfiles_n = 0 - def curl_complete_args(self, urls, timeout=None, options=None, - insecure=False, force_resolve=True): - if not isinstance(urls, list): - urls = [urls] - u = urlparse(urls[0]) - #assert u.hostname, f"hostname not in url: {urls[0]}" - headerfile = f"{self.gen_dir}/curl.headers.{self._curl_headerfiles_n}" - self._curl_headerfiles_n += 1 + def curl_resolve_args(self, url, insecure=False, force_resolve=True, options=None): + u = urlparse(url) args = [ - self._curl, "-s", "--path-as-is", "-D", headerfile, ] if u.scheme == 'http': pass @@ -660,19 +718,33 @@ class HttpdTestEnv: if ca_pem: args.extend(["--cacert", ca_pem]) - if self._current_test is not None: - args.extend(["-H", f'AP-Test-Name: {self._current_test}']) - if force_resolve and u.hostname and u.hostname != 'localhost' \ and u.hostname != self._httpd_addr \ and not re.match(r'^(\d+|\[|:).*', u.hostname): - assert u.port, f"port not in url: {urls[0]}" + assert u.port, f"port not in url: {url}" args.extend(["--resolve", f"{u.hostname}:{u.port}:{self._httpd_addr}"]) + return args + + def curl_complete_args(self, urls, stdout_list=False, + timeout=None, options=None, + insecure=False, force_resolve=True): + headerfile = f"{self.gen_dir}/curl.headers.{self._curl_headerfiles_n}" + self._curl_headerfiles_n += 1 + + args = [ + self._curl, "-s", "--path-as-is", "-D", headerfile, + ] + args.extend(self.curl_resolve_args(urls[0], insecure=insecure, + force_resolve=force_resolve, + options=options)) + if stdout_list: + args.extend(['-w', '%{stdout}' + HttpdTestSetup.CURL_STDOUT_SEPARATOR]) + if self._current_test is not None: + args.extend(["-H", f'AP-Test-Name: {self._current_test}']) if timeout is not None and int(timeout) > 0: args.extend(["--connect-timeout", str(int(timeout))]) if options: args.extend(options) - args += urls return args, headerfile def curl_parse_headerfile(self, headerfile: str, r: ExecResult = None) -> ExecResult: @@ -730,16 +802,24 @@ class HttpdTestEnv: return r def curl_raw(self, urls, timeout=10, options=None, insecure=False, - force_resolve=True): + force_resolve=True, no_stdout_list=False): + if not isinstance(urls, list): + urls = [urls] + stdout_list = False + if len(urls) > 1 and not no_stdout_list: + stdout_list = True args, headerfile = self.curl_complete_args( - urls=urls, timeout=timeout, options=options, insecure=insecure, + urls=urls, stdout_list=stdout_list, + timeout=timeout, options=options, insecure=insecure, force_resolve=force_resolve) - r = self.run(args) + args += urls + r = self.run(args, stdout_list=stdout_list) if r.exit_code == 0: self.curl_parse_headerfile(headerfile, r=r) if r.json: r.response["json"] = r.json - os.remove(headerfile) + if os.path.isfile(headerfile): + os.remove(headerfile) return r def curl_get(self, url, insecure=False, options=None): @@ -801,3 +881,18 @@ class HttpdTestEnv: } run.add_results({"h2load": stats}) return run + + def make_data_file(self, indir: str, fname: str, fsize: int) -> str: + fpath = os.path.join(indir, fname) + s10 = "0123456789" + s = (101 * s10) + s10[0:3] + with open(fpath, 'w') as fd: + for i in range(int(fsize / 1024)): + fd.write(f"{i:09d}-{s}\n") + remain = int(fsize % 1024) + if remain != 0: + i = int(fsize / 1024) + 1 + s = f"{i:09d}-{s}\n" + fd.write(s[0:remain]) + return fpath + diff --git a/test/pyhttpd/nghttp.py b/test/pyhttpd/nghttp.py index fe4a1ae..43721f5 100644 --- a/test/pyhttpd/nghttp.py +++ b/test/pyhttpd/nghttp.py @@ -37,6 +37,7 @@ class Nghttp: "id": sid, "body": b'' }, + "data_lengths": [], "paddings": [], "promises": [] } @@ -131,12 +132,13 @@ class Nghttp: s = self.get_stream(streams, m.group(3)) blen = int(m.group(2)) if s: - print("stream %d: %d DATA bytes added" % (s["id"], blen)) + print(f'stream {s["id"]}: {blen} DATA bytes added via "{l}"') padlen = 0 if len(lines) > lidx + 2: mpad = re.match(r' +\(padlen=(\d+)\)', lines[lidx+2]) if mpad: padlen = int(mpad.group(1)) + s["data_lengths"].append(blen) s["paddings"].append(padlen) blen -= padlen s["response"]["body"] += body[-blen:].encode() @@ -196,6 +198,7 @@ class Nghttp: if main_stream in streams: output["response"] = streams[main_stream]["response"] output["paddings"] = streams[main_stream]["paddings"] + output["data_lengths"] = streams[main_stream]["data_lengths"] return output def _raw(self, url, timeout, options): @@ -244,14 +247,16 @@ class Nghttp: def post_name(self, url, name, timeout=5, options=None): reqbody = ("%s/nghttp.req.body" % self.TMP_DIR) with open(reqbody, 'w') as f: - f.write("--DSAJKcd9876\n") - f.write("Content-Disposition: form-data; name=\"value\"; filename=\"xxxxx\"\n") - f.write("Content-Type: text/plain\n") - f.write("\n%s\n" % name) - f.write("--DSAJKcd9876\n") + f.write("--DSAJKcd9876\r\n") + f.write("Content-Disposition: form-data; name=\"value\"; filename=\"xxxxx\"\r\n") + f.write("Content-Type: text/plain\r\n") + f.write(f"\r\n{name}") + f.write("\r\n--DSAJKcd9876\r\n") if not options: options = [] - options.extend(["--data=%s" % reqbody]) + options.extend([ + "--data=%s" % reqbody, + "-HContent-Type: multipart/form-data; boundary=DSAJKcd9876"]) return self._raw(url, timeout, options) def upload(self, url, fpath, timeout=5, options=None): @@ -265,20 +270,23 @@ class Nghttp: reqbody = ("%s/nghttp.req.body" % self.TMP_DIR) with open(fpath, 'rb') as fin: with open(reqbody, 'wb') as f: - f.write(("""--DSAJKcd9876 -Content-Disposition: form-data; name="xxx"; filename="xxxxx" -Content-Type: text/plain - -testing mod_h2 ---DSAJKcd9876 -Content-Disposition: form-data; name="file"; filename="%s" -Content-Type: application/octet-stream -Content-Transfer-Encoding: binary - -""" % fname).encode('utf-8')) + preamble = [ + '--DSAJKcd9876', + 'Content-Disposition: form-data; name="xxx"; filename="xxxxx"', + 'Content-Type: text/plain', + '', + 'testing mod_h2', + '\r\n--DSAJKcd9876', + f'Content-Disposition: form-data; name="file"; filename="{fname}"', + 'Content-Type: application/octet-stream', + 'Content-Transfer-Encoding: binary', + '', '' + ] + f.write('\r\n'.join(preamble).encode('utf-8')) f.write(fin.read()) - f.write(""" ---DSAJKcd9876""".encode('utf-8')) + f.write('\r\n'.join([ + '\r\n--DSAJKcd9876', '' + ]).encode('utf-8')) if not options: options = [] options.extend([ diff --git a/test/pyhttpd/result.py b/test/pyhttpd/result.py index 04ea825..4bf9ff2 100644 --- a/test/pyhttpd/result.py +++ b/test/pyhttpd/result.py @@ -6,7 +6,9 @@ from typing import Optional, Dict, List class ExecResult: def __init__(self, args: List[str], exit_code: int, - stdout: bytes, stderr: bytes = None, duration: timedelta = None): + stdout: bytes, stderr: bytes = None, + stdout_as_list: List[bytes] = None, + duration: timedelta = None): self._args = args self._exit_code = exit_code self._stdout = stdout if stdout is not None else b'' @@ -17,13 +19,23 @@ class ExecResult: self._assets = [] # noinspection PyBroadException try: - out = self._stdout.decode() + if stdout_as_list is None: + out = self._stdout.decode() + else: + out = "[" + ','.join(stdout_as_list) + "]" self._json_out = json.loads(out) except: self._json_out = None def __repr__(self): - return f"ExecResult[code={self.exit_code}, args={self._args}, stdout={self._stdout}, stderr={self._stderr}]" + out = [ + f"ExecResult[code={self.exit_code}, args={self._args}\n", + "----stdout---------------------------------------\n", + self._stdout.decode(), + "----stderr---------------------------------------\n", + self._stderr.decode() + ] + return ''.join(out) @property def exit_code(self) -> int: diff --git a/test/pyhttpd/ws_util.py b/test/pyhttpd/ws_util.py new file mode 100644 index 0000000..38a3cf7 --- /dev/null +++ b/test/pyhttpd/ws_util.py @@ -0,0 +1,137 @@ +import logging +import struct + + +log = logging.getLogger(__name__) + + +class WsFrame: + + CONT = 0 + TEXT = 1 + BINARY = 2 + RSVD3 = 3 + RSVD4 = 4 + RSVD5 = 5 + RSVD6 = 6 + RSVD7 = 7 + CLOSE = 8 + PING = 9 + PONG = 10 + RSVD11 = 11 + RSVD12 = 12 + RSVD13 = 13 + RSVD14 = 14 + RSVD15 = 15 + + OP_NAMES = [ + "CONT", + "TEXT", + "BINARY", + "RSVD3", + "RSVD4", + "RSVD5", + "RSVD6", + "RSVD7", + "CLOSE", + "PING", + "PONG", + "RSVD11", + "RSVD12", + "RSVD13", + "RSVD14", + "RSVD15", + ] + + def __init__(self, opcode: int, fin: bool, mask: bytes, data: bytes): + self.opcode = opcode + self.fin = fin + self.mask = mask + self.data = data + self.length = len(data) + + def __repr__(self): + return f'WsFrame[{self.OP_NAMES[self.opcode]} fin={self.fin}, mask={self.mask}, len={len(self.data)}]' + + @property + def data_len(self) -> int: + return len(self.data) if self.data else 0 + + def to_network(self) -> bytes: + nd = bytearray() + h1 = self.opcode + if self.fin: + h1 |= 0x80 + nd.extend(struct.pack("!B", h1)) + mask_bit = 0x80 if self.mask is not None else 0x0 + h2 = self.data_len + if h2 > 65535: + nd.extend(struct.pack("!BQ", 127|mask_bit, h2)) + elif h2 > 126: + nd.extend(struct.pack("!BH", 126|mask_bit, h2)) + else: + nd.extend(struct.pack("!B", h2|mask_bit)) + if self.mask is not None: + nd.extend(self.mask) + if self.data is not None: + nd.extend(self.data) + return nd + + @classmethod + def client_ping(cls, data: bytes, mask: bytes = None) -> 'WsFrame': + if mask is None: + mask = bytes.fromhex('00 00 00 00') + return WsFrame(opcode=WsFrame.PING, fin=True, mask=mask, data=data) + + @classmethod + def client_close(cls, code: int, reason: str = None, + mask: bytes = None) -> 'WsFrame': + data = bytearray(struct.pack("!H", code)) + if reason is not None: + data.extend(reason.encode()) + if mask is None: + mask = bytes.fromhex('00 00 00 00') + return WsFrame(opcode=WsFrame.CLOSE, fin=True, mask=mask, data=data) + + +class WsFrameReader: + + def __init__(self, data: bytes): + self.data = data + + def _read(self, n: int): + if len(self.data) < n: + raise EOFError(f'have {len(self.data)} bytes left, but {n} requested') + elif n == 0: + return b'' + chunk = self.data[:n] + del self.data[:n] + return chunk + + def next_frame(self): + data = self._read(2) + h1, h2 = struct.unpack("!BB", data) + log.debug(f'parsed h1={h1} h2={h2} from {data}') + fin = True if h1 & 0x80 else False + opcode = h1 & 0xf + has_mask = True if h2 & 0x80 else False + mask = None + dlen = h2 & 0x7f + if dlen == 126: + (dlen,) = struct.unpack("!H", self._read(2)) + elif dlen == 127: + (dlen,) = struct.unpack("!Q", self._read(8)) + if has_mask: + mask = self._read(4) + return WsFrame(opcode=opcode, fin=fin, mask=mask, data=self._read(dlen)) + + def eof(self): + return len(self.data) == 0 + + @classmethod + def parse(cls, data: bytes): + frames = [] + reader = WsFrameReader(data=data) + while not reader.eof(): + frames.append(reader.next_frame()) + return frames |