diff options
Diffstat (limited to 'test/pyhttpd')
37 files changed, 4480 insertions, 0 deletions
diff --git a/test/pyhttpd/__init__.py b/test/pyhttpd/__init__.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/test/pyhttpd/__init__.py diff --git a/test/pyhttpd/certs.py b/test/pyhttpd/certs.py new file mode 100644 index 0000000..5519f16 --- /dev/null +++ b/test/pyhttpd/certs.py @@ -0,0 +1,476 @@ +import os +import re +from datetime import timedelta, datetime +from typing import List, Any, Optional + +from cryptography import x509 +from cryptography.hazmat.backends import default_backend +from cryptography.hazmat.primitives import hashes +from cryptography.hazmat.primitives.asymmetric import ec, rsa +from cryptography.hazmat.primitives.asymmetric.ec import EllipticCurvePrivateKey +from cryptography.hazmat.primitives.asymmetric.rsa import RSAPrivateKey +from cryptography.hazmat.primitives.serialization import Encoding, PrivateFormat, NoEncryption, load_pem_private_key +from cryptography.x509 import ExtendedKeyUsageOID, NameOID + + +EC_SUPPORTED = {} +EC_SUPPORTED.update([(curve.name.upper(), curve) for curve in [ + ec.SECP192R1, + ec.SECP224R1, + ec.SECP256R1, + ec.SECP384R1, +]]) + + +def _private_key(key_type): + if isinstance(key_type, str): + key_type = key_type.upper() + m = re.match(r'^(RSA)?(\d+)$', key_type) + if m: + key_type = int(m.group(2)) + + if isinstance(key_type, int): + return rsa.generate_private_key( + public_exponent=65537, + key_size=key_type, + backend=default_backend() + ) + if not isinstance(key_type, ec.EllipticCurve) and key_type in EC_SUPPORTED: + key_type = EC_SUPPORTED[key_type] + return ec.generate_private_key( + curve=key_type, + backend=default_backend() + ) + + +class CertificateSpec: + + def __init__(self, name: str = None, domains: List[str] = None, + email: str = None, + key_type: str = None, single_file: bool = False, + valid_from: timedelta = timedelta(days=-1), + valid_to: timedelta = timedelta(days=89), + client: bool = False, + sub_specs: List['CertificateSpec'] = None): + self._name = name + self.domains = domains + self.client = client + self.email = email + self.key_type = key_type + self.single_file = single_file + self.valid_from = valid_from + self.valid_to = valid_to + self.sub_specs = sub_specs + + @property + def name(self) -> Optional[str]: + if self._name: + return self._name + elif self.domains: + return self.domains[0] + return None + + @property + def type(self) -> Optional[str]: + if self.domains and len(self.domains): + return "server" + elif self.client: + return "client" + elif self.name: + return "ca" + return None + + +class Credentials: + + def __init__(self, name: str, cert: Any, pkey: Any, issuer: 'Credentials' = None): + self._name = name + self._cert = cert + self._pkey = pkey + self._issuer = issuer + self._cert_file = None + self._pkey_file = None + self._store = None + + @property + def name(self) -> str: + return self._name + + @property + def subject(self) -> x509.Name: + return self._cert.subject + + @property + def key_type(self): + if isinstance(self._pkey, RSAPrivateKey): + return f"rsa{self._pkey.key_size}" + elif isinstance(self._pkey, EllipticCurvePrivateKey): + return f"{self._pkey.curve.name}" + else: + raise Exception(f"unknown key type: {self._pkey}") + + @property + def private_key(self) -> Any: + return self._pkey + + @property + def certificate(self) -> Any: + return self._cert + + @property + def cert_pem(self) -> bytes: + return self._cert.public_bytes(Encoding.PEM) + + @property + def pkey_pem(self) -> bytes: + return self._pkey.private_bytes( + Encoding.PEM, + PrivateFormat.TraditionalOpenSSL if self.key_type.startswith('rsa') else PrivateFormat.PKCS8, + NoEncryption()) + + @property + def issuer(self) -> Optional['Credentials']: + return self._issuer + + def set_store(self, store: 'CertStore'): + self._store = store + + def set_files(self, cert_file: str, pkey_file: str = None): + self._cert_file = cert_file + self._pkey_file = pkey_file + + @property + def cert_file(self) -> str: + return self._cert_file + + @property + def pkey_file(self) -> Optional[str]: + return self._pkey_file + + def get_first(self, name) -> Optional['Credentials']: + creds = self._store.get_credentials_for_name(name) if self._store else [] + return creds[0] if len(creds) else None + + def get_credentials_for_name(self, name) -> List['Credentials']: + return self._store.get_credentials_for_name(name) if self._store else [] + + def issue_certs(self, specs: List[CertificateSpec], + chain: List['Credentials'] = None) -> List['Credentials']: + return [self.issue_cert(spec=spec, chain=chain) for spec in specs] + + def issue_cert(self, spec: CertificateSpec, chain: List['Credentials'] = None) -> 'Credentials': + key_type = spec.key_type if spec.key_type else self.key_type + creds = None + if self._store: + creds = self._store.load_credentials( + name=spec.name, key_type=key_type, single_file=spec.single_file, issuer=self) + if creds is None: + creds = HttpdTestCA.create_credentials(spec=spec, issuer=self, key_type=key_type, + valid_from=spec.valid_from, valid_to=spec.valid_to) + if self._store: + self._store.save(creds, single_file=spec.single_file) + if spec.type == "ca": + self._store.save_chain(creds, "ca", with_root=True) + + if spec.sub_specs: + if self._store: + sub_store = CertStore(fpath=os.path.join(self._store.path, creds.name)) + creds.set_store(sub_store) + subchain = chain.copy() if chain else [] + subchain.append(self) + creds.issue_certs(spec.sub_specs, chain=subchain) + return creds + + +class CertStore: + + def __init__(self, fpath: str): + self._store_dir = fpath + if not os.path.exists(self._store_dir): + os.makedirs(self._store_dir) + self._creds_by_name = {} + + @property + def path(self) -> str: + return self._store_dir + + def save(self, creds: Credentials, name: str = None, + chain: List[Credentials] = None, + single_file: bool = False) -> None: + name = name if name is not None else creds.name + cert_file = self.get_cert_file(name=name, key_type=creds.key_type) + pkey_file = self.get_pkey_file(name=name, key_type=creds.key_type) + if single_file: + pkey_file = None + with open(cert_file, "wb") as fd: + fd.write(creds.cert_pem) + if chain: + for c in chain: + fd.write(c.cert_pem) + if pkey_file is None: + fd.write(creds.pkey_pem) + if pkey_file is not None: + with open(pkey_file, "wb") as fd: + fd.write(creds.pkey_pem) + creds.set_files(cert_file, pkey_file) + self._add_credentials(name, creds) + + def save_chain(self, creds: Credentials, infix: str, with_root=False): + name = creds.name + chain = [creds] + while creds.issuer is not None: + creds = creds.issuer + chain.append(creds) + if not with_root and len(chain) > 1: + chain = chain[:-1] + chain_file = os.path.join(self._store_dir, f'{name}-{infix}.pem') + with open(chain_file, "wb") as fd: + for c in chain: + fd.write(c.cert_pem) + + def _add_credentials(self, name: str, creds: Credentials): + if name not in self._creds_by_name: + self._creds_by_name[name] = [] + self._creds_by_name[name].append(creds) + + def get_credentials_for_name(self, name) -> List[Credentials]: + return self._creds_by_name[name] if name in self._creds_by_name else [] + + def get_cert_file(self, name: str, key_type=None) -> str: + key_infix = ".{0}".format(key_type) if key_type is not None else "" + return os.path.join(self._store_dir, f'{name}{key_infix}.cert.pem') + + def get_pkey_file(self, name: str, key_type=None) -> str: + key_infix = ".{0}".format(key_type) if key_type is not None else "" + return os.path.join(self._store_dir, f'{name}{key_infix}.pkey.pem') + + def load_pem_cert(self, fpath: str) -> x509.Certificate: + with open(fpath) as fd: + return x509.load_pem_x509_certificate("".join(fd.readlines()).encode()) + + def load_pem_pkey(self, fpath: str): + with open(fpath) as fd: + return load_pem_private_key("".join(fd.readlines()).encode(), password=None) + + def load_credentials(self, name: str, key_type=None, single_file: bool = False, issuer: Credentials = None): + cert_file = self.get_cert_file(name=name, key_type=key_type) + pkey_file = cert_file if single_file else self.get_pkey_file(name=name, key_type=key_type) + if os.path.isfile(cert_file) and os.path.isfile(pkey_file): + cert = self.load_pem_cert(cert_file) + pkey = self.load_pem_pkey(pkey_file) + creds = Credentials(name=name, cert=cert, pkey=pkey, issuer=issuer) + creds.set_store(self) + creds.set_files(cert_file, pkey_file) + self._add_credentials(name, creds) + return creds + return None + + +class HttpdTestCA: + + @classmethod + def create_root(cls, name: str, store_dir: str, key_type: str = "rsa2048") -> Credentials: + store = CertStore(fpath=store_dir) + creds = store.load_credentials(name="ca", key_type=key_type, issuer=None) + if creds is None: + creds = HttpdTestCA._make_ca_credentials(name=name, key_type=key_type) + store.save(creds, name="ca") + creds.set_store(store) + return creds + + @staticmethod + def create_credentials(spec: CertificateSpec, issuer: Credentials, key_type: Any, + valid_from: timedelta = timedelta(days=-1), + valid_to: timedelta = timedelta(days=89), + ) -> Credentials: + """Create a certificate signed by this CA for the given domains. + :returns: the certificate and private key PEM file paths + """ + if spec.domains and len(spec.domains): + creds = HttpdTestCA._make_server_credentials(name=spec.name, domains=spec.domains, + issuer=issuer, valid_from=valid_from, + valid_to=valid_to, key_type=key_type) + elif spec.client: + creds = HttpdTestCA._make_client_credentials(name=spec.name, issuer=issuer, + email=spec.email, valid_from=valid_from, + valid_to=valid_to, key_type=key_type) + elif spec.name: + creds = HttpdTestCA._make_ca_credentials(name=spec.name, issuer=issuer, + valid_from=valid_from, valid_to=valid_to, + key_type=key_type) + else: + raise Exception(f"unrecognized certificate specification: {spec}") + return creds + + @staticmethod + def _make_x509_name(org_name: str = None, common_name: str = None, parent: x509.Name = None) -> x509.Name: + name_pieces = [] + if org_name: + oid = NameOID.ORGANIZATIONAL_UNIT_NAME if parent else NameOID.ORGANIZATION_NAME + name_pieces.append(x509.NameAttribute(oid, org_name)) + elif common_name: + name_pieces.append(x509.NameAttribute(NameOID.COMMON_NAME, common_name)) + if parent: + name_pieces.extend([rdn for rdn in parent]) + return x509.Name(name_pieces) + + @staticmethod + def _make_csr( + subject: x509.Name, + pkey: Any, + issuer_subject: Optional[Credentials], + valid_from_delta: timedelta = None, + valid_until_delta: timedelta = None + ): + pubkey = pkey.public_key() + issuer_subject = issuer_subject if issuer_subject is not None else subject + + valid_from = datetime.now() + if valid_until_delta is not None: + valid_from += valid_from_delta + valid_until = datetime.now() + if valid_until_delta is not None: + valid_until += valid_until_delta + + return ( + x509.CertificateBuilder() + .subject_name(subject) + .issuer_name(issuer_subject) + .public_key(pubkey) + .not_valid_before(valid_from) + .not_valid_after(valid_until) + .serial_number(x509.random_serial_number()) + .add_extension( + x509.SubjectKeyIdentifier.from_public_key(pubkey), + critical=False, + ) + ) + + @staticmethod + def _add_ca_usages(csr: Any) -> Any: + return csr.add_extension( + x509.BasicConstraints(ca=True, path_length=9), + critical=True, + ).add_extension( + x509.KeyUsage( + digital_signature=True, + content_commitment=False, + key_encipherment=False, + data_encipherment=False, + key_agreement=False, + key_cert_sign=True, + crl_sign=True, + encipher_only=False, + decipher_only=False), + critical=True + ).add_extension( + x509.ExtendedKeyUsage([ + ExtendedKeyUsageOID.CLIENT_AUTH, + ExtendedKeyUsageOID.SERVER_AUTH, + ExtendedKeyUsageOID.CODE_SIGNING, + ]), + critical=True + ) + + @staticmethod + def _add_leaf_usages(csr: Any, domains: List[str], issuer: Credentials) -> Any: + return csr.add_extension( + x509.BasicConstraints(ca=False, path_length=None), + critical=True, + ).add_extension( + x509.AuthorityKeyIdentifier.from_issuer_subject_key_identifier( + issuer.certificate.extensions.get_extension_for_class( + x509.SubjectKeyIdentifier).value), + critical=False + ).add_extension( + x509.SubjectAlternativeName([x509.DNSName(domain) for domain in domains]), + critical=True, + ).add_extension( + x509.ExtendedKeyUsage([ + ExtendedKeyUsageOID.SERVER_AUTH, + ]), + critical=True + ) + + @staticmethod + def _add_client_usages(csr: Any, issuer: Credentials, rfc82name: str = None) -> Any: + cert = csr.add_extension( + x509.BasicConstraints(ca=False, path_length=None), + critical=True, + ).add_extension( + x509.AuthorityKeyIdentifier.from_issuer_subject_key_identifier( + issuer.certificate.extensions.get_extension_for_class( + x509.SubjectKeyIdentifier).value), + critical=False + ) + if rfc82name: + cert.add_extension( + x509.SubjectAlternativeName([x509.RFC822Name(rfc82name)]), + critical=True, + ) + cert.add_extension( + x509.ExtendedKeyUsage([ + ExtendedKeyUsageOID.CLIENT_AUTH, + ]), + critical=True + ) + return cert + + @staticmethod + def _make_ca_credentials(name, key_type: Any, + issuer: Credentials = None, + valid_from: timedelta = timedelta(days=-1), + valid_to: timedelta = timedelta(days=89), + ) -> Credentials: + pkey = _private_key(key_type=key_type) + if issuer is not None: + issuer_subject = issuer.certificate.subject + issuer_key = issuer.private_key + else: + issuer_subject = None + issuer_key = pkey + subject = HttpdTestCA._make_x509_name(org_name=name, parent=issuer.subject if issuer else None) + csr = HttpdTestCA._make_csr(subject=subject, + issuer_subject=issuer_subject, pkey=pkey, + valid_from_delta=valid_from, valid_until_delta=valid_to) + csr = HttpdTestCA._add_ca_usages(csr) + cert = csr.sign(private_key=issuer_key, + algorithm=hashes.SHA256(), + backend=default_backend()) + return Credentials(name=name, cert=cert, pkey=pkey, issuer=issuer) + + @staticmethod + def _make_server_credentials(name: str, domains: List[str], issuer: Credentials, + key_type: Any, + valid_from: timedelta = timedelta(days=-1), + valid_to: timedelta = timedelta(days=89), + ) -> Credentials: + name = name + pkey = _private_key(key_type=key_type) + subject = HttpdTestCA._make_x509_name(common_name=name, parent=issuer.subject) + csr = HttpdTestCA._make_csr(subject=subject, + issuer_subject=issuer.certificate.subject, pkey=pkey, + valid_from_delta=valid_from, valid_until_delta=valid_to) + csr = HttpdTestCA._add_leaf_usages(csr, domains=domains, issuer=issuer) + cert = csr.sign(private_key=issuer.private_key, + algorithm=hashes.SHA256(), + backend=default_backend()) + return Credentials(name=name, cert=cert, pkey=pkey, issuer=issuer) + + @staticmethod + def _make_client_credentials(name: str, + issuer: Credentials, email: Optional[str], + key_type: Any, + valid_from: timedelta = timedelta(days=-1), + valid_to: timedelta = timedelta(days=89), + ) -> Credentials: + pkey = _private_key(key_type=key_type) + subject = HttpdTestCA._make_x509_name(common_name=name, parent=issuer.subject) + csr = HttpdTestCA._make_csr(subject=subject, + issuer_subject=issuer.certificate.subject, pkey=pkey, + valid_from_delta=valid_from, valid_until_delta=valid_to) + csr = HttpdTestCA._add_client_usages(csr, issuer=issuer, rfc82name=email) + cert = csr.sign(private_key=issuer.private_key, + algorithm=hashes.SHA256(), + backend=default_backend()) + return Credentials(name=name, cert=cert, pkey=pkey, issuer=issuer) diff --git a/test/pyhttpd/conf.py b/test/pyhttpd/conf.py new file mode 100644 index 0000000..cd3363f --- /dev/null +++ b/test/pyhttpd/conf.py @@ -0,0 +1,188 @@ +from typing import Dict, Any + +from pyhttpd.env import HttpdTestEnv + + +class HttpdConf(object): + + def __init__(self, env: HttpdTestEnv, extras: Dict[str, Any] = None): + """ Create a new httpd configuration. + :param env: then environment this operates in + :param extras: extra configuration directive with ServerName as key and + 'base' as special key for global configuration additions. + """ + self.env = env + self._indents = 0 + self._lines = [] + self._extras = extras.copy() if extras else {} + if 'base' in self._extras: + self.add(self._extras['base']) + self._tls_engine_ports = set() + + def __repr__(self): + s = '\n'.join(self._lines) + return f"HttpdConf[{s}]" + + def install(self): + self.env.install_test_conf(self._lines) + + def add(self, line: Any): + if isinstance(line, str): + if self._indents > 0: + line = f"{' ' * self._indents}{line}" + self._lines.append(line) + else: + if self._indents > 0: + line = [f"{' ' * self._indents}{l}" for l in line] + self._lines.extend(line) + return self + + def add_certificate(self, cert_file, key_file, ssl_module=None): + if ssl_module is None: + ssl_module = self.env.ssl_module + if ssl_module == 'mod_ssl': + self.add([ + f"SSLCertificateFile {cert_file}", + f"SSLCertificateKeyFile {key_file if key_file else cert_file}", + ]) + elif ssl_module == 'mod_tls': + self.add(f"TLSCertificate {cert_file} {key_file if key_file else ''}") + elif ssl_module == 'mod_gnutls': + self.add([ + f"GnuTLSCertificateFile {cert_file}", + f"GnuTLSKeyFile {key_file if key_file else cert_file}", + ]) + else: + raise Exception(f"unsupported ssl module: {ssl_module}") + + def add_vhost(self, domains, port=None, doc_root="htdocs", with_ssl=None, + with_certificates=None, ssl_module=None): + self.start_vhost(domains=domains, port=port, doc_root=doc_root, + with_ssl=with_ssl, with_certificates=with_certificates, + ssl_module=ssl_module) + self.end_vhost() + return self + + def start_vhost(self, domains, port=None, doc_root="htdocs", with_ssl=None, + ssl_module=None, with_certificates=None): + if not isinstance(domains, list): + domains = [domains] + if port is None: + port = self.env.https_port + if ssl_module is None: + ssl_module = self.env.ssl_module + if with_ssl is None: + with_ssl = self.env.https_port == port + if with_ssl and ssl_module == 'mod_tls' and port not in self._tls_engine_ports: + self.add(f"TLSEngine {port}") + self._tls_engine_ports.add(port) + self.add("") + self.add(f"<VirtualHost *:{port}>") + self._indents += 1 + self.add(f"ServerName {domains[0]}") + for alias in domains[1:]: + self.add(f"ServerAlias {alias}") + self.add(f"DocumentRoot {doc_root}") + if with_ssl: + if ssl_module == 'mod_ssl': + self.add("SSLEngine on") + elif ssl_module == 'mod_gnutls': + self.add("GnuTLSEnable on") + if with_certificates is not False: + for cred in self.env.get_credentials_for_name(domains[0]): + self.add_certificate(cred.cert_file, cred.pkey_file, ssl_module=ssl_module) + if domains[0] in self._extras: + self.add(self._extras[domains[0]]) + return self + + def end_vhost(self): + self._indents -= 1 + self.add("</VirtualHost>") + self.add("") + return self + + def add_proxies(self, host, proxy_self=False, h2proxy_self=False): + if proxy_self or h2proxy_self: + self.add("ProxyPreserveHost on") + if proxy_self: + self.add([ + f"ProxyPass /proxy/ http://127.0.0.1:{self.env.http_port}/", + f"ProxyPassReverse /proxy/ http://{host}.{self.env.http_tld}:{self.env.http_port}/", + ]) + if h2proxy_self: + self.add([ + f"ProxyPass /h2proxy/ h2://127.0.0.1:{self.env.https_port}/", + f"ProxyPassReverse /h2proxy/ https://{host}.{self.env.http_tld}:self.env.https_port/", + ]) + return self + + def add_vhost_test1(self, proxy_self=False, h2proxy_self=False): + domain = f"test1.{self.env.http_tld}" + self.start_vhost(domains=[domain, f"www1.{self.env.http_tld}"], + port=self.env.http_port, doc_root="htdocs/test1") + self.end_vhost() + self.start_vhost(domains=[domain, f"www1.{self.env.http_tld}"], + port=self.env.https_port, doc_root="htdocs/test1") + self.add([ + "<Location /006>", + " Options +Indexes", + "</Location>", + ]) + self.add_proxies("test1", proxy_self, h2proxy_self) + self.end_vhost() + return self + + def add_vhost_test2(self): + domain = f"test2.{self.env.http_tld}" + self.start_vhost(domains=[domain, f"www2.{self.env.http_tld}"], + port=self.env.http_port, doc_root="htdocs/test2") + self.end_vhost() + self.start_vhost(domains=[domain, f"www2.{self.env.http_tld}"], + port=self.env.https_port, doc_root="htdocs/test2") + self.add([ + "<Location /006>", + " Options +Indexes", + "</Location>", + ]) + self.end_vhost() + return self + + def add_vhost_cgi(self, proxy_self=False, h2proxy_self=False): + domain = f"cgi.{self.env.http_tld}" + if proxy_self: + self.add(["ProxyStatus on", "ProxyTimeout 5", + "SSLProxyEngine on", "SSLProxyVerify none"]) + if h2proxy_self: + self.add(["SSLProxyEngine on", "SSLProxyCheckPeerName off"]) + self.start_vhost(domains=[domain, f"cgi-alias.{self.env.http_tld}"], + port=self.env.https_port, doc_root="htdocs/cgi") + self.add_proxies("cgi", proxy_self=proxy_self, h2proxy_self=h2proxy_self) + self.end_vhost() + self.start_vhost(domains=[domain, f"cgi-alias.{self.env.http_tld}"], + port=self.env.http_port, doc_root="htdocs/cgi") + self.add("AddHandler cgi-script .py") + self.add_proxies("cgi", proxy_self=proxy_self, h2proxy_self=h2proxy_self) + self.end_vhost() + return self + + @staticmethod + def merge_extras(e1: Dict[str, Any], e2: Dict[str, Any]) -> Dict[str, Any]: + def _concat(v1, v2): + if isinstance(v1, str): + v1 = [v1] + if isinstance(v2, str): + v2 = [v2] + v1.extend(v2) + return v1 + + if e1 is None: + return e2.copy() if e2 else None + if e2 is None: + return e1.copy() + e3 = e1.copy() + for name, val in e2.items(): + if name in e3: + e3[name] = _concat(e3[name], val) + else: + e3[name] = val + return e3 diff --git a/test/pyhttpd/conf/httpd.conf.template b/test/pyhttpd/conf/httpd.conf.template new file mode 100644 index 0000000..255b88a --- /dev/null +++ b/test/pyhttpd/conf/httpd.conf.template @@ -0,0 +1,60 @@ +ServerName localhost +ServerRoot "${server_dir}" + +Include "conf/modules.conf" + +DocumentRoot "${server_dir}/htdocs" + +<IfModule log_config_module> + LogFormat "{ \"request\": \"%r\", \"status\": %>s, \"bytes_resp_B\": %B, \"bytes_tx_O\": %O, \"bytes_rx_I\": %I, \"bytes_rx_tx_S\": %S, \"time_taken\": %D }" combined + LogFormat "%h %l %u %t \"%r\" %>s %b" common + CustomLog "logs/access_log" combined + +</IfModule> + +TypesConfig "${gen_dir}/apache/conf/mime.types" + +Listen ${http_port} +Listen ${https_port} + +<IfModule mod_ssl.c> + # provide some default + SSLSessionCache "shmcb:ssl_gcache_data(32000)" +</IfModule> + +# Insert our test specific configuration before the first vhost, +# so that its vhosts can be the default one. This is relevant in +# certain behaviours, such as protocol selection during SSL ALPN +# negotiation. +# +Include "conf/test.conf" + +RequestReadTimeout header=10 body=10 + +<IfModule deflate_module> + AddOutputFilterByType DEFLATE text/html text/plain text/xml text/css +</IfModule> +<IfModule brotli_module> + AddOutputFilterByType BROTLI_COMPRESS text/html text/plain text/xml text/css +</IfModule> + +<VirtualHost *:${http_port}> + ServerName ${http_tld} + ServerAlias www.${http_tld} + <IfModule ssl_module> + SSLEngine off + </IfModule> + DocumentRoot "${server_dir}/htdocs" +</VirtualHost> + +<Directory "${server_dir}/htdocs/cgi"> + Options Indexes FollowSymLinks + AllowOverride None + Require all granted + + AddHandler cgi-script .py + AddHandler cgi-script .cgi + Options +ExecCGI +</Directory> + + diff --git a/test/pyhttpd/conf/mime.types b/test/pyhttpd/conf/mime.types new file mode 100644 index 0000000..2db5c09 --- /dev/null +++ b/test/pyhttpd/conf/mime.types @@ -0,0 +1,1588 @@ +# This file maps Internet media types to unique file extension(s). +# Although created for httpd, this file is used by many software systems +# and has been placed in the public domain for unlimited redistribution. +# +# The table below contains both registered and (common) unregistered types. +# A type that has no unique extension can be ignored -- they are listed +# here to guide configurations toward known types and to make it easier to +# identify "new" types. File extensions are also commonly used to indicate +# content languages and encodings, so choose them carefully. +# +# Internet media types should be registered as described in RFC 4288. +# The registry is at <http://www.iana.org/assignments/media-types/>. +# +# MIME type (lowercased) Extensions +# ============================================ ========== +# application/1d-interleaved-parityfec +# application/3gpp-ims+xml +# application/activemessage +application/andrew-inset ez +# application/applefile +application/applixware aw +application/atom+xml atom +application/atomcat+xml atomcat +# application/atomicmail +application/atomsvc+xml atomsvc +# application/auth-policy+xml +# application/batch-smtp +# application/beep+xml +# application/calendar+xml +# application/cals-1840 +# application/ccmp+xml +application/ccxml+xml ccxml +application/cdmi-capability cdmia +application/cdmi-container cdmic +application/cdmi-domain cdmid +application/cdmi-object cdmio +application/cdmi-queue cdmiq +# application/cea-2018+xml +# application/cellml+xml +# application/cfw +# application/cnrp+xml +# application/commonground +# application/conference-info+xml +# application/cpl+xml +# application/csta+xml +# application/cstadata+xml +application/cu-seeme cu +# application/cybercash +application/davmount+xml davmount +# application/dca-rft +# application/dec-dx +# application/dialog-info+xml +# application/dicom +# application/dns +application/docbook+xml dbk +# application/dskpp+xml +application/dssc+der dssc +application/dssc+xml xdssc +# application/dvcs +application/ecmascript ecma +# application/edi-consent +# application/edi-x12 +# application/edifact +application/emma+xml emma +# application/epp+xml +application/epub+zip epub +# application/eshop +# application/example +application/exi exi +# application/fastinfoset +# application/fastsoap +# application/fits +application/font-tdpfr pfr +# application/framework-attributes+xml +application/gml+xml gml +application/gpx+xml gpx +application/gxf gxf +# application/h224 +# application/held+xml +# application/http +application/hyperstudio stk +# application/ibe-key-request+xml +# application/ibe-pkg-reply+xml +# application/ibe-pp-data +# application/iges +# application/im-iscomposing+xml +# application/index +# application/index.cmd +# application/index.obj +# application/index.response +# application/index.vnd +application/inkml+xml ink inkml +# application/iotp +application/ipfix ipfix +# application/ipp +# application/isup +application/java-archive jar +application/java-serialized-object ser +application/java-vm class +application/javascript js +application/json json +application/jsonml+json jsonml +# application/kpml-request+xml +# application/kpml-response+xml +application/lost+xml lostxml +application/mac-binhex40 hqx +application/mac-compactpro cpt +# application/macwriteii +application/mads+xml mads +application/marc mrc +application/marcxml+xml mrcx +application/mathematica ma nb mb +# application/mathml-content+xml +# application/mathml-presentation+xml +application/mathml+xml mathml +# application/mbms-associated-procedure-description+xml +# application/mbms-deregister+xml +# application/mbms-envelope+xml +# application/mbms-msk+xml +# application/mbms-msk-response+xml +# application/mbms-protection-description+xml +# application/mbms-reception-report+xml +# application/mbms-register+xml +# application/mbms-register-response+xml +# application/mbms-user-service-description+xml +application/mbox mbox +# application/media_control+xml +application/mediaservercontrol+xml mscml +application/metalink+xml metalink +application/metalink4+xml meta4 +application/mets+xml mets +# application/mikey +application/mods+xml mods +# application/moss-keys +# application/moss-signature +# application/mosskey-data +# application/mosskey-request +application/mp21 m21 mp21 +application/mp4 mp4s +# application/mpeg4-generic +# application/mpeg4-iod +# application/mpeg4-iod-xmt +# application/msc-ivr+xml +# application/msc-mixer+xml +application/msword doc dot +application/mxf mxf +# application/nasdata +# application/news-checkgroups +# application/news-groupinfo +# application/news-transmission +# application/nss +# application/ocsp-request +# application/ocsp-response +application/octet-stream bin dms lrf mar so dist distz pkg bpk dump elc deploy +application/oda oda +application/oebps-package+xml opf +application/ogg ogx +application/omdoc+xml omdoc +application/onenote onetoc onetoc2 onetmp onepkg +application/oxps oxps +# application/parityfec +application/patch-ops-error+xml xer +application/pdf pdf +application/pgp-encrypted pgp +# application/pgp-keys +application/pgp-signature asc sig +application/pics-rules prf +# application/pidf+xml +# application/pidf-diff+xml +application/pkcs10 p10 +application/pkcs7-mime p7m p7c +application/pkcs7-signature p7s +application/pkcs8 p8 +application/pkix-attr-cert ac +application/pkix-cert cer +application/pkix-crl crl +application/pkix-pkipath pkipath +application/pkixcmp pki +application/pls+xml pls +# application/poc-settings+xml +application/postscript ai eps ps +# application/prs.alvestrand.titrax-sheet +application/prs.cww cww +# application/prs.nprend +# application/prs.plucker +# application/prs.rdf-xml-crypt +# application/prs.xsf+xml +application/pskc+xml pskcxml +# application/qsig +application/rdf+xml rdf +application/reginfo+xml rif +application/relax-ng-compact-syntax rnc +# application/remote-printing +application/resource-lists+xml rl +application/resource-lists-diff+xml rld +# application/riscos +# application/rlmi+xml +application/rls-services+xml rs +application/rpki-ghostbusters gbr +application/rpki-manifest mft +application/rpki-roa roa +# application/rpki-updown +application/rsd+xml rsd +application/rss+xml rss +application/rtf rtf +# application/rtx +# application/samlassertion+xml +# application/samlmetadata+xml +application/sbml+xml sbml +application/scvp-cv-request scq +application/scvp-cv-response scs +application/scvp-vp-request spq +application/scvp-vp-response spp +application/sdp sdp +# application/set-payment +application/set-payment-initiation setpay +# application/set-registration +application/set-registration-initiation setreg +# application/sgml +# application/sgml-open-catalog +application/shf+xml shf +# application/sieve +# application/simple-filter+xml +# application/simple-message-summary +# application/simplesymbolcontainer +# application/slate +# application/smil +application/smil+xml smi smil +# application/soap+fastinfoset +# application/soap+xml +application/sparql-query rq +application/sparql-results+xml srx +# application/spirits-event+xml +application/srgs gram +application/srgs+xml grxml +application/sru+xml sru +application/ssdl+xml ssdl +application/ssml+xml ssml +# application/tamp-apex-update +# application/tamp-apex-update-confirm +# application/tamp-community-update +# application/tamp-community-update-confirm +# application/tamp-error +# application/tamp-sequence-adjust +# application/tamp-sequence-adjust-confirm +# application/tamp-status-query +# application/tamp-status-response +# application/tamp-update +# application/tamp-update-confirm +application/tei+xml tei teicorpus +application/thraud+xml tfi +# application/timestamp-query +# application/timestamp-reply +application/timestamped-data tsd +# application/tve-trigger +# application/ulpfec +# application/vcard+xml +# application/vemmi +# application/vividence.scriptfile +# application/vnd.3gpp.bsf+xml +application/vnd.3gpp.pic-bw-large plb +application/vnd.3gpp.pic-bw-small psb +application/vnd.3gpp.pic-bw-var pvb +# application/vnd.3gpp.sms +# application/vnd.3gpp2.bcmcsinfo+xml +# application/vnd.3gpp2.sms +application/vnd.3gpp2.tcap tcap +application/vnd.3m.post-it-notes pwn +application/vnd.accpac.simply.aso aso +application/vnd.accpac.simply.imp imp +application/vnd.acucobol acu +application/vnd.acucorp atc acutc +application/vnd.adobe.air-application-installer-package+zip air +application/vnd.adobe.formscentral.fcdt fcdt +application/vnd.adobe.fxp fxp fxpl +# application/vnd.adobe.partial-upload +application/vnd.adobe.xdp+xml xdp +application/vnd.adobe.xfdf xfdf +# application/vnd.aether.imp +# application/vnd.ah-barcode +application/vnd.ahead.space ahead +application/vnd.airzip.filesecure.azf azf +application/vnd.airzip.filesecure.azs azs +application/vnd.amazon.ebook azw +application/vnd.americandynamics.acc acc +application/vnd.amiga.ami ami +# application/vnd.amundsen.maze+xml +application/vnd.android.package-archive apk +application/vnd.anser-web-certificate-issue-initiation cii +application/vnd.anser-web-funds-transfer-initiation fti +application/vnd.antix.game-component atx +application/vnd.apple.installer+xml mpkg +application/vnd.apple.mpegurl m3u8 +# application/vnd.arastra.swi +application/vnd.aristanetworks.swi swi +application/vnd.astraea-software.iota iota +application/vnd.audiograph aep +# application/vnd.autopackage +# application/vnd.avistar+xml +application/vnd.blueice.multipass mpm +# application/vnd.bluetooth.ep.oob +application/vnd.bmi bmi +application/vnd.businessobjects rep +# application/vnd.cab-jscript +# application/vnd.canon-cpdl +# application/vnd.canon-lips +# application/vnd.cendio.thinlinc.clientconf +application/vnd.chemdraw+xml cdxml +application/vnd.chipnuts.karaoke-mmd mmd +application/vnd.cinderella cdy +# application/vnd.cirpack.isdn-ext +application/vnd.claymore cla +application/vnd.cloanto.rp9 rp9 +application/vnd.clonk.c4group c4g c4d c4f c4p c4u +application/vnd.cluetrust.cartomobile-config c11amc +application/vnd.cluetrust.cartomobile-config-pkg c11amz +# application/vnd.collection+json +# application/vnd.commerce-battelle +application/vnd.commonspace csp +application/vnd.contact.cmsg cdbcmsg +application/vnd.cosmocaller cmc +application/vnd.crick.clicker clkx +application/vnd.crick.clicker.keyboard clkk +application/vnd.crick.clicker.palette clkp +application/vnd.crick.clicker.template clkt +application/vnd.crick.clicker.wordbank clkw +application/vnd.criticaltools.wbs+xml wbs +application/vnd.ctc-posml pml +# application/vnd.ctct.ws+xml +# application/vnd.cups-pdf +# application/vnd.cups-postscript +application/vnd.cups-ppd ppd +# application/vnd.cups-raster +# application/vnd.cups-raw +# application/vnd.curl +application/vnd.curl.car car +application/vnd.curl.pcurl pcurl +# application/vnd.cybank +application/vnd.dart dart +application/vnd.data-vision.rdz rdz +application/vnd.dece.data uvf uvvf uvd uvvd +application/vnd.dece.ttml+xml uvt uvvt +application/vnd.dece.unspecified uvx uvvx +application/vnd.dece.zip uvz uvvz +application/vnd.denovo.fcselayout-link fe_launch +# application/vnd.dir-bi.plate-dl-nosuffix +application/vnd.dna dna +application/vnd.dolby.mlp mlp +# application/vnd.dolby.mobile.1 +# application/vnd.dolby.mobile.2 +application/vnd.dpgraph dpg +application/vnd.dreamfactory dfac +application/vnd.ds-keypoint kpxx +application/vnd.dvb.ait ait +# application/vnd.dvb.dvbj +# application/vnd.dvb.esgcontainer +# application/vnd.dvb.ipdcdftnotifaccess +# application/vnd.dvb.ipdcesgaccess +# application/vnd.dvb.ipdcesgaccess2 +# application/vnd.dvb.ipdcesgpdd +# application/vnd.dvb.ipdcroaming +# application/vnd.dvb.iptv.alfec-base +# application/vnd.dvb.iptv.alfec-enhancement +# application/vnd.dvb.notif-aggregate-root+xml +# application/vnd.dvb.notif-container+xml +# application/vnd.dvb.notif-generic+xml +# application/vnd.dvb.notif-ia-msglist+xml +# application/vnd.dvb.notif-ia-registration-request+xml +# application/vnd.dvb.notif-ia-registration-response+xml +# application/vnd.dvb.notif-init+xml +# application/vnd.dvb.pfr +application/vnd.dvb.service svc +# application/vnd.dxr +application/vnd.dynageo geo +# application/vnd.easykaraoke.cdgdownload +# application/vnd.ecdis-update +application/vnd.ecowin.chart mag +# application/vnd.ecowin.filerequest +# application/vnd.ecowin.fileupdate +# application/vnd.ecowin.series +# application/vnd.ecowin.seriesrequest +# application/vnd.ecowin.seriesupdate +# application/vnd.emclient.accessrequest+xml +application/vnd.enliven nml +# application/vnd.eprints.data+xml +application/vnd.epson.esf esf +application/vnd.epson.msf msf +application/vnd.epson.quickanime qam +application/vnd.epson.salt slt +application/vnd.epson.ssf ssf +# application/vnd.ericsson.quickcall +application/vnd.eszigno3+xml es3 et3 +# application/vnd.etsi.aoc+xml +# application/vnd.etsi.cug+xml +# application/vnd.etsi.iptvcommand+xml +# application/vnd.etsi.iptvdiscovery+xml +# application/vnd.etsi.iptvprofile+xml +# application/vnd.etsi.iptvsad-bc+xml +# application/vnd.etsi.iptvsad-cod+xml +# application/vnd.etsi.iptvsad-npvr+xml +# application/vnd.etsi.iptvservice+xml +# application/vnd.etsi.iptvsync+xml +# application/vnd.etsi.iptvueprofile+xml +# application/vnd.etsi.mcid+xml +# application/vnd.etsi.overload-control-policy-dataset+xml +# application/vnd.etsi.sci+xml +# application/vnd.etsi.simservs+xml +# application/vnd.etsi.tsl+xml +# application/vnd.etsi.tsl.der +# application/vnd.eudora.data +application/vnd.ezpix-album ez2 +application/vnd.ezpix-package ez3 +# application/vnd.f-secure.mobile +application/vnd.fdf fdf +application/vnd.fdsn.mseed mseed +application/vnd.fdsn.seed seed dataless +# application/vnd.ffsns +# application/vnd.fints +application/vnd.flographit gph +application/vnd.fluxtime.clip ftc +# application/vnd.font-fontforge-sfd +application/vnd.framemaker fm frame maker book +application/vnd.frogans.fnc fnc +application/vnd.frogans.ltf ltf +application/vnd.fsc.weblaunch fsc +application/vnd.fujitsu.oasys oas +application/vnd.fujitsu.oasys2 oa2 +application/vnd.fujitsu.oasys3 oa3 +application/vnd.fujitsu.oasysgp fg5 +application/vnd.fujitsu.oasysprs bh2 +# application/vnd.fujixerox.art-ex +# application/vnd.fujixerox.art4 +# application/vnd.fujixerox.hbpl +application/vnd.fujixerox.ddd ddd +application/vnd.fujixerox.docuworks xdw +application/vnd.fujixerox.docuworks.binder xbd +# application/vnd.fut-misnet +application/vnd.fuzzysheet fzs +application/vnd.genomatix.tuxedo txd +# application/vnd.geocube+xml +application/vnd.geogebra.file ggb +application/vnd.geogebra.tool ggt +application/vnd.geometry-explorer gex gre +application/vnd.geonext gxt +application/vnd.geoplan g2w +application/vnd.geospace g3w +# application/vnd.globalplatform.card-content-mgt +# application/vnd.globalplatform.card-content-mgt-response +application/vnd.gmx gmx +application/vnd.google-earth.kml+xml kml +application/vnd.google-earth.kmz kmz +application/vnd.grafeq gqf gqs +# application/vnd.gridmp +application/vnd.groove-account gac +application/vnd.groove-help ghf +application/vnd.groove-identity-message gim +application/vnd.groove-injector grv +application/vnd.groove-tool-message gtm +application/vnd.groove-tool-template tpl +application/vnd.groove-vcard vcg +# application/vnd.hal+json +application/vnd.hal+xml hal +application/vnd.handheld-entertainment+xml zmm +application/vnd.hbci hbci +# application/vnd.hcl-bireports +application/vnd.hhe.lesson-player les +application/vnd.hp-hpgl hpgl +application/vnd.hp-hpid hpid +application/vnd.hp-hps hps +application/vnd.hp-jlyt jlt +application/vnd.hp-pcl pcl +application/vnd.hp-pclxl pclxl +# application/vnd.httphone +application/vnd.hydrostatix.sof-data sfd-hdstx +# application/vnd.hzn-3d-crossword +# application/vnd.ibm.afplinedata +# application/vnd.ibm.electronic-media +application/vnd.ibm.minipay mpy +application/vnd.ibm.modcap afp listafp list3820 +application/vnd.ibm.rights-management irm +application/vnd.ibm.secure-container sc +application/vnd.iccprofile icc icm +application/vnd.igloader igl +application/vnd.immervision-ivp ivp +application/vnd.immervision-ivu ivu +# application/vnd.informedcontrol.rms+xml +# application/vnd.informix-visionary +# application/vnd.infotech.project +# application/vnd.infotech.project+xml +# application/vnd.innopath.wamp.notification +application/vnd.insors.igm igm +application/vnd.intercon.formnet xpw xpx +application/vnd.intergeo i2g +# application/vnd.intertrust.digibox +# application/vnd.intertrust.nncp +application/vnd.intu.qbo qbo +application/vnd.intu.qfx qfx +# application/vnd.iptc.g2.conceptitem+xml +# application/vnd.iptc.g2.knowledgeitem+xml +# application/vnd.iptc.g2.newsitem+xml +# application/vnd.iptc.g2.newsmessage+xml +# application/vnd.iptc.g2.packageitem+xml +# application/vnd.iptc.g2.planningitem+xml +application/vnd.ipunplugged.rcprofile rcprofile +application/vnd.irepository.package+xml irp +application/vnd.is-xpr xpr +application/vnd.isac.fcs fcs +application/vnd.jam jam +# application/vnd.japannet-directory-service +# application/vnd.japannet-jpnstore-wakeup +# application/vnd.japannet-payment-wakeup +# application/vnd.japannet-registration +# application/vnd.japannet-registration-wakeup +# application/vnd.japannet-setstore-wakeup +# application/vnd.japannet-verification +# application/vnd.japannet-verification-wakeup +application/vnd.jcp.javame.midlet-rms rms +application/vnd.jisp jisp +application/vnd.joost.joda-archive joda +application/vnd.kahootz ktz ktr +application/vnd.kde.karbon karbon +application/vnd.kde.kchart chrt +application/vnd.kde.kformula kfo +application/vnd.kde.kivio flw +application/vnd.kde.kontour kon +application/vnd.kde.kpresenter kpr kpt +application/vnd.kde.kspread ksp +application/vnd.kde.kword kwd kwt +application/vnd.kenameaapp htke +application/vnd.kidspiration kia +application/vnd.kinar kne knp +application/vnd.koan skp skd skt skm +application/vnd.kodak-descriptor sse +application/vnd.las.las+xml lasxml +# application/vnd.liberty-request+xml +application/vnd.llamagraphics.life-balance.desktop lbd +application/vnd.llamagraphics.life-balance.exchange+xml lbe +application/vnd.lotus-1-2-3 123 +application/vnd.lotus-approach apr +application/vnd.lotus-freelance pre +application/vnd.lotus-notes nsf +application/vnd.lotus-organizer org +application/vnd.lotus-screencam scm +application/vnd.lotus-wordpro lwp +application/vnd.macports.portpkg portpkg +# application/vnd.marlin.drm.actiontoken+xml +# application/vnd.marlin.drm.conftoken+xml +# application/vnd.marlin.drm.license+xml +# application/vnd.marlin.drm.mdcf +application/vnd.mcd mcd +application/vnd.medcalcdata mc1 +application/vnd.mediastation.cdkey cdkey +# application/vnd.meridian-slingshot +application/vnd.mfer mwf +application/vnd.mfmp mfm +application/vnd.micrografx.flo flo +application/vnd.micrografx.igx igx +application/vnd.mif mif +# application/vnd.minisoft-hp3000-save +# application/vnd.mitsubishi.misty-guard.trustweb +application/vnd.mobius.daf daf +application/vnd.mobius.dis dis +application/vnd.mobius.mbk mbk +application/vnd.mobius.mqy mqy +application/vnd.mobius.msl msl +application/vnd.mobius.plc plc +application/vnd.mobius.txf txf +application/vnd.mophun.application mpn +application/vnd.mophun.certificate mpc +# application/vnd.motorola.flexsuite +# application/vnd.motorola.flexsuite.adsi +# application/vnd.motorola.flexsuite.fis +# application/vnd.motorola.flexsuite.gotap +# application/vnd.motorola.flexsuite.kmr +# application/vnd.motorola.flexsuite.ttc +# application/vnd.motorola.flexsuite.wem +# application/vnd.motorola.iprm +application/vnd.mozilla.xul+xml xul +application/vnd.ms-artgalry cil +# application/vnd.ms-asf +application/vnd.ms-cab-compressed cab +# application/vnd.ms-color.iccprofile +application/vnd.ms-excel xls xlm xla xlc xlt xlw +application/vnd.ms-excel.addin.macroenabled.12 xlam +application/vnd.ms-excel.sheet.binary.macroenabled.12 xlsb +application/vnd.ms-excel.sheet.macroenabled.12 xlsm +application/vnd.ms-excel.template.macroenabled.12 xltm +application/vnd.ms-fontobject eot +application/vnd.ms-htmlhelp chm +application/vnd.ms-ims ims +application/vnd.ms-lrm lrm +# application/vnd.ms-office.activex+xml +application/vnd.ms-officetheme thmx +# application/vnd.ms-opentype +# application/vnd.ms-package.obfuscated-opentype +application/vnd.ms-pki.seccat cat +application/vnd.ms-pki.stl stl +# application/vnd.ms-playready.initiator+xml +application/vnd.ms-powerpoint ppt pps pot +application/vnd.ms-powerpoint.addin.macroenabled.12 ppam +application/vnd.ms-powerpoint.presentation.macroenabled.12 pptm +application/vnd.ms-powerpoint.slide.macroenabled.12 sldm +application/vnd.ms-powerpoint.slideshow.macroenabled.12 ppsm +application/vnd.ms-powerpoint.template.macroenabled.12 potm +# application/vnd.ms-printing.printticket+xml +application/vnd.ms-project mpp mpt +# application/vnd.ms-tnef +# application/vnd.ms-wmdrm.lic-chlg-req +# application/vnd.ms-wmdrm.lic-resp +# application/vnd.ms-wmdrm.meter-chlg-req +# application/vnd.ms-wmdrm.meter-resp +application/vnd.ms-word.document.macroenabled.12 docm +application/vnd.ms-word.template.macroenabled.12 dotm +application/vnd.ms-works wps wks wcm wdb +application/vnd.ms-wpl wpl +application/vnd.ms-xpsdocument xps +application/vnd.mseq mseq +# application/vnd.msign +# application/vnd.multiad.creator +# application/vnd.multiad.creator.cif +# application/vnd.music-niff +application/vnd.musician mus +application/vnd.muvee.style msty +application/vnd.mynfc taglet +# application/vnd.ncd.control +# application/vnd.ncd.reference +# application/vnd.nervana +# application/vnd.netfpx +application/vnd.neurolanguage.nlu nlu +application/vnd.nitf ntf nitf +application/vnd.noblenet-directory nnd +application/vnd.noblenet-sealer nns +application/vnd.noblenet-web nnw +# application/vnd.nokia.catalogs +# application/vnd.nokia.conml+wbxml +# application/vnd.nokia.conml+xml +# application/vnd.nokia.isds-radio-presets +# application/vnd.nokia.iptv.config+xml +# application/vnd.nokia.landmark+wbxml +# application/vnd.nokia.landmark+xml +# application/vnd.nokia.landmarkcollection+xml +# application/vnd.nokia.n-gage.ac+xml +application/vnd.nokia.n-gage.data ngdat +application/vnd.nokia.n-gage.symbian.install n-gage +# application/vnd.nokia.ncd +# application/vnd.nokia.pcd+wbxml +# application/vnd.nokia.pcd+xml +application/vnd.nokia.radio-preset rpst +application/vnd.nokia.radio-presets rpss +application/vnd.novadigm.edm edm +application/vnd.novadigm.edx edx +application/vnd.novadigm.ext ext +# application/vnd.ntt-local.file-transfer +# application/vnd.ntt-local.sip-ta_remote +# application/vnd.ntt-local.sip-ta_tcp_stream +application/vnd.oasis.opendocument.chart odc +application/vnd.oasis.opendocument.chart-template otc +application/vnd.oasis.opendocument.database odb +application/vnd.oasis.opendocument.formula odf +application/vnd.oasis.opendocument.formula-template odft +application/vnd.oasis.opendocument.graphics odg +application/vnd.oasis.opendocument.graphics-template otg +application/vnd.oasis.opendocument.image odi +application/vnd.oasis.opendocument.image-template oti +application/vnd.oasis.opendocument.presentation odp +application/vnd.oasis.opendocument.presentation-template otp +application/vnd.oasis.opendocument.spreadsheet ods +application/vnd.oasis.opendocument.spreadsheet-template ots +application/vnd.oasis.opendocument.text odt +application/vnd.oasis.opendocument.text-master odm +application/vnd.oasis.opendocument.text-template ott +application/vnd.oasis.opendocument.text-web oth +# application/vnd.obn +# application/vnd.oftn.l10n+json +# application/vnd.oipf.contentaccessdownload+xml +# application/vnd.oipf.contentaccessstreaming+xml +# application/vnd.oipf.cspg-hexbinary +# application/vnd.oipf.dae.svg+xml +# application/vnd.oipf.dae.xhtml+xml +# application/vnd.oipf.mippvcontrolmessage+xml +# application/vnd.oipf.pae.gem +# application/vnd.oipf.spdiscovery+xml +# application/vnd.oipf.spdlist+xml +# application/vnd.oipf.ueprofile+xml +# application/vnd.oipf.userprofile+xml +application/vnd.olpc-sugar xo +# application/vnd.oma-scws-config +# application/vnd.oma-scws-http-request +# application/vnd.oma-scws-http-response +# application/vnd.oma.bcast.associated-procedure-parameter+xml +# application/vnd.oma.bcast.drm-trigger+xml +# application/vnd.oma.bcast.imd+xml +# application/vnd.oma.bcast.ltkm +# application/vnd.oma.bcast.notification+xml +# application/vnd.oma.bcast.provisioningtrigger +# application/vnd.oma.bcast.sgboot +# application/vnd.oma.bcast.sgdd+xml +# application/vnd.oma.bcast.sgdu +# application/vnd.oma.bcast.simple-symbol-container +# application/vnd.oma.bcast.smartcard-trigger+xml +# application/vnd.oma.bcast.sprov+xml +# application/vnd.oma.bcast.stkm +# application/vnd.oma.cab-address-book+xml +# application/vnd.oma.cab-feature-handler+xml +# application/vnd.oma.cab-pcc+xml +# application/vnd.oma.cab-user-prefs+xml +# application/vnd.oma.dcd +# application/vnd.oma.dcdc +application/vnd.oma.dd2+xml dd2 +# application/vnd.oma.drm.risd+xml +# application/vnd.oma.group-usage-list+xml +# application/vnd.oma.pal+xml +# application/vnd.oma.poc.detailed-progress-report+xml +# application/vnd.oma.poc.final-report+xml +# application/vnd.oma.poc.groups+xml +# application/vnd.oma.poc.invocation-descriptor+xml +# application/vnd.oma.poc.optimized-progress-report+xml +# application/vnd.oma.push +# application/vnd.oma.scidm.messages+xml +# application/vnd.oma.xcap-directory+xml +# application/vnd.omads-email+xml +# application/vnd.omads-file+xml +# application/vnd.omads-folder+xml +# application/vnd.omaloc-supl-init +application/vnd.openofficeorg.extension oxt +# application/vnd.openxmlformats-officedocument.custom-properties+xml +# application/vnd.openxmlformats-officedocument.customxmlproperties+xml +# application/vnd.openxmlformats-officedocument.drawing+xml +# application/vnd.openxmlformats-officedocument.drawingml.chart+xml +# application/vnd.openxmlformats-officedocument.drawingml.chartshapes+xml +# application/vnd.openxmlformats-officedocument.drawingml.diagramcolors+xml +# application/vnd.openxmlformats-officedocument.drawingml.diagramdata+xml +# application/vnd.openxmlformats-officedocument.drawingml.diagramlayout+xml +# application/vnd.openxmlformats-officedocument.drawingml.diagramstyle+xml +# application/vnd.openxmlformats-officedocument.extended-properties+xml +# application/vnd.openxmlformats-officedocument.presentationml.commentauthors+xml +# application/vnd.openxmlformats-officedocument.presentationml.comments+xml +# application/vnd.openxmlformats-officedocument.presentationml.handoutmaster+xml +# application/vnd.openxmlformats-officedocument.presentationml.notesmaster+xml +# application/vnd.openxmlformats-officedocument.presentationml.notesslide+xml +application/vnd.openxmlformats-officedocument.presentationml.presentation pptx +# application/vnd.openxmlformats-officedocument.presentationml.presentation.main+xml +# application/vnd.openxmlformats-officedocument.presentationml.presprops+xml +application/vnd.openxmlformats-officedocument.presentationml.slide sldx +# application/vnd.openxmlformats-officedocument.presentationml.slide+xml +# application/vnd.openxmlformats-officedocument.presentationml.slidelayout+xml +# application/vnd.openxmlformats-officedocument.presentationml.slidemaster+xml +application/vnd.openxmlformats-officedocument.presentationml.slideshow ppsx +# application/vnd.openxmlformats-officedocument.presentationml.slideshow.main+xml +# application/vnd.openxmlformats-officedocument.presentationml.slideupdateinfo+xml +# application/vnd.openxmlformats-officedocument.presentationml.tablestyles+xml +# application/vnd.openxmlformats-officedocument.presentationml.tags+xml +application/vnd.openxmlformats-officedocument.presentationml.template potx +# application/vnd.openxmlformats-officedocument.presentationml.template.main+xml +# application/vnd.openxmlformats-officedocument.presentationml.viewprops+xml +# application/vnd.openxmlformats-officedocument.spreadsheetml.calcchain+xml +# application/vnd.openxmlformats-officedocument.spreadsheetml.chartsheet+xml +# application/vnd.openxmlformats-officedocument.spreadsheetml.comments+xml +# application/vnd.openxmlformats-officedocument.spreadsheetml.connections+xml +# application/vnd.openxmlformats-officedocument.spreadsheetml.dialogsheet+xml +# application/vnd.openxmlformats-officedocument.spreadsheetml.externallink+xml +# application/vnd.openxmlformats-officedocument.spreadsheetml.pivotcachedefinition+xml +# application/vnd.openxmlformats-officedocument.spreadsheetml.pivotcacherecords+xml +# application/vnd.openxmlformats-officedocument.spreadsheetml.pivottable+xml +# application/vnd.openxmlformats-officedocument.spreadsheetml.querytable+xml +# application/vnd.openxmlformats-officedocument.spreadsheetml.revisionheaders+xml +# application/vnd.openxmlformats-officedocument.spreadsheetml.revisionlog+xml +# application/vnd.openxmlformats-officedocument.spreadsheetml.sharedstrings+xml +application/vnd.openxmlformats-officedocument.spreadsheetml.sheet xlsx +# application/vnd.openxmlformats-officedocument.spreadsheetml.sheet.main+xml +# application/vnd.openxmlformats-officedocument.spreadsheetml.sheetmetadata+xml +# application/vnd.openxmlformats-officedocument.spreadsheetml.styles+xml +# application/vnd.openxmlformats-officedocument.spreadsheetml.table+xml +# application/vnd.openxmlformats-officedocument.spreadsheetml.tablesinglecells+xml +application/vnd.openxmlformats-officedocument.spreadsheetml.template xltx +# application/vnd.openxmlformats-officedocument.spreadsheetml.template.main+xml +# application/vnd.openxmlformats-officedocument.spreadsheetml.usernames+xml +# application/vnd.openxmlformats-officedocument.spreadsheetml.volatiledependencies+xml +# application/vnd.openxmlformats-officedocument.spreadsheetml.worksheet+xml +# application/vnd.openxmlformats-officedocument.theme+xml +# application/vnd.openxmlformats-officedocument.themeoverride+xml +# application/vnd.openxmlformats-officedocument.vmldrawing +# application/vnd.openxmlformats-officedocument.wordprocessingml.comments+xml +application/vnd.openxmlformats-officedocument.wordprocessingml.document docx +# application/vnd.openxmlformats-officedocument.wordprocessingml.document.glossary+xml +# application/vnd.openxmlformats-officedocument.wordprocessingml.document.main+xml +# application/vnd.openxmlformats-officedocument.wordprocessingml.endnotes+xml +# application/vnd.openxmlformats-officedocument.wordprocessingml.fonttable+xml +# application/vnd.openxmlformats-officedocument.wordprocessingml.footer+xml +# application/vnd.openxmlformats-officedocument.wordprocessingml.footnotes+xml +# application/vnd.openxmlformats-officedocument.wordprocessingml.numbering+xml +# application/vnd.openxmlformats-officedocument.wordprocessingml.settings+xml +# application/vnd.openxmlformats-officedocument.wordprocessingml.styles+xml +application/vnd.openxmlformats-officedocument.wordprocessingml.template dotx +# application/vnd.openxmlformats-officedocument.wordprocessingml.template.main+xml +# application/vnd.openxmlformats-officedocument.wordprocessingml.websettings+xml +# application/vnd.openxmlformats-package.core-properties+xml +# application/vnd.openxmlformats-package.digital-signature-xmlsignature+xml +# application/vnd.openxmlformats-package.relationships+xml +# application/vnd.quobject-quoxdocument +# application/vnd.osa.netdeploy +application/vnd.osgeo.mapguide.package mgp +# application/vnd.osgi.bundle +application/vnd.osgi.dp dp +application/vnd.osgi.subsystem esa +# application/vnd.otps.ct-kip+xml +application/vnd.palm pdb pqa oprc +# application/vnd.paos.xml +application/vnd.pawaafile paw +application/vnd.pg.format str +application/vnd.pg.osasli ei6 +# application/vnd.piaccess.application-licence +application/vnd.picsel efif +application/vnd.pmi.widget wg +# application/vnd.poc.group-advertisement+xml +application/vnd.pocketlearn plf +application/vnd.powerbuilder6 pbd +# application/vnd.powerbuilder6-s +# application/vnd.powerbuilder7 +# application/vnd.powerbuilder7-s +# application/vnd.powerbuilder75 +# application/vnd.powerbuilder75-s +# application/vnd.preminet +application/vnd.previewsystems.box box +application/vnd.proteus.magazine mgz +application/vnd.publishare-delta-tree qps +application/vnd.pvi.ptid1 ptid +# application/vnd.pwg-multiplexed +# application/vnd.pwg-xhtml-print+xml +# application/vnd.qualcomm.brew-app-res +application/vnd.quark.quarkxpress qxd qxt qwd qwt qxl qxb +# application/vnd.radisys.moml+xml +# application/vnd.radisys.msml+xml +# application/vnd.radisys.msml-audit+xml +# application/vnd.radisys.msml-audit-conf+xml +# application/vnd.radisys.msml-audit-conn+xml +# application/vnd.radisys.msml-audit-dialog+xml +# application/vnd.radisys.msml-audit-stream+xml +# application/vnd.radisys.msml-conf+xml +# application/vnd.radisys.msml-dialog+xml +# application/vnd.radisys.msml-dialog-base+xml +# application/vnd.radisys.msml-dialog-fax-detect+xml +# application/vnd.radisys.msml-dialog-fax-sendrecv+xml +# application/vnd.radisys.msml-dialog-group+xml +# application/vnd.radisys.msml-dialog-speech+xml +# application/vnd.radisys.msml-dialog-transform+xml +# application/vnd.rainstor.data +# application/vnd.rapid +application/vnd.realvnc.bed bed +application/vnd.recordare.musicxml mxl +application/vnd.recordare.musicxml+xml musicxml +# application/vnd.renlearn.rlprint +application/vnd.rig.cryptonote cryptonote +application/vnd.rim.cod cod +application/vnd.rn-realmedia rm +application/vnd.rn-realmedia-vbr rmvb +application/vnd.route66.link66+xml link66 +# application/vnd.rs-274x +# application/vnd.ruckus.download +# application/vnd.s3sms +application/vnd.sailingtracker.track st +# application/vnd.sbm.cid +# application/vnd.sbm.mid2 +# application/vnd.scribus +# application/vnd.sealed.3df +# application/vnd.sealed.csf +# application/vnd.sealed.doc +# application/vnd.sealed.eml +# application/vnd.sealed.mht +# application/vnd.sealed.net +# application/vnd.sealed.ppt +# application/vnd.sealed.tiff +# application/vnd.sealed.xls +# application/vnd.sealedmedia.softseal.html +# application/vnd.sealedmedia.softseal.pdf +application/vnd.seemail see +application/vnd.sema sema +application/vnd.semd semd +application/vnd.semf semf +application/vnd.shana.informed.formdata ifm +application/vnd.shana.informed.formtemplate itp +application/vnd.shana.informed.interchange iif +application/vnd.shana.informed.package ipk +application/vnd.simtech-mindmapper twd twds +application/vnd.smaf mmf +# application/vnd.smart.notebook +application/vnd.smart.teacher teacher +# application/vnd.software602.filler.form+xml +# application/vnd.software602.filler.form-xml-zip +application/vnd.solent.sdkm+xml sdkm sdkd +application/vnd.spotfire.dxp dxp +application/vnd.spotfire.sfs sfs +# application/vnd.sss-cod +# application/vnd.sss-dtf +# application/vnd.sss-ntf +application/vnd.stardivision.calc sdc +application/vnd.stardivision.draw sda +application/vnd.stardivision.impress sdd +application/vnd.stardivision.math smf +application/vnd.stardivision.writer sdw vor +application/vnd.stardivision.writer-global sgl +application/vnd.stepmania.package smzip +application/vnd.stepmania.stepchart sm +# application/vnd.street-stream +application/vnd.sun.xml.calc sxc +application/vnd.sun.xml.calc.template stc +application/vnd.sun.xml.draw sxd +application/vnd.sun.xml.draw.template std +application/vnd.sun.xml.impress sxi +application/vnd.sun.xml.impress.template sti +application/vnd.sun.xml.math sxm +application/vnd.sun.xml.writer sxw +application/vnd.sun.xml.writer.global sxg +application/vnd.sun.xml.writer.template stw +# application/vnd.sun.wadl+xml +application/vnd.sus-calendar sus susp +application/vnd.svd svd +# application/vnd.swiftview-ics +application/vnd.symbian.install sis sisx +application/vnd.syncml+xml xsm +application/vnd.syncml.dm+wbxml bdm +application/vnd.syncml.dm+xml xdm +# application/vnd.syncml.dm.notification +# application/vnd.syncml.ds.notification +application/vnd.tao.intent-module-archive tao +application/vnd.tcpdump.pcap pcap cap dmp +application/vnd.tmobile-livetv tmo +application/vnd.trid.tpt tpt +application/vnd.triscape.mxs mxs +application/vnd.trueapp tra +# application/vnd.truedoc +# application/vnd.ubisoft.webplayer +application/vnd.ufdl ufd ufdl +application/vnd.uiq.theme utz +application/vnd.umajin umj +application/vnd.unity unityweb +application/vnd.uoml+xml uoml +# application/vnd.uplanet.alert +# application/vnd.uplanet.alert-wbxml +# application/vnd.uplanet.bearer-choice +# application/vnd.uplanet.bearer-choice-wbxml +# application/vnd.uplanet.cacheop +# application/vnd.uplanet.cacheop-wbxml +# application/vnd.uplanet.channel +# application/vnd.uplanet.channel-wbxml +# application/vnd.uplanet.list +# application/vnd.uplanet.list-wbxml +# application/vnd.uplanet.listcmd +# application/vnd.uplanet.listcmd-wbxml +# application/vnd.uplanet.signal +application/vnd.vcx vcx +# application/vnd.vd-study +# application/vnd.vectorworks +# application/vnd.verimatrix.vcas +# application/vnd.vidsoft.vidconference +application/vnd.visio vsd vst vss vsw +application/vnd.visionary vis +# application/vnd.vividence.scriptfile +application/vnd.vsf vsf +# application/vnd.wap.sic +# application/vnd.wap.slc +application/vnd.wap.wbxml wbxml +application/vnd.wap.wmlc wmlc +application/vnd.wap.wmlscriptc wmlsc +application/vnd.webturbo wtb +# application/vnd.wfa.wsc +# application/vnd.wmc +# application/vnd.wmf.bootstrap +# application/vnd.wolfram.mathematica +# application/vnd.wolfram.mathematica.package +application/vnd.wolfram.player nbp +application/vnd.wordperfect wpd +application/vnd.wqd wqd +# application/vnd.wrq-hp3000-labelled +application/vnd.wt.stf stf +# application/vnd.wv.csp+wbxml +# application/vnd.wv.csp+xml +# application/vnd.wv.ssp+xml +application/vnd.xara xar +application/vnd.xfdl xfdl +# application/vnd.xfdl.webform +# application/vnd.xmi+xml +# application/vnd.xmpie.cpkg +# application/vnd.xmpie.dpkg +# application/vnd.xmpie.plan +# application/vnd.xmpie.ppkg +# application/vnd.xmpie.xlim +application/vnd.yamaha.hv-dic hvd +application/vnd.yamaha.hv-script hvs +application/vnd.yamaha.hv-voice hvp +application/vnd.yamaha.openscoreformat osf +application/vnd.yamaha.openscoreformat.osfpvg+xml osfpvg +# application/vnd.yamaha.remote-setup +application/vnd.yamaha.smaf-audio saf +application/vnd.yamaha.smaf-phrase spf +# application/vnd.yamaha.through-ngn +# application/vnd.yamaha.tunnel-udpencap +application/vnd.yellowriver-custom-menu cmp +application/vnd.zul zir zirz +application/vnd.zzazz.deck+xml zaz +application/voicexml+xml vxml +# application/vq-rtcpxr +# application/watcherinfo+xml +# application/whoispp-query +# application/whoispp-response +application/widget wgt +application/winhlp hlp +# application/wita +# application/wordperfect5.1 +application/wsdl+xml wsdl +application/wspolicy+xml wspolicy +application/x-7z-compressed 7z +application/x-abiword abw +application/x-ace-compressed ace +# application/x-amf +application/x-apple-diskimage dmg +application/x-authorware-bin aab x32 u32 vox +application/x-authorware-map aam +application/x-authorware-seg aas +application/x-bcpio bcpio +application/x-bittorrent torrent +application/x-blorb blb blorb +application/x-bzip bz +application/x-bzip2 bz2 boz +application/x-cbr cbr cba cbt cbz cb7 +application/x-cdlink vcd +application/x-cfs-compressed cfs +application/x-chat chat +application/x-chess-pgn pgn +application/x-conference nsc +# application/x-compress +application/x-cpio cpio +application/x-csh csh +application/x-debian-package deb udeb +application/x-dgc-compressed dgc +application/x-director dir dcr dxr cst cct cxt w3d fgd swa +application/x-doom wad +application/x-dtbncx+xml ncx +application/x-dtbook+xml dtb +application/x-dtbresource+xml res +application/x-dvi dvi +application/x-envoy evy +application/x-eva eva +application/x-font-bdf bdf +# application/x-font-dos +# application/x-font-framemaker +application/x-font-ghostscript gsf +# application/x-font-libgrx +application/x-font-linux-psf psf +application/x-font-otf otf +application/x-font-pcf pcf +application/x-font-snf snf +# application/x-font-speedo +# application/x-font-sunos-news +application/x-font-ttf ttf ttc +application/x-font-type1 pfa pfb pfm afm +application/x-font-woff woff +# application/x-font-vfont +application/x-freearc arc +application/x-futuresplash spl +application/x-gca-compressed gca +application/x-glulx ulx +application/x-gnumeric gnumeric +application/x-gramps-xml gramps +application/x-gtar gtar +# application/x-gzip +application/x-hdf hdf +application/x-install-instructions install +application/x-iso9660-image iso +application/x-java-jnlp-file jnlp +application/x-latex latex +application/x-lzh-compressed lzh lha +application/x-mie mie +application/x-mobipocket-ebook prc mobi +application/x-ms-application application +application/x-ms-shortcut lnk +application/x-ms-wmd wmd +application/x-ms-wmz wmz +application/x-ms-xbap xbap +application/x-msaccess mdb +application/x-msbinder obd +application/x-mscardfile crd +application/x-msclip clp +application/x-msdownload exe dll com bat msi +application/x-msmediaview mvb m13 m14 +application/x-msmetafile wmf wmz emf emz +application/x-msmoney mny +application/x-mspublisher pub +application/x-msschedule scd +application/x-msterminal trm +application/x-mswrite wri +application/x-netcdf nc cdf +application/x-nzb nzb +application/x-pkcs12 p12 pfx +application/x-pkcs7-certificates p7b spc +application/x-pkcs7-certreqresp p7r +application/x-rar-compressed rar +application/x-research-info-systems ris +application/x-sh sh +application/x-shar shar +application/x-shockwave-flash swf +application/x-silverlight-app xap +application/x-sql sql +application/x-stuffit sit +application/x-stuffitx sitx +application/x-subrip srt +application/x-sv4cpio sv4cpio +application/x-sv4crc sv4crc +application/x-t3vm-image t3 +application/x-tads gam +application/x-tar tar +application/x-tcl tcl +application/x-tex tex +application/x-tex-tfm tfm +application/x-texinfo texinfo texi +application/x-tgif obj +application/x-ustar ustar +application/x-wais-source src +application/x-x509-ca-cert der crt +application/x-xfig fig +application/x-xliff+xml xlf +application/x-xpinstall xpi +application/x-xz xz +application/x-zmachine z1 z2 z3 z4 z5 z6 z7 z8 +# application/x400-bp +application/xaml+xml xaml +# application/xcap-att+xml +# application/xcap-caps+xml +application/xcap-diff+xml xdf +# application/xcap-el+xml +# application/xcap-error+xml +# application/xcap-ns+xml +# application/xcon-conference-info-diff+xml +# application/xcon-conference-info+xml +application/xenc+xml xenc +application/xhtml+xml xhtml xht +# application/xhtml-voice+xml +application/xml xml xsl +application/xml-dtd dtd +# application/xml-external-parsed-entity +# application/xmpp+xml +application/xop+xml xop +application/xproc+xml xpl +application/xslt+xml xslt +application/xspf+xml xspf +application/xv+xml mxml xhvml xvml xvm +application/yang yang +application/yin+xml yin +application/zip zip +# audio/1d-interleaved-parityfec +# audio/32kadpcm +# audio/3gpp +# audio/3gpp2 +# audio/ac3 +audio/adpcm adp +# audio/amr +# audio/amr-wb +# audio/amr-wb+ +# audio/asc +# audio/atrac-advanced-lossless +# audio/atrac-x +# audio/atrac3 +audio/basic au snd +# audio/bv16 +# audio/bv32 +# audio/clearmode +# audio/cn +# audio/dat12 +# audio/dls +# audio/dsr-es201108 +# audio/dsr-es202050 +# audio/dsr-es202211 +# audio/dsr-es202212 +# audio/dv +# audio/dvi4 +# audio/eac3 +# audio/evrc +# audio/evrc-qcp +# audio/evrc0 +# audio/evrc1 +# audio/evrcb +# audio/evrcb0 +# audio/evrcb1 +# audio/evrcwb +# audio/evrcwb0 +# audio/evrcwb1 +# audio/example +# audio/fwdred +# audio/g719 +# audio/g722 +# audio/g7221 +# audio/g723 +# audio/g726-16 +# audio/g726-24 +# audio/g726-32 +# audio/g726-40 +# audio/g728 +# audio/g729 +# audio/g7291 +# audio/g729d +# audio/g729e +# audio/gsm +# audio/gsm-efr +# audio/gsm-hr-08 +# audio/ilbc +# audio/ip-mr_v2.5 +# audio/isac +# audio/l16 +# audio/l20 +# audio/l24 +# audio/l8 +# audio/lpc +audio/midi mid midi kar rmi +# audio/mobile-xmf +audio/mp4 mp4a +# audio/mp4a-latm +# audio/mpa +# audio/mpa-robust +audio/mpeg mpga mp2 mp2a mp3 m2a m3a +# audio/mpeg4-generic +# audio/musepack +audio/ogg oga ogg spx +# audio/opus +# audio/parityfec +# audio/pcma +# audio/pcma-wb +# audio/pcmu-wb +# audio/pcmu +# audio/prs.sid +# audio/qcelp +# audio/red +# audio/rtp-enc-aescm128 +# audio/rtp-midi +# audio/rtx +audio/s3m s3m +audio/silk sil +# audio/smv +# audio/smv0 +# audio/smv-qcp +# audio/sp-midi +# audio/speex +# audio/t140c +# audio/t38 +# audio/telephone-event +# audio/tone +# audio/uemclip +# audio/ulpfec +# audio/vdvi +# audio/vmr-wb +# audio/vnd.3gpp.iufp +# audio/vnd.4sb +# audio/vnd.audiokoz +# audio/vnd.celp +# audio/vnd.cisco.nse +# audio/vnd.cmles.radio-events +# audio/vnd.cns.anp1 +# audio/vnd.cns.inf1 +audio/vnd.dece.audio uva uvva +audio/vnd.digital-winds eol +# audio/vnd.dlna.adts +# audio/vnd.dolby.heaac.1 +# audio/vnd.dolby.heaac.2 +# audio/vnd.dolby.mlp +# audio/vnd.dolby.mps +# audio/vnd.dolby.pl2 +# audio/vnd.dolby.pl2x +# audio/vnd.dolby.pl2z +# audio/vnd.dolby.pulse.1 +audio/vnd.dra dra +audio/vnd.dts dts +audio/vnd.dts.hd dtshd +# audio/vnd.dvb.file +# audio/vnd.everad.plj +# audio/vnd.hns.audio +audio/vnd.lucent.voice lvp +audio/vnd.ms-playready.media.pya pya +# audio/vnd.nokia.mobile-xmf +# audio/vnd.nortel.vbk +audio/vnd.nuera.ecelp4800 ecelp4800 +audio/vnd.nuera.ecelp7470 ecelp7470 +audio/vnd.nuera.ecelp9600 ecelp9600 +# audio/vnd.octel.sbc +# audio/vnd.qcelp +# audio/vnd.rhetorex.32kadpcm +audio/vnd.rip rip +# audio/vnd.sealedmedia.softseal.mpeg +# audio/vnd.vmx.cvsd +# audio/vorbis +# audio/vorbis-config +audio/webm weba +audio/x-aac aac +audio/x-aiff aif aiff aifc +audio/x-caf caf +audio/x-flac flac +audio/x-matroska mka +audio/x-mpegurl m3u +audio/x-ms-wax wax +audio/x-ms-wma wma +audio/x-pn-realaudio ram ra +audio/x-pn-realaudio-plugin rmp +# audio/x-tta +audio/x-wav wav +audio/xm xm +chemical/x-cdx cdx +chemical/x-cif cif +chemical/x-cmdf cmdf +chemical/x-cml cml +chemical/x-csml csml +# chemical/x-pdb +chemical/x-xyz xyz +image/bmp bmp +image/cgm cgm +# image/example +# image/fits +image/g3fax g3 +image/gif gif +image/ief ief +# image/jp2 +image/jpeg jpeg jpg jpe +# image/jpm +# image/jpx +image/ktx ktx +# image/naplps +image/png png +image/prs.btif btif +# image/prs.pti +image/sgi sgi +image/svg+xml svg svgz +# image/t38 +image/tiff tiff tif +# image/tiff-fx +image/vnd.adobe.photoshop psd +# image/vnd.cns.inf2 +image/vnd.dece.graphic uvi uvvi uvg uvvg +image/vnd.dvb.subtitle sub +image/vnd.djvu djvu djv +image/vnd.dwg dwg +image/vnd.dxf dxf +image/vnd.fastbidsheet fbs +image/vnd.fpx fpx +image/vnd.fst fst +image/vnd.fujixerox.edmics-mmr mmr +image/vnd.fujixerox.edmics-rlc rlc +# image/vnd.globalgraphics.pgb +# image/vnd.microsoft.icon +# image/vnd.mix +image/vnd.ms-modi mdi +image/vnd.ms-photo wdp +image/vnd.net-fpx npx +# image/vnd.radiance +# image/vnd.sealed.png +# image/vnd.sealedmedia.softseal.gif +# image/vnd.sealedmedia.softseal.jpg +# image/vnd.svf +image/vnd.wap.wbmp wbmp +image/vnd.xiff xif +image/webp webp +image/x-3ds 3ds +image/x-cmu-raster ras +image/x-cmx cmx +image/x-freehand fh fhc fh4 fh5 fh7 +image/x-icon ico +image/x-mrsid-image sid +image/x-pcx pcx +image/x-pict pic pct +image/x-portable-anymap pnm +image/x-portable-bitmap pbm +image/x-portable-graymap pgm +image/x-portable-pixmap ppm +image/x-rgb rgb +image/x-tga tga +image/x-xbitmap xbm +image/x-xpixmap xpm +image/x-xwindowdump xwd +# message/cpim +# message/delivery-status +# message/disposition-notification +# message/example +# message/external-body +# message/feedback-report +# message/global +# message/global-delivery-status +# message/global-disposition-notification +# message/global-headers +# message/http +# message/imdn+xml +# message/news +# message/partial +message/rfc822 eml mime +# message/s-http +# message/sip +# message/sipfrag +# message/tracking-status +# message/vnd.si.simp +# model/example +model/iges igs iges +model/mesh msh mesh silo +model/vnd.collada+xml dae +model/vnd.dwf dwf +# model/vnd.flatland.3dml +model/vnd.gdl gdl +# model/vnd.gs-gdl +# model/vnd.gs.gdl +model/vnd.gtw gtw +# model/vnd.moml+xml +model/vnd.mts mts +# model/vnd.parasolid.transmit.binary +# model/vnd.parasolid.transmit.text +model/vnd.vtu vtu +model/vrml wrl vrml +model/x3d+binary x3db x3dbz +model/x3d+vrml x3dv x3dvz +model/x3d+xml x3d x3dz +# multipart/alternative +# multipart/appledouble +# multipart/byteranges +# multipart/digest +# multipart/encrypted +# multipart/example +# multipart/form-data +# multipart/header-set +# multipart/mixed +# multipart/parallel +# multipart/related +# multipart/report +# multipart/signed +# multipart/voice-message +# text/1d-interleaved-parityfec +text/cache-manifest appcache +text/calendar ics ifb +text/css css +text/csv csv +# text/directory +# text/dns +# text/ecmascript +# text/enriched +# text/example +# text/fwdred +text/html html htm +# text/javascript +text/n3 n3 +# text/parityfec +text/plain txt text conf def list log in +# text/prs.fallenstein.rst +text/prs.lines.tag dsc +# text/vnd.radisys.msml-basic-layout +# text/red +# text/rfc822-headers +text/richtext rtx +# text/rtf +# text/rtp-enc-aescm128 +# text/rtx +text/sgml sgml sgm +# text/t140 +text/tab-separated-values tsv +text/troff t tr roff man me ms +text/turtle ttl +# text/ulpfec +text/uri-list uri uris urls +text/vcard vcard +# text/vnd.abc +text/vnd.curl curl +text/vnd.curl.dcurl dcurl +text/vnd.curl.scurl scurl +text/vnd.curl.mcurl mcurl +# text/vnd.dmclientscript +text/vnd.dvb.subtitle sub +# text/vnd.esmertec.theme-descriptor +text/vnd.fly fly +text/vnd.fmi.flexstor flx +text/vnd.graphviz gv +text/vnd.in3d.3dml 3dml +text/vnd.in3d.spot spot +# text/vnd.iptc.newsml +# text/vnd.iptc.nitf +# text/vnd.latex-z +# text/vnd.motorola.reflex +# text/vnd.ms-mediapackage +# text/vnd.net2phone.commcenter.command +# text/vnd.si.uricatalogue +text/vnd.sun.j2me.app-descriptor jad +# text/vnd.trolltech.linguist +# text/vnd.wap.si +# text/vnd.wap.sl +text/vnd.wap.wml wml +text/vnd.wap.wmlscript wmls +text/x-asm s asm +text/x-c c cc cxx cpp h hh dic +text/x-fortran f for f77 f90 +text/x-java-source java +text/x-opml opml +text/x-pascal p pas +text/x-nfo nfo +text/x-setext etx +text/x-sfv sfv +text/x-uuencode uu +text/x-vcalendar vcs +text/x-vcard vcf +# text/xml +# text/xml-external-parsed-entity +# video/1d-interleaved-parityfec +video/3gpp 3gp +# video/3gpp-tt +video/3gpp2 3g2 +# video/bmpeg +# video/bt656 +# video/celb +# video/dv +# video/example +video/h261 h261 +video/h263 h263 +# video/h263-1998 +# video/h263-2000 +video/h264 h264 +# video/h264-rcdo +# video/h264-svc +video/jpeg jpgv +# video/jpeg2000 +video/jpm jpm jpgm +video/mj2 mj2 mjp2 +# video/mp1s +# video/mp2p +# video/mp2t +video/mp4 mp4 mp4v mpg4 +# video/mp4v-es +video/mpeg mpeg mpg mpe m1v m2v +# video/mpeg4-generic +# video/mpv +# video/nv +video/ogg ogv +# video/parityfec +# video/pointer +video/quicktime qt mov +# video/raw +# video/rtp-enc-aescm128 +# video/rtx +# video/smpte292m +# video/ulpfec +# video/vc1 +# video/vnd.cctv +video/vnd.dece.hd uvh uvvh +video/vnd.dece.mobile uvm uvvm +# video/vnd.dece.mp4 +video/vnd.dece.pd uvp uvvp +video/vnd.dece.sd uvs uvvs +video/vnd.dece.video uvv uvvv +# video/vnd.directv.mpeg +# video/vnd.directv.mpeg-tts +# video/vnd.dlna.mpeg-tts +video/vnd.dvb.file dvb +video/vnd.fvt fvt +# video/vnd.hns.video +# video/vnd.iptvforum.1dparityfec-1010 +# video/vnd.iptvforum.1dparityfec-2005 +# video/vnd.iptvforum.2dparityfec-1010 +# video/vnd.iptvforum.2dparityfec-2005 +# video/vnd.iptvforum.ttsavc +# video/vnd.iptvforum.ttsmpeg2 +# video/vnd.motorola.video +# video/vnd.motorola.videop +video/vnd.mpegurl mxu m4u +video/vnd.ms-playready.media.pyv pyv +# video/vnd.nokia.interleaved-multimedia +# video/vnd.nokia.videovoip +# video/vnd.objectvideo +# video/vnd.sealed.mpeg1 +# video/vnd.sealed.mpeg4 +# video/vnd.sealed.swf +# video/vnd.sealedmedia.softseal.mov +video/vnd.uvvu.mp4 uvu uvvu +video/vnd.vivo viv +video/webm webm +video/x-f4v f4v +video/x-fli fli +video/x-flv flv +video/x-m4v m4v +video/x-matroska mkv mk3d mks +video/x-mng mng +video/x-ms-asf asf asx +video/x-ms-vob vob +video/x-ms-wm wm +video/x-ms-wmv wmv +video/x-ms-wmx wmx +video/x-ms-wvx wvx +video/x-msvideo avi +video/x-sgi-movie movie +video/x-smv smv +x-conference/x-cooltalk ice diff --git a/test/pyhttpd/conf/stop.conf.template b/test/pyhttpd/conf/stop.conf.template new file mode 100644 index 0000000..21bae84 --- /dev/null +++ b/test/pyhttpd/conf/stop.conf.template @@ -0,0 +1,46 @@ +# a config safe to use for stopping the server +# this allows us to stop the server even when+ +# the config in the file is borked (as test cases may try to do that) +# +ServerName localhost +ServerRoot "${server_dir}" + +Include "conf/modules.conf" + +DocumentRoot "${server_dir}/htdocs" + +<IfModule log_config_module> + LogFormat "%h %l %u %t \"%r\" %>s %O \"%{Referer}i\" \"%{User-Agent}i\" %k" combined + LogFormat "%h %l %u %t \"%r\" %>s %b" common + CustomLog "logs/access_log" combined + +</IfModule> + +TypesConfig "${gen_dir}/apache/conf/mime.types" + +Listen ${http_port} +Listen ${https_port} + +<IfModule mod_ssl.c> + # provide some default + SSLSessionCache "shmcb:ssl_gcache_data(32000)" +</IfModule> + +<VirtualHost *:${http_port}> + ServerName ${http_tld} + ServerAlias www.${http_tld} + <IfModule ssl_module> + SSLEngine off + </IfModule> + DocumentRoot "${server_dir}/htdocs" +</VirtualHost> + +<Directory "${server_dir}/htdocs/cgi"> + Options Indexes FollowSymLinks + AllowOverride None + Require all granted + + AddHandler cgi-script .py + AddHandler cgi-script .cgi + Options +ExecCGI +</Directory> diff --git a/test/pyhttpd/conf/test.conf b/test/pyhttpd/conf/test.conf new file mode 100644 index 0000000..7534af6 --- /dev/null +++ b/test/pyhttpd/conf/test.conf @@ -0,0 +1 @@ +# empty placeholder for test specific configurations diff --git a/test/pyhttpd/config.ini.in b/test/pyhttpd/config.ini.in new file mode 100644 index 0000000..3f42248 --- /dev/null +++ b/test/pyhttpd/config.ini.in @@ -0,0 +1,32 @@ +[global] +curl_bin = curl +nghttp = nghttp +h2load = h2load + +prefix = @prefix@ +exec_prefix = @exec_prefix@ +bindir = @bindir@ +sbindir = @sbindir@ +libdir = @libdir@ +libexecdir = @libexecdir@ + +apr_bindir = @APR_BINDIR@ +apxs = @bindir@/apxs +apachectl = @sbindir@/apachectl + +[httpd] +version = @HTTPD_VERSION@ +name = @progname@ +dso_modules = @DSO_MODULES@ +mpm_modules = @MPM_MODULES@ + +[test] +gen_dir = @abs_srcdir@/../gen +http_port = 5002 +https_port = 5001 +proxy_port = 5003 +http_port2 = 5004 +ws_port = 5100 +http_tld = tests.httpd.apache.org +test_dir = @abs_srcdir@ +test_src_dir = @abs_srcdir@ diff --git a/test/pyhttpd/curl.py b/test/pyhttpd/curl.py new file mode 100644 index 0000000..5a215cd --- /dev/null +++ b/test/pyhttpd/curl.py @@ -0,0 +1,138 @@ +import datetime +import re +import subprocess +import sys +import time +from threading import Thread + +from .env import HttpdTestEnv + + +class CurlPiper: + + def __init__(self, env: HttpdTestEnv, url: str): + self.env = env + self.url = url + self.proc = None + self.args = None + self.headerfile = None + self._stderr = [] + self._stdout = [] + self.stdout_thread = None + self.stderr_thread = None + self._exitcode = -1 + self._r = None + + @property + def exitcode(self): + return self._exitcode + + @property + def response(self): + return self._r.response if self._r else None + + def __repr__(self): + return f'CurlPiper[exitcode={self._exitcode}, stderr={self._stderr}, stdout={self._stdout}]' + + def start(self): + self.args, self.headerfile = self.env.curl_complete_args([self.url], timeout=5, options=[ + "-T", "-", "-X", "POST", "--trace-ascii", "%", "--trace-time" + ]) + self.args.append(self.url) + sys.stderr.write("starting: {0}\n".format(self.args)) + self.proc = subprocess.Popen(self.args, stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + bufsize=0) + + def read_output(fh, buffer): + while True: + chunk = fh.read() + if not chunk: + break + buffer.append(chunk.decode()) + + # collect all stdout and stderr until we are done + # use separate threads to not block ourself + self._stderr = [] + self._stdout = [] + if self.proc.stderr: + self.stderr_thread = Thread(target=read_output, args=(self.proc.stderr, self._stderr)) + self.stderr_thread.start() + if self.proc.stdout: + self.stdout_thread = Thread(target=read_output, args=(self.proc.stdout, self._stdout)) + self.stdout_thread.start() + return self.proc + + def send(self, data: str): + self.proc.stdin.write(data.encode()) + self.proc.stdin.flush() + + def close(self) -> ([str], [str]): + self.proc.stdin.close() + self.stdout_thread.join() + self.stderr_thread.join() + self._end() + return self._stdout, self._stderr + + def _end(self): + if self.proc: + # noinspection PyBroadException + try: + if self.proc.stdin: + # noinspection PyBroadException + try: + self.proc.stdin.close() + except Exception: + pass + if self.proc.stdout: + self.proc.stdout.close() + if self.proc.stderr: + self.proc.stderr.close() + except Exception: + self.proc.terminate() + finally: + self.proc.wait() + self.stdout_thread = None + self.stderr_thread = None + self._exitcode = self.proc.returncode + self.proc = None + self._r = self.env.curl_parse_headerfile(self.headerfile) + + def stutter_check(self, chunks: [str], stutter: datetime.timedelta): + if not self.proc: + self.start() + for chunk in chunks: + self.send(chunk) + time.sleep(stutter.total_seconds()) + recv_out, recv_err = self.close() + # assert we got everything back + assert "".join(chunks) == "".join(recv_out) + # now the tricky part: check *when* we got everything back + recv_times = [] + for line in "".join(recv_err).split('\n'): + m = re.match(r'^\s*(\d+:\d+:\d+(\.\d+)?) <= Recv data, (\d+) bytes.*', line) + if m: + recv_times.append(datetime.time.fromisoformat(m.group(1))) + # received as many chunks as we sent + assert len(chunks) == len(recv_times), "received response not in {0} chunks, but {1}".format( + len(chunks), len(recv_times)) + + def microsecs(tdelta): + return ((tdelta.hour * 60 + tdelta.minute) * 60 + tdelta.second) * 1000000 + tdelta.microsecond + + recv_deltas = [] + last_mics = microsecs(recv_times[0]) + for ts in recv_times[1:]: + mics = microsecs(ts) + delta_mics = mics - last_mics + if delta_mics < 0: + delta_mics += datetime.time(23, 59, 59, 999999) + recv_deltas.append(datetime.timedelta(microseconds=delta_mics)) + last_mics = mics + stutter_td = datetime.timedelta(seconds=stutter.total_seconds() * 0.75) # 25% leeway + # TODO: the first two chunks are often close together, it seems + # there still is a little buffering delay going on + for idx, td in enumerate(recv_deltas[1:]): + assert stutter_td < td, \ + f"chunk {idx} arrived too early \n{recv_deltas}\nafter {td}\n{recv_err}" diff --git a/test/pyhttpd/env.py b/test/pyhttpd/env.py new file mode 100644 index 0000000..1d4e8b1 --- /dev/null +++ b/test/pyhttpd/env.py @@ -0,0 +1,898 @@ +import importlib +import inspect +import logging +import re +import os +import shutil +import stat +import subprocess +import sys +import time +from datetime import datetime, timedelta +from string import Template +from typing import List, Optional + +from configparser import ConfigParser, ExtendedInterpolation +from urllib.parse import urlparse + +from .certs import Credentials, HttpdTestCA, CertificateSpec +from .log import HttpdErrorLog +from .nghttp import Nghttp +from .result import ExecResult + + +log = logging.getLogger(__name__) + + +class Dummy: + pass + + +class HttpdTestSetup: + + # the modules we want to load + MODULES = [ + "log_config", + "logio", + "unixd", + "version", + "watchdog", + "authn_core", + "authz_host", + "authz_groupfile", + "authz_user", + "authz_core", + "access_compat", + "auth_basic", + "cache", + "cache_disk", + "cache_socache", + "socache_shmcb", + "dumpio", + "reqtimeout", + "filter", + "mime", + "env", + "headers", + "setenvif", + "slotmem_shm", + "status", + "dir", + "alias", + "rewrite", + "deflate", + "proxy", + "proxy_http", + ] + + CURL_STDOUT_SEPARATOR = "===CURL_STDOUT_SEPARATOR===" + + def __init__(self, env: 'HttpdTestEnv'): + self.env = env + self._source_dirs = [os.path.dirname(inspect.getfile(HttpdTestSetup))] + self._modules = HttpdTestSetup.MODULES.copy() + self._optional_modules = [] + + def add_source_dir(self, source_dir): + self._source_dirs.append(source_dir) + + def add_modules(self, modules: List[str]): + self._modules.extend(modules) + + def add_optional_modules(self, modules: List[str]): + self._optional_modules.extend(modules) + + def make(self): + self._make_dirs() + self._make_conf() + if self.env.mpm_module is not None \ + and self.env.mpm_module in self.env.mpm_modules: + self.add_modules([self.env.mpm_module]) + if self.env.ssl_module is not None: + self.add_modules([self.env.ssl_module]) + self._make_modules_conf() + self._make_htdocs() + self._add_aptest() + self.env.clear_curl_headerfiles() + + def _make_dirs(self): + if not os.path.exists(self.env.gen_dir): + os.makedirs(self.env.gen_dir) + if not os.path.exists(self.env.server_logs_dir): + os.makedirs(self.env.server_logs_dir) + + def _make_conf(self): + # remove anything from another run/test suite + conf_dest_dir = os.path.join(self.env.server_dir, 'conf') + if os.path.isdir(conf_dest_dir): + shutil.rmtree(conf_dest_dir) + for d in self._source_dirs: + conf_src_dir = os.path.join(d, 'conf') + if os.path.isdir(conf_src_dir): + if not os.path.exists(conf_dest_dir): + os.makedirs(conf_dest_dir) + for name in os.listdir(conf_src_dir): + src_path = os.path.join(conf_src_dir, name) + m = re.match(r'(.+).template', name) + if m: + self._make_template(src_path, os.path.join(conf_dest_dir, m.group(1))) + elif os.path.isfile(src_path): + shutil.copy(src_path, os.path.join(conf_dest_dir, name)) + + def _make_template(self, src, dest): + var_map = dict() + for name, value in HttpdTestEnv.__dict__.items(): + if isinstance(value, property): + var_map[name] = value.fget(self.env) + t = Template(''.join(open(src).readlines())) + with open(dest, 'w') as fd: + fd.write(t.substitute(var_map)) + + def _make_modules_conf(self): + loaded = set() + modules_conf = os.path.join(self.env.server_dir, 'conf/modules.conf') + with open(modules_conf, 'w') as fd: + # issue load directives for all modules we want that are shared + missing_mods = list() + for m in self._modules: + match = re.match(r'^mod_(.+)$', m) + if match: + m = match.group(1) + if m in loaded: + continue + mod_path = os.path.join(self.env.libexec_dir, f"mod_{m}.so") + if os.path.isfile(mod_path): + fd.write(f"LoadModule {m}_module \"{mod_path}\"\n") + elif m in self.env.dso_modules: + missing_mods.append(m) + else: + fd.write(f"#built static: LoadModule {m}_module \"{mod_path}\"\n") + loaded.add(m) + for m in self._optional_modules: + match = re.match(r'^mod_(.+)$', m) + if match: + m = match.group(1) + if m in loaded: + continue + mod_path = os.path.join(self.env.libexec_dir, f"mod_{m}.so") + if os.path.isfile(mod_path): + fd.write(f"LoadModule {m}_module \"{mod_path}\"\n") + loaded.add(m) + if len(missing_mods) > 0: + raise Exception(f"Unable to find modules: {missing_mods} " + f"DSOs: {self.env.dso_modules}") + + def _make_htdocs(self): + if not os.path.exists(self.env.server_docs_dir): + os.makedirs(self.env.server_docs_dir) + dest_dir = os.path.join(self.env.server_dir, 'htdocs') + # remove anything from another run/test suite + if os.path.isdir(dest_dir): + shutil.rmtree(dest_dir) + for d in self._source_dirs: + srcdocs = os.path.join(d, 'htdocs') + if os.path.isdir(srcdocs): + shutil.copytree(srcdocs, dest_dir, dirs_exist_ok=True) + # make all contained .py scripts executable + for dirpath, _dirnames, filenames in os.walk(dest_dir): + for fname in filenames: + if re.match(r'.+\.py', fname): + py_file = os.path.join(dirpath, fname) + st = os.stat(py_file) + os.chmod(py_file, st.st_mode | stat.S_IEXEC) + + def _add_aptest(self): + local_dir = os.path.dirname(inspect.getfile(HttpdTestSetup)) + p = subprocess.run([self.env.apxs, '-c', 'mod_aptest.c'], + capture_output=True, + cwd=os.path.join(local_dir, 'mod_aptest')) + rv = p.returncode + if rv != 0: + log.error(f"compiling mod_aptest failed: {p.stderr}") + raise Exception(f"compiling mod_aptest failed: {p.stderr}") + + modules_conf = os.path.join(self.env.server_dir, 'conf/modules.conf') + with open(modules_conf, 'a') as fd: + # load our test module which is not installed + fd.write(f"LoadModule aptest_module \"{local_dir}/mod_aptest/.libs/mod_aptest.so\"\n") + + +class HttpdTestEnv: + + LIBEXEC_DIR = None + + @classmethod + def has_python_package(cls, name: str) -> bool: + if name in sys.modules: + # already loaded + return True + elif (spec := importlib.util.find_spec(name)) is not None: + module = importlib.util.module_from_spec(spec) + sys.modules[name] = module + spec.loader.exec_module(module) + return True + else: + return False + + @classmethod + def get_ssl_module(cls): + return os.environ['SSL'] if 'SSL' in os.environ else 'mod_ssl' + + @classmethod + def has_shared_module(cls, name): + if cls.LIBEXEC_DIR is None: + env = HttpdTestEnv() # will initialized it + path = os.path.join(cls.LIBEXEC_DIR, f"mod_{name}.so") + return os.path.isfile(path) + + def __init__(self, pytestconfig=None): + self._our_dir = os.path.dirname(inspect.getfile(Dummy)) + self.config = ConfigParser(interpolation=ExtendedInterpolation()) + self.config.read(os.path.join(self._our_dir, 'config.ini')) + + self._bin_dir = self.config.get('global', 'bindir') + self._apxs = self.config.get('global', 'apxs') + self._prefix = self.config.get('global', 'prefix') + self._apachectl = self.config.get('global', 'apachectl') + if HttpdTestEnv.LIBEXEC_DIR is None: + HttpdTestEnv.LIBEXEC_DIR = self._libexec_dir = self.get_apxs_var('LIBEXECDIR') + self._curl = self.config.get('global', 'curl_bin') + if 'CURL' in os.environ: + self._curl = os.environ['CURL'] + self._nghttp = self.config.get('global', 'nghttp') + if self._nghttp is None: + self._nghttp = 'nghttp' + self._h2load = self.config.get('global', 'h2load') + if self._h2load is None: + self._h2load = 'h2load' + + self._http_port = int(self.config.get('test', 'http_port')) + self._http_port2 = int(self.config.get('test', 'http_port2')) + self._https_port = int(self.config.get('test', 'https_port')) + self._proxy_port = int(self.config.get('test', 'proxy_port')) + self._ws_port = int(self.config.get('test', 'ws_port')) + self._http_tld = self.config.get('test', 'http_tld') + self._test_dir = self.config.get('test', 'test_dir') + self._clients_dir = os.path.join(os.path.dirname(self._test_dir), 'clients') + self._gen_dir = self.config.get('test', 'gen_dir') + self._server_dir = os.path.join(self._gen_dir, 'apache') + self._server_conf_dir = os.path.join(self._server_dir, "conf") + self._server_docs_dir = os.path.join(self._server_dir, "htdocs") + self._server_logs_dir = os.path.join(self.server_dir, "logs") + self._server_access_log = os.path.join(self._server_logs_dir, "access_log") + self._error_log = HttpdErrorLog(os.path.join(self._server_logs_dir, "error_log")) + self._apachectl_stderr = None + + self._dso_modules = self.config.get('httpd', 'dso_modules').split(' ') + self._mpm_modules = self.config.get('httpd', 'mpm_modules').split(' ') + self._mpm_module = f"mpm_{os.environ['MPM']}" if 'MPM' in os.environ else 'mpm_event' + self._ssl_module = self.get_ssl_module() + if len(self._ssl_module.strip()) == 0: + self._ssl_module = None + + self._httpd_addr = "127.0.0.1" + self._http_base = f"http://{self._httpd_addr}:{self.http_port}" + self._https_base = f"https://{self._httpd_addr}:{self.https_port}" + + self._verbosity = pytestconfig.option.verbose if pytestconfig is not None else 0 + self._test_conf = os.path.join(self._server_conf_dir, "test.conf") + self._httpd_base_conf = [] + self._httpd_log_modules = ['aptest'] + self._log_interesting = None + self._setup = None + + self._ca = None + self._cert_specs = [CertificateSpec(domains=[ + f"test1.{self._http_tld}", + f"test2.{self._http_tld}", + f"test3.{self._http_tld}", + f"cgi.{self._http_tld}", + ], key_type='rsa4096')] + + self._verify_certs = False + self._curl_headerfiles_n = 0 + self._curl_version = None + self._h2load_version = None + self._current_test = None + + def add_httpd_conf(self, lines: List[str]): + self._httpd_base_conf.extend(lines) + + def add_httpd_log_modules(self, modules: List[str]): + self._httpd_log_modules.extend(modules) + + def issue_certs(self): + if self._ca is None: + self._ca = HttpdTestCA.create_root(name=self.http_tld, + store_dir=os.path.join(self.server_dir, 'ca'), + key_type="rsa4096") + self._ca.issue_certs(self._cert_specs) + + def setup_httpd(self, setup: HttpdTestSetup = None): + """Create the server environment with config, htdocs and certificates""" + self._setup = setup if setup is not None else HttpdTestSetup(env=self) + self._setup.make() + self.issue_certs() + if self._httpd_log_modules: + if self._verbosity >= 2: + log_level = "trace2" + elif self._verbosity >= 1: + log_level = "debug" + else: + log_level = "info" + self._log_interesting = "LogLevel" + for name in self._httpd_log_modules: + self._log_interesting += f" {name}:{log_level}" + + @property + def curl(self) -> str: + return self._curl + + @property + def apxs(self) -> str: + return self._apxs + + @property + def verbosity(self) -> int: + return self._verbosity + + @property + def prefix(self) -> str: + return self._prefix + + @property + def mpm_module(self) -> str: + return self._mpm_module + + @property + def ssl_module(self) -> str: + return self._ssl_module + + @property + def http_addr(self) -> str: + return self._httpd_addr + + @property + def http_port(self) -> int: + return self._http_port + + @property + def http_port2(self) -> int: + return self._http_port2 + + @property + def https_port(self) -> int: + return self._https_port + + @property + def proxy_port(self) -> int: + return self._proxy_port + + @property + def ws_port(self) -> int: + return self._ws_port + + @property + def http_tld(self) -> str: + return self._http_tld + + @property + def http_base_url(self) -> str: + return self._http_base + + @property + def https_base_url(self) -> str: + return self._https_base + + @property + def bin_dir(self) -> str: + return self._bin_dir + + @property + def gen_dir(self) -> str: + return self._gen_dir + + @property + def test_dir(self) -> str: + return self._test_dir + + @property + def clients_dir(self) -> str: + return self._clients_dir + + @property + def server_dir(self) -> str: + return self._server_dir + + @property + def server_logs_dir(self) -> str: + return self._server_logs_dir + + @property + def libexec_dir(self) -> str: + return HttpdTestEnv.LIBEXEC_DIR + + @property + def dso_modules(self) -> List[str]: + return self._dso_modules + + @property + def mpm_modules(self) -> List[str]: + return self._mpm_modules + + @property + def server_conf_dir(self) -> str: + return self._server_conf_dir + + @property + def server_docs_dir(self) -> str: + return self._server_docs_dir + + @property + def httpd_error_log(self) -> HttpdErrorLog: + return self._error_log + + def htdocs_src(self, path): + return os.path.join(self._our_dir, 'htdocs', path) + + @property + def h2load(self) -> str: + return self._h2load + + @property + def ca(self) -> Credentials: + return self._ca + + @property + def current_test_name(self) -> str: + return self._current_test + + def set_current_test_name(self, val) -> None: + self._current_test = val + + @property + def apachectl_stderr(self): + return self._apachectl_stderr + + def add_cert_specs(self, specs: List[CertificateSpec]): + self._cert_specs.extend(specs) + + def get_credentials_for_name(self, dns_name) -> List['Credentials']: + for spec in [s for s in self._cert_specs if s.domains is not None]: + if dns_name in spec.domains: + return self.ca.get_credentials_for_name(spec.domains[0]) + return [] + + def _versiontuple(self, v): + v = re.sub(r'(\d+\.\d+(\.\d+)?)(-\S+)?', r'\1', v) + return tuple(map(int, v.split('.'))) + + def httpd_is_at_least(self, minv): + hv = self._versiontuple(self.get_httpd_version()) + return hv >= self._versiontuple(minv) + + def has_h2load(self): + return self._h2load != "" + + def h2load_is_at_least(self, minv): + if not self.has_h2load(): + return False + if self._h2load_version is None: + p = subprocess.run([self._h2load, '--version'], capture_output=True, text=True) + if p.returncode != 0: + return False + s = p.stdout.strip() + m = re.match(r'h2load nghttp2/(\S+)', s) + if m: + self._h2load_version = self._versiontuple(m.group(1)) + if self._h2load_version is not None: + return self._h2load_version >= self._versiontuple(minv) + return False + + def curl_is_at_least(self, minv): + if self._curl_version is None: + p = subprocess.run([self._curl, '-V'], capture_output=True, text=True) + if p.returncode != 0: + return False + for l in p.stdout.splitlines(): + m = re.match(r'curl ([0-9.]+)[- ].*', l) + if m: + self._curl_version = self._versiontuple(m.group(1)) + break + if self._curl_version is not None: + return self._curl_version >= self._versiontuple(minv) + return False + + def curl_is_less_than(self, version): + if self._curl_version is None: + p = subprocess.run([self._curl, '-V'], capture_output=True, text=True) + if p.returncode != 0: + return False + for l in p.stdout.splitlines(): + m = re.match(r'curl ([0-9.]+)[- ].*', l) + if m: + self._curl_version = self._versiontuple(m.group(1)) + break + if self._curl_version is not None: + return self._curl_version < self._versiontuple(version) + return False + + def has_nghttp(self): + return self._nghttp != "" + + def has_nghttp_get_assets(self): + if not self.has_nghttp(): + return False + args = [self._nghttp, "-a"] + p = subprocess.run(args, stderr=subprocess.PIPE, stdout=subprocess.PIPE) + rv = p.returncode + if rv != 0: + return False + return p.stderr == "" + + def get_apxs_var(self, name: str) -> str: + p = subprocess.run([self._apxs, "-q", name], capture_output=True, text=True) + if p.returncode != 0: + return "" + return p.stdout.strip() + + def get_httpd_version(self) -> str: + return self.get_apxs_var("HTTPD_VERSION") + + def mkpath(self, path): + if not os.path.exists(path): + return os.makedirs(path) + + def run(self, args, stdout_list=False, intext=None, inbytes=None, debug_log=True): + if debug_log: + log.debug(f"run: {args}") + start = datetime.now() + if intext is not None: + inbytes = intext.encode() + p = subprocess.run(args, stderr=subprocess.PIPE, stdout=subprocess.PIPE, + input=inbytes) + stdout_as_list = None + if stdout_list: + try: + out = p.stdout.decode() + if HttpdTestSetup.CURL_STDOUT_SEPARATOR in out: + stdout_as_list = out.split(HttpdTestSetup.CURL_STDOUT_SEPARATOR) + if not stdout_as_list[len(stdout_as_list) - 1]: + stdout_as_list.pop() + p.stdout.replace(HttpdTestSetup.CURL_STDOUT_SEPARATOR.encode(), b'') + except: + pass + return ExecResult(args=args, exit_code=p.returncode, + stdout=p.stdout, stderr=p.stderr, + stdout_as_list=stdout_as_list, + duration=datetime.now() - start) + + def mkurl(self, scheme, hostname, path='/'): + port = self.https_port if scheme == 'https' else self.http_port + return f"{scheme}://{hostname}.{self.http_tld}:{port}{path}" + + def install_test_conf(self, lines: List[str]): + with open(self._test_conf, 'w') as fd: + fd.write('\n'.join(self._httpd_base_conf)) + fd.write('\n') + fd.write(f"CoreDumpDirectory {self._server_dir}\n") + if self._verbosity >= 2: + fd.write(f"LogLevel core:trace5 {self.mpm_module}:trace5 http:trace5\n") + if self._verbosity >= 3: + fd.write(f"LogLevel dumpio:trace7\n") + fd.write(f"DumpIoOutput on\n") + fd.write(f"DumpIoInput on\n") + if self._log_interesting: + fd.write(self._log_interesting) + fd.write('\n\n') + fd.write('\n'.join(lines)) + fd.write('\n') + + def is_live(self, url: str = None, timeout: timedelta = None): + if url is None: + url = self._http_base + if timeout is None: + timeout = timedelta(seconds=5) + try_until = datetime.now() + timeout + last_err = "" + while datetime.now() < try_until: + # noinspection PyBroadException + try: + r = self.curl_get(url, insecure=True) + if r.exit_code == 0: + return True + time.sleep(.1) + except ConnectionRefusedError: + log.debug("connection refused") + time.sleep(.1) + except: + if last_err != str(sys.exc_info()[0]): + last_err = str(sys.exc_info()[0]) + log.debug("Unexpected error: %s", last_err) + time.sleep(.1) + log.debug(f"Unable to contact server after {timeout}") + return False + + def is_dead(self, url: str = None, timeout: timedelta = None): + if url is None: + url = self._http_base + if timeout is None: + timeout = timedelta(seconds=5) + try_until = datetime.now() + timeout + last_err = None + while datetime.now() < try_until: + # noinspection PyBroadException + try: + r = self.curl_get(url) + if r.exit_code != 0: + return True + time.sleep(.1) + except ConnectionRefusedError: + log.debug("connection refused") + return True + except: + if last_err != str(sys.exc_info()[0]): + last_err = str(sys.exc_info()[0]) + log.debug("Unexpected error: %s", last_err) + time.sleep(.1) + log.debug(f"Server still responding after {timeout}") + return False + + def _run_apachectl(self, cmd) -> ExecResult: + conf_file = 'stop.conf' if cmd == 'stop' else 'httpd.conf' + args = [self._apachectl, + "-d", self.server_dir, + "-f", os.path.join(self._server_dir, f'conf/{conf_file}'), + "-k", cmd] + r = self.run(args) + self._apachectl_stderr = r.stderr + if r.exit_code != 0: + log.warning(f"failed: {r}") + return r + + def apache_reload(self): + r = self._run_apachectl("graceful") + if r.exit_code == 0: + timeout = timedelta(seconds=10) + return 0 if self.is_live(self._http_base, timeout=timeout) else -1 + return r.exit_code + + def apache_restart(self): + self.apache_stop() + r = self._run_apachectl("start") + if r.exit_code == 0: + timeout = timedelta(seconds=10) + return 0 if self.is_live(self._http_base, timeout=timeout) else -1 + return r.exit_code + + def apache_stop(self): + r = self._run_apachectl("stop") + if r.exit_code == 0: + timeout = timedelta(seconds=10) + return 0 if self.is_dead(self._http_base, timeout=timeout) else -1 + return r + + def apache_graceful_stop(self): + log.debug("stop apache") + self._run_apachectl("graceful-stop") + return 0 if self.is_dead() else -1 + + def apache_fail(self): + log.debug("expect apache fail") + self._run_apachectl("stop") + rv = self._run_apachectl("start") + if rv == 0: + rv = 0 if self.is_dead() else -1 + else: + rv = 0 + return rv + + def apache_access_log_clear(self): + if os.path.isfile(self._server_access_log): + os.remove(self._server_access_log) + + def get_ca_pem_file(self, hostname: str) -> Optional[str]: + if len(self.get_credentials_for_name(hostname)) > 0: + return self.ca.cert_file + return None + + def clear_curl_headerfiles(self): + for fname in os.listdir(path=self.gen_dir): + if re.match(r'curl\.headers\.\d+', fname): + os.remove(os.path.join(self.gen_dir, fname)) + self._curl_headerfiles_n = 0 + + def curl_resolve_args(self, url, insecure=False, force_resolve=True, options=None): + u = urlparse(url) + + args = [ + ] + if u.scheme == 'http': + pass + elif insecure: + args.append('--insecure') + elif options and "--cacert" in options: + pass + elif u.hostname: + ca_pem = self.get_ca_pem_file(u.hostname) + if ca_pem: + args.extend(["--cacert", ca_pem]) + + if force_resolve and u.hostname and u.hostname != 'localhost' \ + and u.hostname != self._httpd_addr \ + and not re.match(r'^(\d+|\[|:).*', u.hostname): + assert u.port, f"port not in url: {url}" + args.extend(["--resolve", f"{u.hostname}:{u.port}:{self._httpd_addr}"]) + return args + + def curl_complete_args(self, urls, stdout_list=False, + timeout=None, options=None, + insecure=False, force_resolve=True): + headerfile = f"{self.gen_dir}/curl.headers.{self._curl_headerfiles_n}" + self._curl_headerfiles_n += 1 + + args = [ + self._curl, "-s", "--path-as-is", "-D", headerfile, + ] + args.extend(self.curl_resolve_args(urls[0], insecure=insecure, + force_resolve=force_resolve, + options=options)) + if stdout_list: + args.extend(['-w', '%{stdout}' + HttpdTestSetup.CURL_STDOUT_SEPARATOR]) + if self._current_test is not None: + args.extend(["-H", f'AP-Test-Name: {self._current_test}']) + if timeout is not None and int(timeout) > 0: + args.extend(["--connect-timeout", str(int(timeout))]) + if options: + args.extend(options) + return args, headerfile + + def curl_parse_headerfile(self, headerfile: str, r: ExecResult = None) -> ExecResult: + lines = open(headerfile).readlines() + if r is None: + r = ExecResult(args=[], exit_code=0, stdout=b'', stderr=b'') + + response = None + def fin_response(response): + if response: + r.add_response(response) + + expected = ['status'] + for line in lines: + if re.match(r'^$', line): + if 'trailer' in expected: + # end of trailers + fin_response(response) + response = None + expected = ['status'] + elif 'header' in expected: + # end of header, another status or trailers might follow + expected = ['status', 'trailer'] + else: + assert False, f"unexpected line: {line}" + continue + if 'status' in expected: + log.debug("reading 1st response line: %s", line) + m = re.match(r'^(\S+) (\d+) (.*)$', line) + if m: + fin_response(response) + response = { + "protocol": m.group(1), + "status": int(m.group(2)), + "description": m.group(3), + "header": {}, + "trailer": {}, + "body": r.outraw + } + expected = ['header'] + continue + if 'trailer' in expected: + m = re.match(r'^([^:]+):\s*(.*)$', line) + if m: + response['trailer'][m.group(1).lower()] = m.group(2) + continue + if 'header' in expected: + m = re.match(r'^([^:]+):\s*(.*)$', line) + if m: + response['header'][m.group(1).lower()] = m.group(2) + continue + assert False, f"unexpected line: {line}" + + fin_response(response) + return r + + def curl_raw(self, urls, timeout=10, options=None, insecure=False, + force_resolve=True, no_stdout_list=False): + if not isinstance(urls, list): + urls = [urls] + stdout_list = False + if len(urls) > 1 and not no_stdout_list: + stdout_list = True + args, headerfile = self.curl_complete_args( + urls=urls, stdout_list=stdout_list, + timeout=timeout, options=options, insecure=insecure, + force_resolve=force_resolve) + args += urls + r = self.run(args, stdout_list=stdout_list) + if r.exit_code == 0: + self.curl_parse_headerfile(headerfile, r=r) + if r.json: + r.response["json"] = r.json + if os.path.isfile(headerfile): + os.remove(headerfile) + return r + + def curl_get(self, url, insecure=False, options=None): + return self.curl_raw([url], insecure=insecure, options=options) + + def curl_upload(self, url, fpath, timeout=5, options=None): + if not options: + options = [] + options.extend([ + "--form", ("file=@%s" % fpath) + ]) + return self.curl_raw(urls=[url], timeout=timeout, options=options) + + def curl_post_data(self, url, data="", timeout=5, options=None): + if not options: + options = [] + options.extend(["--data", "%s" % data]) + return self.curl_raw(url, timeout, options) + + def curl_post_value(self, url, key, value, timeout=5, options=None): + if not options: + options = [] + options.extend(["--form", "{0}={1}".format(key, value)]) + return self.curl_raw(url, timeout, options) + + def curl_protocol_version(self, url, timeout=5, options=None): + if not options: + options = [] + options.extend(["-w", "%{http_version}\n", "-o", "/dev/null"]) + r = self.curl_raw(url, timeout=timeout, options=options) + if r.exit_code == 0 and r.response: + return r.response["body"].decode('utf-8').rstrip() + return -1 + + def nghttp(self): + return Nghttp(self._nghttp, connect_addr=self._httpd_addr, + tmp_dir=self.gen_dir, test_name=self._current_test) + + def h2load_status(self, run: ExecResult): + stats = {} + m = re.search( + r'requests: (\d+) total, (\d+) started, (\d+) done, (\d+) succeeded' + r', (\d+) failed, (\d+) errored, (\d+) timeout', run.stdout) + if m: + stats["requests"] = { + "total": int(m.group(1)), + "started": int(m.group(2)), + "done": int(m.group(3)), + "succeeded": int(m.group(4)) + } + m = re.search(r'status codes: (\d+) 2xx, (\d+) 3xx, (\d+) 4xx, (\d+) 5xx', + run.stdout) + if m: + stats["status"] = { + "2xx": int(m.group(1)), + "3xx": int(m.group(2)), + "4xx": int(m.group(3)), + "5xx": int(m.group(4)) + } + run.add_results({"h2load": stats}) + return run + + def make_data_file(self, indir: str, fname: str, fsize: int) -> str: + fpath = os.path.join(indir, fname) + s10 = "0123456789" + s = (101 * s10) + s10[0:3] + with open(fpath, 'w') as fd: + for i in range(int(fsize / 1024)): + fd.write(f"{i:09d}-{s}\n") + remain = int(fsize % 1024) + if remain != 0: + i = int(fsize / 1024) + 1 + s = f"{i:09d}-{s}\n" + fd.write(s[0:remain]) + return fpath + diff --git a/test/pyhttpd/htdocs/alive.json b/test/pyhttpd/htdocs/alive.json new file mode 100644 index 0000000..2239ee2 --- /dev/null +++ b/test/pyhttpd/htdocs/alive.json @@ -0,0 +1,4 @@ +{ + "host" : "generic", + "alive" : true +} diff --git a/test/pyhttpd/htdocs/forbidden.html b/test/pyhttpd/htdocs/forbidden.html new file mode 100644 index 0000000..e186310 --- /dev/null +++ b/test/pyhttpd/htdocs/forbidden.html @@ -0,0 +1,11 @@ +<html> + <head> + <title>403 - Forbidden</title> + </head> + <body> + <h1>403 - Forbidden</h1> + <p> + An example of an error document. + </p> + </body> +</html> diff --git a/test/pyhttpd/htdocs/index.html b/test/pyhttpd/htdocs/index.html new file mode 100644 index 0000000..3c07626 --- /dev/null +++ b/test/pyhttpd/htdocs/index.html @@ -0,0 +1,9 @@ +<html> + <head> + <title>mod_h2 test site generic</title> + </head> + <body> + <h1>mod_h2 test site generic</h1> + </body> +</html> + diff --git a/test/pyhttpd/htdocs/test1/001.html b/test/pyhttpd/htdocs/test1/001.html new file mode 100644 index 0000000..184952d --- /dev/null +++ b/test/pyhttpd/htdocs/test1/001.html @@ -0,0 +1,10 @@ +<!DOCTYPE HTML>
+ <html>
+ <head>
+ <title>HTML/2.0 Test File: 001</title>
+ </head>
+ <body>
+ <p><h1>HTML/2.0 Test File: 001</h1></p>
+ <p>This file only contains a simple HTML structure with plain text.</p>
+ </body>
+</html>
diff --git a/test/pyhttpd/htdocs/test1/002.jpg b/test/pyhttpd/htdocs/test1/002.jpg Binary files differnew file mode 100644 index 0000000..3feefb0 --- /dev/null +++ b/test/pyhttpd/htdocs/test1/002.jpg diff --git a/test/pyhttpd/htdocs/test1/003.html b/test/pyhttpd/htdocs/test1/003.html new file mode 100644 index 0000000..d5b08c5 --- /dev/null +++ b/test/pyhttpd/htdocs/test1/003.html @@ -0,0 +1,11 @@ +<!DOCTYPE HTML>
+ <html>
+ <head>
+ <title>HTML/2.0 Test File: 003</title>
+ </head>
+ <body>
+ <p><h1>HTML/2.0 Test File: 003</h1></p>
+ <p>This is a text HTML file with a big image:</p>
+ <p><img src="003/003_img.jpg" alt="GSMA Logo" style="width:269px;height:249px"></p>
+ </body>
+</html>
diff --git a/test/pyhttpd/htdocs/test1/003/003_img.jpg b/test/pyhttpd/htdocs/test1/003/003_img.jpg Binary files differnew file mode 100644 index 0000000..3feefb0 --- /dev/null +++ b/test/pyhttpd/htdocs/test1/003/003_img.jpg diff --git a/test/pyhttpd/htdocs/test1/004.html b/test/pyhttpd/htdocs/test1/004.html new file mode 100644 index 0000000..768cb82 --- /dev/null +++ b/test/pyhttpd/htdocs/test1/004.html @@ -0,0 +1,23 @@ +<html>
+ <head>
+ <title>HTML/2.0 Test File: 004</title>
+ </head>
+ <body>
+ <p><h1>HTML/2.0 Test File: 004</h1>
+ This file contains plain text with a bunch of images.<br>
+ <img src="004/gophertiles_142.jpg" height="32" width="32"><img src="004/gophertiles_084.jpg" height="32" width="32"><img src="004/gophertiles_052.jpg" height="32" width="32"><img src="004/gophertiles_077.jpg" height="32" width="32"><img src="004/gophertiles_030.jpg" height="32" width="32"><img src="004/gophertiles_027.jpg" height="32" width="32"><img src="004/gophertiles_039.jpg" height="32" width="32"><img src="004/gophertiles_025.jpg" height="32" width="32"><img src="004/gophertiles_017.jpg" height="32" width="32"><img src="004/gophertiles_179.jpg" height="32" width="32"><img src="004/gophertiles_032.jpg" height="32" width="32"><img src="004/gophertiles_161.jpg" height="32" width="32"><img src="004/gophertiles_088.jpg" height="32" width="32"><img src="004/gophertiles_022.jpg" height="32" width="32"><img src="004/gophertiles_146.jpg" height="32" width="32"><br>
+ <img src="004/gophertiles_102.jpg" height="32" width="32"><img src="004/gophertiles_009.jpg" height="32" width="32"><img src="004/gophertiles_132.jpg" height="32" width="32"><img src="004/gophertiles_137.jpg" height="32" width="32"><img src="004/gophertiles_055.jpg" height="32" width="32"><img src="004/gophertiles_036.jpg" height="32" width="32"><img src="004/gophertiles_127.jpg" height="32" width="32"><img src="004/gophertiles_145.jpg" height="32" width="32"><img src="004/gophertiles_147.jpg" height="32" width="32"><img src="004/gophertiles_153.jpg" height="32" width="32"><img src="004/gophertiles_105.jpg" height="32" width="32"><img src="004/gophertiles_103.jpg" height="32" width="32"><img src="004/gophertiles_033.jpg" height="32" width="32"><img src="004/gophertiles_054.jpg" height="32" width="32"><img src="004/gophertiles_015.jpg" height="32" width="32"><br>
+ <img src="004/gophertiles_016.jpg" height="32" width="32"><img src="004/gophertiles_072.jpg" height="32" width="32"><img src="004/gophertiles_115.jpg" height="32" width="32"><img src="004/gophertiles_108.jpg" height="32" width="32"><img src="004/gophertiles_148.jpg" height="32" width="32"><img src="004/gophertiles_070.jpg" height="32" width="32"><img src="004/gophertiles_083.jpg" height="32" width="32"><img src="004/gophertiles_118.jpg" height="32" width="32"><img src="004/gophertiles_053.jpg" height="32" width="32"><img src="004/gophertiles_021.jpg" height="32" width="32"><img src="004/gophertiles_059.jpg" height="32" width="32"><img src="004/gophertiles_130.jpg" height="32" width="32"><img src="004/gophertiles_163.jpg" height="32" width="32"><img src="004/gophertiles_098.jpg" height="32" width="32"><img src="004/gophertiles_064.jpg" height="32" width="32"><br>
+ <img src="004/gophertiles_018.jpg" height="32" width="32"><img src="004/gophertiles_058.jpg" height="32" width="32"><img src="004/gophertiles_167.jpg" height="32" width="32"><img src="004/gophertiles_082.jpg" height="32" width="32"><img src="004/gophertiles_056.jpg" height="32" width="32"><img src="004/gophertiles_180.jpg" height="32" width="32"><img src="004/gophertiles_046.jpg" height="32" width="32"><img src="004/gophertiles_093.jpg" height="32" width="32"><img src="004/gophertiles_106.jpg" height="32" width="32"><img src="004/gophertiles_065.jpg" height="32" width="32"><img src="004/gophertiles_175.jpg" height="32" width="32"><img src="004/gophertiles_139.jpg" height="32" width="32"><img src="004/gophertiles_101.jpg" height="32" width="32"><img src="004/gophertiles_099.jpg" height="32" width="32"><img src="004/gophertiles_051.jpg" height="32" width="32"><br>
+ <img src="004/gophertiles_140.jpg" height="32" width="32"><img src="004/gophertiles_134.jpg" height="32" width="32"><img src="004/gophertiles_149.jpg" height="32" width="32"><img src="004/gophertiles_049.jpg" height="32" width="32"><img src="004/gophertiles_095.jpg" height="32" width="32"><img src="004/gophertiles_075.jpg" height="32" width="32"><img src="004/gophertiles_066.jpg" height="32" width="32"><img src="004/gophertiles_090.jpg" height="32" width="32"><img src="004/gophertiles_035.jpg" height="32" width="32"><img src="004/gophertiles_114.jpg" height="32" width="32"><img src="004/gophertiles_160.jpg" height="32" width="32"><img src="004/gophertiles_079.jpg" height="32" width="32"><img src="004/gophertiles_062.jpg" height="32" width="32"><img src="004/gophertiles_096.jpg" height="32" width="32"><img src="004/gophertiles_100.jpg" height="32" width="32"><br>
+ <img src="004/gophertiles_104.jpg" height="32" width="32"><img src="004/gophertiles_057.jpg" height="32" width="32"><img src="004/gophertiles_037.jpg" height="32" width="32"><img src="004/gophertiles_086.jpg" height="32" width="32"><img src="004/gophertiles_168.jpg" height="32" width="32"><img src="004/gophertiles_138.jpg" height="32" width="32"><img src="004/gophertiles_045.jpg" height="32" width="32"><img src="004/gophertiles_141.jpg" height="32" width="32"><img src="004/gophertiles_029.jpg" height="32" width="32"><img src="004/gophertiles_165.jpg" height="32" width="32"><img src="004/gophertiles_110.jpg" height="32" width="32"><img src="004/gophertiles_063.jpg" height="32" width="32"><img src="004/gophertiles_158.jpg" height="32" width="32"><img src="004/gophertiles_122.jpg" height="32" width="32"><img src="004/gophertiles_068.jpg" height="32" width="32"><br>
+ <img src="004/gophertiles_170.jpg" height="32" width="32"><img src="004/gophertiles_120.jpg" height="32" width="32"><img src="004/gophertiles_117.jpg" height="32" width="32"><img src="004/gophertiles_031.jpg" height="32" width="32"><img src="004/gophertiles_113.jpg" height="32" width="32"><img src="004/gophertiles_074.jpg" height="32" width="32"><img src="004/gophertiles_129.jpg" height="32" width="32"><img src="004/gophertiles_019.jpg" height="32" width="32"><img src="004/gophertiles_060.jpg" height="32" width="32"><img src="004/gophertiles_109.jpg" height="32" width="32"><img src="004/gophertiles_080.jpg" height="32" width="32"><img src="004/gophertiles_097.jpg" height="32" width="32"><img src="004/gophertiles_116.jpg" height="32" width="32"><img src="004/gophertiles_085.jpg" height="32" width="32"><img src="004/gophertiles_050.jpg" height="32" width="32"><br>
+ <img src="004/gophertiles_151.jpg" height="32" width="32"><img src="004/gophertiles_094.jpg" height="32" width="32"><img src="004/gophertiles_067.jpg" height="32" width="32"><img src="004/gophertiles_128.jpg" height="32" width="32"><img src="004/gophertiles_034.jpg" height="32" width="32"><img src="004/gophertiles_135.jpg" height="32" width="32"><img src="004/gophertiles_012.jpg" height="32" width="32"><img src="004/gophertiles_010.jpg" height="32" width="32"><img src="004/gophertiles_152.jpg" height="32" width="32"><img src="004/gophertiles_171.jpg" height="32" width="32"><img src="004/gophertiles_087.jpg" height="32" width="32"><img src="004/gophertiles_126.jpg" height="32" width="32"><img src="004/gophertiles_048.jpg" height="32" width="32"><img src="004/gophertiles_023.jpg" height="32" width="32"><img src="004/gophertiles_078.jpg" height="32" width="32"><br>
+ <img src="004/gophertiles_071.jpg" height="32" width="32"><img src="004/gophertiles_131.jpg" height="32" width="32"><img src="004/gophertiles_073.jpg" height="32" width="32"><img src="004/gophertiles_143.jpg" height="32" width="32"><img src="004/gophertiles_173.jpg" height="32" width="32"><img src="004/gophertiles_154.jpg" height="32" width="32"><img src="004/gophertiles_061.jpg" height="32" width="32"><img src="004/gophertiles_178.jpg" height="32" width="32"><img src="004/gophertiles_013.jpg" height="32" width="32"><img src="004/gophertiles_028.jpg" height="32" width="32"><img src="004/gophertiles_157.jpg" height="32" width="32"><img src="004/gophertiles_038.jpg" height="32" width="32"><img src="004/gophertiles_069.jpg" height="32" width="32"><img src="004/gophertiles_174.jpg" height="32" width="32"><img src="004/gophertiles_076.jpg" height="32" width="32"><br>
+ <img src="004/gophertiles_155.jpg" height="32" width="32"><img src="004/gophertiles_107.jpg" height="32" width="32"><img src="004/gophertiles_136.jpg" height="32" width="32"><img src="004/gophertiles_144.jpg" height="32" width="32"><img src="004/gophertiles_091.jpg" height="32" width="32"><img src="004/gophertiles_024.jpg" height="32" width="32"><img src="004/gophertiles_014.jpg" height="32" width="32"><img src="004/gophertiles_159.jpg" height="32" width="32"><img src="004/gophertiles_011.jpg" height="32" width="32"><img src="004/gophertiles_176.jpg" height="32" width="32"><img src="004/gophertiles_162.jpg" height="32" width="32"><img src="004/gophertiles_156.jpg" height="32" width="32"><img src="004/gophertiles_081.jpg" height="32" width="32"><img src="004/gophertiles_119.jpg" height="32" width="32"><img src="004/gophertiles_026.jpg" height="32" width="32"><br>
+ <img src="004/gophertiles_133.jpg" height="32" width="32"><img src="004/gophertiles_020.jpg" height="32" width="32"><img src="004/gophertiles_044.jpg" height="32" width="32"><img src="004/gophertiles_125.jpg" height="32" width="32"><img src="004/gophertiles_150.jpg" height="32" width="32"><img src="004/gophertiles_172.jpg" height="32" width="32"><img src="004/gophertiles_002.jpg" height="32" width="32"><img src="004/gophertiles_169.jpg" height="32" width="32"><img src="004/gophertiles_007.jpg" height="32" width="32"><img src="004/gophertiles_008.jpg" height="32" width="32"><img src="004/gophertiles_042.jpg" height="32" width="32"><img src="004/gophertiles_041.jpg" height="32" width="32"><img src="004/gophertiles_166.jpg" height="32" width="32"><img src="004/gophertiles_005.jpg" height="32" width="32"><img src="004/gophertiles_089.jpg" height="32" width="32"><br>
+ <img src="004/gophertiles_177.jpg" height="32" width="32"><img src="004/gophertiles_092.jpg" height="32" width="32"><img src="004/gophertiles_043.jpg" height="32" width="32"><img src="004/gophertiles_111.jpg" height="32" width="32"><img src="004/gophertiles_047.jpg" height="32" width="32"><img src="004/gophertiles.jpg" height="32" width="32"><img src="004/gophertiles_006.jpg" height="32" width="32"><img src="004/gophertiles_121.jpg" height="32" width="32"><img src="004/gophertiles_004.jpg" height="32" width="32"><img src="004/gophertiles_124.jpg" height="32" width="32"><img src="004/gophertiles_123.jpg" height="32" width="32"><img src="004/gophertiles_112.jpg" height="32" width="32"><img src="004/gophertiles_040.jpg" height="32" width="32"><img src="004/gophertiles_164.jpg" height="32" width="32"><img src="004/gophertiles_003.jpg" height="32" width="32"><br>
+ <hr>This page is developed using this template:<a href="https://http2.golang.org/">HTTP/2 demo server</a>
+ </p>
+ </body>
+</html>
\ No newline at end of file diff --git a/test/pyhttpd/htdocs/test1/004/gophertiles.jpg b/test/pyhttpd/htdocs/test1/004/gophertiles.jpg Binary files differnew file mode 100644 index 0000000..e45ac3b --- /dev/null +++ b/test/pyhttpd/htdocs/test1/004/gophertiles.jpg diff --git a/test/pyhttpd/htdocs/test1/006.html b/test/pyhttpd/htdocs/test1/006.html new file mode 100644 index 0000000..6b73025 --- /dev/null +++ b/test/pyhttpd/htdocs/test1/006.html @@ -0,0 +1,23 @@ +<!DOCTYPE HTML>
+ <html>
+ <head>
+ <title>HTML/2.0 Test File: 006</title>
+ <link rel="stylesheet" type="text/css" href="006/006.css">
+ <script type="text/javascript" src="006/006.js"></script>
+ </head>
+ <body>
+ <h1>HTML/2.0 Test File: 006</h1>
+ <div class="listTitle">This page contains:
+ <ul class="listElements">
+ <li>HTML
+ <li>CSS
+ <li>JavaScript
+ </ul>
+ </div>
+ <div class="listTitle">
+ <script type="text/javascript">
+ mainJavascript();
+ </script>
+ </div>
+ </body>
+</html>
\ No newline at end of file diff --git a/test/pyhttpd/htdocs/test1/006/006.css b/test/pyhttpd/htdocs/test1/006/006.css new file mode 100644 index 0000000..de6aa5f --- /dev/null +++ b/test/pyhttpd/htdocs/test1/006/006.css @@ -0,0 +1,21 @@ +@CHARSET "ISO-8859-1";
+body{
+ background:HoneyDew;
+}
+p{
+color:#0000FF;
+text-align:left;
+}
+
+h1{
+color:#FF0000;
+text-align:center;
+}
+
+.listTitle{
+ font-size:large;
+}
+
+.listElements{
+ color:#3366FF
+}
\ No newline at end of file diff --git a/test/pyhttpd/htdocs/test1/006/006.js b/test/pyhttpd/htdocs/test1/006/006.js new file mode 100644 index 0000000..b450067 --- /dev/null +++ b/test/pyhttpd/htdocs/test1/006/006.js @@ -0,0 +1,31 @@ +/**
+ * JavaScript Functions File
+ */
+function returnDate()
+{
+ var currentDate;
+ currentDate=new Date();
+ var dateString=(currentDate.getMonth()+1)+'/'+currentDate.getDate()+'/'+currentDate.getFullYear();
+ return dateString;
+}
+
+function returnHour()
+{
+ var currentDate;
+ currentDate=new Date();
+ var hourString=currentDate.getHours()+':'+currentDate.getMinutes()+':'+currentDate.getSeconds();
+ return hourString;
+}
+
+function javaScriptMessage(){
+ return 'This section is generated under JavaScript:<br>';
+}
+
+function mainJavascript(){
+ document.write(javaScriptMessage())
+ document.write('<ul class="listElements">');
+ document.write('<li>Current date (dd/mm/yyyy): ' + returnDate());
+ document.write('<br>');
+ document.write('<li>Current time (hh:mm:ss): '+returnHour());
+ document.write('</ul>');
+}
\ No newline at end of file diff --git a/test/pyhttpd/htdocs/test1/006/header.html b/test/pyhttpd/htdocs/test1/006/header.html new file mode 100644 index 0000000..bace20e --- /dev/null +++ b/test/pyhttpd/htdocs/test1/006/header.html @@ -0,0 +1 @@ +<title>My Header Title</title> diff --git a/test/pyhttpd/htdocs/test1/007.html b/test/pyhttpd/htdocs/test1/007.html new file mode 100644 index 0000000..4db93e4 --- /dev/null +++ b/test/pyhttpd/htdocs/test1/007.html @@ -0,0 +1,21 @@ +<!DOCTYPE html>
+<html>
+<head>
+<meta charset="ISO-8859-1">
+<title>HTML/2.0 Test File: 007</title>
+</head>
+<body>
+ <h1>HTML/2.0 Test File: 007</h1>
+ <div><p>This page is used to send data from the client to the server:</p>
+ <FORM ACTION="007/007.py" METHOD="post" ENCTYPE="multipart/form-data">
+ <input type="hidden" name="pageName" value="007.html">
+ Name:<input type="text" name="pName" value="Write your name here." size="30" maxlength="30"><br>
+ Age:<input type="text" name="pAge" value="00" size="2" maxlength="2"><br>
+ Gender: Male<input type="radio" name="pGender" VALUE="Male">
+ Female<input type="radio" name="pGender" VALUE="Female"><br>
+ <input type="submit" name="userForm" value="Send">
+ <input type="reset" value="Clear">
+ </FORM>
+ </div>
+</body>
+</html>
\ No newline at end of file diff --git a/test/pyhttpd/htdocs/test1/007/007.py b/test/pyhttpd/htdocs/test1/007/007.py new file mode 100644 index 0000000..02b5466 --- /dev/null +++ b/test/pyhttpd/htdocs/test1/007/007.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +import cgi, sys +import cgitb; cgitb.enable() + +print "Content-Type: text/html;charset=UTF-8" +print + +print """\ + <!DOCTYPE html><html><head> + <title>HTML/2.0 Test File: 007 (received data)</title></head> + <body><h1>HTML/2.0 Test File: 007</h1>""" + +# alternative output: parsed form params <-> plain POST body +parseContent = True # <-> False + +if parseContent: + print '<h2>Data processed:</h2><ul>' + form = cgi.FieldStorage() + for name in form: + print '<li>', name, ': ', form[name].value, '</li>' + print '</ul>' +else: + print '<h2>POST data output:</h2><div><pre>' + data = sys.stdin.read() + print data + print '</pre></div>' + +print '</body></html>'
\ No newline at end of file diff --git a/test/pyhttpd/htdocs/test1/009.py b/test/pyhttpd/htdocs/test1/009.py new file mode 100644 index 0000000..8fd9095 --- /dev/null +++ b/test/pyhttpd/htdocs/test1/009.py @@ -0,0 +1,21 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +import cgi, sys, time +import cgitb; cgitb.enable() + +print "Content-Type: text/html;charset=UTF-8" +print + +print """\ + <!DOCTYPE html><html><head> + <title>HTML/2.0 Test File: 009 (server time)</title></head> + <body><h1>HTML/2.0 Test File: 009</h1> + <p>60 seconds of server time, one by one.</p>""" + +for i in range(60): + s = time.strftime("%Y-%m-%d %H:%M:%S") + print "<div>", s, "</div>" + sys.stdout.flush() + time.sleep(1) + +print "<p>done.</p></body></html>"
\ No newline at end of file diff --git a/test/pyhttpd/htdocs/test1/alive.json b/test/pyhttpd/htdocs/test1/alive.json new file mode 100644 index 0000000..93e7f95 --- /dev/null +++ b/test/pyhttpd/htdocs/test1/alive.json @@ -0,0 +1,5 @@ +{ + "host" : "test1", + "alive" : true +} + diff --git a/test/pyhttpd/htdocs/test1/index.html b/test/pyhttpd/htdocs/test1/index.html new file mode 100644 index 0000000..9f752b5 --- /dev/null +++ b/test/pyhttpd/htdocs/test1/index.html @@ -0,0 +1,46 @@ +<html> + <head> + <title>mod_h2 test site</title> + </head> + <body> + <h1>mod_h2 test site</h1> + <p></p> + <h2>served directly</h2> + <ul> + <li><a href="001.html">01: html</a></li> + <li><a href="002.jpg">02: image</a></li> + <li><a href="003.html">03: html+image</a></li> + <li><a href="004.html">04: tiled image</a></li> + <li><a href="005.txt">05: large text</a></li> + <li><a href="006.html">06: html/js/css</a></li> + <li><a href="007.html">07: form submit</a></li> + <li><a href="upload.py">08: upload</a></li> + <li><a href="009.py">09: small chunks</a></li> + </ul> + <h2>mod_proxyied</h2> + <ul> + <li><a href="proxy/001.html">01: html</a></li> + <li><a href="proxy/002.jpg">02: image</a></li> + <li><a href="proxy/003.html">03: html+image</a></li> + <li><a href="proxy/004.html">04: tiled image</a></li> + <li><a href="proxy/005.txt">05: large text</a></li> + <li><a href="proxy/006.html">06: html/js/css</a></li> + <li><a href="proxy/007.html">07: form submit</a></li> + <li><a href="proxy/upload.py">08: upload</a></li> + <li><a href="proxy/009.py">09: small chunks</a></li> + </ul> + <h2>mod_rewritten</h2> + <ul> + <li><a href="rewrite/001.html">01: html</a></li> + <li><a href="rewrite/002.jpg">02: image</a></li> + <li><a href="rewrite/003.html">03: html+image</a></li> + <li><a href="rewrite/004.html">04: tiled image</a></li> + <li><a href="rewrite/005.txt">05: large text</a></li> + <li><a href="rewrite/006.html">06: html/js/css</a></li> + <li><a href="rewrite/007.html">07: form submit</a></li> + <li><a href="rewrite/upload.py">08: upload</a></li> + <li><a href="rewrite/009.py">09: small chunks</a></li> + </ul> + </body> +</html> + diff --git a/test/pyhttpd/htdocs/test2/006/006.css b/test/pyhttpd/htdocs/test2/006/006.css new file mode 100755 index 0000000..de6aa5f --- /dev/null +++ b/test/pyhttpd/htdocs/test2/006/006.css @@ -0,0 +1,21 @@ +@CHARSET "ISO-8859-1";
+body{
+ background:HoneyDew;
+}
+p{
+color:#0000FF;
+text-align:left;
+}
+
+h1{
+color:#FF0000;
+text-align:center;
+}
+
+.listTitle{
+ font-size:large;
+}
+
+.listElements{
+ color:#3366FF
+}
\ No newline at end of file diff --git a/test/pyhttpd/htdocs/test2/10%abnormal.txt b/test/pyhttpd/htdocs/test2/10%abnormal.txt new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/test/pyhttpd/htdocs/test2/10%abnormal.txt diff --git a/test/pyhttpd/htdocs/test2/alive.json b/test/pyhttpd/htdocs/test2/alive.json new file mode 100644 index 0000000..6a74223 --- /dev/null +++ b/test/pyhttpd/htdocs/test2/alive.json @@ -0,0 +1,4 @@ +{ + "host" : "test2", + "alive" : true +} diff --git a/test/pyhttpd/htdocs/test2/x%2f.test b/test/pyhttpd/htdocs/test2/x%2f.test new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/test/pyhttpd/htdocs/test2/x%2f.test diff --git a/test/pyhttpd/log.py b/test/pyhttpd/log.py new file mode 100644 index 0000000..dff7623 --- /dev/null +++ b/test/pyhttpd/log.py @@ -0,0 +1,163 @@ +import os +import re +import time +from datetime import datetime, timedelta +from io import SEEK_END +from typing import List, Tuple, Any + + +class HttpdErrorLog: + """Checking the httpd error log for errors and warnings, including + limiting checks from a last known position forward. + """ + + RE_ERRLOG_ERROR = re.compile(r'.*\[(?P<module>[^:]+):error].*') + RE_ERRLOG_WARN = re.compile(r'.*\[(?P<module>[^:]+):warn].*') + RE_APLOGNO = re.compile(r'.*\[(?P<module>[^:]+):(error|warn)].* (?P<aplogno>AH\d+): .+') + RE_SSL_LIB_ERR = re.compile(r'.*\[ssl:error].* SSL Library Error: error:(?P<errno>\S+):.+') + + def __init__(self, path: str): + self._path = path + self._ignored_modules = [] + self._ignored_lognos = set() + self._ignored_patterns = [] + # remember the file position we started with + self._start_pos = 0 + if os.path.isfile(self._path): + with open(self._path) as fd: + self._start_pos = fd.seek(0, SEEK_END) + self._last_pos = self._start_pos + self._last_errors = [] + self._last_warnings = [] + self._observed_erros = set() + self._observed_warnings = set() + + def __repr__(self): + return f"HttpdErrorLog[{self._path}, errors: {' '.join(self._last_errors)}, " \ + f"warnings: {' '.join(self._last_warnings)}]" + + @property + def path(self) -> str: + return self._path + + def clear_log(self): + if os.path.isfile(self.path): + os.remove(self.path) + self._start_pos = 0 + self._last_pos = self._start_pos + self._last_errors = [] + self._last_warnings = [] + self._observed_erros = set() + self._observed_warnings = set() + + def set_ignored_modules(self, modules: List[str]): + self._ignored_modules = modules.copy() if modules else [] + + def set_ignored_lognos(self, lognos: List[str]): + if lognos: + for l in lognos: + self._ignored_lognos.add(l) + + def add_ignored_patterns(self, patterns: List[Any]): + self._ignored_patterns.extend(patterns) + + def _is_ignored(self, line: str) -> bool: + for p in self._ignored_patterns: + if p.match(line): + return True + m = self.RE_APLOGNO.match(line) + if m and m.group('aplogno') in self._ignored_lognos: + return True + return False + + def get_recent(self, advance=True) -> Tuple[List[str], List[str]]: + """Collect error and warning from the log since the last remembered position + :param advance: advance the position to the end of the log afterwards + :return: list of error and list of warnings as tuple + """ + self._last_errors = [] + self._last_warnings = [] + if os.path.isfile(self._path): + with open(self._path) as fd: + fd.seek(self._last_pos, os.SEEK_SET) + for line in fd: + if self._is_ignored(line): + continue + m = self.RE_ERRLOG_ERROR.match(line) + if m and m.group('module') not in self._ignored_modules: + self._last_errors.append(line) + continue + m = self.RE_ERRLOG_WARN.match(line) + if m: + if m and m.group('module') not in self._ignored_modules: + self._last_warnings.append(line) + continue + if advance: + self._last_pos = fd.tell() + self._observed_erros.update(set(self._last_errors)) + self._observed_warnings.update(set(self._last_warnings)) + return self._last_errors, self._last_warnings + + def get_recent_count(self, advance=True): + errors, warnings = self.get_recent(advance=advance) + return len(errors), len(warnings) + + def ignore_recent(self): + """After a test case triggered errors/warnings on purpose, add + those to our 'observed' list so the do not get reported as 'missed'. + """ + self._last_errors = [] + self._last_warnings = [] + if os.path.isfile(self._path): + with open(self._path) as fd: + fd.seek(self._last_pos, os.SEEK_SET) + for line in fd: + if self._is_ignored(line): + continue + m = self.RE_ERRLOG_ERROR.match(line) + if m and m.group('module') not in self._ignored_modules: + self._observed_erros.add(line) + continue + m = self.RE_ERRLOG_WARN.match(line) + if m: + if m and m.group('module') not in self._ignored_modules: + self._observed_warnings.add(line) + continue + self._last_pos = fd.tell() + + def get_missed(self) -> Tuple[List[str], List[str]]: + errors = [] + warnings = [] + if os.path.isfile(self._path): + with open(self._path) as fd: + fd.seek(self._start_pos, os.SEEK_SET) + for line in fd: + if self._is_ignored(line): + continue + m = self.RE_ERRLOG_ERROR.match(line) + if m and m.group('module') not in self._ignored_modules \ + and line not in self._observed_erros: + errors.append(line) + continue + m = self.RE_ERRLOG_WARN.match(line) + if m: + if m and m.group('module') not in self._ignored_modules \ + and line not in self._observed_warnings: + warnings.append(line) + continue + return errors, warnings + + def scan_recent(self, pattern: re, timeout=10): + if not os.path.isfile(self.path): + return False + with open(self.path) as fd: + end = datetime.now() + timedelta(seconds=timeout) + while True: + fd.seek(self._last_pos, os.SEEK_SET) + for line in fd: + if pattern.match(line): + return True + if datetime.now() > end: + raise TimeoutError(f"pattern not found in error log after {timeout} seconds") + time.sleep(.1) + return False diff --git a/test/pyhttpd/mod_aptest/mod_aptest.c b/test/pyhttpd/mod_aptest/mod_aptest.c new file mode 100644 index 0000000..d1a8e05 --- /dev/null +++ b/test/pyhttpd/mod_aptest/mod_aptest.c @@ -0,0 +1,66 @@ +/* Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <apr_optional.h> +#include <apr_optional_hooks.h> +#include <apr_strings.h> +#include <apr_cstr.h> +#include <apr_want.h> + +#include <httpd.h> +#include <http_protocol.h> +#include <http_request.h> +#include <http_log.h> + +static void aptest_hooks(apr_pool_t *pool); + +AP_DECLARE_MODULE(aptest) = { + STANDARD20_MODULE_STUFF, + NULL, /* func to create per dir config */ + NULL, /* func to merge per dir config */ + NULL, /* func to create per server config */ + NULL, /* func to merge per server config */ + NULL, /* command handlers */ + aptest_hooks, +#if defined(AP_MODULE_FLAG_NONE) + AP_MODULE_FLAG_ALWAYS_MERGE +#endif +}; + + +static int aptest_post_read_request(request_rec *r) +{ + const char *test_name = apr_table_get(r->headers_in, "AP-Test-Name"); + if (test_name) { + ap_log_rerror(APLOG_MARK, APLOG_INFO, 0, r, "test[%s]: %s", + test_name, r->the_request); + } + return DECLINED; +} + +/* Install this module into the apache2 infrastructure. + */ +static void aptest_hooks(apr_pool_t *pool) +{ + ap_log_perror(APLOG_MARK, APLOG_TRACE1, 0, pool, + "installing hooks and handlers"); + + /* test case monitoring */ + ap_hook_post_read_request(aptest_post_read_request, NULL, + NULL, APR_HOOK_MIDDLE); + +} + diff --git a/test/pyhttpd/nghttp.py b/test/pyhttpd/nghttp.py new file mode 100644 index 0000000..43721f5 --- /dev/null +++ b/test/pyhttpd/nghttp.py @@ -0,0 +1,304 @@ +import re +import os +import subprocess +from datetime import datetime +from typing import Dict + +from urllib.parse import urlparse + +from .result import ExecResult + + +def _get_path(x): + return x["path"] + + +class Nghttp: + + def __init__(self, path, connect_addr=None, tmp_dir="/tmp", + test_name: str = None): + self.NGHTTP = path + self.CONNECT_ADDR = connect_addr + self.TMP_DIR = tmp_dir + self._test_name = test_name + + @staticmethod + def get_stream(streams, sid): + sid = int(sid) + if sid not in streams: + streams[sid] = { + "id": sid, + "header": {}, + "request": { + "id": sid, + "body": b'' + }, + "response": { + "id": sid, + "body": b'' + }, + "data_lengths": [], + "paddings": [], + "promises": [] + } + return streams[sid] if sid in streams else None + + def run(self, urls, timeout, options): + return self._baserun(urls, timeout, options) + + def complete_args(self, url, _timeout, options: [str]) -> [str]: + if not isinstance(url, list): + url = [url] + u = urlparse(url[0]) + args = [self.NGHTTP] + if self.CONNECT_ADDR: + connect_host = self.CONNECT_ADDR + args.append("--header=host: %s:%s" % (u.hostname, u.port)) + else: + connect_host = u.hostname + if options: + args.extend(options) + for xurl in url: + xu = urlparse(xurl) + nurl = "%s://%s:%s/%s" % (u.scheme, connect_host, xu.port, xu.path) + if xu.query: + nurl = "%s?%s" % (nurl, xu.query) + args.append(nurl) + return args + + def _baserun(self, url, timeout, options): + return self._run(self.complete_args(url, timeout, options)) + + def parse_output(self, btext) -> Dict: + # getting meta data and response body out of nghttp's output + # is a bit tricky. Without '-v' we just get the body. With '-v' meta + # data and timings in both directions are listed. + # We rely on response :status: to be unique and on + # response body not starting with space. + # Something not good enough for general purpose, but for these tests. + output = {} + body = '' + streams = {} + skip_indents = True + # chunk output into lines. nghttp mixes text + # meta output with bytes from the response body. + lines = [l.decode() for l in btext.split(b'\n')] + + for lidx, l in enumerate(lines): + if len(l) == 0: + body += '\n' + continue + m = re.match(r'(.*)\[.*] recv \(stream_id=(\d+)\) (\S+): (\S*)', l) + if m: + body += m.group(1) + s = self.get_stream(streams, m.group(2)) + hname = m.group(3) + hval = m.group(4) + print("stream %d header %s: %s" % (s["id"], hname, hval)) + header = s["header"] + if hname in header: + header[hname] += ", %s" % hval + else: + header[hname] = hval + continue + + m = re.match(r'(.*)\[.*] recv HEADERS frame <.* stream_id=(\d+)>', l) + if m: + body += m.group(1) + s = self.get_stream(streams, m.group(2)) + if s: + print("stream %d: recv %d header" % (s["id"], len(s["header"]))) + response = s["response"] + hkey = "header" + if "header" in response: + h = response["header"] + if ":status" in h and int(h[":status"]) >= 200: + hkey = "trailer" + else: + prev = { + "header": h + } + if "previous" in response: + prev["previous"] = response["previous"] + response["previous"] = prev + response[hkey] = s["header"] + s["header"] = {} + body = '' + continue + + m = re.match(r'(.*)\[.*] recv DATA frame <length=(\d+), .*stream_id=(\d+)>', l) + if m: + body += m.group(1) + s = self.get_stream(streams, m.group(3)) + blen = int(m.group(2)) + if s: + print(f'stream {s["id"]}: {blen} DATA bytes added via "{l}"') + padlen = 0 + if len(lines) > lidx + 2: + mpad = re.match(r' +\(padlen=(\d+)\)', lines[lidx+2]) + if mpad: + padlen = int(mpad.group(1)) + s["data_lengths"].append(blen) + s["paddings"].append(padlen) + blen -= padlen + s["response"]["body"] += body[-blen:].encode() + body = '' + skip_indents = True + continue + + m = re.match(r'(.*)\[.*] recv PUSH_PROMISE frame <.* stream_id=(\d+)>', l) + if m: + body += m.group(1) + s = self.get_stream(streams, m.group(2)) + if s: + # headers we have are request headers for the PUSHed stream + # these have been received on the originating stream, the promised + # stream id it mentioned in the following lines + print("stream %d: %d PUSH_PROMISE header" % (s["id"], len(s["header"]))) + if len(lines) > lidx+2: + m2 = re.match(r'\s+\(.*promised_stream_id=(\d+)\)', lines[lidx+2]) + if m2: + s2 = self.get_stream(streams, m2.group(1)) + s2["request"]["header"] = s["header"] + s["promises"].append(s2) + s["header"] = {} + continue + + m = re.match(r'(.*)\[.*] recv (\S+) frame <length=(\d+), .*stream_id=(\d+)>', l) + if m: + print("recv frame %s on stream %s" % (m.group(2), m.group(4))) + body += m.group(1) + skip_indents = True + continue + + m = re.match(r'(.*)\[.*] send (\S+) frame <length=(\d+), .*stream_id=(\d+)>', l) + if m: + print("send frame %s on stream %s" % (m.group(2), m.group(4))) + body += m.group(1) + skip_indents = True + continue + + if skip_indents and l.startswith(' '): + continue + + if '[' != l[0]: + skip_indents = None + body += l + '\n' + + # the main request is done on the lowest odd numbered id + main_stream = 99999999999 + for sid in streams: + s = streams[sid] + if "header" in s["response"] and ":status" in s["response"]["header"]: + s["response"]["status"] = int(s["response"]["header"][":status"]) + if (sid % 2) == 1 and sid < main_stream: + main_stream = sid + + output["streams"] = streams + if main_stream in streams: + output["response"] = streams[main_stream]["response"] + output["paddings"] = streams[main_stream]["paddings"] + output["data_lengths"] = streams[main_stream]["data_lengths"] + return output + + def _raw(self, url, timeout, options): + args = ["-v"] + if self._test_name is not None: + args.append(f'--header=AP-Test-Name: {self._test_name}') + if options: + args.extend(options) + r = self._baserun(url, timeout, args) + if 0 == r.exit_code: + r.add_results(self.parse_output(r.outraw)) + return r + + def get(self, url, timeout=5, options=None): + return self._raw(url, timeout, options) + + def assets(self, url, timeout=5, options=None): + if not options: + options = [] + options.extend(["-ans"]) + r = self._baserun(url, timeout, options) + assets = [] + if 0 == r.exit_code: + lines = re.findall(r'[^\n]*\n', r.stdout, re.MULTILINE) + for lidx, l in enumerate(lines): + m = re.match(r'\s*(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+/(.*)', l) + if m: + assets.append({ + "path": m.group(7), + "status": int(m.group(5)), + "size": m.group(6) + }) + assets.sort(key=_get_path) + r.add_assets(assets) + return r + + def post_data(self, url, data, timeout=5, options=None): + reqbody = ("%s/nghttp.req.body" % self.TMP_DIR) + with open(reqbody, 'wb') as f: + f.write(data.encode('utf-8')) + if not options: + options = [] + options.extend(["--data=%s" % reqbody]) + return self._raw(url, timeout, options) + + def post_name(self, url, name, timeout=5, options=None): + reqbody = ("%s/nghttp.req.body" % self.TMP_DIR) + with open(reqbody, 'w') as f: + f.write("--DSAJKcd9876\r\n") + f.write("Content-Disposition: form-data; name=\"value\"; filename=\"xxxxx\"\r\n") + f.write("Content-Type: text/plain\r\n") + f.write(f"\r\n{name}") + f.write("\r\n--DSAJKcd9876\r\n") + if not options: + options = [] + options.extend([ + "--data=%s" % reqbody, + "-HContent-Type: multipart/form-data; boundary=DSAJKcd9876"]) + return self._raw(url, timeout, options) + + def upload(self, url, fpath, timeout=5, options=None): + if not options: + options = [] + options.extend(["--data=%s" % fpath]) + return self._raw(url, timeout, options) + + def upload_file(self, url, fpath, timeout=5, options=None): + fname = os.path.basename(fpath) + reqbody = ("%s/nghttp.req.body" % self.TMP_DIR) + with open(fpath, 'rb') as fin: + with open(reqbody, 'wb') as f: + preamble = [ + '--DSAJKcd9876', + 'Content-Disposition: form-data; name="xxx"; filename="xxxxx"', + 'Content-Type: text/plain', + '', + 'testing mod_h2', + '\r\n--DSAJKcd9876', + f'Content-Disposition: form-data; name="file"; filename="{fname}"', + 'Content-Type: application/octet-stream', + 'Content-Transfer-Encoding: binary', + '', '' + ] + f.write('\r\n'.join(preamble).encode('utf-8')) + f.write(fin.read()) + f.write('\r\n'.join([ + '\r\n--DSAJKcd9876', '' + ]).encode('utf-8')) + if not options: + options = [] + options.extend([ + "--data=%s" % reqbody, + "--expect-continue", + "-HContent-Type: multipart/form-data; boundary=DSAJKcd9876"]) + return self._raw(url, timeout, options) + + def _run(self, args) -> ExecResult: + print(("execute: %s" % " ".join(args))) + start = datetime.now() + p = subprocess.run(args, capture_output=True, text=False) + return ExecResult(args=args, exit_code=p.returncode, + stdout=p.stdout, stderr=p.stderr, + duration=datetime.now() - start) diff --git a/test/pyhttpd/result.py b/test/pyhttpd/result.py new file mode 100644 index 0000000..4bf9ff2 --- /dev/null +++ b/test/pyhttpd/result.py @@ -0,0 +1,92 @@ +import json +from datetime import timedelta +from typing import Optional, Dict, List + + +class ExecResult: + + def __init__(self, args: List[str], exit_code: int, + stdout: bytes, stderr: bytes = None, + stdout_as_list: List[bytes] = None, + duration: timedelta = None): + self._args = args + self._exit_code = exit_code + self._stdout = stdout if stdout is not None else b'' + self._stderr = stderr if stderr is not None else b'' + self._duration = duration if duration is not None else timedelta() + self._response = None + self._results = {} + self._assets = [] + # noinspection PyBroadException + try: + if stdout_as_list is None: + out = self._stdout.decode() + else: + out = "[" + ','.join(stdout_as_list) + "]" + self._json_out = json.loads(out) + except: + self._json_out = None + + def __repr__(self): + out = [ + f"ExecResult[code={self.exit_code}, args={self._args}\n", + "----stdout---------------------------------------\n", + self._stdout.decode(), + "----stderr---------------------------------------\n", + self._stderr.decode() + ] + return ''.join(out) + + @property + def exit_code(self) -> int: + return self._exit_code + + @property + def args(self) -> List[str]: + return self._args + + @property + def outraw(self) -> bytes: + return self._stdout + + @property + def stdout(self) -> str: + return self._stdout.decode() + + @property + def json(self) -> Optional[Dict]: + """Output as JSON dictionary or None if not parseable.""" + return self._json_out + + @property + def stderr(self) -> str: + return self._stderr.decode() + + @property + def duration(self) -> timedelta: + return self._duration + + @property + def response(self) -> Optional[Dict]: + return self._response + + @property + def results(self) -> Dict: + return self._results + + @property + def assets(self) -> List: + return self._assets + + def add_response(self, resp: Dict): + if self._response: + resp['previous'] = self._response + self._response = resp + + def add_results(self, results: Dict): + self._results.update(results) + if 'response' in results: + self.add_response(results['response']) + + def add_assets(self, assets: List): + self._assets.extend(assets) diff --git a/test/pyhttpd/ws_util.py b/test/pyhttpd/ws_util.py new file mode 100644 index 0000000..38a3cf7 --- /dev/null +++ b/test/pyhttpd/ws_util.py @@ -0,0 +1,137 @@ +import logging +import struct + + +log = logging.getLogger(__name__) + + +class WsFrame: + + CONT = 0 + TEXT = 1 + BINARY = 2 + RSVD3 = 3 + RSVD4 = 4 + RSVD5 = 5 + RSVD6 = 6 + RSVD7 = 7 + CLOSE = 8 + PING = 9 + PONG = 10 + RSVD11 = 11 + RSVD12 = 12 + RSVD13 = 13 + RSVD14 = 14 + RSVD15 = 15 + + OP_NAMES = [ + "CONT", + "TEXT", + "BINARY", + "RSVD3", + "RSVD4", + "RSVD5", + "RSVD6", + "RSVD7", + "CLOSE", + "PING", + "PONG", + "RSVD11", + "RSVD12", + "RSVD13", + "RSVD14", + "RSVD15", + ] + + def __init__(self, opcode: int, fin: bool, mask: bytes, data: bytes): + self.opcode = opcode + self.fin = fin + self.mask = mask + self.data = data + self.length = len(data) + + def __repr__(self): + return f'WsFrame[{self.OP_NAMES[self.opcode]} fin={self.fin}, mask={self.mask}, len={len(self.data)}]' + + @property + def data_len(self) -> int: + return len(self.data) if self.data else 0 + + def to_network(self) -> bytes: + nd = bytearray() + h1 = self.opcode + if self.fin: + h1 |= 0x80 + nd.extend(struct.pack("!B", h1)) + mask_bit = 0x80 if self.mask is not None else 0x0 + h2 = self.data_len + if h2 > 65535: + nd.extend(struct.pack("!BQ", 127|mask_bit, h2)) + elif h2 > 126: + nd.extend(struct.pack("!BH", 126|mask_bit, h2)) + else: + nd.extend(struct.pack("!B", h2|mask_bit)) + if self.mask is not None: + nd.extend(self.mask) + if self.data is not None: + nd.extend(self.data) + return nd + + @classmethod + def client_ping(cls, data: bytes, mask: bytes = None) -> 'WsFrame': + if mask is None: + mask = bytes.fromhex('00 00 00 00') + return WsFrame(opcode=WsFrame.PING, fin=True, mask=mask, data=data) + + @classmethod + def client_close(cls, code: int, reason: str = None, + mask: bytes = None) -> 'WsFrame': + data = bytearray(struct.pack("!H", code)) + if reason is not None: + data.extend(reason.encode()) + if mask is None: + mask = bytes.fromhex('00 00 00 00') + return WsFrame(opcode=WsFrame.CLOSE, fin=True, mask=mask, data=data) + + +class WsFrameReader: + + def __init__(self, data: bytes): + self.data = data + + def _read(self, n: int): + if len(self.data) < n: + raise EOFError(f'have {len(self.data)} bytes left, but {n} requested') + elif n == 0: + return b'' + chunk = self.data[:n] + del self.data[:n] + return chunk + + def next_frame(self): + data = self._read(2) + h1, h2 = struct.unpack("!BB", data) + log.debug(f'parsed h1={h1} h2={h2} from {data}') + fin = True if h1 & 0x80 else False + opcode = h1 & 0xf + has_mask = True if h2 & 0x80 else False + mask = None + dlen = h2 & 0x7f + if dlen == 126: + (dlen,) = struct.unpack("!H", self._read(2)) + elif dlen == 127: + (dlen,) = struct.unpack("!Q", self._read(8)) + if has_mask: + mask = self._read(4) + return WsFrame(opcode=opcode, fin=fin, mask=mask, data=self._read(dlen)) + + def eof(self): + return len(self.data) == 0 + + @classmethod + def parse(cls, data: bytes): + frames = [] + reader = WsFrameReader(data=data) + while not reader.eof(): + frames.append(reader.next_frame()) + return frames |
