diff options
| author | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-04-28 14:29:10 +0000 |
|---|---|---|
| committer | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-04-28 14:29:10 +0000 |
| commit | 2aa4a82499d4becd2284cdb482213d541b8804dd (patch) | |
| tree | b80bf8bf13c3766139fbacc530efd0dd9d54394c /testing/mozbase/mozproxy | |
| parent | Initial commit. (diff) | |
| download | firefox-2aa4a82499d4becd2284cdb482213d541b8804dd.tar.xz firefox-2aa4a82499d4becd2284cdb482213d541b8804dd.zip | |
Adding upstream version 86.0.1.upstream/86.0.1upstream
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'testing/mozbase/mozproxy')
45 files changed, 2986 insertions, 0 deletions
diff --git a/testing/mozbase/mozproxy/MANIFEST.in b/testing/mozbase/mozproxy/MANIFEST.in new file mode 100644 index 0000000000..c3527dc755 --- /dev/null +++ b/testing/mozbase/mozproxy/MANIFEST.in @@ -0,0 +1 @@ +recursive-include mozproxy * diff --git a/testing/mozbase/mozproxy/mozproxy/__init__.py b/testing/mozbase/mozproxy/mozproxy/__init__.py new file mode 100644 index 0000000000..4790eb73bf --- /dev/null +++ b/testing/mozbase/mozproxy/mozproxy/__init__.py @@ -0,0 +1,45 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import +import sys +import os + + +def path_join(*args): + path = os.path.join(*args) + return os.path.abspath(path) + + +mozproxy_src_dir = os.path.dirname(os.path.realpath(__file__)) +mozproxy_dir = path_join(mozproxy_src_dir, "..") +mozbase_dir = path_join(mozproxy_dir, "..") + +# needed so unit tests can find their imports +if os.environ.get("SCRIPTSPATH", None) is not None: + # in production it is env SCRIPTS_PATH + mozharness_dir = os.environ["SCRIPTSPATH"] +else: + # locally it's in source tree + mozharness_dir = path_join(mozbase_dir, "..", "mozharness") + + +def get_playback(config): + """Returns an instance of the right Playback class""" + sys.path.insert(0, mozharness_dir) + sys.path.insert(0, mozproxy_dir) + sys.path.insert(0, mozproxy_src_dir) + + from .server import get_backend + from .utils import LOG + + tool_name = config.get("playback_tool", None) + if tool_name is None: + LOG.critical("playback_tool name not found in config") + return None + try: + return get_backend(tool_name, config) + except KeyError: + LOG.critical("specified playback tool is unsupported: %s" % tool_name) + return None diff --git a/testing/mozbase/mozproxy/mozproxy/backends/__init__.py b/testing/mozbase/mozproxy/mozproxy/backends/__init__.py new file mode 100644 index 0000000000..6fbe8159b2 --- /dev/null +++ b/testing/mozbase/mozproxy/mozproxy/backends/__init__.py @@ -0,0 +1,3 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. diff --git a/testing/mozbase/mozproxy/mozproxy/backends/base.py b/testing/mozbase/mozproxy/mozproxy/backends/base.py new file mode 100644 index 0000000000..dad6383e0d --- /dev/null +++ b/testing/mozbase/mozproxy/mozproxy/backends/base.py @@ -0,0 +1,37 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import +import six + +from abc import ABCMeta, abstractmethod + + +# abstract class for all playback tools +@six.add_metaclass(ABCMeta) +class Playback(object): + def __init__(self, config): + self.config = config + self.host = None + self.port = None + + @abstractmethod + def download(self): + pass + + @abstractmethod + def setup(self): + pass + + @abstractmethod + def start(self): + pass + + @abstractmethod + def stop(self): + pass + + @abstractmethod + def confidence(self): + pass diff --git a/testing/mozbase/mozproxy/mozproxy/backends/mitm/__init__.py b/testing/mozbase/mozproxy/mozproxy/backends/mitm/__init__.py new file mode 100644 index 0000000000..e82b658292 --- /dev/null +++ b/testing/mozbase/mozproxy/mozproxy/backends/mitm/__init__.py @@ -0,0 +1,8 @@ +# flake8: noqa +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import + +from .mitm import * diff --git a/testing/mozbase/mozproxy/mozproxy/backends/mitm/android.py b/testing/mozbase/mozproxy/mozproxy/backends/mitm/android.py new file mode 100644 index 0000000000..bdd2921728 --- /dev/null +++ b/testing/mozbase/mozproxy/mozproxy/backends/mitm/android.py @@ -0,0 +1,228 @@ +"""Functions to download, install, setup, and use the mitmproxy playback tool""" +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +from __future__ import absolute_import + +import glob +import os +import subprocess +import sys +from subprocess import PIPE + +import mozinfo +from mozproxy.backends.mitm.mitm import Mitmproxy +from mozproxy.utils import download_file_from_url, tooltool_download, LOG + +# path for mitmproxy certificate, generated auto after mitmdump is started +# on local machine it is 'HOME', however it is different on production machines +try: + DEFAULT_CERT_PATH = os.path.join( + os.getenv("HOME"), ".mitmproxy", "mitmproxy-ca-cert.cer" + ) +except Exception: + DEFAULT_CERT_PATH = os.path.join( + os.getenv("HOMEDRIVE"), + os.getenv("HOMEPATH"), + ".mitmproxy", + "mitmproxy-ca-cert.cer", + ) + +# On Windows, deal with mozilla-build having forward slashes in $HOME: +if os.name == "nt" and "/" in DEFAULT_CERT_PATH: + DEFAULT_CERT_PATH = DEFAULT_CERT_PATH.replace("/", "\\") + + +class MitmproxyAndroid(Mitmproxy): + def setup(self): + self.download_and_install_host_utils() + self.install_mitmproxy_cert(self.browser_path) + + def download_and_install_host_utils(self): + """ + If running locally: + 1. Will use the `certutil` tool from the local Firefox desktop build + + If running in production: + 1. Get the tooltools manifest file for downloading hostutils (contains certutil) + 2. Get the `certutil` tool by downloading hostutils using the tooltool manifest + """ + if self.config["run_local"]: + # when running locally, it is found in the Firefox desktop build (..obj../dist/bin) + self.certutil_path = os.path.join(os.environ["MOZ_HOST_BIN"], "certutil") + if not ( + os.path.isfile(self.certutil_path) + and os.access(self.certutil_path, os.X_OK) + ): + raise Exception( + "Abort: unable to execute certutil: {}".format(self.certutil_path) + ) + self.certutil_path = os.environ["MOZ_HOST_BIN"] + os.environ["LD_LIBRARY_PATH"] = self.certutil_path + else: + # must download certutil inside hostutils via tooltool; use this manifest: + # mozilla-central/testing/config/tooltool-manifests/linux64/hostutils.manifest + # after it will be found here inside the worker/bitbar container: + # /builds/worker/workspace/build/hostutils/host-utils-66.0a1.en-US.linux-x86_64 + LOG.info("downloading certutil binary (hostutils)") + + # get path to the hostutils tooltool manifest; was set earlier in + # mozharness/configs/raptor/android_hw_config.py, to the path i.e. + # mozilla-central/testing/config/tooltool-manifests/linux64/hostutils.manifest + # the bitbar container is always linux64 + if os.environ.get("GECKO_HEAD_REPOSITORY", None) is None: + raise Exception("Abort: unable to get GECKO_HEAD_REPOSITORY") + + if os.environ.get("GECKO_HEAD_REV", None) is None: + raise Exception("Abort: unable to get GECKO_HEAD_REV") + + if os.environ.get("HOSTUTILS_MANIFEST_PATH", None) is not None: + manifest_url = os.path.join( + os.environ["GECKO_HEAD_REPOSITORY"], + "raw-file", + os.environ["GECKO_HEAD_REV"], + os.environ["HOSTUTILS_MANIFEST_PATH"], + ) + else: + raise Exception("Abort: unable to get HOSTUTILS_MANIFEST_PATH!") + + # first need to download the hostutils tooltool manifest file itself + _dest = os.path.join(self.mozproxy_dir, "hostutils.manifest") + have_manifest = download_file_from_url(manifest_url, _dest) + if not have_manifest: + raise Exception("failed to download the hostutils tooltool manifest") + + # now use the manifest to download hostutils so we can get certutil + tooltool_download(_dest, self.config["run_local"], self.mozproxy_dir) + + # the production bitbar container host is always linux + self.certutil_path = glob.glob( + os.path.join(self.mozproxy_dir, "host-utils*[!z|checksum]") + )[0] + + # must add hostutils/certutil to the path + os.environ["LD_LIBRARY_PATH"] = self.certutil_path + + bin_suffix = mozinfo.info.get("bin_suffix", "") + self.certutil_path = os.path.join(self.certutil_path, "certutil" + bin_suffix) + if os.path.isfile(self.certutil_path): + LOG.info("certutil is found at: %s" % self.certutil_path) + else: + raise Exception("unable to find certutil at %s" % self.certutil_path) + + def install_mitmproxy_cert(self, browser_path): + """Install the CA certificate generated by mitmproxy, into geckoview android + + 1. Create an NSS certificate database in the geckoview browser profile dir, only + if it doesn't already exist. Use this certutil command: + `certutil -N -d sql:<path to profile> --empty-password` + 2. Import the mitmproxy certificate into the database, i.e.: + `certutil -A -d sql:<path to profile> -n "some nickname" -t TC,, -a -i <path to CA.pem>` + """ + + cert_db_location = "sql:%s/" % self.config["local_profile_dir"] + + if not self.cert_db_exists(cert_db_location): + self.create_cert_db(cert_db_location) + + # DEFAULT_CERT_PATH has local path and name of mitmproxy cert i.e. + # /home/cltbld/.mitmproxy/mitmproxy-ca-cert.cer + self.import_certificate_in_cert_db(cert_db_location, DEFAULT_CERT_PATH) + + # cannot continue if failed to add CA cert to Firefox, need to check + if not self.is_mitmproxy_cert_installed(cert_db_location): + LOG.error("Aborting: failed to install mitmproxy CA cert into Firefox") + self.stop_mitmproxy_playback() + sys.exit() + + def import_certificate_in_cert_db(self, cert_db_location, local_cert_path): + # import mitmproxy cert into the db + args = [ + "-A", + "-d", + cert_db_location, + "-n", + "mitmproxy-cert", + "-t", + "TC,,", + "-a", + "-i", + local_cert_path, + ] + LOG.info("importing mitmproxy cert into db using command") + self.certutil(args) + + def create_cert_db(self, cert_db_location): + # create cert db if it doesn't already exist; it may exist already + # if a previous pageload test ran in the same test suite + args = ["-d", cert_db_location, "-N", "--empty-password"] + + LOG.info("creating nss cert database") + + self.certutil(args) + + if not self.cert_db_exists(cert_db_location): + raise Exception("nss cert db creation command failed. Cert db not created.") + + def cert_db_exists(self, cert_db_location): + # check if the nss ca cert db already exists in the device profile + LOG.info( + "checking if the nss cert db already exists in the android browser profile" + ) + + args = ["-d", cert_db_location, "-L"] + cert_db_exists = self.certutil(args, raise_exception=False) + + if cert_db_exists: + LOG.info("the nss cert db exists") + return True + else: + LOG.info("nss cert db doesn't exist yet.") + return False + + def is_mitmproxy_cert_installed(self, cert_db_location): + """Verify mitmxproy CA cert was added to Firefox on android""" + LOG.info("verifying that the mitmproxy ca cert is installed on android") + + # list the certifcates that are in the nss cert db (inside the browser profile dir) + LOG.info( + "getting the list of certs in the nss cert db in the android browser profile" + ) + args = ["-d", cert_db_location, "-L"] + + cmd_output = self.certutil(args) + + if "mitmproxy-cert" in cmd_output: + LOG.info( + "verfied the mitmproxy-cert is installed in the nss cert db on android" + ) + return True + return False + + def certutil(self, args, raise_exception=True): + cmd = [self.certutil_path] + list(args) + LOG.info("Certutil: Running command: %s" % " ".join(cmd)) + try: + cmd_proc = subprocess.Popen( + cmd, stdout=PIPE, stderr=PIPE, env=os.environ.copy() + ) + + cmd_output, errs = cmd_proc.communicate() + except subprocess.SubprocessError: + LOG.critical("could not run the certutil command") + raise + + if cmd_proc.returncode == 0: + # Debug purpose only remove if stable + LOG.info("Certutil returncode: %s" % cmd_proc.returncode) + LOG.info("Certutil output: %s" % cmd_output) + return cmd_output + else: + if raise_exception: + LOG.critical("Certutil command failed!!") + LOG.info("Certutil returncode: %s" % cmd_proc.returncode) + LOG.info("Certutil output: %s" % cmd_output) + LOG.info("Certutil error: %s" % errs) + raise Exception("Certutil command failed!!") + else: + return False diff --git a/testing/mozbase/mozproxy/mozproxy/backends/mitm/desktop.py b/testing/mozbase/mozproxy/mozproxy/backends/mitm/desktop.py new file mode 100644 index 0000000000..b8011edf2d --- /dev/null +++ b/testing/mozbase/mozproxy/mozproxy/backends/mitm/desktop.py @@ -0,0 +1,162 @@ +"""Functions to download, install, setup, and use the mitmproxy playback tool""" +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +from __future__ import absolute_import + +import os +import sys + +import mozinfo +from mozproxy.backends.mitm.mitm import Mitmproxy +from mozproxy.utils import LOG + +# to install mitmproxy certificate into Firefox and turn on/off proxy +POLICIES_CONTENT_ON = """{ + "policies": { + "Certificates": { + "Install": ["%(cert)s"] + }, + "Proxy": { + "Mode": "manual", + "HTTPProxy": "%(host)s:%(port)d", + "SSLProxy": "%(host)s:%(port)d", + "Passthrough": "%(host)s", + "Locked": true + } + } +}""" + +POLICIES_CONTENT_OFF = """{ + "policies": { + "Proxy": { + "Mode": "none", + "Locked": false + } + } +}""" + +# path for mitmproxy certificate, generated auto after mitmdump is started +# on local machine it is 'HOME', however it is different on production machines +try: + DEFAULT_CERT_PATH = os.path.join( + os.getenv("HOME"), ".mitmproxy", "mitmproxy-ca-cert.cer" + ) +except Exception: + DEFAULT_CERT_PATH = os.path.join( + os.getenv("HOMEDRIVE"), + os.getenv("HOMEPATH"), + ".mitmproxy", + "mitmproxy-ca-cert.cer", + ) + +# On Windows, deal with mozilla-build having forward slashes in $HOME: +if os.name == "nt" and "/" in DEFAULT_CERT_PATH: + DEFAULT_CERT_PATH = DEFAULT_CERT_PATH.replace("/", "\\") + + +class MitmproxyDesktop(Mitmproxy): + def setup(self): + """ + Installs certificates. + + For Firefox we need to install the generated mitmproxy CA cert. For + Chromium this is not necessary as it will be started with the + --ignore-certificate-errors cmd line arg. + """ + if not self.config["app"] == "firefox": + return + # install the generated CA certificate into Firefox desktop + self.install_mitmproxy_cert(self.browser_path) + + def install_mitmproxy_cert(self, browser_path): + """Install the CA certificate generated by mitmproxy, into Firefox + 1. Create a dir called 'distribution' in the same directory as the Firefox executable + 2. Create the policies.json file inside that folder; which points to the certificate + location, and turns on the the browser proxy settings + """ + LOG.info("Installing mitmproxy CA certficate into Firefox") + + # browser_path is the exe, we want the folder + self.policies_dir = os.path.dirname(browser_path) + # on macosx we need to remove the last folders 'MacOS' + # and the policies json needs to go in ../Content/Resources/ + if "mac" in self.config["platform"]: + self.policies_dir = os.path.join(self.policies_dir[:-6], "Resources") + # for all platforms the policies json goes in a 'distribution' dir + self.policies_dir = os.path.join(self.policies_dir, "distribution") + + self.cert_path = DEFAULT_CERT_PATH + # for windows only + if mozinfo.os == "win": + self.cert_path = self.cert_path.replace("\\", "\\\\") + + if not os.path.exists(self.policies_dir): + LOG.info("creating folder: %s" % self.policies_dir) + os.makedirs(self.policies_dir) + else: + LOG.info("folder already exists: %s" % self.policies_dir) + + self.write_policies_json( + self.policies_dir, + policies_content=POLICIES_CONTENT_ON + % {"cert": self.cert_path, "host": self.host, "port": self.port}, + ) + + # cannot continue if failed to add CA cert to Firefox, need to check + if not self.is_mitmproxy_cert_installed(): + LOG.error( + "Aborting: failed to install mitmproxy CA cert into Firefox desktop" + ) + self.stop_mitmproxy_playback() + sys.exit() + + def write_policies_json(self, location, policies_content): + policies_file = os.path.join(location, "policies.json") + LOG.info("writing: %s" % policies_file) + + with open(policies_file, "w") as fd: + fd.write(policies_content) + + def read_policies_json(self, location): + policies_file = os.path.join(location, "policies.json") + LOG.info("reading: %s" % policies_file) + + with open(policies_file, "r") as fd: + return fd.read() + + def is_mitmproxy_cert_installed(self): + """Verify mitmxproy CA cert was added to Firefox""" + try: + # read autoconfig file, confirm mitmproxy cert is in there + contents = self.read_policies_json(self.policies_dir) + LOG.info("Firefox policies file contents:") + LOG.info(contents) + if ( + POLICIES_CONTENT_ON + % {"cert": self.cert_path, "host": self.host, "port": self.port} + ) in contents: + LOG.info("Verified mitmproxy CA certificate is installed in Firefox") + else: + + return False + except Exception as e: + LOG.info("failed to read Firefox policies file, exeption: %s" % e) + return False + return True + + def stop(self): + LOG.info("MitmproxyDesktop stop!!") + super(MitmproxyDesktop, self).stop() + self.turn_off_browser_proxy() + + def turn_off_browser_proxy(self): + """Turn off the browser proxy that was used for mitmproxy playback. In Firefox + we need to change the autoconfig files to revert the proxy; for Chromium the proxy + was setup on the cmd line, so nothing is required here.""" + if self.config["app"] == "firefox" and self.policies_dir is not None: + LOG.info("Turning off the browser proxy") + + self.write_policies_json( + self.policies_dir, policies_content=POLICIES_CONTENT_OFF + ) diff --git a/testing/mozbase/mozproxy/mozproxy/backends/mitm/manifests/mitmproxy-rel-bin-4.0.4-linux64.manifest b/testing/mozbase/mozproxy/mozproxy/backends/mitm/manifests/mitmproxy-rel-bin-4.0.4-linux64.manifest new file mode 100644 index 0000000000..24419f001d --- /dev/null +++ b/testing/mozbase/mozproxy/mozproxy/backends/mitm/manifests/mitmproxy-rel-bin-4.0.4-linux64.manifest @@ -0,0 +1,10 @@ +[ + { + "size": 21805767, + "visibility": "public", + "digest": "69b314f9189b1c09e5412680869c03d07510b98b34e309f0f1b6961e5d8963f935994fe2f5ca86827f08631cbb271292a72aba2f0973b144832c2e7049a464a8", + "algorithm": "sha512", + "filename": "mitmproxy-4.0.4-linux.tar.gz", + "unpack": true + } +] diff --git a/testing/mozbase/mozproxy/mozproxy/backends/mitm/manifests/mitmproxy-rel-bin-4.0.4-osx.manifest b/testing/mozbase/mozproxy/mozproxy/backends/mitm/manifests/mitmproxy-rel-bin-4.0.4-osx.manifest new file mode 100644 index 0000000000..b809e5bbdb --- /dev/null +++ b/testing/mozbase/mozproxy/mozproxy/backends/mitm/manifests/mitmproxy-rel-bin-4.0.4-osx.manifest @@ -0,0 +1,10 @@ +[ + { + "size": 36939029, + "visibility": "public", + "digest": "3f933b142d7afb7cd409f436bc1151cc44c6f207368644ebe9fc9015607448b2c2d52dad4e5c71595bb6aa8accf4869e534f8db0ff6144725fad8f98daf50b40", + "algorithm": "sha512", + "filename": "mitmproxy-4.0.4-osx.tar.gz", + "unpack": true + } +] diff --git a/testing/mozbase/mozproxy/mozproxy/backends/mitm/manifests/mitmproxy-rel-bin-4.0.4-win.manifest b/testing/mozbase/mozproxy/mozproxy/backends/mitm/manifests/mitmproxy-rel-bin-4.0.4-win.manifest new file mode 100644 index 0000000000..f090ca3a6f --- /dev/null +++ b/testing/mozbase/mozproxy/mozproxy/backends/mitm/manifests/mitmproxy-rel-bin-4.0.4-win.manifest @@ -0,0 +1,10 @@ +[ + { + "size": 35903223, + "visibility": "public", + "digest": "cf5a2bd056ae655d4db0a953ec1bf80229990f449672ae9b636568540fae192a1c361c7742f1e2a8b560426a11e69955358b6a37445f37117dfcac154ef84713", + "algorithm": "sha512", + "filename": "mitmproxy-4.0.4-windows.zip", + "unpack": true + } +] diff --git a/testing/mozbase/mozproxy/mozproxy/backends/mitm/manifests/mitmproxy-rel-bin-5.0.1-linux64.manifest b/testing/mozbase/mozproxy/mozproxy/backends/mitm/manifests/mitmproxy-rel-bin-5.0.1-linux64.manifest new file mode 100644 index 0000000000..cfbbd667fd --- /dev/null +++ b/testing/mozbase/mozproxy/mozproxy/backends/mitm/manifests/mitmproxy-rel-bin-5.0.1-linux64.manifest @@ -0,0 +1,10 @@ +[ + { + "size": 76345093, + "visibility": "public", + "digest": "4d653c0c74a8677e8e78cd72d109b1b54c75ef57b2e2ce980c5cfd602966c310065cf0e95c35f4fbfb1fe817f062ac6cf9cc129d72d42184b0237fb9b0bde081", + "algorithm": "sha512", + "filename": "mitmproxy-5.0.1-linux.tar.gz", + "unpack": true + } +] diff --git a/testing/mozbase/mozproxy/mozproxy/backends/mitm/manifests/mitmproxy-rel-bin-5.0.1-osx.manifest b/testing/mozbase/mozproxy/mozproxy/backends/mitm/manifests/mitmproxy-rel-bin-5.0.1-osx.manifest new file mode 100644 index 0000000000..2a97c84eed --- /dev/null +++ b/testing/mozbase/mozproxy/mozproxy/backends/mitm/manifests/mitmproxy-rel-bin-5.0.1-osx.manifest @@ -0,0 +1,10 @@ +[ + { + "size": 41341221, + "visibility": "public", + "digest": "4624bc26638cd7f3ab8c8d2ee8ab44ab81018114714a2b82614883adbfad540de73a97455fb228a3003006b4d03a8a9a597d70f8b27ccf1718f78380f911bdd8", + "algorithm": "sha512", + "filename": "mitmproxy-5.0.1-osx.tar.gz", + "unpack": true + } +] diff --git a/testing/mozbase/mozproxy/mozproxy/backends/mitm/manifests/mitmproxy-rel-bin-5.0.1-win.manifest b/testing/mozbase/mozproxy/mozproxy/backends/mitm/manifests/mitmproxy-rel-bin-5.0.1-win.manifest new file mode 100644 index 0000000000..2d7d60715a --- /dev/null +++ b/testing/mozbase/mozproxy/mozproxy/backends/mitm/manifests/mitmproxy-rel-bin-5.0.1-win.manifest @@ -0,0 +1,10 @@ +[ + { + "size": 37875625, + "visibility": "public", + "digest": "d66234c9ca692d03412dd194b3f47c098e8ee1b17b178034fc86e0d8ada4d4f6cd1fbcfb62b7e7016539a878b1274ef83451a0acaca7011efaacb291fa52918d", + "algorithm": "sha512", + "filename": "mitmproxy-5.0.1-windows.zip", + "unpack": true + } +] diff --git a/testing/mozbase/mozproxy/mozproxy/backends/mitm/manifests/mitmproxy-rel-bin-5.1.1-linux64.manifest b/testing/mozbase/mozproxy/mozproxy/backends/mitm/manifests/mitmproxy-rel-bin-5.1.1-linux64.manifest new file mode 100644 index 0000000000..3022d190bb --- /dev/null +++ b/testing/mozbase/mozproxy/mozproxy/backends/mitm/manifests/mitmproxy-rel-bin-5.1.1-linux64.manifest @@ -0,0 +1,10 @@ +[ + { + "size": 24989250, + "visibility": "public", + "digest": "75ea9d024cc9138e5bc993c292ec65c2a40ce4e382f01c41ef865ddfac6f1553ee0e73e3ff2003234d6ecab2d3cdbbd38809cc7e62bbcabaed521a623575e2b8", + "algorithm": "sha512", + "filename": "mitmproxy-5.1.1-linux.tar.gz", + "unpack": true + } +] diff --git a/testing/mozbase/mozproxy/mozproxy/backends/mitm/manifests/mitmproxy-rel-bin-5.1.1-osx.manifest b/testing/mozbase/mozproxy/mozproxy/backends/mitm/manifests/mitmproxy-rel-bin-5.1.1-osx.manifest new file mode 100644 index 0000000000..a039c71d27 --- /dev/null +++ b/testing/mozbase/mozproxy/mozproxy/backends/mitm/manifests/mitmproxy-rel-bin-5.1.1-osx.manifest @@ -0,0 +1,10 @@ +[ + { + "size": 13178683, + "visibility": "public", + "digest": "efbc90674a85165e9065e70a8f0c2d4e42688a045130546c2d5721cdb4c54637f7d2b9f4e6a6953a37f94d0edb5bf128c25d7a628246a75d4ad7ba6c884589a9", + "algorithm": "sha512", + "filename": "mitmproxy-5.1.1-osx.tar.gz", + "unpack": true + } +] diff --git a/testing/mozbase/mozproxy/mozproxy/backends/mitm/manifests/mitmproxy-rel-bin-5.1.1-win.manifest b/testing/mozbase/mozproxy/mozproxy/backends/mitm/manifests/mitmproxy-rel-bin-5.1.1-win.manifest new file mode 100644 index 0000000000..d0937a1bed --- /dev/null +++ b/testing/mozbase/mozproxy/mozproxy/backends/mitm/manifests/mitmproxy-rel-bin-5.1.1-win.manifest @@ -0,0 +1,10 @@ +[ + { + "size": 18019253, + "visibility": "public", + "digest": "69470f6a58c50912072ab6a911ac3e266d5ec8c9410c8aa4bad76cd7f61bca640e748fd682379ef1523f12da2a8c7a9c67d5bbae5f6d6fa164c2c6b9765b79c1", + "algorithm": "sha512", + "filename": "mitmproxy-5.1.1-windows.zip", + "unpack": true + } +] diff --git a/testing/mozbase/mozproxy/mozproxy/backends/mitm/mitm.py b/testing/mozbase/mozproxy/mozproxy/backends/mitm/mitm.py new file mode 100644 index 0000000000..991b4ebced --- /dev/null +++ b/testing/mozbase/mozproxy/mozproxy/backends/mitm/mitm.py @@ -0,0 +1,490 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +from __future__ import absolute_import + +import json +import os +import signal +import socket +import sys +import time + +import mozinfo +import six +from mozprocess import ProcessHandler +from mozproxy.backends.base import Playback +from mozproxy.recordings import RecordingFile +from mozproxy.utils import ( + download_file_from_url, + transform_platform, + tooltool_download, + get_available_port, + LOG, +) + +here = os.path.dirname(__file__) +mitm_folder = os.path.dirname(os.path.realpath(__file__)) + + +def normalize_path(path): + path = os.path.normpath(path) + if mozinfo.os == "win": + return path.replace("\\", "\\\\\\") + return path + + +# maximal allowed runtime of a mitmproxy command +MITMDUMP_COMMAND_TIMEOUT = 30 + + +class Mitmproxy(Playback): + def __init__(self, config): + self.config = config + + self.host = ( + "127.0.0.1" if "localhost" in self.config["host"] else self.config["host"] + ) + self.port = None + self.mitmproxy_proc = None + self.mitmdump_path = None + self.record_mode = config.get("record", False) + self.recording = None + self.playback_files = [] + + self.browser_path = "" + if config.get("binary", None): + self.browser_path = os.path.normpath(config.get("binary")) + + self.policies_dir = None + self.ignore_mitmdump_exit_failure = config.get( + "ignore_mitmdump_exit_failure", False + ) + + if self.record_mode: + if "recording_file" not in self.config: + LOG.error( + "recording_file value was not provided. Proxy service wont' start " + ) + raise Exception("Please provide a playback_files list.") + + if not isinstance(self.config.get("recording_file"), six.string_types): + LOG.error("recording_file argument type is not str!") + raise Exception("recording_file argument type invalid!") + + if not os.path.splitext(self.config.get("recording_file"))[1] == ".zip": + LOG.error( + "Recording file type (%s) should be a zip. " + "Please provide a valid file type!" + % self.config.get("recording_file") + ) + raise Exception("Recording file type should be a zip") + + if os.path.exists(self.config.get("recording_file")): + LOG.error( + "Recording file (%s) already exists." + "Please provide a valid file path!" + % self.config.get("recording_file") + ) + raise Exception("Recording file already exists.") + + if self.config.get("playback_files", False): + LOG.error("Record mode is True and playback_files where provided!") + raise Exception("playback_files specified during record!") + + if self.config.get("playback_version") is None: + LOG.error( + "mitmproxy was not provided with a 'playback_version' " + "Please provide a valid playback version" + ) + raise Exception("playback_version not specified!") + + # mozproxy_dir is where we will download all mitmproxy required files + # when running locally it comes from obj_path via mozharness/mach + if self.config.get("obj_path") is not None: + self.mozproxy_dir = self.config.get("obj_path") + else: + # in production it is ../tasks/task_N/build/, in production that dir + # is not available as an envvar, however MOZ_UPLOAD_DIR is set as + # ../tasks/task_N/build/blobber_upload_dir so take that and go up 1 level + self.mozproxy_dir = os.path.dirname( + os.path.dirname(os.environ["MOZ_UPLOAD_DIR"]) + ) + + self.mozproxy_dir = os.path.join(self.mozproxy_dir, "testing", "mozproxy") + self.upload_dir = os.environ.get("MOZ_UPLOAD_DIR", self.mozproxy_dir) + + LOG.info( + "mozproxy_dir used for mitmproxy downloads and exe files: %s" + % self.mozproxy_dir + ) + # setting up the MOZPROXY_DIR env variable so custom scripts know + # where to get the data + os.environ["MOZPROXY_DIR"] = self.mozproxy_dir + + LOG.info("Playback tool: %s" % self.config["playback_tool"]) + LOG.info("Playback tool version: %s" % self.config["playback_version"]) + + def download_mitm_bin(self): + # Download and setup mitm binaries + + manifest = os.path.join( + here, + "manifests", + "mitmproxy-rel-bin-%s-{platform}.manifest" + % self.config["playback_version"], + ) + transformed_manifest = transform_platform(manifest, self.config["platform"]) + + # generate the mitmdump_path + self.mitmdump_path = os.path.normpath( + os.path.join( + self.mozproxy_dir, + "mitmdump-%s" % self.config["playback_version"], + "mitmdump", + ) + ) + + # Check if mitmproxy bin exists + if os.path.exists(self.mitmdump_path): + LOG.info("mitmproxy binary already exists. Skipping download") + else: + # Download and unpack mitmproxy binary + download_path = os.path.dirname(self.mitmdump_path) + LOG.info("create mitmproxy %s dir" % self.config["playback_version"]) + if not os.path.exists(download_path): + os.makedirs(download_path) + + LOG.info("downloading mitmproxy binary") + tooltool_download( + transformed_manifest, self.config["run_local"], download_path + ) + + def download_manifest_file(self, manifest_path): + # Manifest File + # we use one pageset for all platforms + LOG.info("downloading mitmproxy pageset") + + tooltool_download(manifest_path, self.config["run_local"], self.mozproxy_dir) + + with open(manifest_path) as manifest_file: + manifest = json.load(manifest_file) + for file in manifest: + zip_path = os.path.join(self.mozproxy_dir, file["filename"]) + LOG.info("Adding %s to recording list" % zip_path) + self.playback_files.append(RecordingFile(zip_path)) + + def download_playback_files(self): + # Detect type of file from playback_files and download accordingly + if "playback_files" not in self.config: + LOG.error( + "playback_files value was not provided. Proxy service wont' start " + ) + raise Exception("Please provide a playback_files list.") + + if not isinstance(self.config["playback_files"], list): + LOG.error("playback_files should be a list") + raise Exception("playback_files should be a list") + + for playback_file in self.config["playback_files"]: + + if playback_file.startswith("https://") and "mozilla.com" in playback_file: + # URL provided + dest = os.path.join(self.mozproxy_dir, os.path.basename(playback_file)) + download_file_from_url(playback_file, self.mozproxy_dir, extract=False) + # Add Downloaded file to playback_files list + LOG.info("Adding %s to recording list" % dest) + self.playback_files.append(RecordingFile(dest)) + continue + + if not os.path.exists(playback_file): + LOG.error( + "Zip or manifest file path (%s) does not exist. Please provide a valid path!" + % playback_file + ) + raise Exception("Zip or manifest file path does not exist") + + if os.path.splitext(playback_file)[1] == ".zip": + # zip file path provided + LOG.info("Adding %s to recording list" % playback_file) + self.playback_files.append(RecordingFile(playback_file)) + elif os.path.splitext(playback_file)[1] == ".manifest": + # manifest file path provided + self.download_manifest_file(playback_file) + + def download(self): + """Download and unpack mitmproxy binary and pageset using tooltool""" + if not os.path.exists(self.mozproxy_dir): + os.makedirs(self.mozproxy_dir) + + self.download_mitm_bin() + + if self.record_mode: + self.recording = RecordingFile(self.config["recording_file"]) + else: + self.download_playback_files() + + def stop(self): + LOG.info("Mitmproxy stop!!") + self.stop_mitmproxy_playback() + if self.record_mode: + LOG.info("Record mode ON. Generating zip file ") + self.recording.generate_zip_file() + + def wait(self, timeout=1): + """Wait until the mitmproxy process has terminated.""" + # We wait using this method to allow Windows to respond to the Ctrl+Break + # signal so that we can exit cleanly from the command-line driver. + while True: + returncode = self.mitmproxy_proc.wait(timeout) + if returncode is not None: + return returncode + + def start(self): + # go ahead and download and setup mitmproxy + self.download() + + # mitmproxy must be started before setup, so that the CA cert is available + self.start_mitmproxy_playback(self.mitmdump_path, self.browser_path) + + # In case the setup fails, we want to stop the process before raising. + try: + self.setup() + except Exception: + try: + self.stop() + except Exception: + LOG.error("MitmProxy failed to STOP.", exc_info=True) + LOG.error("Setup of MitmProxy failed.", exc_info=True) + raise + + def start_mitmproxy_playback(self, mitmdump_path, browser_path): + """Startup mitmproxy and replay the specified flow file""" + if self.mitmproxy_proc is not None: + raise Exception("Proxy already started.") + self.port = get_available_port() + + LOG.info("mitmdump path: %s" % mitmdump_path) + LOG.info("browser path: %s" % browser_path) + + # mitmproxy needs some DLL's that are a part of Firefox itself, so add to path + env = os.environ.copy() + env["PATH"] = os.path.dirname(browser_path) + os.pathsep + env["PATH"] + command = [mitmdump_path] + + # add proxy host and port options + command.extend(["--listen-host", self.host, "--listen-port", str(self.port)]) + + # record mode + if self.record_mode: + + # generate recording script paths + inject_deterministic = os.path.join( + mitm_folder, + "scripts", + "inject-deterministic.py", + ) + http_protocol_extractor = os.path.join( + mitm_folder, + "scripts", + "http_protocol_extractor.py", + ) + + args = [ + "--save-stream-file", + normalize_path(self.recording.recording_path), + "--set", + "websocket=false", + "--scripts", + inject_deterministic, + "--scripts", + http_protocol_extractor, + ] + command.extend(args) + else: + # playback mode + if len(self.playback_files) > 0: + script = os.path.join( + mitm_folder, + "scripts", + "alternate-server-replay.py", + ) + + if self.config["playback_version"] in ["4.0.4", "5.1.1"]: + args = [ + "-v", # Verbose mode + "--set", + "upstream_cert=false", + "--set", + "upload_dir=" + normalize_path(self.upload_dir), + "--set", + "websocket=false", + "--set", + "server_replay_files={}".format( + ",".join( + [ + normalize_path(playback_file.recording_path) + for playback_file in self.playback_files + ] + ) + ), + "--scripts", + normalize_path(script), + ] + command.extend(args) + else: + raise Exception("Mitmproxy version is unknown!") + + else: + raise Exception( + "Mitmproxy can't start playback! Playback settings missing." + ) + + # mitmproxy needs some DLL's that are a part of Firefox itself, so add to path + env = os.environ.copy() + if not os.path.dirname(self.browser_path) in env["PATH"]: + env["PATH"] = os.path.dirname(self.browser_path) + os.pathsep + env["PATH"] + + LOG.info("Starting mitmproxy playback using env path: %s" % env["PATH"]) + LOG.info("Starting mitmproxy playback using command: %s" % " ".join(command)) + # to turn off mitmproxy log output, use these params for Popen: + # Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env) + self.mitmproxy_proc = ProcessHandler( + command, + logfile=os.path.join(self.upload_dir, "mitmproxy.log"), + env=env, + processStderrLine=LOG.error, + storeOutput=False, + ) + self.mitmproxy_proc.run() + end_time = time.time() + MITMDUMP_COMMAND_TIMEOUT + + ready = False + while time.time() < end_time: + ready = self.check_proxy(host=self.host, port=self.port) + if ready: + LOG.info( + "Mitmproxy playback successfully started on %s:%d as pid %d" + % (self.host, self.port, self.mitmproxy_proc.pid) + ) + return + time.sleep(0.25) + + # cannot continue as we won't be able to playback the pages + LOG.error("Aborting: Mitmproxy process did not startup") + self.stop_mitmproxy_playback() + sys.exit(1) # XXX why do we need to do that? a raise is not enough? + + def stop_mitmproxy_playback(self): + """Stop the mitproxy server playback""" + if self.mitmproxy_proc is None or self.mitmproxy_proc.poll() is not None: + return + LOG.info( + "Stopping mitmproxy playback, killing process %d" % self.mitmproxy_proc.pid + ) + # On Windows, mozprocess brutally kills mitmproxy with TerminateJobObject + # The process has no chance to gracefully shutdown. + # Here, we send the process a break event to give it a chance to wrapup. + # See the signal handler in the alternate-server-replay-4.0.4.py script + if mozinfo.os == "win": + LOG.info("Sending CTRL_BREAK_EVENT to mitmproxy") + os.kill(self.mitmproxy_proc.pid, signal.CTRL_BREAK_EVENT) + time.sleep(2) + + exit_code = self.mitmproxy_proc.kill() + self.mitmproxy_proc = None + + if exit_code != 0: + if exit_code is None: + LOG.error("Failed to kill the mitmproxy playback process") + return + + if mozinfo.os == "win": + from mozprocess.winprocess import ERROR_CONTROL_C_EXIT # noqa + + if exit_code == ERROR_CONTROL_C_EXIT: + LOG.info( + "Successfully killed the mitmproxy playback process" + " with exit code %d" % exit_code + ) + return + log_func = LOG.error + if self.ignore_mitmdump_exit_failure: + log_func = LOG.info + log_func("Mitmproxy exited with error code %d" % exit_code) + else: + LOG.info("Successfully killed the mitmproxy playback process") + + def check_proxy(self, host, port): + """Check that mitmproxy process is working by doing a socket call using the proxy settings + :param host: Host of the proxy server + :param port: Port of the proxy server + :return: True if the proxy service is working + """ + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + try: + s.connect((host, port)) + s.shutdown(socket.SHUT_RDWR) + s.close() + return True + except socket.error: + return False + + def confidence(self): + """Extract confidence metrics from the netlocs file + and convert them to perftest results + """ + if len(self.playback_files) == 0: + LOG.warning( + "Proxy service did not load a recording file. " + "Confidence metrics will nt be generated" + ) + return + + file_name = ( + "mitm_netlocs_%s.json" + % os.path.splitext(os.path.basename(self.playback_files[0].recording_path))[ + 0 + ] + ) + path = os.path.normpath(os.path.join(self.upload_dir, file_name)) + if os.path.exists(path): + try: + LOG.info("Reading confidence values from: %s" % path) + with open(path, "r") as f: + data = json.load(f) + return { + "replay-confidence": { + "values": data["replay-confidence"], + "subtest-prefix-type": False, + "unit": "%", + "shouldAlert": False, + "lowerIsBetter": False, + }, + "recording-proportion-used": { + "values": data["recording-proportion-used"], + "subtest-prefix-type": False, + "unit": "%", + "shouldAlert": False, + "lowerIsBetter": False, + }, + "not-replayed": { + "values": data["not-replayed"], + "subtest-prefix-type": False, + "shouldAlert": False, + "unit": "a.u.", + }, + "replayed": { + "values": data["replayed"], + "subtest-prefix-type": False, + "unit": "a.u.", + "shouldAlert": False, + "lowerIsBetter": False, + }, + } + except Exception: + LOG.info("Can't read netlocs file!", exc_info=True) + return None + else: + LOG.info("Netlocs file is not available! Cant find %s" % path) + return None diff --git a/testing/mozbase/mozproxy/mozproxy/backends/mitm/mitmproxy_requirements.txt b/testing/mozbase/mozproxy/mozproxy/backends/mitm/mitmproxy_requirements.txt new file mode 100644 index 0000000000..7d9842b7d6 --- /dev/null +++ b/testing/mozbase/mozproxy/mozproxy/backends/mitm/mitmproxy_requirements.txt @@ -0,0 +1,35 @@ +argh==0.26.2 +asn1crypto==0.22.0 +blinker==1.4 +pycparser==2.17 +cffi==1.10.0 +brotlipy==0.6.0 +certifi==2017.4.17 +click==6.7 +construct==2.8.12 +cryptography==1.8.2 +cssutils==1.0.2 +EditorConfig==0.12.1 +h2==2.6.2 +hpack==3.0.0 +html2text==2016.9.19 +hyperframe==4.0.2 +idna==2.5 +jsbeautifier==1.6.12 +kaitaistruct==0.6 +mitmproxy==4.0.4 +packaging==16.8 +passlib==1.7.1 +pathtools==0.1.2 +pyasn1==0.2.3 +pyOpenSSL==16.2.0 +pyparsing==2.2.0 +pyperclip==1.5.27 +PyYAML==3.12 +requests==2.13.0 +ruamel.yaml==0.13.14 +six==1.13.0 +sortedcontainers==1.5.7 +tornado==4.4.3 +urwid==1.3.1 +watchdog==0.8.3 diff --git a/testing/mozbase/mozproxy/mozproxy/backends/mitm/scripts/__init__.py b/testing/mozbase/mozproxy/mozproxy/backends/mitm/scripts/__init__.py new file mode 100644 index 0000000000..0499f3a93f --- /dev/null +++ b/testing/mozbase/mozproxy/mozproxy/backends/mitm/scripts/__init__.py @@ -0,0 +1,4 @@ +#!/usr/bin/env python +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. diff --git a/testing/mozbase/mozproxy/mozproxy/backends/mitm/scripts/alternate-server-replay.py b/testing/mozbase/mozproxy/mozproxy/backends/mitm/scripts/alternate-server-replay.py new file mode 100644 index 0000000000..2fe64a66a7 --- /dev/null +++ b/testing/mozbase/mozproxy/mozproxy/backends/mitm/scripts/alternate-server-replay.py @@ -0,0 +1,308 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# This file was copied from mitmproxy/mitmproxy/addons/serverplayback.py release tag 4.0.4 +# and modified by Florin Strugariu + +# Altered features: +# * returns 404 rather than dropping the whole HTTP/2 connection on the floor +# * remove the replay packages that don't have any content in their response package +from __future__ import absolute_import, print_function + +import os +import json +import hashlib +from collections import defaultdict +import time +import signal + +import typing + +from six.moves import urllib +from mitmproxy import ctx, http +from mitmproxy import exceptions +from mitmproxy import io + +# PATCHING AREA - ALLOWS HTTP/2 WITH NO CERT SNIFFING +from mitmproxy.proxy.protocol import tls +from mitmproxy.proxy.protocol.http2 import Http2Layer, SafeH2Connection + +_PROTO = {} + + +@property +def _alpn(self): + proto = _PROTO.get(self.server_sni) + if proto is None: + return self.server_conn.get_alpn_proto_negotiated() + if proto.startswith("HTTP/2"): + return b"h2" + elif proto.startswith("HTTP/1"): + return b"h1" + return b"" + + +tls.TlsLayer.alpn_for_client_connection = _alpn + + +def _server_conn(self): + if not self.server_conn.connected() and self.server_conn not in self.connections: + # we can't use ctx.log in this layer + print("Ignored CONNECT call on upstream server") + return + if self.server_conn.connected(): + import h2.config + + config = h2.config.H2Configuration( + client_side=True, + header_encoding=False, + validate_outbound_headers=False, + validate_inbound_headers=False, + ) + self.connections[self.server_conn] = SafeH2Connection( + self.server_conn, config=config + ) + self.connections[self.server_conn].initiate_connection() + self.server_conn.send(self.connections[self.server_conn].data_to_send()) + + +Http2Layer._initiate_server_conn = _server_conn + + +def _remote_settings_changed(self, event, other_conn): + if other_conn not in self.connections: + # we can't use ctx.log in this layer + print("Ignored remote settings upstream") + return True + new_settings = dict( + [(key, cs.new_value) for (key, cs) in event.changed_settings.items()] + ) + self.connections[other_conn].safe_update_settings(new_settings) + return True + + +Http2Layer._handle_remote_settings_changed = _remote_settings_changed +# END OF PATCHING + + +class AlternateServerPlayback: + def __init__(self): + ctx.master.addons.remove(ctx.master.addons.get("serverplayback")) + self.flowmap = {} + self.configured = False + self.netlocs = defaultdict(lambda: defaultdict(int)) + self.calls = [] + self._done = False + self._replayed = 0 + self._not_replayed = 0 + self._recordings_used = 0 + self.mitm_version = ctx.mitmproxy.version.VERSION + + ctx.log.info("MitmProxy version: %s" % self.mitm_version) + + def load(self, loader): + loader.add_option( + "server_replay_files", + typing.Sequence[str], + [], + "Replay server responses from a saved file.", + ) + loader.add_option( + "upload_dir", + str, + "", + "Upload directory", + ) + + def load_flows(self, flows): + """ + Replay server responses from flows. + """ + for i in flows: + if i.type == "websocket": + # Mitmproxy can't replay WebSocket packages. + ctx.log.info( + "Recorded response is a WebSocketFlow. Removing from recording list as" + " WebSockets are disabled" + ) + elif i.response and self.mitm_version in ("4.0.2", "4.0.4", "5.1.1"): + # see: https://github.com/mitmproxy/mitmproxy/issues/3856 + f = self.flowmap.setdefault( + self._hash(i), {"flow": None, "reply_count": 0} + ) + # overwrite with new flow if already hashed + f["flow"] = i + else: + ctx.log.info( + "Recorded request %s has no response. Removing from recording list" + % i.request.url + ) + ctx.master.addons.trigger("update", []) + + def load_files(self, paths): + try: + if "," in paths[0]: + paths = paths[0].split(",") + for path in paths: + ctx.log.info("Loading flows from %s" % path) + if not os.path.exists(path): + raise Exception("File does not exist!") + try: + flows = io.read_flows_from_paths([path]) + except exceptions.FlowReadException as e: + raise exceptions.CommandError(str(e)) + self.load_flows(flows) + proto = os.path.join(os.path.dirname(path), "metadata.json") + if os.path.exists(proto): + ctx.log.info("Loading proto info from %s" % proto) + with open(proto) as f: + recording_info = json.loads(f.read()) + if recording_info.get("http_protocol", False): + ctx.log.info( + "Replaying file {} recorded on {}".format( + path, recording_info["recording_date"] + ) + ) + _PROTO.update(recording_info["http_protocol"]) + else: + ctx.log.warn( + "Replaying file {} has no http_protocol info.".format(proto) + ) + except Exception as e: + ctx.log.error("Could not load recording file! Stopping playback process!") + ctx.log.error(str(e)) + ctx.master.shutdown() + + def _hash(self, flow): + """ + Calculates a loose hash of the flow request. + """ + r = flow.request + + # unquote url + # See Bug 1509835 + _, _, path, _, query, _ = urllib.parse.urlparse(urllib.parse.unquote(r.url)) + queriesArray = urllib.parse.parse_qsl(query, keep_blank_values=True) + + key = [str(r.port), str(r.scheme), str(r.method), str(path)] + key.append(str(r.raw_content)) + key.append(r.host) + + for p in queriesArray: + key.append(p[0]) + key.append(p[1]) + + return hashlib.sha256(repr(key).encode("utf8", "surrogateescape")).digest() + + def next_flow(self, request): + """ + Returns the next flow object, or None if no matching flow was + found. + """ + hsh = self._hash(request) + if hsh in self.flowmap: + if self.flowmap[hsh]["reply_count"] == 0: + self._recordings_used += 1 + self.flowmap[hsh]["reply_count"] += 1 + # return the most recently added flow with this hash + return self.flowmap[hsh]["flow"] + + def configure(self, updated): + if not self.configured and ctx.options.server_replay_files: + self.configured = True + self.load_files(ctx.options.server_replay_files) + + def done(self): + if self._done or not ctx.options.upload_dir: + return + + replay_confidence = float(self._replayed) / ( + self._replayed + self._not_replayed + ) + recording_proportion_used = ( + 0 + if self._recordings_used == 0 + else float(self._recordings_used) / len(self.flowmap) + ) + stats = { + "totals": dict(self.netlocs), + "calls": self.calls, + "replayed": self._replayed, + "not-replayed": self._not_replayed, + "replay-confidence": int(replay_confidence * 100), + "recording-proportion-used": int(recording_proportion_used * 100), + } + file_name = ( + "mitm_netlocs_%s.json" + % os.path.splitext(os.path.basename(ctx.options.server_replay_files[0]))[0] + ) + path = os.path.normpath(os.path.join(ctx.options.upload_dir, file_name)) + try: + with open(path, "w") as f: + f.write(json.dumps(stats, indent=2, sort_keys=True)) + finally: + self._done = True + + def request(self, f): + if self.flowmap: + try: + rflow = self.next_flow(f) + if rflow: + response = rflow.response.copy() + response.is_replay = True + # Refresh server replay responses by adjusting date, expires and + # last-modified headers, as well as adjusting cookie expiration. + response.refresh() + + f.response = response + self._replayed += 1 + else: + # returns 404 rather than dropping the whole HTTP/2 connection + ctx.log.warn( + "server_playback: killed non-replay request {}".format( + f.request.url + ) + ) + f.response = http.HTTPResponse.make( + 404, b"", {"content-type": "text/plain"} + ) + self._not_replayed += 1 + + # collecting stats only if we can dump them (see .done()) + if ctx.options.upload_dir: + parsed_url = urllib.parse.urlparse( + urllib.parse.unquote(f.request.url) + ) + self.netlocs[parsed_url.netloc][f.response.status_code] += 1 + self.calls.append( + { + "time": str(time.time()), + "url": f.request.url, + "response_status": f.response.status_code, + } + ) + except Exception as e: + ctx.log.error("Could not generate response! Stopping playback process!") + ctx.log.info(e) + ctx.master.shutdown() + + else: + ctx.log.error("Playback library is empty! Stopping playback process!") + ctx.master.shutdown() + return + + +playback = AlternateServerPlayback() + +if hasattr(signal, "SIGBREAK"): + # allows the addon to dump the stats even if mitmproxy + # does not call done() like on windows termination + # for this, the parent process sends CTRL_BREAK_EVENT which + # is received as an SIGBREAK event + def _shutdown(sig, frame): + ctx.master.shutdown() + + signal.signal(signal.SIGBREAK, _shutdown) + +addons = [playback] diff --git a/testing/mozbase/mozproxy/mozproxy/backends/mitm/scripts/catapult/LICENSE b/testing/mozbase/mozproxy/mozproxy/backends/mitm/scripts/catapult/LICENSE new file mode 100644 index 0000000000..c992fe483c --- /dev/null +++ b/testing/mozbase/mozproxy/mozproxy/backends/mitm/scripts/catapult/LICENSE @@ -0,0 +1,27 @@ +Copyright 2015 The Chromium Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +* Neither the name of catapult nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + diff --git a/testing/mozbase/mozproxy/mozproxy/backends/mitm/scripts/catapult/deterministic.js b/testing/mozbase/mozproxy/mozproxy/backends/mitm/scripts/catapult/deterministic.js new file mode 100644 index 0000000000..d2e818ba00 --- /dev/null +++ b/testing/mozbase/mozproxy/mozproxy/backends/mitm/scripts/catapult/deterministic.js @@ -0,0 +1,71 @@ +// Copyright 2017 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + + +'use strict'; + +(function () { + var random_count = 0; + var random_count_threshold = 25; + var random_seed = 0.462; + Math.random = function() { + random_count++; + if (random_count > random_count_threshold){ + random_seed += 0.1; + random_count = 1; + } + return (random_seed % 1); + }; + if (typeof(crypto) == 'object' && + typeof(crypto.getRandomValues) == 'function') { + crypto.getRandomValues = function(arr) { + var scale = Math.pow(256, arr.BYTES_PER_ELEMENT); + for (var i = 0; i < arr.length; i++) { + arr[i] = Math.floor(Math.random() * scale); + } + return arr; + }; + } +})(); +(function () { + var date_count = 0; + var date_count_threshold = 25; + var orig_date = Date; + // Time since epoch in milliseconds. This is replaced by script injector with + // the date when the recording is done. + var time_seed = REPLACE_LOAD_TIMESTAMP; + Date = function() { + if (this instanceof Date) { + date_count++; + if (date_count > date_count_threshold){ + time_seed += 50; + date_count = 1; + } + switch (arguments.length) { + case 0: return new orig_date(time_seed); + case 1: return new orig_date(arguments[0]); + default: return new orig_date(arguments[0], arguments[1], + arguments.length >= 3 ? arguments[2] : 1, + arguments.length >= 4 ? arguments[3] : 0, + arguments.length >= 5 ? arguments[4] : 0, + arguments.length >= 6 ? arguments[5] : 0, + arguments.length >= 7 ? arguments[6] : 0); + } + } + return new Date().toString(); + }; + Date.__proto__ = orig_date; + Date.prototype = orig_date.prototype; + Date.prototype.constructor = Date; + orig_date.now = function() { + return new Date().getTime(); + }; + orig_date.prototype.getTimezoneOffset = function() { + var dst2010Start = 1268560800000; + var dst2010End = 1289120400000; + if (this.getTime() >= dst2010Start && this.getTime() < dst2010End) + return 420; + return 480; + }; +})(); diff --git a/testing/mozbase/mozproxy/mozproxy/backends/mitm/scripts/http_protocol_extractor.py b/testing/mozbase/mozproxy/mozproxy/backends/mitm/scripts/http_protocol_extractor.py new file mode 100644 index 0000000000..b2bc308bd8 --- /dev/null +++ b/testing/mozbase/mozproxy/mozproxy/backends/mitm/scripts/http_protocol_extractor.py @@ -0,0 +1,85 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +from __future__ import absolute_import, print_function + +import datetime +import hashlib +import json +import os +import urllib +from urllib import parse + + +class HttpProtocolExtractor: + def get_ctx(self): + from mitmproxy import ctx + + return ctx + + def load(self, loader): + self.ctx = self.get_ctx() + + self.request_protocol = {} + self.hashes = [] + self.request_count = 0 + + self.ctx.log.info("Init Http Protocol extractor JS") + + def _hash(self, flow): + """ + Calculates a loose hash of the flow request. + """ + r = flow.request + + # unquote url + # See Bug 1509835 + _, _, path, _, query, _ = urllib.parse.urlparse(parse.unquote(r.url)) + queriesArray = urllib.parse.parse_qsl(query, keep_blank_values=True) + + key = [str(r.port), str(r.scheme), str(r.method), str(path)] + key.append(str(r.raw_content)) + key.append(r.host) + + for p in queriesArray: + key.append(p[0]) + key.append(p[1]) + + return hashlib.sha256(repr(key).encode("utf8", "surrogateescape")).digest() + + def response(self, flow): + self.request_count += 1 + hash = self._hash(flow) + if hash not in self.hashes: + self.hashes.append(hash) + + if flow.type == "websocket": + self.ctx.log.info("Response is a WebSocketFlow. Bug 1559117") + else: + self.ctx.log.info( + "Response using protocol: %s" % flow.response.data.http_version + ) + self.request_protocol[ + urllib.parse.urlparse(flow.request.url).netloc + ] = flow.response.data.http_version.decode("utf-8") + + def done(self): + output_json = {} + + output_json["recording_date"] = str(datetime.datetime.now()) + output_json["http_protocol"] = self.request_protocol + output_json["recorded_requests"] = self.request_count + output_json["recorded_requests_unique"] = len(self.hashes) + + recording_file_name = self.ctx.options.save_stream_file + + json_file_name = os.path.join( + os.path.dirname(recording_file_name), + "%s.json" % os.path.basename(recording_file_name).split(".")[0], + ) + print("Saving response protocol data to %s" % json_file_name) + with open(json_file_name, "w") as file: + file.write(json.dumps(output_json)) + + +addons = [HttpProtocolExtractor()] diff --git a/testing/mozbase/mozproxy/mozproxy/backends/mitm/scripts/inject-deterministic.py b/testing/mozbase/mozproxy/mozproxy/backends/mitm/scripts/inject-deterministic.py new file mode 100644 index 0000000000..06bc017b1f --- /dev/null +++ b/testing/mozbase/mozproxy/mozproxy/backends/mitm/scripts/inject-deterministic.py @@ -0,0 +1,208 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +from __future__ import absolute_import, print_function +import base64 +import hashlib +import re +import time +from os import path + +from mitmproxy import ctx + + +class AddDeterministic: + def __init__(self): + self.millis = int(round(time.time() * 1000)) + ctx.log.info("Init Deterministic JS") + + def load(self, loader): + ctx.log.info("Load Deterministic JS") + + def get_csp_directives(self, test_header, headers): + csp = headers.get(test_header, "") + return [d.strip() for d in csp.split(";")] + + def get_csp_script_sources(self, test_header, headers): + sources = [] + for directive in self.get_csp_directives(test_header, headers): + if directive.startswith("script-src "): + sources = directive.split()[1:] + return sources + + def get_nonce_from_headers(self, test_header, headers): + """ + get_nonce_from_headers returns the nonce token from a + Content-Security-Policy (CSP) header's script source directive. + + Note: + For more background information on CSP and nonce, please refer to + https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/ + Content-Security-Policy/script-src + https://developers.google.com/web/fundamentals/security/csp/ + """ + + for source in self.get_csp_script_sources(test_header, headers) or []: + if source.startswith("'nonce-"): + return source.partition("'nonce-")[-1][:-1] + + def get_script_with_nonce(self, script, nonce=None): + """ + Given a nonce, get_script_with_nonce returns the injected script text with the nonce. + + If nonce None, get_script_with_nonce returns the script block + without attaching a nonce attribute. + + Note: + Some responses may specify a nonce inside their Content-Security-Policy, + script-src directive. + The script injector needs to set the injected script's nonce attribute to + open execute permission for the injected script. + """ + + if nonce: + return '<script nonce="{}">{}</script>'.format(nonce, script) + return "<script>{}</script>".format(script) + + def update_csp_script_src(self, test_header, headers, sha256): + """ + Update the CSP script directives with appropriate information + + Without this permissions a page with a + restrictive CSP will not execute injected scripts. + + Note: + For more background information on CSP, please refer to + https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/ + Content-Security-Policy/script-src + https://developers.google.com/web/fundamentals/security/csp/ + """ + + sources = self.get_csp_script_sources(test_header, headers) + add_unsafe = True + + for token in sources: + if token == "'unsafe-inline'": + add_unsafe = False + ctx.log.info("Contains unsafe-inline") + elif token.startswith("'sha"): + sources.append("'sha256-{}'".format(sha256)) + add_unsafe = False + ctx.log.info("Add sha hash directive") + break + + if add_unsafe: + ctx.log.info("Add unsafe") + sources.append("'unsafe-inline'") + + return "script-src {}".format(" ".join(sources)) + + def get_new_csp_header(self, test_header, headers, updated_csp_script): + """ + get_new_csp_header generates a new header object containing + the updated elements from new_csp_script_directives + """ + + if updated_csp_script: + directives = self.get_csp_directives(test_header, headers) + for index, directive in enumerate(directives): + if directive.startswith("script-src "): + directives[index] = updated_csp_script + + ctx.log.info("Original Header %s \n" % headers["Content-Security-Policy"]) + headers["Content-Security-Policy"] = "; ".join(directives) + ctx.log.info("Updated Header %s \n" % headers["Content-Security-Policy"]) + + return headers + + def response(self, flow): + # pylint: disable=W1633 + if "content-type" in flow.response.headers: + if "text/html" in flow.response.headers["content-type"]: + ctx.log.info( + "Working on {}".format(flow.response.headers["content-type"]) + ) + + flow.response.decode() + html = flow.response.text + + with open( + path.join(path.dirname(__file__), "catapult/deterministic.js"), "r" + ) as jsfile: + + js = jsfile.read().replace( + "REPLACE_LOAD_TIMESTAMP", str(self.millis) + ) + + if js not in html: + script_index = re.search("(?i).*?<head.*?>", html) + if script_index is None: + script_index = re.search("(?i).*?<html.*?>", html) + if script_index is None: + script_index = re.search("(?i).*?<!doctype html>", html) + if script_index is None: + ctx.log.info( + "No start tags found in request {}. Skip injecting".format( + flow.request.url + ) + ) + return + script_index = script_index.end() + + nonce = None + for test_header in [ + "Content-Security-Policy", + "Content-Security-Policy-Report-Only", + ]: + if flow.response.headers.get(test_header, False): + nonce = self.get_nonce_from_headers( + test_header, flow.response.headers + ) + ctx.log.info("nonce : %s" % nonce) + + if ( + self.get_csp_script_sources( + test_header, flow.response.headers + ) + and not nonce + ): + # generate sha256 for the script + hash_object = hashlib.sha256(js.encode("utf-8")) + script_sha256 = base64.b64encode( + hash_object.digest() + ).decode("utf-8") + + # generate the new response headers + updated_script_sources = self.update_csp_script_src( + test_header, + flow.response.headers, + script_sha256, + ) + flow.response.headers = self.get_new_csp_header( + test_header, + flow.response.headers, + updated_script_sources, + ) + + # generate new html file + new_html = ( + html[:script_index] + + self.get_script_with_nonce(js, nonce) + + html[script_index:] + ) + flow.response.text = new_html + + ctx.log.info( + "In request {} injected deterministic JS".format( + flow.request.url + ) + ) + else: + ctx.log.info( + "Script already injected in request {}".format( + flow.request.url + ) + ) + + +addons = [AddDeterministic()] diff --git a/testing/mozbase/mozproxy/mozproxy/driver.py b/testing/mozbase/mozproxy/mozproxy/driver.py new file mode 100644 index 0000000000..aa56c5142a --- /dev/null +++ b/testing/mozbase/mozproxy/mozproxy/driver.py @@ -0,0 +1,117 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +from __future__ import absolute_import + +import argparse +import os +import signal +import sys + +import mozinfo +import mozlog.commandline + +from . import get_playback +from .utils import LOG, TOOLTOOL_PATHS + +EXIT_SUCCESS = 0 +EXIT_EARLY_TERMINATE = 3 +EXIT_EXCEPTION = 4 + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument( + "--local", action="store_true", help="run this locally (i.e. not in production)" + ) + parser.add_argument("--record", default=False, help="generate a proxy recording") + parser.add_argument( + "--tool", + default="mitmproxy", + help="the playback tool to use (default: %(default)s)", + ) + parser.add_argument( + "--tool-version", + default="4.0.4", + help="the playback tool version to use (default: %(default)s)", + ) + parser.add_argument( + "--host", default="localhost", help="the host to use for the proxy server" + ) + parser.add_argument( + "--binary", + required=True, + help=("the path to the binary being tested (typically " "firefox)"), + ) + parser.add_argument( + "--topsrcdir", + required=True, + help="the top of the source directory for this project", + ) + parser.add_argument( + "--objdir", required=True, help="the object directory for this build" + ) + parser.add_argument( + "--app", default="firefox", help="the app being tested (default: %(default)s)" + ) + parser.add_argument( + "playback", + nargs="*", + help="The playback files to use. " + "It can be any combination of the following: zip file, manifest file," + "or a URL to zip/manifest file.", + ) + + mozlog.commandline.add_logging_group(parser) + + args = parser.parse_args() + mozlog.commandline.setup_logging("mozproxy", args, {"raw": sys.stdout}) + + TOOLTOOL_PATHS.append( + os.path.join( + args.topsrcdir, "python", "mozbuild", "mozbuild", "action", "tooltool.py" + ) + ) + + if hasattr(signal, "SIGBREAK"): + # Terminating on windows is slightly different than other platforms. + # On POSIX, we just let Python's default SIGINT handler raise a + # KeyboardInterrupt. This doesn't work on Windows, so instead we wait + # for a Ctrl+Break event and raise our own KeyboardInterrupt. + def handle_sigbreak(sig, frame): + raise KeyboardInterrupt() + + signal.signal(signal.SIGBREAK, handle_sigbreak) + + try: + playback = get_playback( + { + "run_local": args.local, + "host": args.host, + "binary": args.binary, + "obj_path": args.objdir, + "platform": mozinfo.os, + "playback_tool": args.tool, + "playback_version": args.tool_version, + "playback_files": args.playback, + "record": args.record, + "app": args.app, + } + ) + playback.start() + LOG.info("Proxy running on port %d" % playback.port) + # Wait for a keyboard interrupt from the caller so we know when to + # terminate. + playback.wait() + return EXIT_EARLY_TERMINATE + except KeyboardInterrupt: + LOG.info("Terminating mozproxy") + playback.stop() + return EXIT_SUCCESS + except Exception as e: + LOG.error(str(e), exc_info=True) + return EXIT_EXCEPTION + + +if __name__ == "__main__": + main() diff --git a/testing/mozbase/mozproxy/mozproxy/recordings.py b/testing/mozbase/mozproxy/mozproxy/recordings.py new file mode 100644 index 0000000000..1f36d8d6fb --- /dev/null +++ b/testing/mozbase/mozproxy/mozproxy/recordings.py @@ -0,0 +1,163 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +from __future__ import absolute_import + +import json +import os +import shutil +from datetime import datetime +from shutil import copyfile +from zipfile import ZipFile + +from .utils import LOG + + +class RecordingFile: + def __init__(self, path_to_zip_file): + self._recording_zip_path = path_to_zip_file + + self._base_name = os.path.splitext(os.path.basename(self._recording_zip_path))[ + 0 + ] + if not os.path.splitext(path_to_zip_file)[1] == ".zip": + LOG.error( + "Wrong file type! The provided recording should be a zip file. %s" + % path_to_zip_file + ) + raise Exception( + "Wrong file type! The provided recording should be a zip file." + ) + + # create a temp dir + self._mozproxy_dir = os.environ["MOZPROXY_DIR"] + self._tempdir = os.path.join(self._mozproxy_dir, self._base_name) + + if os.path.exists(self._tempdir): + LOG.info( + "The recording dir already exists! Resetting the existing dir and data." + ) + shutil.rmtree(self._tempdir) + os.mkdir(self._tempdir) + + self._metadata_path = self._get_temp_path("metadata.json") + self._recording = self._get_temp_path("dump.mp") + + if os.path.exists(path_to_zip_file): + with ZipFile(path_to_zip_file, "r") as zipObj: + # Extract all the contents of zip file in different directory + zipObj.extractall(self._tempdir) + + if not os.path.exists(self._recording): + self._convert_to_new_format() + + if not os.path.exists(self._metadata_path): + LOG.error("metadata file is missing!") + raise Exception("metadata file is missing!") + + with open(self._metadata_path) as json_file: + self._metadata = json.load(json_file) + self.validate_recording() + + else: + LOG.info("Recording file does not exists!!! Generating base structure") + self._metadata = {"content": [], "recording_date": str(datetime.now())} + + def _convert_to_new_format(self): + # Convert zip recording to new format + + LOG.info("Convert zip recording to new format") + + for tmp_file in os.listdir(self._tempdir): + if tmp_file.endswith(".mp"): + LOG.info("Renaming %s to dump.mp file" % tmp_file) + os.rename(self._get_temp_path(tmp_file), self._get_temp_path("dump.mp")) + elif tmp_file.endswith(".json"): + if tmp_file.startswith("mitm_netlocs_"): + LOG.info("Renaming %s to netlocs.json file" % tmp_file) + os.rename( + self._get_temp_path("%s.json" % os.path.splitext(tmp_file)[0]), + self._get_temp_path("netlocs.json"), + ) + else: + LOG.info("Renaming %s to metadata.json file" % tmp_file) + os.rename( + self._get_temp_path("%s.json" % os.path.splitext(tmp_file)[0]), + self._get_temp_path("metadata.json"), + ) + elif tmp_file.endswith(".png"): + LOG.info("Renaming %s to screenshot.png file" % tmp_file) + os.rename( + self._get_temp_path("%s.png" % os.path.splitext(tmp_file)[0]), + self._get_temp_path("screenshot.png"), + ) + + def _get_temp_path(self, file_name): + return os.path.join(self._tempdir, file_name) + + def validate_recording(self): + # Validates that minimum zip file content exists + if not os.path.exists(self._recording): + LOG.error("Recording file is missing!") + raise Exception("Recording file is missing!") + + if not os.path.exists(self._metadata_path): + LOG.error("Metadata file is missing!") + raise Exception("Metadata file is missing!") + + if "content" in self._metadata: + # check that all extra files specified in the recording are present + for content_file in self._metadata["content"]: + if not os.path.exists(self._get_temp_path(content_file)): + LOG.error("File %s does not exist!!" % content_file) + raise Exception("Recording file is missing!") + else: + LOG.info("Old file type! Not confirming content!") + + def metadata(self, name): + # Return metadata value + return self._metadata[name] + + def set_metadata(self, entry, value): + # Set metadata value + self._metadata[entry] = value + + @property + def recording_path(self): + # Returns the path of the recoring.mp file included in the zip + return self._recording + + def get_file(self, file_name): + # Returns the path to a specified file included in the recording zip + return self._get_temp_path(file_name) + + def add_file(self, path): + # Adds file to Zip + if os.path.exists(path): + copyfile(path, self._tempdir) + self._metadata["content"].append(os.path.basename(path)) + else: + LOG.error("Target file %s does not exist!!" % path) + raise Exception("File does not exist!!!") + + def update_metadata(self): + # Update metadata with information generated by HttpProtocolExtractor plugin + # This data is geerated after closing the MitMPtoxy process + + dump_file = self._get_temp_path("dump.json") + if os.path.exists(dump_file): + with open(dump_file) as dump: + dump_info = json.load(dump) + self._metadata.update(dump_info) + + def generate_zip_file(self): + self.update_metadata() + with open(self._get_temp_path(self._metadata_path), "w") as metadata_file: + json.dump(self._metadata, metadata_file) + + with ZipFile(self._recording_zip_path, "w") as zf: + zf.write(self._metadata_path, "metadata.json") + zf.write(self.recording_path, "dump.mp") + for file in self._metadata["content"]: + zf.write(self._get_temp_path(file), file) + LOG.info("Generated new recording file at: %s" % self._recording_zip_path) diff --git a/testing/mozbase/mozproxy/mozproxy/server.py b/testing/mozbase/mozproxy/mozproxy/server.py new file mode 100644 index 0000000000..60ec44e146 --- /dev/null +++ b/testing/mozbase/mozproxy/mozproxy/server.py @@ -0,0 +1,17 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import +from mozproxy.backends.mitm.android import MitmproxyAndroid +from mozproxy.backends.mitm.desktop import MitmproxyDesktop + +_BACKENDS = {"mitmproxy": MitmproxyDesktop, "mitmproxy-android": MitmproxyAndroid} + + +def get_backend(name, *args, **kw): + """Returns the class that implements the backend. + + Raises KeyError in case the backend does not exists. + """ + return _BACKENDS[name](*args, **kw) diff --git a/testing/mozbase/mozproxy/mozproxy/utils.py b/testing/mozbase/mozproxy/mozproxy/utils.py new file mode 100644 index 0000000000..1ef5b5d8cc --- /dev/null +++ b/testing/mozbase/mozproxy/mozproxy/utils.py @@ -0,0 +1,247 @@ +"""Utility functions for Raptor""" +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import + +import subprocess +import time +import bz2 +import gzip +import os +import signal +import sys +import socket + +from six.moves.urllib.request import urlretrieve +from redo import retriable, retry + +try: + import zstandard +except ImportError: + zstandard = None +try: + import lzma +except ImportError: + lzma = None + +from mozlog import get_proxy_logger +from mozprocess import ProcessHandler +from mozproxy import mozharness_dir + + +LOG = get_proxy_logger(component="mozproxy") + +# running locally via mach +TOOLTOOL_PATHS = [os.path.join(mozharness_dir, "external_tools", "tooltool.py")] + +if "MOZ_UPLOAD_DIR" in os.environ: + TOOLTOOL_PATHS.append( + os.path.join( + os.environ["MOZ_UPLOAD_DIR"], + "..", + "..", + "mozharness", + "external_tools", + "tooltool.py", + ) + ) + + +def transform_platform(str_to_transform, config_platform, config_processor=None): + """Transform platform name i.e. 'mitmproxy-rel-bin-{platform}.manifest' + + transforms to 'mitmproxy-rel-bin-osx.manifest'. + Also transform '{x64}' if needed for 64 bit / win 10 + """ + if "{platform}" not in str_to_transform and "{x64}" not in str_to_transform: + return str_to_transform + + if "win" in config_platform: + platform_id = "win" + elif config_platform == "mac": + platform_id = "osx" + else: + platform_id = "linux64" + + if "{platform}" in str_to_transform: + str_to_transform = str_to_transform.replace("{platform}", platform_id) + + if "{x64}" in str_to_transform and config_processor is not None: + if "x86_64" in config_processor: + str_to_transform = str_to_transform.replace("{x64}", "_x64") + else: + str_to_transform = str_to_transform.replace("{x64}", "") + + return str_to_transform + + +@retriable(sleeptime=2) +def tooltool_download(manifest, run_local, raptor_dir): + """Download a file from tooltool using the provided tooltool manifest""" + + def outputHandler(line): + LOG.info(line) + + tooltool_path = None + + for path in TOOLTOOL_PATHS: + if os.path.exists(os.path.dirname(path)): + tooltool_path = path + break + + if tooltool_path is None: + raise Exception("Could not find tooltool path!") + + if run_local: + command = [sys.executable, tooltool_path, "fetch", "-o", "-m", manifest] + else: + # Attempt to determine the tooltool cache path: + # - TOOLTOOLCACHE is used by Raptor tests + # - TOOLTOOL_CACHE is automatically set for taskcluster jobs + # - fallback to a hardcoded path + _cache = next( + x + for x in ( + os.environ.get("TOOLTOOLCACHE"), + os.environ.get("TOOLTOOL_CACHE"), + "/builds/tooltool_cache", + ) + if x is not None + ) + + command = [ + sys.executable, + tooltool_path, + "fetch", + "-o", + "-m", + manifest, + "-c", + _cache, + ] + + try: + proc = ProcessHandler( + command, processOutputLine=outputHandler, storeOutput=False, cwd=raptor_dir + ) + proc.run() + if proc.wait() != 0: + raise Exception("Command failed") + except Exception as e: + LOG.critical( + "Error while downloading {} from tooltool:{}".format(manifest, str(e)) + ) + if proc.poll() is None: + proc.kill(signal.SIGTERM) + raise + + +def archive_type(path): + filename, extension = os.path.splitext(path) + filename, extension2 = os.path.splitext(filename) + if extension2 != "": + extension = extension2 + if extension == ".tar": + return "tar" + elif extension == ".zip": + return "zip" + return None + + +def extract_archive(path, dest_dir, typ): + """Extract an archive to a destination directory.""" + + # Resolve paths to absolute variants. + path = os.path.abspath(path) + dest_dir = os.path.abspath(dest_dir) + suffix = os.path.splitext(path)[-1] + + # We pipe input to the decompressor program so that we can apply + # custom decompressors that the program may not know about. + if typ == "tar": + if suffix == ".bz2": + ifh = bz2.open(str(path), "rb") + elif suffix == ".gz": + ifh = gzip.open(str(path), "rb") + elif suffix == ".xz": + if not lzma: + raise ValueError("lzma Python package not available") + ifh = lzma.open(str(path), "rb") + elif suffix == ".zst": + if not zstandard: + raise ValueError("zstandard Python package not available") + dctx = zstandard.ZstdDecompressor() + ifh = dctx.stream_reader(path.open("rb")) + elif suffix == ".tar": + ifh = path.open("rb") + else: + raise ValueError("unknown archive format for tar file: %s" % path) + args = ["tar", "xf", "-"] + pipe_stdin = True + elif typ == "zip": + # unzip from stdin has wonky behavior. We don't use a pipe for it. + ifh = open(os.devnull, "rb") + args = ["unzip", "-o", str(path)] + pipe_stdin = False + else: + raise ValueError("unknown archive format: %s" % path) + + LOG.info("Extracting %s to %s using %r" % (path, dest_dir, args)) + t0 = time.time() + with ifh: + p = subprocess.Popen(args, cwd=str(dest_dir), bufsize=0, stdin=subprocess.PIPE) + while True: + if not pipe_stdin: + break + chunk = ifh.read(131072) + if not chunk: + break + p.stdin.write(chunk) + # make sure we wait for the command to finish + p.communicate() + + if p.returncode: + raise Exception("%r exited %d" % (args, p.returncode)) + LOG.info("%s extracted in %.3fs" % (path, time.time() - t0)) + + +def download_file_from_url(url, local_dest, extract=False): + """Receive a file in a URL and download it, i.e. for the hostutils tooltool manifest + the url received would be formatted like this: + config/tooltool-manifests/linux64/hostutils.manifest""" + if os.path.exists(local_dest): + LOG.info("file already exists at: %s" % local_dest) + if not extract: + return True + else: + LOG.info("downloading: %s to %s" % (url, local_dest)) + try: + retry(urlretrieve, args=(url, local_dest), attempts=3, sleeptime=5) + except Exception: + LOG.error("Failed to download file: %s" % local_dest, exc_info=True) + if os.path.exists(local_dest): + # delete partial downloaded file + os.remove(local_dest) + return False + + if not extract: + return os.path.exists(local_dest) + + typ = archive_type(local_dest) + if typ is None: + LOG.info("Not able to determine archive type for: %s" % local_dest) + return False + + extract_archive(local_dest, os.path.dirname(local_dest), typ) + return True + + +def get_available_port(): + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + s.bind(("", 0)) + s.listen(1) + port = s.getsockname()[1] + s.close() + return port diff --git a/testing/mozbase/mozproxy/setup.py b/testing/mozbase/mozproxy/setup.py new file mode 100644 index 0000000000..9fc4ef4526 --- /dev/null +++ b/testing/mozbase/mozproxy/setup.py @@ -0,0 +1,39 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import + +from setuptools import setup + +PACKAGE_NAME = "mozproxy" +PACKAGE_VERSION = "1.0" + +# dependencies +deps = ["redo", "mozinfo", "mozlog >= 6.0"] + +setup( + name=PACKAGE_NAME, + version=PACKAGE_VERSION, + description="Proxy for playback", + long_description="see https://firefox-source-docs.mozilla.org/mozbase/index.html", + classifiers=[ + "Programming Language :: Python :: 2.7", + "Programming Language :: Python :: 3.5", + ], + # Get strings from http://pypi.python.org/pypi?%3Aaction=list_classifiers + keywords="mozilla", + author="Mozilla Automation and Tools team", + author_email="tools@lists.mozilla.org", + url="https://wiki.mozilla.org/Auto-tools/Projects/Mozbase", + license="MPL", + packages=["mozproxy"], + install_requires=deps, + entry_points={ + "console_scripts": [ + "mozproxy=mozproxy.driver:main", + ], + }, + include_package_data=True, + zip_safe=False, +) diff --git a/testing/mozbase/mozproxy/tests/__init__.py b/testing/mozbase/mozproxy/tests/__init__.py new file mode 100644 index 0000000000..792d600548 --- /dev/null +++ b/testing/mozbase/mozproxy/tests/__init__.py @@ -0,0 +1 @@ +# diff --git a/testing/mozbase/mozproxy/tests/archive.tar.gz b/testing/mozbase/mozproxy/tests/archive.tar.gz Binary files differnew file mode 100644 index 0000000000..b4f9461b09 --- /dev/null +++ b/testing/mozbase/mozproxy/tests/archive.tar.gz diff --git a/testing/mozbase/mozproxy/tests/example.dump b/testing/mozbase/mozproxy/tests/example.dump Binary files differnew file mode 100644 index 0000000000..aee6249ac7 --- /dev/null +++ b/testing/mozbase/mozproxy/tests/example.dump diff --git a/testing/mozbase/mozproxy/tests/files/mitm5-linux-firefox-amazon.manifest b/testing/mozbase/mozproxy/tests/files/mitm5-linux-firefox-amazon.manifest new file mode 100644 index 0000000000..0060b25393 --- /dev/null +++ b/testing/mozbase/mozproxy/tests/files/mitm5-linux-firefox-amazon.manifest @@ -0,0 +1,10 @@ +[ + { + "algorithm": "sha512", + "digest": "d801dc23873ef5fac668aa58fa948f5de0d9f3ccc53d6773fb5a137515bd04e72cc8c0c7975c6e1fc19c72b3d721effb5432fce78b0ca6f3a90f2d6467ee5b68", + "filename": "mitm5-linux-firefox-amazon.zip", + "size": 6588776, + "unpack": true, + "visibility": "public" + } +] diff --git a/testing/mozbase/mozproxy/tests/files/mitm5-linux-firefox-amazon.zip b/testing/mozbase/mozproxy/tests/files/mitm5-linux-firefox-amazon.zip Binary files differnew file mode 100644 index 0000000000..8c724762d3 --- /dev/null +++ b/testing/mozbase/mozproxy/tests/files/mitm5-linux-firefox-amazon.zip diff --git a/testing/mozbase/mozproxy/tests/files/recording.zip b/testing/mozbase/mozproxy/tests/files/recording.zip Binary files differnew file mode 100644 index 0000000000..7cea81a5e6 --- /dev/null +++ b/testing/mozbase/mozproxy/tests/files/recording.zip diff --git a/testing/mozbase/mozproxy/tests/firefox b/testing/mozbase/mozproxy/tests/firefox new file mode 100644 index 0000000000..4a938f06f7 --- /dev/null +++ b/testing/mozbase/mozproxy/tests/firefox @@ -0,0 +1 @@ +# I am firefox diff --git a/testing/mozbase/mozproxy/tests/manifest.ini b/testing/mozbase/mozproxy/tests/manifest.ini new file mode 100644 index 0000000000..51817ac77a --- /dev/null +++ b/testing/mozbase/mozproxy/tests/manifest.ini @@ -0,0 +1,10 @@ +[DEFAULT] +subsuite = mozbase +[test_proxy.py] +[test_utils.py] +[test_command_line.py] +run-if = python == 3 # The mozproxy command line interface is designed to run on Python 3. +[test_recordings.py] +[test_recording.py] +[test_mitm_addons.py] +run-if = python == 3 # The mitm addons are designed to run on Python 3. diff --git a/testing/mozbase/mozproxy/tests/paypal.mp b/testing/mozbase/mozproxy/tests/paypal.mp new file mode 100644 index 0000000000..8e48bd50de --- /dev/null +++ b/testing/mozbase/mozproxy/tests/paypal.mp @@ -0,0 +1 @@ +# fake recorded playback diff --git a/testing/mozbase/mozproxy/tests/support.py b/testing/mozbase/mozproxy/tests/support.py new file mode 100644 index 0000000000..6471c080ef --- /dev/null +++ b/testing/mozbase/mozproxy/tests/support.py @@ -0,0 +1,15 @@ +from __future__ import absolute_import, print_function +import shutil +import contextlib +import tempfile + + +# This helper can be replaced by pytest tmpdir fixture +# once Bug 1536029 lands (@mock.patch disturbs pytest fixtures) +@contextlib.contextmanager +def tempdir(): + dest_dir = tempfile.mkdtemp() + try: + yield dest_dir + finally: + shutil.rmtree(dest_dir, ignore_errors=True) diff --git a/testing/mozbase/mozproxy/tests/test_command_line.py b/testing/mozbase/mozproxy/tests/test_command_line.py new file mode 100644 index 0000000000..aef9d94e3e --- /dev/null +++ b/testing/mozbase/mozproxy/tests/test_command_line.py @@ -0,0 +1,137 @@ +#!/usr/bin/env python +from __future__ import absolute_import, print_function +import json +import os +import re +import signal +import threading +import time + +import mozunit +import pytest +from mozbuild.base import MozbuildObject +from mozprocess import ProcessHandler + +here = os.path.dirname(__file__) + + +# This is copied from <python/mozperftest/mozperftest/utils.py>. It's copied +# instead of imported since mozperfest is Python 3, and this file is +# (currently) Python 2. +def _install_package(virtualenv_manager, package): + from pip._internal.req.constructors import install_req_from_line + + req = install_req_from_line(package) + req.check_if_exists(use_user_site=False) + # already installed, check if it's in our venv + if req.satisfied_by is not None: + venv_site_lib = os.path.abspath( + os.path.join(virtualenv_manager.bin_path, "..", "lib") + ) + site_packages = os.path.abspath(req.satisfied_by.location) + if site_packages.startswith(venv_site_lib): + # already installed in this venv, we can skip + return + virtualenv_manager._run_pip(["install", package]) + + +def _kill_mozproxy(pid): + kill_signal = getattr(signal, "CTRL_BREAK_EVENT", signal.SIGINT) + os.kill(pid, kill_signal) + + +class OutputHandler(object): + def __init__(self): + self.port = None + self.port_event = threading.Event() + + def __call__(self, line): + if not line.strip(): + return + line = line.decode("utf-8", errors="replace") + # Print the output we received so we have useful logs if a test fails. + print(line) + + try: + data = json.loads(line) + except ValueError: + return + + if isinstance(data, dict) and "action" in data: + # Retrieve the port number for the proxy server from the logs of + # our subprocess. + m = re.match(r"Proxy running on port (\d+)", data.get("message", "")) + if m: + self.port = m.group(1) + self.port_event.set() + + def finished(self): + self.port_event.set() + + +@pytest.fixture(scope="module") +def install_mozproxy(): + build = MozbuildObject.from_environment(cwd=here, virtualenv_name="python-test") + build.virtualenv_manager.activate() + + mozbase = os.path.join(build.topsrcdir, "testing", "mozbase") + mozproxy_deps = ["mozinfo", "mozlog", "mozproxy"] + for i in mozproxy_deps: + _install_package(build.virtualenv_manager, os.path.join(mozbase, i)) + return build + + +def test_help(install_mozproxy): + p = ProcessHandler(["mozproxy", "--help"]) + p.run() + assert p.wait() == 0 + + +def test_run(install_mozproxy): + build = install_mozproxy + output_handler = OutputHandler() + p = ProcessHandler( + [ + "mozproxy", + "--local", + "--binary=firefox", + "--topsrcdir=" + build.topsrcdir, + "--objdir=" + build.topobjdir, + os.path.join(here, "files", "mitm5-linux-firefox-amazon.zip"), + ], + processOutputLine=output_handler, + onFinish=output_handler.finished, + ) + p.run() + # The first time we run mozproxy, we need to fetch mitmproxy, which can + # take a while... + assert output_handler.port_event.wait(120) is True + # Give mitmproxy a bit of time to start up so we can verify that it's + # actually running before we kill mozproxy. + time.sleep(5) + _kill_mozproxy(p.pid) + + assert p.wait(10) == 0 + assert output_handler.port is not None + + +def test_failure(install_mozproxy): + output_handler = OutputHandler() + p = ProcessHandler( + [ + "mozproxy", + "--local", + # Exclude some options here to trigger a command-line error. + os.path.join(here, "files", "mitm5-linux-firefox-amazon.zip"), + ], + processOutputLine=output_handler, + onFinish=output_handler.finished, + ) + p.run() + assert output_handler.port_event.wait(10) is True + assert p.wait(10) == 2 + assert output_handler.port is None + + +if __name__ == "__main__": + mozunit.main(runwith="pytest") diff --git a/testing/mozbase/mozproxy/tests/test_mitm_addons.py b/testing/mozbase/mozproxy/tests/test_mitm_addons.py new file mode 100644 index 0000000000..bcde35fc81 --- /dev/null +++ b/testing/mozbase/mozproxy/tests/test_mitm_addons.py @@ -0,0 +1,91 @@ +#!/usr/bin/env python +from __future__ import absolute_import, print_function + +import json +import os + +import mock +import mozunit + +here = os.path.dirname(__file__) +os.environ["MOZPROXY_DIR"] = os.path.join(here, "files") + +protocol = { + "http_protocol": {"aax-us-iad.amazon.com": "HTTP/1.1"}, + "recorded_requests": 4, + "recorded_requests_unique": 1, +} + + +@mock.patch( + "mozproxy.backends.mitm.scripts.http_protocol_extractor.HttpProtocolExtractor.get_ctx" +) +def test_http_protocol_generate_json_file(ctx_mock): + ctx_mock.return_value.options.save_stream_file = os.path.join( + os.environ["MOZPROXY_DIR"], "http_protocol_recording_done.mp" + ) + + from mozproxy.backends.mitm.scripts.http_protocol_extractor import ( + HttpProtocolExtractor, + ) + + test_http_protocol = HttpProtocolExtractor() + test_http_protocol.ctx = test_http_protocol.get_ctx() + + # test data + test_http_protocol.request_protocol = protocol["http_protocol"] + test_http_protocol.hashes = ["Hash string"] + test_http_protocol.request_count = protocol["recorded_requests"] + + test_http_protocol.done() + + json_path = os.path.join( + os.environ["MOZPROXY_DIR"], "http_protocol_recording_done.json" + ) + assert os.path.exists(json_path) + with open(json_path) as json_file: + output_data = json.load(json_file) + + assert output_data["recorded_requests"] == protocol["recorded_requests"] + assert ( + output_data["recorded_requests_unique"] + == protocol["recorded_requests_unique"] + ) + assert output_data["http_protocol"] == protocol["http_protocol"] + + +@mock.patch( + "mozproxy.backends.mitm.scripts.http_protocol_extractor.HttpProtocolExtractor.get_ctx" +) +def test_http_protocol_response(ctx_mock): + ctx_mock.return_value.options.save_stream_file = os.path.join( + os.environ["MOZPROXY_DIR"], "http_protocol_recording_done.mp" + ) + + from mozproxy.backends.mitm.scripts.http_protocol_extractor import ( + HttpProtocolExtractor, + ) + + test_http_protocol = HttpProtocolExtractor() + test_http_protocol.ctx = test_http_protocol.get_ctx() + + # test data + flow = mock.MagicMock() + flow.type = "http" + flow.request.url = "https://www.google.com/complete/search" + flow.request.port = 33 + flow.response.data.http_version = b"HTTP/1.1" + + test_http_protocol.request_protocol = {} + test_http_protocol.hashes = [] + test_http_protocol.request_count = 0 + + test_http_protocol.response(flow) + + assert test_http_protocol.request_count == 1 + assert len(test_http_protocol.hashes) == 1 + assert test_http_protocol.request_protocol["www.google.com"] == "HTTP/1.1" + + +if __name__ == "__main__": + mozunit.main(runwith="pytest") diff --git a/testing/mozbase/mozproxy/tests/test_proxy.py b/testing/mozbase/mozproxy/tests/test_proxy.py new file mode 100644 index 0000000000..f9b80aeb8a --- /dev/null +++ b/testing/mozbase/mozproxy/tests/test_proxy.py @@ -0,0 +1,197 @@ +#!/usr/bin/env python +from __future__ import absolute_import, print_function +import os + +import mock +import mozunit +import mozinfo +import requests +from mozproxy import get_playback +from support import tempdir + +here = os.path.dirname(__file__) + + +class Process: + def __init__(self, *args, **kw): + pass + + def run(self): + print("I am running something") + + def poll(self): + return None + + def wait(self): + return 0 + + def kill(self, sig=9): + pass + + proc = object() + pid = 1234 + stderr = stdout = None + returncode = 0 + + +_RETRY = 0 + + +class ProcessWithRetry(Process): + def __init__(self, *args, **kw): + Process.__init__(self, *args, **kw) + + def wait(self): + global _RETRY + _RETRY += 1 + if _RETRY >= 2: + _RETRY = 0 + return 0 + return -1 + + +def kill(pid, signal): + if pid == 1234: + return + return os.kill(pid, signal) + + +def get_status_code(url, playback): + response = requests.get( + url=url, proxies={"http": "http://%s:%s/" % (playback.host, playback.port)} + ) + return response.status_code + + +def test_mitm_check_proxy(*args): + # test setup + pageset_name = os.path.join(here, "files", "mitm5-linux-firefox-amazon.manifest") + + config = { + "playback_tool": "mitmproxy", + "playback_files": [os.path.join(here, "files", pageset_name)], + "playback_version": "5.1.1", + "platform": mozinfo.os, + "run_local": "MOZ_AUTOMATION" not in os.environ, + "binary": "firefox", + "app": "firefox", + "host": "127.0.0.1", + } + + with tempdir() as obj_path: + config["obj_path"] = obj_path + playback = get_playback(config) + assert playback is not None + + try: + playback.start() + + url = "https://m.media-amazon.com/images/G/01/csm/showads.v2.js" + assert get_status_code(url, playback) == 200 + + url = "http://mozproxy/checkProxy" + assert get_status_code(url, playback) == 404 + finally: + playback.stop() + + +@mock.patch("mozproxy.backends.mitm.Mitmproxy.check_proxy") +@mock.patch("mozproxy.backends.mitm.mitm.ProcessHandler", new=Process) +@mock.patch("mozproxy.utils.ProcessHandler", new=Process) +@mock.patch("os.kill", new=kill) +def test_mitm(*args): + pageset_name = os.path.join(here, "files", "mitm5-linux-firefox-amazon.manifest") + + config = { + "playback_tool": "mitmproxy", + "playback_files": [pageset_name], + "playback_version": "5.1.1", + "platform": mozinfo.os, + "run_local": True, + "binary": "firefox", + "app": "firefox", + "host": "example.com", + } + + with tempdir() as obj_path: + config["obj_path"] = obj_path + playback = get_playback(config) + assert playback is not None + try: + playback.start() + finally: + playback.stop() + + +@mock.patch("mozproxy.backends.mitm.Mitmproxy.check_proxy") +@mock.patch("mozproxy.backends.mitm.mitm.ProcessHandler", new=Process) +@mock.patch("mozproxy.utils.ProcessHandler", new=Process) +@mock.patch("os.kill", new=kill) +def test_playback_setup_failed(*args): + class SetupFailed(Exception): + pass + + def setup(*args, **kw): + def _s(self): + raise SetupFailed("Failed") + + return _s + + pageset_name = os.path.join(here, "files", "mitm5-linux-firefox-amazon.manifest") + + config = { + "playback_tool": "mitmproxy", + "playback_files": [pageset_name], + "playback_version": "4.0.4", + "platform": mozinfo.os, + "run_local": True, + "binary": "firefox", + "app": "firefox", + "host": "example.com", + } + + prefix = "mozproxy.backends.mitm.desktop.MitmproxyDesktop." + + with tempdir() as obj_path: + config["obj_path"] = obj_path + with mock.patch(prefix + "setup", new_callable=setup): + with mock.patch(prefix + "stop_mitmproxy_playback") as p: + try: + pb = get_playback(config) + pb.start() + except SetupFailed: + assert p.call_count == 1 + except Exception: + raise + + +@mock.patch("mozproxy.backends.mitm.Mitmproxy.check_proxy") +@mock.patch("mozproxy.backends.mitm.mitm.ProcessHandler", new=ProcessWithRetry) +@mock.patch("mozproxy.utils.ProcessHandler", new=ProcessWithRetry) +@mock.patch("os.kill", new=kill) +def test_mitm_with_retry(*args): + pageset_name = os.path.join(here, "files", "mitm5-linux-firefox-amazon.manifest") + + config = { + "playback_tool": "mitmproxy", + "playback_files": [pageset_name], + "playback_version": "5.1.1", + "platform": mozinfo.os, + "run_local": True, + "binary": "firefox", + "app": "firefox", + "host": "example.com", + } + + with tempdir() as obj_path: + config["obj_path"] = obj_path + playback = get_playback(config) + assert playback is not None + try: + playback.start() + finally: + playback.stop() + + +if __name__ == "__main__": + mozunit.main(runwith="pytest") diff --git a/testing/mozbase/mozproxy/tests/test_recording.py b/testing/mozbase/mozproxy/tests/test_recording.py new file mode 100644 index 0000000000..d47a1714b0 --- /dev/null +++ b/testing/mozbase/mozproxy/tests/test_recording.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python +from __future__ import absolute_import, print_function + +import os + +import mozinfo +import mozunit +import requests +from mozproxy import get_playback +from support import tempdir + +here = os.path.dirname(__file__) +os.environ["MOZPROXY_DIR"] = os.path.join(here, "files") + + +def get_status_code(url, playback): + response = requests.get( + url=url, proxies={"http": "http://%s:%s/" % (playback.host, playback.port)} + ) + return response.status_code + + +def test_record_and_replay(*args): + # test setup + recording_file = os.path.join(here, "files", "new_recoding.zip") + + # Record part + config = { + "playback_tool": "mitmproxy", + "recording_file": recording_file, + "playback_version": "5.1.1", + "platform": mozinfo.os, + "run_local": "MOZ_AUTOMATION" not in os.environ, + "binary": "firefox", + "app": "firefox", + "host": "127.0.0.1", + "record": True, + } + + with tempdir() as obj_path: + config["obj_path"] = obj_path + record = get_playback(config) + assert record is not None + + try: + record.start() + + url = "https://m.media-amazon.com/images/G/01/csm/showads.v2.js" + assert get_status_code(url, record) == 200 + finally: + record.stop() + + # playback part + config["record"] = False + config["recording_file"] = None + config["playback_files"] = [recording_file] + playback = get_playback(config) + assert playback is not None + try: + playback.start() + + url = "https://m.media-amazon.com/images/G/01/csm/showads.v2.js" + assert get_status_code(url, playback) == 200 + finally: + playback.stop() + + +if __name__ == "__main__": + mozunit.main(runwith="pytest") diff --git a/testing/mozbase/mozproxy/tests/test_recordings.py b/testing/mozbase/mozproxy/tests/test_recordings.py new file mode 100644 index 0000000000..fe321bcf13 --- /dev/null +++ b/testing/mozbase/mozproxy/tests/test_recordings.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python +from __future__ import absolute_import, print_function + +import os + +import mozunit +from mozproxy.recordings import RecordingFile + +here = os.path.dirname(__file__) +os.environ["MOZPROXY_DIR"] = os.path.join(here, "files") + + +def test_recording_generation(*args): + test_file = os.path.join(here, "files", "new_file.zip") + file = RecordingFile(test_file) + with open(file.recording_path, "w") as recording: + recording.write("This is a recording") + + file.set_metadata("test_file", True) + file.generate_zip_file() + + assert os.path.exists(test_file) + os.remove(test_file) + + +def test_recording_content(*args): + test_file = os.path.join(here, "files", "recording.zip") + file = RecordingFile(test_file) + + assert file.metadata("test_file") is True + assert os.path.exists(file.recording_path) + + +if __name__ == "__main__": + mozunit.main(runwith="pytest") diff --git a/testing/mozbase/mozproxy/tests/test_utils.py b/testing/mozbase/mozproxy/tests/test_utils.py new file mode 100644 index 0000000000..967517c0f1 --- /dev/null +++ b/testing/mozbase/mozproxy/tests/test_utils.py @@ -0,0 +1,34 @@ +#!/usr/bin/env python +from __future__ import absolute_import, print_function + +import os +import shutil +import mock +import mozunit + +from mozproxy.utils import download_file_from_url +from support import tempdir + +here = os.path.dirname(__file__) + + +def urlretrieve(*args, **kw): + def _urlretrieve(url, local_dest): + # simply copy over our tarball + shutil.copyfile(os.path.join(here, "archive.tar.gz"), local_dest) + return local_dest, {} + + return _urlretrieve + + +@mock.patch("mozproxy.utils.urlretrieve", new_callable=urlretrieve) +def test_download_file(*args): + with tempdir() as dest_dir: + dest = os.path.join(dest_dir, "archive.tar.gz") + download_file_from_url("http://example.com/archive.tar.gz", dest, extract=True) + # archive.tar.gz contains hey.txt, if it worked we should see it + assert os.path.exists(os.path.join(dest_dir, "hey.txt")) + + +if __name__ == "__main__": + mozunit.main(runwith="pytest") |