diff options
Diffstat (limited to 'testing/mozbase/mozproxy/tests')
16 files changed, 699 insertions, 0 deletions
diff --git a/testing/mozbase/mozproxy/tests/__init__.py b/testing/mozbase/mozproxy/tests/__init__.py new file mode 100644 index 0000000000..792d600548 --- /dev/null +++ b/testing/mozbase/mozproxy/tests/__init__.py @@ -0,0 +1 @@ +# diff --git a/testing/mozbase/mozproxy/tests/archive.tar.gz b/testing/mozbase/mozproxy/tests/archive.tar.gz Binary files differnew file mode 100644 index 0000000000..b4f9461b09 --- /dev/null +++ b/testing/mozbase/mozproxy/tests/archive.tar.gz diff --git a/testing/mozbase/mozproxy/tests/example.dump b/testing/mozbase/mozproxy/tests/example.dump Binary files differnew file mode 100644 index 0000000000..aee6249ac7 --- /dev/null +++ b/testing/mozbase/mozproxy/tests/example.dump diff --git a/testing/mozbase/mozproxy/tests/files/mitm5-linux-firefox-amazon.manifest b/testing/mozbase/mozproxy/tests/files/mitm5-linux-firefox-amazon.manifest new file mode 100644 index 0000000000..0060b25393 --- /dev/null +++ b/testing/mozbase/mozproxy/tests/files/mitm5-linux-firefox-amazon.manifest @@ -0,0 +1,10 @@ +[ + { + "algorithm": "sha512", + "digest": "d801dc23873ef5fac668aa58fa948f5de0d9f3ccc53d6773fb5a137515bd04e72cc8c0c7975c6e1fc19c72b3d721effb5432fce78b0ca6f3a90f2d6467ee5b68", + "filename": "mitm5-linux-firefox-amazon.zip", + "size": 6588776, + "unpack": true, + "visibility": "public" + } +] diff --git a/testing/mozbase/mozproxy/tests/files/mitm5-linux-firefox-amazon.zip b/testing/mozbase/mozproxy/tests/files/mitm5-linux-firefox-amazon.zip Binary files differnew file mode 100644 index 0000000000..8c724762d3 --- /dev/null +++ b/testing/mozbase/mozproxy/tests/files/mitm5-linux-firefox-amazon.zip diff --git a/testing/mozbase/mozproxy/tests/files/recording.zip b/testing/mozbase/mozproxy/tests/files/recording.zip Binary files differnew file mode 100644 index 0000000000..7cea81a5e6 --- /dev/null +++ b/testing/mozbase/mozproxy/tests/files/recording.zip diff --git a/testing/mozbase/mozproxy/tests/firefox b/testing/mozbase/mozproxy/tests/firefox new file mode 100644 index 0000000000..4a938f06f7 --- /dev/null +++ b/testing/mozbase/mozproxy/tests/firefox @@ -0,0 +1 @@ +# I am firefox diff --git a/testing/mozbase/mozproxy/tests/manifest.ini b/testing/mozbase/mozproxy/tests/manifest.ini new file mode 100644 index 0000000000..51817ac77a --- /dev/null +++ b/testing/mozbase/mozproxy/tests/manifest.ini @@ -0,0 +1,10 @@ +[DEFAULT] +subsuite = mozbase +[test_proxy.py] +[test_utils.py] +[test_command_line.py] +run-if = python == 3 # The mozproxy command line interface is designed to run on Python 3. +[test_recordings.py] +[test_recording.py] +[test_mitm_addons.py] +run-if = python == 3 # The mitm addons are designed to run on Python 3. diff --git a/testing/mozbase/mozproxy/tests/paypal.mp b/testing/mozbase/mozproxy/tests/paypal.mp new file mode 100644 index 0000000000..8e48bd50de --- /dev/null +++ b/testing/mozbase/mozproxy/tests/paypal.mp @@ -0,0 +1 @@ +# fake recorded playback diff --git a/testing/mozbase/mozproxy/tests/support.py b/testing/mozbase/mozproxy/tests/support.py new file mode 100644 index 0000000000..a3367852ad --- /dev/null +++ b/testing/mozbase/mozproxy/tests/support.py @@ -0,0 +1,14 @@ +import contextlib +import shutil +import tempfile + + +# This helper can be replaced by pytest tmpdir fixture +# once Bug 1536029 lands (@mock.patch disturbs pytest fixtures) +@contextlib.contextmanager +def tempdir(): + dest_dir = tempfile.mkdtemp() + try: + yield dest_dir + finally: + shutil.rmtree(dest_dir, ignore_errors=True) diff --git a/testing/mozbase/mozproxy/tests/test_command_line.py b/testing/mozbase/mozproxy/tests/test_command_line.py new file mode 100644 index 0000000000..27f797efa5 --- /dev/null +++ b/testing/mozbase/mozproxy/tests/test_command_line.py @@ -0,0 +1,233 @@ +#!/usr/bin/env python +import json +import os +import re +import signal +import subprocess +import sys +import threading +import time + +import mozunit +from buildconfig import topobjdir, topsrcdir +from mozprocess import ProcessHandler + +here = os.path.dirname(__file__) + + +# This is copied from <python/mozperftest/mozperftest/utils.py>. It's copied +# instead of imported since mozperfest is Python 3, and this file is +# (currently) Python 2. +def _install_package(virtualenv_manager, package): + from pip._internal.req.constructors import install_req_from_line + + req = install_req_from_line(package) + req.check_if_exists(use_user_site=False) + # already installed, check if it's in our venv + if req.satisfied_by is not None: + venv_site_lib = os.path.abspath( + os.path.join(virtualenv_manager.bin_path, "..", "lib") + ) + site_packages = os.path.abspath(req.satisfied_by.location) + if site_packages.startswith(venv_site_lib): + # already installed in this venv, we can skip + return + + subprocess.check_call( + [ + virtualenv_manager.python_path, + "-m", + "pip", + "install", + package, + ] + ) + + +def _kill_mozproxy(pid): + kill_signal = getattr(signal, "CTRL_BREAK_EVENT", signal.SIGINT) + os.kill(pid, kill_signal) + + +class OutputHandler(object): + def __init__(self): + self.port = None + self.port_event = threading.Event() + + def __call__(self, line): + if not line.strip(): + return + line = line.decode("utf-8", errors="replace") + # Print the output we received so we have useful logs if a test fails. + print(line) + + try: + data = json.loads(line) + except ValueError: + return + + if isinstance(data, dict) and "action" in data: + # Retrieve the port number for the proxy server from the logs of + # our subprocess. + m = re.match(r"Proxy running on port (\d+)", data.get("message", "")) + if m: + self.port = m.group(1) + self.port_event.set() + + def finished(self): + self.port_event.set() + + +def test_help(): + p = ProcessHandler([sys.executable, "-m", "mozproxy", "--help"]) + p.run() + assert p.wait() == 0 + + +def test_run_record_no_files(): + output_handler = OutputHandler() + p = ProcessHandler( + [ + sys.executable, + "-m", + "mozproxy", + "--local", + "--mode=record", + "--binary=firefox", + "--topsrcdir=" + topsrcdir, + "--objdir=" + topobjdir, + ], + processOutputLine=output_handler, + onFinish=output_handler.finished, + ) + + p.run() + # The first time we run mozproxy, we need to fetch mitmproxy, which can + # take a while... + assert output_handler.port_event.wait(120) is True + # Give mitmproxy a bit of time to start up so we can verify that it's + # actually running before we kill mozproxy. + time.sleep(5) + _kill_mozproxy(p.pid) + + # Assert process raises error + assert p.wait(10) == 2 + assert output_handler.port is None + + +def test_run_record_multiple_files(): + output_handler = OutputHandler() + p = ProcessHandler( + [ + sys.executable, + "-m", + "mozproxy", + "--local", + "--mode=record", + "--binary=firefox", + "--topsrcdir=" + topsrcdir, + "--objdir=" + topobjdir, + os.path.join(here, "files", "new_record.zip"), + os.path.join(here, "files", "new_record2.zip"), + ], + processOutputLine=output_handler, + onFinish=output_handler.finished, + ) + + p.run() + # The first time we run mozproxy, we need to fetch mitmproxy, which can + # take a while... + assert output_handler.port_event.wait(120) is True + # Give mitmproxy a bit of time to start up so we can verify that it's + # actually running before we kill mozproxy. + time.sleep(5) + _kill_mozproxy(p.pid) + + assert p.wait(10) == 4 + assert output_handler.port is None + + +def test_run_record(): + output_handler = OutputHandler() + p = ProcessHandler( + [ + sys.executable, + "-m", + "mozproxy", + "--local", + "--mode=record", + "--binary=firefox", + "--topsrcdir=" + topsrcdir, + "--objdir=" + topobjdir, + os.path.join(here, "files", "record.zip"), + ], + processOutputLine=output_handler, + onFinish=output_handler.finished, + ) + try: + p.run() + # The first time we run mozproxy, we need to fetch mitmproxy, which can + # take a while... + assert output_handler.port_event.wait(120) is True + # Give mitmproxy a bit of time to start up so we can verify that it's + # actually running before we kill mozproxy. + time.sleep(5) + _kill_mozproxy(p.pid) + + assert p.wait(10) == 0 + assert output_handler.port is not None + finally: + os.remove(os.path.join(here, "files", "record.zip")) + + +def test_run_playback(): + output_handler = OutputHandler() + p = ProcessHandler( + [ + sys.executable, + "-m", + "mozproxy", + "--local", + "--binary=firefox", + "--topsrcdir=" + topsrcdir, + "--objdir=" + topobjdir, + os.path.join(here, "files", "mitm5-linux-firefox-amazon.zip"), + ], + processOutputLine=output_handler, + onFinish=output_handler.finished, + ) + p.run() + # The first time we run mozproxy, we need to fetch mitmproxy, which can + # take a while... + assert output_handler.port_event.wait(120) is True + # Give mitmproxy a bit of time to start up so we can verify that it's + # actually running before we kill mozproxy. + time.sleep(5) + _kill_mozproxy(p.pid) + + assert p.wait(10) == 0 + assert output_handler.port is not None + + +def test_failure(): + output_handler = OutputHandler() + p = ProcessHandler( + [ + sys.executable, + "-m", + "mozproxy", + "--local", + # Exclude some options here to trigger a command-line error. + os.path.join(here, "files", "mitm5-linux-firefox-amazon.zip"), + ], + processOutputLine=output_handler, + onFinish=output_handler.finished, + ) + p.run() + assert output_handler.port_event.wait(10) is True + assert p.wait(10) == 2 + assert output_handler.port is None + + +if __name__ == "__main__": + mozunit.main(runwith="pytest") diff --git a/testing/mozbase/mozproxy/tests/test_mitm_addons.py b/testing/mozbase/mozproxy/tests/test_mitm_addons.py new file mode 100644 index 0000000000..ed3805ef9d --- /dev/null +++ b/testing/mozbase/mozproxy/tests/test_mitm_addons.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python +import json +import os +from unittest import mock + +import mozunit + +here = os.path.dirname(__file__) +os.environ["MOZPROXY_DIR"] = os.path.join(here, "files") + +protocol = { + "http_protocol": {"aax-us-iad.amazon.com": "HTTP/1.1"}, + "recorded_requests": 4, + "recorded_requests_unique": 1, +} + + +@mock.patch( + "mozproxy.backends.mitm.scripts.http_protocol_extractor.HttpProtocolExtractor.get_ctx" +) +def test_http_protocol_generate_json_file(ctx_mock): + ctx_mock.return_value.options.save_stream_file = os.path.join( + os.environ["MOZPROXY_DIR"], "http_protocol_recording_done.mp" + ) + + from mozproxy.backends.mitm.scripts.http_protocol_extractor import ( + HttpProtocolExtractor, + ) + + test_http_protocol = HttpProtocolExtractor() + test_http_protocol.ctx = test_http_protocol.get_ctx() + + # test data + test_http_protocol.request_protocol = protocol["http_protocol"] + test_http_protocol.hashes = ["Hash string"] + test_http_protocol.request_count = protocol["recorded_requests"] + + test_http_protocol.done() + + json_path = os.path.join( + os.environ["MOZPROXY_DIR"], "http_protocol_recording_done.json" + ) + assert os.path.exists(json_path) + with open(json_path) as json_file: + output_data = json.load(json_file) + + assert output_data["recorded_requests"] == protocol["recorded_requests"] + assert ( + output_data["recorded_requests_unique"] + == protocol["recorded_requests_unique"] + ) + assert output_data["http_protocol"] == protocol["http_protocol"] + + +@mock.patch( + "mozproxy.backends.mitm.scripts.http_protocol_extractor.HttpProtocolExtractor.get_ctx" +) +def test_http_protocol_response(ctx_mock): + ctx_mock.return_value.options.save_stream_file = os.path.join( + os.environ["MOZPROXY_DIR"], "http_protocol_recording_done.mp" + ) + + from mozproxy.backends.mitm.scripts.http_protocol_extractor import ( + HttpProtocolExtractor, + ) + + test_http_protocol = HttpProtocolExtractor() + test_http_protocol.ctx = test_http_protocol.get_ctx() + + # test data + flow = mock.MagicMock() + flow.type = "http" + flow.request.url = "https://www.google.com/complete/search" + flow.request.port = 33 + flow.response.data.http_version = b"HTTP/1.1" + + test_http_protocol.request_protocol = {} + test_http_protocol.hashes = [] + test_http_protocol.request_count = 0 + + test_http_protocol.response(flow) + + assert test_http_protocol.request_count == 1 + assert len(test_http_protocol.hashes) == 1 + assert test_http_protocol.request_protocol["www.google.com"] == "HTTP/1.1" + + +if __name__ == "__main__": + mozunit.main(runwith="pytest") diff --git a/testing/mozbase/mozproxy/tests/test_proxy.py b/testing/mozbase/mozproxy/tests/test_proxy.py new file mode 100644 index 0000000000..47da3117ba --- /dev/null +++ b/testing/mozbase/mozproxy/tests/test_proxy.py @@ -0,0 +1,196 @@ +#!/usr/bin/env python +import os +from unittest import mock + +import mozinfo +import mozunit +import requests +from mozproxy import get_playback +from support import tempdir + +here = os.path.dirname(__file__) + + +class Process: + def __init__(self, *args, **kw): + pass + + def run(self): + print("I am running something") + + def poll(self): + return None + + def wait(self): + return 0 + + def kill(self, sig=9): + pass + + proc = object() + pid = 1234 + stderr = stdout = None + returncode = 0 + + +_RETRY = 0 + + +class ProcessWithRetry(Process): + def __init__(self, *args, **kw): + Process.__init__(self, *args, **kw) + + def wait(self): + global _RETRY + _RETRY += 1 + if _RETRY >= 2: + _RETRY = 0 + return 0 + return -1 + + +def kill(pid, signal): + if pid == 1234: + return + return os.kill(pid, signal) + + +def get_status_code(url, playback): + response = requests.get( + url=url, proxies={"http": "http://%s:%s/" % (playback.host, playback.port)} + ) + return response.status_code + + +def test_mitm_check_proxy(*args): + # test setup + pageset_name = os.path.join(here, "files", "mitm5-linux-firefox-amazon.manifest") + + config = { + "playback_tool": "mitmproxy", + "playback_files": [os.path.join(here, "files", pageset_name)], + "playback_version": "8.1.1", + "platform": mozinfo.os, + "run_local": "MOZ_AUTOMATION" not in os.environ, + "binary": "firefox", + "app": "firefox", + "host": "127.0.0.1", + } + + with tempdir() as obj_path: + config["obj_path"] = obj_path + playback = get_playback(config) + assert playback is not None + + try: + playback.start() + + url = "https://m.media-amazon.com/images/G/01/csm/showads.v2.js" + assert get_status_code(url, playback) == 200 + + url = "http://mozproxy/checkProxy" + assert get_status_code(url, playback) == 404 + finally: + playback.stop() + + +@mock.patch("mozproxy.backends.mitm.Mitmproxy.check_proxy") +@mock.patch("mozproxy.backends.mitm.mitm.ProcessHandler", new=Process) +@mock.patch("mozproxy.utils.ProcessHandler", new=Process) +@mock.patch("os.kill", new=kill) +def test_mitm(*args): + pageset_name = os.path.join(here, "files", "mitm5-linux-firefox-amazon.manifest") + + config = { + "playback_tool": "mitmproxy", + "playback_files": [pageset_name], + "playback_version": "8.1.1", + "platform": mozinfo.os, + "run_local": True, + "binary": "firefox", + "app": "firefox", + "host": "example.com", + } + + with tempdir() as obj_path: + config["obj_path"] = obj_path + playback = get_playback(config) + assert playback is not None + try: + playback.start() + finally: + playback.stop() + + +@mock.patch("mozproxy.backends.mitm.Mitmproxy.check_proxy") +@mock.patch("mozproxy.backends.mitm.mitm.ProcessHandler", new=Process) +@mock.patch("mozproxy.utils.ProcessHandler", new=Process) +@mock.patch("os.kill", new=kill) +def test_playback_setup_failed(*args): + class SetupFailed(Exception): + pass + + def setup(*args, **kw): + def _s(self): + raise SetupFailed("Failed") + + return _s + + pageset_name = os.path.join(here, "files", "mitm5-linux-firefox-amazon.manifest") + + config = { + "playback_tool": "mitmproxy", + "playback_files": [pageset_name], + "playback_version": "4.0.4", + "platform": mozinfo.os, + "run_local": True, + "binary": "firefox", + "app": "firefox", + "host": "example.com", + } + + prefix = "mozproxy.backends.mitm.desktop.MitmproxyDesktop." + + with tempdir() as obj_path: + config["obj_path"] = obj_path + with mock.patch(prefix + "setup", new_callable=setup): + with mock.patch(prefix + "stop_mitmproxy_playback") as p: + try: + pb = get_playback(config) + pb.start() + except SetupFailed: + assert p.call_count == 1 + except Exception: + raise + + +@mock.patch("mozproxy.backends.mitm.Mitmproxy.check_proxy") +@mock.patch("mozproxy.backends.mitm.mitm.ProcessHandler", new=ProcessWithRetry) +@mock.patch("mozproxy.utils.ProcessHandler", new=ProcessWithRetry) +@mock.patch("os.kill", new=kill) +def test_mitm_with_retry(*args): + pageset_name = os.path.join(here, "files", "mitm5-linux-firefox-amazon.manifest") + + config = { + "playback_tool": "mitmproxy", + "playback_files": [pageset_name], + "playback_version": "8.1.1", + "platform": mozinfo.os, + "run_local": True, + "binary": "firefox", + "app": "firefox", + "host": "example.com", + } + + with tempdir() as obj_path: + config["obj_path"] = obj_path + playback = get_playback(config) + assert playback is not None + try: + playback.start() + finally: + playback.stop() + + +if __name__ == "__main__": + mozunit.main(runwith="pytest") diff --git a/testing/mozbase/mozproxy/tests/test_recording.py b/testing/mozbase/mozproxy/tests/test_recording.py new file mode 100644 index 0000000000..5f29fe04cd --- /dev/null +++ b/testing/mozbase/mozproxy/tests/test_recording.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python +import datetime +import os +from builtins import Exception + +import mozinfo +import mozunit +import requests +from mozproxy import get_playback +from support import tempdir + +here = os.path.dirname(__file__) +os.environ["MOZPROXY_DIR"] = os.path.join(here, "files") + + +def get_status_code(url, playback): + response = requests.get( + url=url, proxies={"http": "http://%s:%s/" % (playback.host, playback.port)} + ) + return response.status_code + + +def test_record_and_replay(*args): + # test setup + + basename = "recording" + suffix = datetime.datetime.now().strftime("%y%m%d_%H%M%S") + filename = "_".join([basename, suffix]) + recording_file = os.path.join(here, "files", ".".join([filename, "zip"])) + + # Record part + config = { + "playback_tool": "mitmproxy", + "recording_file": recording_file, + "playback_version": "8.1.1", + "platform": mozinfo.os, + "run_local": "MOZ_AUTOMATION" not in os.environ, + "binary": "firefox", + "app": "firefox", + "host": "127.0.0.1", + "record": True, + } + + with tempdir() as obj_path: + config["obj_path"] = obj_path + record = get_playback(config) + assert record is not None + + try: + record.start() + + url = "https://m.media-amazon.com/images/G/01/csm/showads.v2.js" + assert get_status_code(url, record) == 200 + finally: + record.stop() + + # playback part + config["record"] = False + config["recording_file"] = None + config["playback_files"] = [recording_file] + playback = get_playback(config) + assert playback is not None + try: + playback.start() + + url = "https://m.media-amazon.com/images/G/01/csm/showads.v2.js" + assert get_status_code(url, playback) == 200 + finally: + playback.stop() + + # Cleanup + try: + os.remove(recording_file) + except Exception: + pass + + +if __name__ == "__main__": + mozunit.main(runwith="pytest") diff --git a/testing/mozbase/mozproxy/tests/test_recordings.py b/testing/mozbase/mozproxy/tests/test_recordings.py new file mode 100644 index 0000000000..537a0eb7c2 --- /dev/null +++ b/testing/mozbase/mozproxy/tests/test_recordings.py @@ -0,0 +1,33 @@ +#!/usr/bin/env python +import os + +import mozunit +from mozproxy.recordings import RecordingFile + +here = os.path.dirname(__file__) +os.environ["MOZPROXY_DIR"] = os.path.join(here, "files") + + +def test_recording_generation(*args): + test_file = os.path.join(here, "files", "new_file.zip") + file = RecordingFile(test_file) + with open(file.recording_path, "w") as recording: + recording.write("This is a recording") + + file.set_metadata("test_file", True) + file.generate_zip_file() + + assert os.path.exists(test_file) + os.remove(test_file) + + +def test_recording_content(*args): + test_file = os.path.join(here, "files", "recording.zip") + file = RecordingFile(test_file) + + assert file.metadata("test_file") is True + assert os.path.exists(file.recording_path) + + +if __name__ == "__main__": + mozunit.main(runwith="pytest") diff --git a/testing/mozbase/mozproxy/tests/test_utils.py b/testing/mozbase/mozproxy/tests/test_utils.py new file mode 100644 index 0000000000..7cd661dee2 --- /dev/null +++ b/testing/mozbase/mozproxy/tests/test_utils.py @@ -0,0 +1,32 @@ +#!/usr/bin/env python +import os +import shutil +from unittest import mock + +import mozunit +from mozproxy.utils import download_file_from_url +from support import tempdir + +here = os.path.dirname(__file__) + + +def urlretrieve(*args, **kw): + def _urlretrieve(url, local_dest): + # simply copy over our tarball + shutil.copyfile(os.path.join(here, "archive.tar.gz"), local_dest) + return local_dest, {} + + return _urlretrieve + + +@mock.patch("mozproxy.utils.urlretrieve", new_callable=urlretrieve) +def test_download_file(*args): + with tempdir() as dest_dir: + dest = os.path.join(dest_dir, "archive.tar.gz") + download_file_from_url("http://example.com/archive.tar.gz", dest, extract=True) + # archive.tar.gz contains hey.txt, if it worked we should see it + assert os.path.exists(os.path.join(dest_dir, "hey.txt")) + + +if __name__ == "__main__": + mozunit.main(runwith="pytest") |