diff options
Diffstat (limited to 'testing/raptor')
24 files changed, 634 insertions, 197 deletions
diff --git a/testing/raptor/browsertime/motionmark-1-3.js b/testing/raptor/browsertime/motionmark-1-3.js new file mode 100644 index 0000000000..c240b2dddc --- /dev/null +++ b/testing/raptor/browsertime/motionmark-1-3.js @@ -0,0 +1,107 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* eslint-env node */ + +// index for the CSS selector in developer.html +const suiteSelectorNumber = { + MotionMark: 1, + "HTML suite": 2, +}; + +module.exports = async function (context, commands) { + context.log.info("Starting MotionMark 1.3 test"); + let url = context.options.browsertime.url; + let page_cycles = context.options.browsertime.page_cycles; + let suite_name = context.options.browsertime.suite_name; + let page_cycle_delay = context.options.browsertime.page_cycle_delay; + let post_startup_delay = context.options.browsertime.post_startup_delay; + let page_timeout = context.options.timeouts.pageLoad; + let expose_profiler = context.options.browsertime.expose_profiler; + + context.log.info( + "Waiting for %d ms (post_startup_delay)", + post_startup_delay + ); + await commands.wait.byTime(post_startup_delay); + + for (let count = 0; count < page_cycles; count++) { + context.log.info("Navigating to about:blank"); + await commands.navigate("about:blank"); + + context.log.info("Cycle %d, waiting for %d ms", count, page_cycle_delay); + await commands.wait.byTime(page_cycle_delay); + + context.log.info("Cycle %d, starting the measure", count); + if (expose_profiler === "true") { + context.log.info("Custom profiler start!"); + if (context.options.browser === "firefox") { + await commands.profiler.start(); + } else if (context.options.browser === "chrome") { + await commands.trace.start(); + } + } + await commands.measure.start(url); + + let suite_selector = `#suites > ul > li:nth-child(${suiteSelectorNumber[suite_name]}) > label > input[type="checkbox"]`; + + await commands.mouse.singleClick.bySelector(suite_selector); + await commands.js.runAndWait(` + this.benchmarkController.startBenchmark() + `); + + let data_exists = null; + let starttime = await commands.js.run(`return performance.now();`); + while ( + (data_exists == null || !Object.keys(data_exists).length) && + (await commands.js.run(`return performance.now();`)) - starttime < + page_timeout + ) { + let wait_time = 3000; + context.log.info( + "Waiting %d ms for data from %s...", + wait_time, + suite_name + ); + await commands.wait.byTime(wait_time); + + data_exists = await commands.js.run(` + return window.benchmarkRunnerClient.results.data + `); + } + + if (expose_profiler === "true") { + context.log.info("Custom profiler stop!"); + if (context.options.browser === "firefox") { + await commands.profiler.stop(); + } else if (context.options.browser === "chrome") { + await commands.trace.stop(); + } + } + if ( + !data_exists && + (await commands.js.run(`return performance.now();`)) - starttime >= + page_timeout + ) { + context.log.error("Benchmark timed out. Aborting..."); + return false; + } + + let data = null; + data = await commands.js.run(` + const score = window.benchmarkRunnerClient.results.score; + const results = window.benchmarkRunnerClient.results.results[0].testsResults; + return { + score, + results, + }; + `); + data.suite_name = suite_name; + + commands.measure.addObject({ mm_res: data }); + context.log.info("Value of summarized benchmark data: ", data); + } + + return true; +}; diff --git a/testing/raptor/browsertime/support-scripts/motionmark-1-3.py b/testing/raptor/browsertime/support-scripts/motionmark-1-3.py new file mode 100644 index 0000000000..713935fd3f --- /dev/null +++ b/testing/raptor/browsertime/support-scripts/motionmark-1-3.py @@ -0,0 +1,91 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import filters +from base_python_support import BasePythonSupport + + +class MotionMarkSupport(BasePythonSupport): + def handle_result(self, bt_result, raw_result, **kwargs): + """Parse a result for the required results. + + See base_python_support.py for what's expected from this method. + """ + suite_name = raw_result["extras"][0]["mm_res"]["suite_name"] + score_tracker = { + subtest: [] + for subtest in raw_result["extras"][0]["mm_res"]["results"][ + suite_name + ].keys() + } + + motionmark_overall_score = [] + for res in raw_result["extras"]: + motionmark_overall_score.append(round(res["mm_res"]["score"], 3)) + + for k, v in res["mm_res"]["results"][suite_name].items(): + score_tracker[k].append(v["complexity"]["bootstrap"]["median"]) + + for k, v in score_tracker.items(): + bt_result["measurements"][k] = v + + bt_result["measurements"]["score"] = motionmark_overall_score + + def _build_subtest(self, measurement_name, replicates, test): + unit = test.get("unit", "ms") + if test.get("subtest_unit"): + unit = test.get("subtest_unit") + + lower_is_better = test.get( + "subtest_lower_is_better", test.get("lower_is_better", True) + ) + if "score" in measurement_name: + lower_is_better = False + unit = "score" + + subtest = { + "unit": unit, + "alertThreshold": float(test.get("alert_threshold", 2.0)), + "lowerIsBetter": lower_is_better, + "name": measurement_name, + "replicates": replicates, + "value": round(filters.mean(replicates), 3), + } + + return subtest + + def summarize_test(self, test, suite, **kwargs): + """Summarize the measurements found in the test as a suite with subtests. + + See base_python_support.py for what's expected from this method. + """ + suite["type"] = "benchmark" + if suite["subtests"] == {}: + suite["subtests"] = [] + for measurement_name, replicates in test["measurements"].items(): + if not replicates: + continue + suite["subtests"].append( + self._build_subtest(measurement_name, replicates, test) + ) + suite["subtests"].sort(key=lambda subtest: subtest["name"]) + + score = 0 + for subtest in suite["subtests"]: + if subtest["name"] == "score": + score = subtest["value"] + break + suite["value"] = score + + def modify_command(self, cmd, test): + """Modify the browsertime command to have the appropriate suite name. + + This is necessary to grab the correct CSS selector in the browsertime + script, and later for parsing through the final benchmark data in the + support python script (this file). + + Current options are `MotionMark` and `HTML suite`. + """ + + cmd += ["--browsertime.suite_name", test.get("suite_name")] diff --git a/testing/raptor/browsertime/support-scripts/sample_python_support.py b/testing/raptor/browsertime/support-scripts/sample_python_support.py index a1ec0069a5..b31e890c0a 100644 --- a/testing/raptor/browsertime/support-scripts/sample_python_support.py +++ b/testing/raptor/browsertime/support-scripts/sample_python_support.py @@ -6,7 +6,7 @@ from base_python_support import BasePythonSupport class SamplePythonSupport(BasePythonSupport): - def modify_command(self, cmd): + def modify_command(self, cmd, test): for i, entry in enumerate(cmd): if "{replace-with-constant-value}" in entry: cmd[i] = "25" diff --git a/testing/raptor/mach_commands.py b/testing/raptor/mach_commands.py index b1c5002efc..14ffb1caf2 100644 --- a/testing/raptor/mach_commands.py +++ b/testing/raptor/mach_commands.py @@ -68,6 +68,7 @@ class RaptorRunner(MozbuildObject): self.browsertime_visualmetrics = kwargs["browsertime_visualmetrics"] self.browsertime_node = kwargs["browsertime_node"] self.clean = kwargs["clean"] + self.screenshot_on_failure = kwargs["screenshot_on_failure"] if Conditions.is_android(self) or kwargs["app"] in ANDROID_BROWSERS: self.binary_path = None @@ -122,6 +123,7 @@ class RaptorRunner(MozbuildObject): "browsertime_node": self.browsertime_node, "mozbuild_path": get_state_dir(), "clean": self.clean, + "screenshot_on_failure": self.screenshot_on_failure, } sys.path.insert(0, os.path.join(self.topsrcdir, "tools", "browsertime")) @@ -267,7 +269,7 @@ class RaptorRunner(MozbuildObject): def setup_node(command_context): - """Fetch the latest node-16 binary and install it into the .mozbuild directory.""" + """Fetch the latest node-18 binary and install it into the .mozbuild directory.""" import platform from mozbuild.artifact_commands import artifact_toolchain @@ -276,11 +278,11 @@ def setup_node(command_context): print("Setting up node for browsertime...") state_dir = get_state_dir() - cache_path = os.path.join(state_dir, "browsertime", "node-16") + cache_path = os.path.join(state_dir, "browsertime", "node-18") def __check_for_node(): # Check standard locations first - node_exe = find_node_executable(min_version=Version("16.0.0")) + node_exe = find_node_executable(min_version=Version("18.0.0")) if node_exe and (node_exe[0] is not None): return node_exe[0] if not os.path.exists(cache_path): @@ -293,14 +295,14 @@ def setup_node(command_context): node_exe_path = os.path.join( state_dir, "browsertime", - "node-16", + "node-18", "node", ) else: node_exe_path = os.path.join( state_dir, "browsertime", - "node-16", + "node-18", "node", "bin", ) @@ -313,7 +315,7 @@ def setup_node(command_context): node_exe = __check_for_node() if node_exe is None: - toolchain_job = "{}-node-16" + toolchain_job = "{}-node-18" plat = platform.system() if plat == "Windows": toolchain_job = toolchain_job.format("win64") @@ -326,7 +328,7 @@ def setup_node(command_context): toolchain_job = toolchain_job.format("linux64") print( - "Downloading Node v16 from Taskcluster toolchain {}...".format( + "Downloading Node v18 from Taskcluster toolchain {}...".format( toolchain_job ) ) @@ -351,11 +353,11 @@ def setup_node(command_context): node_exe = __check_for_node() if node_exe is None: - raise Exception("Could not find Node v16 binary for Raptor-Browsertime") + raise Exception("Could not find Node v18 binary for Raptor-Browsertime") - print("Finished downloading Node v16 from Taskcluster") + print("Finished downloading Node v18 from Taskcluster") - print("Node v16+ found at: %s" % node_exe) + print("Node v18+ found at: %s" % node_exe) return node_exe diff --git a/testing/raptor/raptor/base_python_support.py b/testing/raptor/raptor/base_python_support.py index ac1281aadc..68d6651f36 100644 --- a/testing/raptor/raptor/base_python_support.py +++ b/testing/raptor/raptor/base_python_support.py @@ -22,11 +22,15 @@ class BasePythonSupport: """ pass - def modify_command(self, cmd): + def modify_command(self, cmd, test): """Used to modify the Browsertime command before running the test. The `cmd` arg holds the current browsertime command to run. It can be changed directly to change how browsertime runs. + + The `test` arg is the test itself with all of its current settings. + It can be modified as needed to add additional information to the + test that will run. """ pass diff --git a/testing/raptor/raptor/browsertime/android.py b/testing/raptor/raptor/browsertime/android.py index 6ca6faabb3..6d83a5063a 100644 --- a/testing/raptor/raptor/browsertime/android.py +++ b/testing/raptor/raptor/browsertime/android.py @@ -126,6 +126,10 @@ class BrowsertimeAndroid(PerftestAndroid, Browsertime): ] ) + if self.config["app"] == "geckoview": + # This is needed as geckoview is crashing on shutdown and is throwing marionette errors similar to 1768889 + args_list.extend(["--ignoreShutdownFailures", "true"]) + if self.config["app"] == "fenix": # See bug 1768889 args_list.extend(["--ignoreShutdownFailures", "true"]) diff --git a/testing/raptor/raptor/browsertime/base.py b/testing/raptor/raptor/browsertime/base.py index f0e9946ecf..1ddf0a91d5 100644 --- a/testing/raptor/raptor/browsertime/base.py +++ b/testing/raptor/raptor/browsertime/base.py @@ -16,7 +16,7 @@ from copy import deepcopy import mozprocess import six from benchmark import Benchmark -from cmdline import CHROME_ANDROID_APPS +from cmdline import CHROME_ANDROID_APPS, DESKTOP_APPS from logger.logger import RaptorLogger from manifestparser.util import evaluate_list_from_string from perftest import GECKO_PROFILER_APPS, TRACE_APPS, Perftest @@ -825,6 +825,28 @@ class Browsertime(Perftest): os.killpg(proc.pid, signal.SIGKILL) proc.wait() + def get_failure_screenshot(self): + if not ( + self.config.get("screenshot_on_failure") + and self.config["app"] in DESKTOP_APPS + ): + return + + # Bug 1884178 + # Temporarily disable on Windows + Chrom* applications. + if self.config["app"] in TRACE_APPS and "win" in self.config["platform"]: + return + + from mozscreenshot import dump_screen + + obj_dir = os.environ.get("MOZ_DEVELOPER_OBJ_DIR", None) + if obj_dir is None: + build_dir = pathlib.Path(os.environ.get("MOZ_UPLOAD_DIR")).parent + utility_path = pathlib.Path(build_dir, "tests", "bin") + else: + utility_path = os.path.join(obj_dir, "dist", "bin") + dump_screen(utility_path, LOG) + def run_extra_profiler_run( self, test, timeout, proc_timeout, output_timeout, line_handler, env ): @@ -899,7 +921,7 @@ class Browsertime(Perftest): if test.get("support_class", None): LOG.info("Test support class is modifying the command...") - test.get("support_class").modify_command(cmd) + test.get("support_class").modify_command(cmd, test) output_timeout = BROWSERTIME_PAGELOAD_OUTPUT_TIMEOUT if test.get("type", "") == "scenario": @@ -1016,16 +1038,19 @@ class Browsertime(Perftest): ) if self.output_timed_out: + self.get_failure_screenshot() raise Exception( f"Browsertime process timed out after waiting {output_timeout} seconds " "for output" ) if self.timed_out: + self.get_failure_screenshot() raise Exception( f"Browsertime process timed out after {proc_timeout} seconds" ) if self.browsertime_failure: + self.get_failure_screenshot() raise Exception(self.browsertime_failure) # We've run the main browsertime process, now we need to run the diff --git a/testing/raptor/raptor/cmdline.py b/testing/raptor/raptor/cmdline.py index 8278dafd79..3021f080ee 100644 --- a/testing/raptor/raptor/cmdline.py +++ b/testing/raptor/raptor/cmdline.py @@ -64,6 +64,13 @@ GECKO_PROFILER_APPS = (FIREFOX, GECKOVIEW, REFBROW, FENIX) TRACE_APPS = (CHROME, CHROMIUM, CHROMIUM_RELEASE) +APP_BINARIES = { + "fenix": "org.mozilla.fenix", + "focus": "org.mozilla.focus", + "geckoview": "org.mozilla.geckoview_example", + "refbrow": "org.mozilla.reference.browser", +} + def print_all_activities(): all_activities = [] @@ -244,7 +251,7 @@ def create_parser(mach_interface=False): "--post-startup-delay", dest="post_startup_delay", type=int, - default=30000, + default=None, help="How long to wait (ms) after browser start-up before starting the tests", ) add_arg( @@ -520,6 +527,13 @@ def create_parser(mach_interface=False): type=str, help="Repository branch that should be used for a particular benchmark test.", ) + add_arg( + "--screenshot-on-failure", + action="store_true", + dest="screenshot_on_failure", + default=False, + help="Take a screenshot when the test fails.", + ) add_logging_group(parser) return parser @@ -528,7 +542,11 @@ def create_parser(mach_interface=False): def verify_options(parser, args): ctx = vars(args) if args.binary is None and args.app != "chrome-m": - parser.error("--binary is required!") + args.binary = APP_BINARIES.get(args.app, None) + if args.binary is None: + parser.error("--binary is required!") + else: + print(f"Using {args.binary} as default binary argument for {args.app} app") # Debug-mode is disabled in CI (check for attribute in case of mach_interface issues) if hasattr(args, "run_local") and (not args.run_local and args.debug_mode): @@ -604,6 +622,10 @@ def verify_options(parser, args): "When a benchmark repository is provided, a revision is also required." ) + if args.post_startup_delay: + if args.post_startup_delay < 0: + parser.error("--post-startup-delay must be a positive integer (in ms).") + def parse_args(argv=None): parser = create_parser() diff --git a/testing/raptor/raptor/perfdocs/browsertime.rst b/testing/raptor/raptor/perfdocs/browsertime.rst index 4d1975224f..beff3f0ad8 100644 --- a/testing/raptor/raptor/perfdocs/browsertime.rst +++ b/testing/raptor/raptor/perfdocs/browsertime.rst @@ -34,7 +34,7 @@ For example, here's a test on ``https://www.sitespeed.io`` using this custom tes :: - ./mach raptor --browsertime -t browsertime --browsertime-arg test_script=pageload --browsertime-arg browsertime.url=https://www.sitespeed.io --browsertime-arg iterations=3 + ./mach raptor -t browsertime --browsertime-arg test_script=pageload --browsertime-arg browsertime.url=https://www.sitespeed.io --browsertime-arg iterations=3 That test will perform 3 iterations of the given url. Note also that we can use simplified names to make use of test scripts that are built into raptor. You can use ``pageload``, ``interactive``, or provide a path to another test script. @@ -50,7 +50,7 @@ There are two ways to run performance tests through browsertime listed below. :: - ./mach raptor --browsertime -t google-search + ./mach raptor -t google-search * Browsertime-"native": @@ -64,23 +64,25 @@ Benchmark tests :: - ./mach raptor -t raptor-speedometer --browsertime + ./mach raptor -t speedometer Running on Android ------------------ +To run on android, the device needs to have Geckoview or Fenix installed on it. Our tests will only work with physical devices, and `bug 1881570 <https://bugzilla.mozilla.org/show_bug.cgi?id=1881570>`__ tracks progress for enabling virtual devices (emulators). Running either of the commands below will attempt to install locally built APKs to the device **while uninstalling/removing any existing APKs of the package on the device**, but this can be skipped by setting ``MOZ_DISABLE_ADB_INSTALL=1`` in your environment. When that environment variable exists, we expect the APK to be pre-installed on the device. + Running on Raptor-Browsertime (recommended): * Running on Fenix :: - ./mach raptor --browsertime -t amazon --app fenix --binary org.mozilla.fenix + ./mach raptor -t amazon --app fenix --binary org.mozilla.fenix * Running on Geckoview :: - ./mach raptor --browsertime -t amazon --app geckoview --binary org.mozilla.geckoview_example + ./mach raptor -t amazon --app geckoview --binary org.mozilla.geckoview_example Running on vanilla Browsertime: @@ -122,7 +124,7 @@ Or for Raptor-Browsertime (use ``chrome`` for desktop, and ``chrome-m`` for mobi :: - ./mach raptor --browsertime -t amazon --app chrome --browsertime-chromedriver <PATH/TO/CHROMEDRIVER> + ./mach raptor -t amazon --app chrome --browsertime-chromedriver <PATH/TO/CHROMEDRIVER> Running Page-load tests with third party WebExtensions ------------------------------------------------------ @@ -137,14 +139,14 @@ Launch amazon tp6 page-load test on Firefox Desktop: :: - ./mach raptor --browsertime -t amazon --conditioned-profile settled-webext + ./mach raptor -t amazon --conditioned-profile settled-webext Launch amazon tp6 mobile page-load test on Firefox for Android (the apk has to be pre-installed, mach raptor does detect if already installed but it does not install it): :: - ./mach raptor --browsertime -t amazon --app fenix --binary org.mozilla.fenix --conditioned-profile settled-webext + ./mach raptor -t amazon --app fenix --binary org.mozilla.fenix --conditioned-profile settled-webext To run these jobs on try, make sure to select the tp6 jobs that include the string `webextensions`, as an example (add ``--no-push`` to force try fuzzy to only list the jobs selected by the try fuzzy query) to run all tp6 page-load webextensons jobs currently defined: @@ -153,12 +155,14 @@ list the jobs selected by the try fuzzy query) to run all tp6 page-load webexten ./mach try fuzzy -q "'tp6 'webextensions" -Similarly for running tp6m on Firefox for Android builds: +Similarly for running tp6m (equivalent to tp6 but for mobile) on Firefox for Android builds: :: ./mach try fuzzy --full -q "'tp6m 'webextensions" +Note that this can also be done using ``./mach try perf --show-all -q "'tp6m 'webextensions"`` to produce a compare view link of the changes before/after the patch being tested. + The set of extensions installed are the ones listed in the ``"addons"`` property of the condprof customization file `webext.json`_ from the ``testing/condprofile/condprof/customization/`` directory. @@ -196,20 +200,27 @@ Other methods for adding additional arguments are: Running Browsertime on Try -------------------------- -You can run all of our browsertime pageload tests through ``./mach try fuzzy --full``. We use chimera mode in these tests which means that both cold and warm pageload variants are running at the same time. -For example: +You can run all of our browsertime pageload tests through ``./mach try perf`` by selecting the ``Pageload`` category. We use chimera mode in these tests which means that both cold and warm pageload variants are running at the same time. There are a lot of other tests/categories available as well. Documentation about this tool can be found in :ref:`Mach Try Perf`. + +For example, the following will select all ``Pageload`` categories to run on desktop: + +:: + + ./mach try perf -q "'Pageload" + +If you need to target android tasks, include the ``--android`` flag like so (remove the ``'android`` from the query string to target desktop tests at the same time): :: - ./mach try fuzzy -q "'g5 'imdb 'geckoview 'vismet '-wr 'shippable" + ./mach try perf --android -q "'Pageload 'android" -Retriggering Browsertime Visual Metrics Tasks ---------------------------------------------- +If you exclude the ``-q "..."`` option, an interface similar to the fuzzy interface will open, and show all available categories. -You can retrigger Browsertime tasks just like you retrigger any other tasks from Treeherder (using the retrigger buttons, add-new-jobs, retrigger-multiple, etc.). +Visual Metrics +-------------- -The following metrics are collected each time: ``fcp, loadtime, ContentfulSpeedIndex, PerceptualSpeedIndex, SpeedIndex, FirstVisualChange, LastVisualChange`` +The following visual metrics are collected in all page load tests: ``ContentfulSpeedIndex, PerceptualSpeedIndex, SpeedIndex, FirstVisualChange, LastVisualChange`` Further information regarding these metrics can be viewed at `visual-metrics <https://www.sitespeed.io/documentation/sitespeed.io/metrics/#visual-metrics>`_ @@ -220,7 +231,7 @@ To run gecko profiling using Raptor-Browsertime you can add the ``--gecko-profil :: - ./mach raptor --browsertime -t amazon --gecko-profile + ./mach raptor -t amazon --gecko-profile Note that vanilla Browsertime does support Gecko Profiling but **it does not symbolicate the profiles** so it is **not recommended** to use for debugging performance regressions/improvements. @@ -248,6 +259,7 @@ Likewise, for chrome trace you will want to be aware of `these defaults. <https: Upgrading Browsertime In-Tree ----------------------------- + To upgrade the browsertime version used in-tree you can run, then commit the changes made to ``package.json`` and ``package-lock.json``: :: @@ -260,6 +272,7 @@ To test the upgrade, run a raptor test locally (with and without visual-metrics Updating Benchmark Tests ------------------------ + To upgrade any of our benchmark tests, you will need to change the revision used in the test manifest. There are three fields that you have available to use there: ``repository_revision`` to denote the revision, ``repository_branch`` to denote the branch name, and ``repository`` to provide the link of the Github repo that contains the benchmark. For instance, with Speedometer 3 (sp3), we can update the revision `by changing the repository_revision field found here <https://searchfox.org/mozilla-central/rev/aa3ccd258b64abfd4c5ce56c1f512bc7f65b844c/testing/raptor/raptor/tests/benchmarks/speedometer-desktop.ini#29>`_. If the change isn't found on the default branch (master/main branch), then you will need to add an entry for ``repository_branch`` to specify this. @@ -268,6 +281,7 @@ If the path to the test file changes (the file that is invoked to run the test), Finding the Geckodriver Being Used ---------------------------------- + If you're looking for the latest geckodriver being used there are two ways: * Find the latest one from here: https://treeherder.mozilla.org/jobs?repo=mozilla-central&searchStr=geckodriver * Alternatively, if you're trying to figure out which geckodriver a given CI task is using, you can click on the browsertime task in treeherder, and then click on the ``Task`` id in the bottom left of the pop-up interface. Then in the window that opens up, click on `See more` in the task details tab on the left, this will show you the dependent tasks with the latest toolchain-geckodriver being used. There's an Artifacts drop down on the right hand side for the toolchain-geckodriver task that you can find the latest geckodriver in. @@ -287,7 +301,7 @@ Mach Browsertime Setup ---------------------- **WARNING** - Raptor-Browsertime (i.e. ``./mach raptor --browsertime -t <TEST>``) is currently required to be ran first in order to acquire the Node-16 binary. In general, it is also not recommended to use ``./mach browsertime`` for testing as it will be deprecated soon. + Raptor-Browsertime (i.e. ``./mach raptor -t <TEST>``) is currently required to be ran first in order to acquire the Node-16 binary. In general, it is also not recommended to use ``./mach browsertime`` for testing as it will be deprecated soon. Note that if you are running Raptor-Browsertime then it will get installed automatically and also update itself. Otherwise, you can run: @@ -311,8 +325,8 @@ With the replacement of ImageMagick, former cross platform installation issues h -- For other issues, try deleting the ``~/.mozbuild/browsertime`` folder and re-running the browsertime setup command or a Raptor-Browsertime test +- For other issues, try deleting the ``~/.mozbuild/browsertime`` folder and re-running the browsertime setup command or a Raptor-Browsertime test. Alternatively, you may need to delete the ``tools/browsertime/node_modules`` folder. -- If you plan on running Browsertime on Android, your Android device must already be set up (see more below in the :ref: `Running on Android` section) +- If you plan on running Browsertime on Android, your Android device must already be set up (see more above in the :ref: `Running on Android` section) - **If you encounter any issues not mentioned here, please** `file a bug <https://bugzilla.mozilla.org/enter_bug.cgi?product=Testing&component=Raptor>`_ **in the** ``Testing::Raptor`` **component.** diff --git a/testing/raptor/raptor/perfdocs/config.yml b/testing/raptor/raptor/perfdocs/config.yml index 12b3b48c11..a9ab330feb 100644 --- a/testing/raptor/raptor/perfdocs/config.yml +++ b/testing/raptor/raptor/perfdocs/config.yml @@ -28,131 +28,133 @@ suites: desktop: description: "Tests for page-load performance. The links direct to the actual websites that are being tested." tests: - amazon: "BT, FF, CH, CU" - bing-search: "BT, FF, CH, CU" - buzzfeed: "BT, FF, CH, CU" - cnn: "BT, FF, CH, CU" - ebay: "BT, FF, CH, CU" - espn: "BT, FF, CH, CU" - expedia: "BT, FF, CH, CU" - facebook: "BT, FF, CH, CU" - fandom: "BT, FF, CH, CU" - google-docs: "BT, FF, CH, CU" - google-mail: "BT, FF, CH, CU" - google-search: "BT, FF, CH, CU" - google-slides: "BT, FF, CH, CU" - imdb: "BT, FF, CH, CU" - imgur: "BT, FF, CH, CU" - instagram: "BT, FF, CH, CU" - linkedin: "BT, FF, CH, CU" - microsoft: "BT, FF, CH, CU" - netflix: "BT, FF, CH, CU" - nytimes: "BT, FF, CH, CU" - office: "BT, FF, CH, CU" - outlook: "BT, FF, CH, CU" - paypal: "BT, FF, CH, CU" - pinterest: "BT, FF, CH, CU" - reddit: "BT, FF, CH, CU" - tumblr: "BT, FF, CH, CU" - twitch: "BT, FF, CH, CU" - twitter: "BT, FF, CH, CU" - wikia: "BT, FF, CH, CU" - wikipedia: "BT, FF, CH, CU" - yahoo-mail: "BT, FF, CH, CU" - youtube: "BT, FF, CH, CU" + amazon: "" + bing-search: "" + buzzfeed: "" + cnn: "" + ebay: "" + espn: "" + expedia: "" + facebook: "" + fandom: "" + google-docs: "" + google-mail: "" + google-search: "" + google-slides: "" + imdb: "" + imgur: "" + instagram: "" + linkedin: "" + microsoft: "" + netflix: "" + nytimes: "" + office: "" + outlook: "" + paypal: "" + pinterest: "" + reddit: "" + tumblr: "" + twitch: "" + twitter: "" + wikia: "" + wikipedia: "" + yahoo-mail: "" + youtube: "" mobile: description: "Page-load performance test suite on Android. The links direct to the actual websites that are being tested." tests: - amazon: "BT, GV, FE, RB, CH-M" - youtube: "BT, GV, FE, RB, CH-M" - allrecipes: "BT, GV, FE, RB, CH-M" - amazon-search: "BT, GV, FE, RB, CH-M" - bing: "BT, GV, FE, RB, CH-M" - bing-search-restaurants: "BT, GV, FE, RB, CH-M" - booking: "BT, GV, FE, RB, CH-M" - cnn: "BT, GV, FE, RB, CH-M" - cnn-ampstories: "BT, GV, FE, RB, CH-M" - dailymail: "BT, GV, FE, RB, CH-M" - ebay-kleinanzeigen: "BT, GV, FE, RB, CH-M" - ebay-kleinanzeigen-search: "BT, GV, FE, RB, CH-M" - espn: "BT, GV, FE, RB, CH-M" - facebook: "BT, GV, FE, RB, CH-M" - facebook-cristiano: "BT, GV, FE, RB, CH-M" - google: "BT, GV, FE, RB, CH-M" - google-maps: "BT, GV, FE, RB, CH-M" - google-search-restaurants: "BT, GV, FE, RB, CH-M" - imdb: "BT, GV, FE, RB, CH-M" - instagram: "BT, GV, FE, RB, CH-M" - microsoft-support: "BT, GV, FE, RB, CH-M" - reddit: "BT, GV, FE, RB, CH-M" - stackoverflow: "BT, GV, FE, RB, CH-M" - sina: "BT, GV, FE, RB, CH-M" - web-de: "BT, GV, FE, RB, CH-M" - wikipedia: "BT, GV, FE, RB, CH-M" - youtube-watch: "BT, GV, FE, RB, CH-M" + amazon: "" + youtube: "" + allrecipes: "" + amazon-search: "" + bing: "" + bing-search-restaurants: "" + booking: "" + cnn: "" + cnn-ampstories: "" + dailymail: "" + ebay-kleinanzeigen: "" + ebay-kleinanzeigen-search: "" + espn: "" + facebook: "" + facebook-cristiano: "" + google: "" + google-maps: "" + google-search-restaurants: "" + imdb: "" + instagram: "" + microsoft-support: "" + reddit: "" + stackoverflow: "" + sina: "" + bild-de: "" + wikipedia: "" + youtube-watch: "" live: description: "A set of test pages that are run as live sites instead of recorded versions. These tests are available on all browsers, on all platforms." tests: - booking-sf: "GV, FE, RB, CH-M, FF, CH, CU" - discord: "GV, FE, RB, CH-M, FF, CH, CU" - fashionbeans: "GV, FE, RB, CH-M, FF, CH, CU" - google-accounts: "GV, FE, RB, CH-M, FF, CH, CU" - imdb-firefox: "GV, FE, RB, CH-M, FF, CH, CU" - medium-article: "GV, FE, RB, CH-M, FF, CH, CU" - people-article: "GV, FE, RB, CH-M, FF, CH, CU" - reddit-thread: "GV, FE, RB, CH-M, FF, CH, CU" - rumble-fox: "GV, FE, RB, CH-M, FF, CH, CU" - stackoverflow-question: "GV, FE, RB, CH-M, FF, CH, CU" - urbandictionary-define: "GV, FE, RB, CH-M, FF, CH, CU" - wikia-marvel: "GV, FE, RB, CH-M, FF, CH, CU" + booking-sf: "" + discord: "" + fashionbeans: "" + google-accounts: "" + imdb-firefox: "" + medium-article: "" + people-article: "" + reddit-thread: "" + rumble-fox: "" + stackoverflow-question: "" + urbandictionary-define: "" + wikia-marvel: "" benchmarks: description: > Standard benchmarks are third-party tests (i.e. Speedometer) that we have integrated into Raptor to run per-commit in our production CI. To update any of these benchmarks, see `Updating Benchmark Tests <browsertime.html#updating-benchmark-tests>`_. tests: - ares6: "FF, CH, CU" - assorted-dom: "FF, CH, CU" - jetstream2: "FF, CH, CU" - matrix-react-bench: "FF, CH, CU" - motionmark-animometer: "FF, CH, CU, FE, CH-M, GV" - motionmark-animometer-ramp: "FF, CH, CU, FE, CH-M, GV" - motionmark-htmlsuite: "FF, CH, CU, FE, CH-M, GV" - motionmark-htmlsuite-ramp: "FF, CH, CU, FE, CH-M, GV" - speedometer: "FF, CH, CU, FE, GV, RB, CH-M" - speedometer3: "FF, CH, CU, FE, GV, RB, CH-M" - stylebench: "FF, CH, CU" - sunspider: "FF, CH, CU" - twitch-animation: "FF" - unity-webgl: "FF, CH, CU, FE, RB, FE, CH-M" - wasm-godot-baseline: "FF" - wasm-godot-optimizing: "FF" - wasm-godot: "FF, CH, CU" - wasm-misc-baseline: "FF" - wasm-misc-optimizing: "FF" - wasm-misc: "FF, CH, CU" - webaudio: "FF, CH, CU" - youtube-playback: "FF, GV, FE, RB, CH" - youtube-playback-av1-sfr: "FF , GV, FE, RB, CH" - youtube-playback-h264-1080p30: "FF" - youtube-playback-h264-1080p60: "FF" - youtube-playback-h264-full-1080p30: "FF" - youtube-playback-h264-full-1080p60: "FF" - youtube-playback-h264-sfr: "FF , GV, FE, RB, CH" - youtube-playback-hfr: "FF , GV, FE, RB, CH" - youtube-playback-v9-1080p30: "FF" - youtube-playback-v9-1080p60: "FF" - youtube-playback-v9-full-1080p30: "FF" - youtube-playback-v9-full-1080p60: "FF" - youtube-playback-vp9-sfr: "FF , GV, FE, RB, CH" - youtube-playback-widevine-h264-sfr: "FF , GV, FE, RB, CH" - youtube-playback-widevine-hfr: "FF , GV, FE, RB, CH" - youtube-playback-widevine-vp9-sfr: "FF , GV, FE, RB, CH" + ares6: "" + assorted-dom: "" + jetstream2: "" + matrix-react-bench: "" + motionmark-1-3: "" + motionmark-htmlsuite-1-3: "" + motionmark-animometer: "" + motionmark-animometer-ramp: "" + motionmark-htmlsuite: "" + motionmark-htmlsuite-ramp: "" + speedometer: "" + speedometer3: "" + stylebench: "" + sunspider: "" + twitch-animation: "" + unity-webgl: "" + wasm-godot-baseline: "" + wasm-godot-optimizing: "" + wasm-godot: "" + wasm-misc-baseline: "" + wasm-misc-optimizing: "" + wasm-misc: "" + webaudio: "" + youtube-playback: "" + youtube-playback-av1-sfr: "" + youtube-playback-h264-1080p30: "" + youtube-playback-h264-1080p60: "" + youtube-playback-h264-full-1080p30: "" + youtube-playback-h264-full-1080p60: "" + youtube-playback-h264-sfr: "" + youtube-playback-hfr: "" + youtube-playback-v9-1080p30: "" + youtube-playback-v9-1080p60: "" + youtube-playback-v9-full-1080p30: "" + youtube-playback-v9-full-1080p60: "" + youtube-playback-vp9-sfr: "" + youtube-playback-widevine-h264-sfr: "" + youtube-playback-widevine-hfr: "" + youtube-playback-widevine-vp9-sfr: "" scenario: description: "Tests that perform a specific action (a scenario), i.e. idle application, idle application in background, etc." tests: - idle: "FE, GV, RB" - idle-bg: "FE, GV, RB" + idle: "" + idle-bg: "" custom: description: "Browsertime tests that use a custom pageload test script. These use the pageload type, but may have other intentions." tests: @@ -191,7 +193,7 @@ suites: unittests: description: "These tests aren't used in standard testing, they are only used in the Raptor unit tests (they are similar to raptor-tp6 tests though)." tests: - test-page-1: "FF" - test-page-2: "FF" - test-page-3: "FF" - test-page-4: "FF" + test-page-1: "" + test-page-2: "" + test-page-3: "" + test-page-4: "" diff --git a/testing/raptor/raptor/perftest.py b/testing/raptor/raptor/perftest.py index eb46e351af..6e21b6d114 100644 --- a/testing/raptor/raptor/perftest.py +++ b/testing/raptor/raptor/perftest.py @@ -90,7 +90,7 @@ class Perftest(object): live_sites=False, is_release_build=False, debug_mode=False, - post_startup_delay=POST_DELAY_DEFAULT, + post_startup_delay=None, interrupt_handler=None, e10s=True, results_handler_class=RaptorResultsHandler, @@ -110,6 +110,7 @@ class Perftest(object): benchmark_revision=None, benchmark_branch=None, clean=False, + screenshot_on_failure=False, **kwargs ): self._remote_test_root = None @@ -156,6 +157,7 @@ class Perftest(object): "benchmark_revision": benchmark_revision, "benchmark_branch": benchmark_branch, "clean": clean, + "screenshot_on_failure": screenshot_on_failure, } self.firefox_android_apps = FIREFOX_ANDROID_APPS @@ -213,22 +215,28 @@ class Perftest(object): self.run_local = self.config["run_local"] self.debug_mode = debug_mode if self.run_local else False - # For the post startup delay, we want to max it to 1s when using the - # conditioned profiles. - if self.config.get("conditioned_profile"): - self.post_startup_delay = min(post_startup_delay, POST_DELAY_CONDPROF) - elif ( - self.debug_mode - ): # if running debug-mode reduce the pause after browser startup - self.post_startup_delay = min(post_startup_delay, POST_DELAY_DEBUG) + if post_startup_delay is None: + # For the post startup delay, we want to max it to 1s when using the + # conditioned profiles. + if self.config.get("conditioned_profile"): + self.post_startup_delay = POST_DELAY_CONDPROF + elif ( + self.debug_mode + ): # if running debug-mode reduce the pause after browser startup + self.post_startup_delay = POST_DELAY_DEBUG + else: + self.post_startup_delay = POST_DELAY_DEFAULT + + if ( + app in CHROME_ANDROID_APPS + FIREFOX_ANDROID_APPS + and not self.config.get("conditioned_profile") + ): + LOG.info("Mobile non-conditioned profile") + self.post_startup_delay = POST_DELAY_MOBILE else: + # User supplied a custom post_startup_delay value self.post_startup_delay = post_startup_delay - if app in CHROME_ANDROID_APPS + FIREFOX_ANDROID_APPS and not self.config.get( - "conditioned_profile" - ): - LOG.info("Mobile non-conditioned profile") - self.post_startup_delay = POST_DELAY_MOBILE LOG.info("Post startup delay set to %d ms" % self.post_startup_delay) LOG.info("main raptor init, config is: %s" % str(self.config)) diff --git a/testing/raptor/raptor/raptor.py b/testing/raptor/raptor/raptor.py index 9390b530b8..8b9cacda25 100644 --- a/testing/raptor/raptor/raptor.py +++ b/testing/raptor/raptor/raptor.py @@ -124,6 +124,7 @@ def main(args=sys.argv[1:]): benchmark_branch=args.benchmark_branch, page_timeout=args.page_timeout, clean=args.clean, + screenshot_on_failure=args.screenshot_on_failure, ) except Exception: traceback.print_exc() diff --git a/testing/raptor/raptor/raptor.toml b/testing/raptor/raptor/raptor.toml index fa6d0913e8..19277e8cff 100644 --- a/testing/raptor/raptor/raptor.toml +++ b/testing/raptor/raptor/raptor.toml @@ -9,6 +9,10 @@ ["include:tests/benchmarks/matrix-react-bench.toml"] +["include:tests/benchmarks/motionmark-1-3-desktop.toml"] + +["include:tests/benchmarks/motionmark-1-3-mobile.toml"] + ["include:tests/benchmarks/motionmark-animometer-desktop.toml"] ["include:tests/benchmarks/motionmark-animometer-mobile.toml"] diff --git a/testing/raptor/raptor/tests/benchmarks/motionmark-1-3-desktop.toml b/testing/raptor/raptor/tests/benchmarks/motionmark-1-3-desktop.toml new file mode 100644 index 0000000000..3c471d3a17 --- /dev/null +++ b/testing/raptor/raptor/tests/benchmarks/motionmark-1-3-desktop.toml @@ -0,0 +1,28 @@ +[DEFAULT] +alert_threshold = 2.0 +apps = "firefox, chrome, chromium, safari, custom-car" +gecko_profile_interval = 1 +gecko_profile_features = "stackwalk,js,cpu,java,processcpu" +gecko_profile_threads = "GeckoMain,Compositor,Renderer,SwComposite,RenderBackend,SceneBuilder,WrWorker,CanvasWorkers,TextureUpdate" +expose_browser_profiler = true +expose_chrome_trace = true +lower_is_better = false +page_cycles = 1 +page_timeout = 600000 +support_class = "motionmark-1-3.py" +test_script = "motionmark-1-3.js" +owner = "Graphics Team" +type = "benchmark" +unit = "score" +repository = "https://github.com/webkit/motionmark" +repository_revision = "5d9c88136d59c11daf78d539c73e4e3e88c091ab" +test_url="http://<host>:<port>/MotionMark/developer.html?warmup-length=2000&warmup-frame-count=30&first-frame-minimum-length=0&test-interval=30&display=minimal&tiles=big&controller=ramp&system-frame-rate=60&frame-rate=60&time-measurement=performance" +cold = true +browser_cycles = 1 +host_from_parent = false + +[motionmark-1-3] +suite_name = "MotionMark" + +[motionmark-htmlsuite-1-3] +suite_name = "HTML suite" diff --git a/testing/raptor/raptor/tests/benchmarks/motionmark-1-3-mobile.toml b/testing/raptor/raptor/tests/benchmarks/motionmark-1-3-mobile.toml new file mode 100644 index 0000000000..f68f418608 --- /dev/null +++ b/testing/raptor/raptor/tests/benchmarks/motionmark-1-3-mobile.toml @@ -0,0 +1,27 @@ +[DEFAULT] +alert_threshold = 2.0 +apps = "fenix, geckoview, chrome-m, cstm-car-m" +gecko_profile_interval = 1 +gecko_profile_features = "stackwalk,js,cpu,java,processcpu" +gecko_profile_threads = "GeckoMain,Compositor,Renderer,SwComposite,RenderBackend,SceneBuilder,WrWorker,CanvasWorkers,TextureUpdate" +expose_browser_profiler = true +lower_is_better = false +page_cycles = 1 +page_timeout = 600000 +support_class = "motionmark-1-3.py" +test_script = "motionmark-1-3.js" +owner = "Graphics Team" +type = "benchmark" +unit = "score" +repository = "https://github.com/webkit/motionmark" +repository_revision = "5d9c88136d59c11daf78d539c73e4e3e88c091ab" +test_url="http://<host>:<port>/MotionMark/developer.html?warmup-length=2000&warmup-frame-count=30&first-frame-minimum-length=0&test-interval=30&display=minimal&tiles=big&controller=ramp&system-frame-rate=60&frame-rate=60&time-measurement=performance" +cold = true +browser_cycles = 1 +host_from_parent = false + +[motionmark-1-3] +suite_name = "MotionMark" + +[motionmark-htmlsuite-1-3] +suite_name = "HTML suite" diff --git a/testing/raptor/raptor/tests/benchmarks/speedometer-desktop.toml b/testing/raptor/raptor/tests/benchmarks/speedometer-desktop.toml index 6a378b999a..d3bc83e420 100644 --- a/testing/raptor/raptor/tests/benchmarks/speedometer-desktop.toml +++ b/testing/raptor/raptor/tests/benchmarks/speedometer-desktop.toml @@ -20,7 +20,7 @@ browser_cycles = 5 owner = "Performance Team" cold = true repository = "https://github.com/WebKit/Speedometer" -repository_revision = "64a5e7c0ba0f668834c0adf01b933e54d2b0d1f0" +repository_revision = "8d67f28d0281ac4330f283495b7f48286654ad7d" host_from_parent = false support_class = "speedometer3.py" test_script = "speedometer3.js" diff --git a/testing/raptor/raptor/tests/benchmarks/speedometer-mobile.toml b/testing/raptor/raptor/tests/benchmarks/speedometer-mobile.toml index e2e4381303..12bb9b16c4 100644 --- a/testing/raptor/raptor/tests/benchmarks/speedometer-mobile.toml +++ b/testing/raptor/raptor/tests/benchmarks/speedometer-mobile.toml @@ -20,7 +20,7 @@ browser_cycles = 5 owner = "Performance Team" cold = true repository = "https://github.com/WebKit/Speedometer" -repository_revision = "64a5e7c0ba0f668834c0adf01b933e54d2b0d1f0" +repository_revision = "8d67f28d0281ac4330f283495b7f48286654ad7d" host_from_parent = false support_class = "speedometer3.py" test_script = "speedometer3.js" diff --git a/testing/raptor/raptor/tests/tp6/mobile/browsertime-tp6m.toml b/testing/raptor/raptor/tests/tp6/mobile/browsertime-tp6m.toml index b20a13f605..a16e0916be 100644 --- a/testing/raptor/raptor/tests/tp6/mobile/browsertime-tp6m.toml +++ b/testing/raptor/raptor/tests/tp6/mobile/browsertime-tp6m.toml @@ -27,6 +27,10 @@ test_url = "https://www.amazon.com" ["amazon-search"] test_url = "https://www.amazon.com/s/ref=nb_sb_noss_2/139-6317191-5622045?url=search-alias%3Daps&field-keywords=mobile+phone" +["bild-de"] +playback_pageset_manifest = "mitm8-android-fenix-{subtest}.manifest" +test_url = "https://www.bild.de/sport/american-football/nfl/super-bowl-rekordquote-fuer-die-nfl-87139668.bild.html" + ["bing"] test_url = "https://www.bing.com/" @@ -90,15 +94,12 @@ test_url = "https://support.microsoft.com/en-us" test_url = "https://www.reddit.com" ["sina"] +playback_pageset_manifest = "mitm8-android-fenix-{subtest}.manifest" test_url = "https://www.sina.com.cn/" ["stackoverflow"] test_url = "https://stackoverflow.com/" -["web-de"] -playback_pageset_manifest = "mitm7-android-gve-p2-web-de.manifest" -test_url = "https://web.de/magazine/politik/politologe-glaubt-grossen-koalition-herbst-knallen-33563566" - ["wikipedia"] test_url = "https://en.m.wikipedia.org/wiki/Main_Page" diff --git a/testing/raptor/raptor/tooltool-manifests/playback/mitm6-android-fenix-sina.manifest b/testing/raptor/raptor/tooltool-manifests/playback/mitm6-android-fenix-sina.manifest deleted file mode 100644 index b52d47df45..0000000000 --- a/testing/raptor/raptor/tooltool-manifests/playback/mitm6-android-fenix-sina.manifest +++ /dev/null @@ -1,9 +0,0 @@ -[ - { - "algorithm": "sha512", - "digest": "f3b55b61e6c38a342fee68fef04196de39b209053348cf58f5d9cc289100d82f271008f065adfd4ec02df564f6de12726f198e151af24aace9f654bf008a7b86", - "filename": "mitm6-android-fenix-sina.zip", - "size": 14757830, - "visibility": "public" - } -] diff --git a/testing/raptor/raptor/tooltool-manifests/playback/mitm6-android-fenix-web-de.manifest b/testing/raptor/raptor/tooltool-manifests/playback/mitm6-android-fenix-web-de.manifest deleted file mode 100644 index 9bde5db272..0000000000 --- a/testing/raptor/raptor/tooltool-manifests/playback/mitm6-android-fenix-web-de.manifest +++ /dev/null @@ -1,9 +0,0 @@ -[ - { - "algorithm": "sha512", - "digest": "9f10db4616f7415dba946f056658126297928d1229e5d2e525f9f1dd539c954b9a90b7d81fa574002fd1127321d8f918a3102cae12054b218118b059702cde7f", - "filename": "mitm6-android-fenix-web-de.zip", - "size": 1891502, - "visibility": "public" - } -] diff --git a/testing/raptor/raptor/tooltool-manifests/playback/mitm8-android-fenix-bild-de.manifest b/testing/raptor/raptor/tooltool-manifests/playback/mitm8-android-fenix-bild-de.manifest new file mode 100644 index 0000000000..e53c42c38f --- /dev/null +++ b/testing/raptor/raptor/tooltool-manifests/playback/mitm8-android-fenix-bild-de.manifest @@ -0,0 +1,9 @@ +[ + { + "filename": "mitm8-android-fenix-bild-de.zip", + "size": 22367904, + "algorithm": "sha512", + "digest": "982b91675bd21aa89d27168f3964bcfe81e3002b6256da80b67398c711f063d2aca7cd023ef3540891f0a5b1f1762af49c22869e69adbc1515caacd7e8282e7b", + "visibility": "public" + } +] diff --git a/testing/raptor/raptor/tooltool-manifests/playback/mitm8-android-fenix-sina.manifest b/testing/raptor/raptor/tooltool-manifests/playback/mitm8-android-fenix-sina.manifest new file mode 100644 index 0000000000..1985bd7247 --- /dev/null +++ b/testing/raptor/raptor/tooltool-manifests/playback/mitm8-android-fenix-sina.manifest @@ -0,0 +1,9 @@ +[ + { + "filename": "mitm8-android-fenix-sina.zip", + "size": 15119356, + "algorithm": "sha512", + "digest": "56f38396be740dcf7ba0476125fa0f66fa68f5cbc049652a598afcf9492deee4e974d73193ddc4b125f3b136b5179931bb6c8a19bfa82e874a168393ea05a7eb", + "visibility": "public" + } +] diff --git a/testing/raptor/test/test_cmdline.py b/testing/raptor/test/test_cmdline.py index 80ce6cf88a..de88e60c77 100644 --- a/testing/raptor/test/test_cmdline.py +++ b/testing/raptor/test/test_cmdline.py @@ -1,5 +1,6 @@ import os import sys +from unittest import mock import mozunit import pytest @@ -11,7 +12,7 @@ sys.path.insert(0, raptor_dir) from argparse import ArgumentParser, Namespace -from cmdline import verify_options +from cmdline import create_parser, verify_options def test_verify_options(filedir): @@ -33,6 +34,7 @@ def test_verify_options(filedir): benchmark_repository=None, benchmark_revision=None, benchmark_branch=None, + post_startup_delay=None, ) parser = ArgumentParser() @@ -61,6 +63,7 @@ def test_verify_options(filedir): benchmark_repository=None, benchmark_revision=None, benchmark_branch=None, + post_startup_delay=None, ) verify_options(parser, args) # assert no exception @@ -83,6 +86,7 @@ def test_verify_options(filedir): benchmark_repository=None, benchmark_revision=None, benchmark_branch=None, + post_startup_delay=None, ) verify_options(parser, args) # assert no exception @@ -105,6 +109,7 @@ def test_verify_options(filedir): benchmark_repository=None, benchmark_revision=None, benchmark_branch=None, + post_startup_delay=None, ) verify_options(parser, args) # assert no exception @@ -127,6 +132,7 @@ def test_verify_options(filedir): benchmark_repository=None, benchmark_revision=None, benchmark_branch=None, + post_startup_delay=None, ) verify_options(parser, args) # assert no exception @@ -149,11 +155,89 @@ def test_verify_options(filedir): benchmark_repository=None, benchmark_revision=None, benchmark_branch=None, + post_startup_delay=None, ) parser = ArgumentParser() verify_options(parser, args) # also will work as uses default activity +@mock.patch("perftest.Perftest.build_browser_profile", new=mock.MagicMock()) +@pytest.mark.parametrize( + "args,settings_to_check", + [ + # Test that post_startup_delay is 30s as expected + [ + [ + "--test", + "test-page-1", + "--binary", + "invalid/path", + # This gets set automatically from mach_commands, but is set + # to False by default in the Perftest class + "--run-local", + ], + [ + ("post_startup_delay", 30000), + ("run_local", True), + ("debug_mode", False), + ], + ], + # Test that run_local is false by default + [ + [ + "--test", + "test-page-1", + "--binary", + "invalid/path", + ], + [ + ("post_startup_delay", 30000), + ("run_local", False), + ("debug_mode", False), + ], + ], + # Test that debug mode gets set when running locally + [ + [ + "--test", + "test-page-1", + "--binary", + "invalid/path", + "--debug-mode", + "--run-local", + ], + [ + ("post_startup_delay", 3000), + ("run_local", True), + ("debug_mode", True), + ], + ], + # Test that debug mode doesn't get set when we're not running locally + [ + [ + "--test", + "test-page-1", + "--binary", + "invalid/path", + "--debug-mode", + ], + [ + ("post_startup_delay", 30000), + ("run_local", False), + ("debug_mode", False), + ], + ], + ], +) +def test_perftest_setup_with_args(ConcretePerftest, args, settings_to_check): + parser = create_parser() + args = parser.parse_args(args) + + perftest = ConcretePerftest(**vars(args)) + for setting, expected in settings_to_check: + assert getattr(perftest, setting) == expected + + if __name__ == "__main__": mozunit.main() diff --git a/testing/raptor/test/test_raptor.py b/testing/raptor/test/test_raptor.py index 9184fa5e18..3c90b611a5 100644 --- a/testing/raptor/test/test_raptor.py +++ b/testing/raptor/test/test_raptor.py @@ -127,38 +127,51 @@ def test_raptor_venv(ConcretePerftest, options): assert perftest.raptor_venv.endswith("raptor-venv") +@mock.patch("perftest.Perftest.build_browser_profile", new=mock.MagicMock()) @pytest.mark.parametrize( + "app," "run_local," "debug_mode," + "conditioned_profile," "post_startup_delay," "expected_post_startup_delay," "expected_debug_mode", [ - [True, True, 1234, 1234, True], - [True, True, 12345, 3000, True], - [False, False, 1234, 1234, False], - [False, False, 12345, 12345, False], - [True, False, 1234, 1234, False], - [True, False, 12345, 12345, False], - [False, True, 1234, 1234, False], - [False, True, 12345, 12345, False], + ["firefox", True, True, None, 1234, 1234, True], + ["firefox", True, True, None, None, 3000, True], + ["firefox", True, False, None, None, 30000, False], + ["firefox", True, False, "settled", None, 1000, False], + ["fenix", True, False, None, None, 20000, False], + ["fenix", True, False, "settled", None, 1000, False], + ["firefox", False, False, None, 1234, 1234, False], + ["firefox", False, False, None, 12345, 12345, False], + ["firefox", True, False, None, 1234, 1234, False], + ["firefox", True, False, None, 12345, 12345, False], + ["firefox", False, True, None, 1234, 1234, False], + ["firefox", False, True, None, 12345, 12345, False], ], ) def test_post_startup_delay( ConcretePerftest, options, + app, run_local, debug_mode, + conditioned_profile, post_startup_delay, expected_post_startup_delay, expected_debug_mode, ): + options["app"] = app + perftest = ConcretePerftest( run_local=run_local, debug_mode=debug_mode, post_startup_delay=post_startup_delay, + conditioned_profile=conditioned_profile, **options ) + assert perftest.post_startup_delay == expected_post_startup_delay assert perftest.debug_mode == expected_debug_mode |