diff options
Diffstat (limited to 'testing/talos')
21 files changed, 334 insertions, 106 deletions
diff --git a/testing/talos/mach_commands.py b/testing/talos/mach_commands.py index c685219422..0b15af7a2c 100644 --- a/testing/talos/mach_commands.py +++ b/testing/talos/mach_commands.py @@ -12,6 +12,7 @@ import sys import six from mach.decorators import Command +from mach.util import get_state_dir from mozbuild.base import BinaryNotFoundException, MozbuildObject HERE = os.path.dirname(os.path.realpath(__file__)) @@ -82,6 +83,7 @@ class TalosRunner(MozbuildObject): "win32": "python3.manifest", "win64": "python3_x64.manifest", }, + "mozbuild_path": get_state_dir(), } def make_args(self): @@ -117,6 +119,16 @@ def create_parser(): return create_parser(mach_interface=True) +def setup_toolchain_artifacts(args, command_context): + if not any(arg.lower() == "pdfpaint" for arg in args): + return + + from mozbuild.bootstrap import bootstrap_toolchain + + print("Setting up pdfpaint PDFs...") + bootstrap_toolchain("talos-pdfs") + + @Command( "talos-test", category="testing", @@ -127,7 +139,9 @@ def run_talos_test(command_context, **kwargs): talos = command_context._spawn(TalosRunner) try: - return talos.run_test(sys.argv[2:]) + args = sys.argv[2:] + setup_toolchain_artifacts(args, command_context) + return talos.run_test(args) except Exception as e: print(str(e)) return 1 diff --git a/testing/talos/perfdocs/config.yml b/testing/talos/perfdocs/config.yml index bbc71a74aa..cebfed5571 100644 --- a/testing/talos/perfdocs/config.yml +++ b/testing/talos/perfdocs/config.yml @@ -607,7 +607,13 @@ suites: - source: - type: `Page load`_ - reporting: time from *performance.timing.navigationStart* to *pagerendered* event in ms (lower is better) - - data: load a PDF 20 times + - data: loads a PDF 5 times + - description: + | Runs through a set of chunks. Each chunk runs 100 PDFs with 5 iterations each. + If --pdfPaintChunk is not used when running the test locally, all PDFs will be tested + by default with only 1 cycle each. The PDFs that are run are found in the Mozilla pdf.js + repository, and this test pulls those in for testing locally through a toolchain artifact + called talos-pdfs. perf_reftest: > - contact: :emilio and css/layout team - source: `perf-reftest <https://dxr.mozilla.org/mozilla-central/source/testing/talos/talos/tests/perf-reftest>`__ diff --git a/testing/talos/talos.json b/testing/talos/talos.json index 32904f87a3..1446abd82a 100644 --- a/testing/talos/talos.json +++ b/testing/talos/talos.json @@ -19,7 +19,6 @@ "tabpaint", "cpstartup", "startup_about_home_paint", - "pdfpaint", "cross_origin_pageload", "startup_about_home_paint_cached" ] @@ -121,6 +120,9 @@ "realworld-webextensions": { "tests": ["startup_about_home_paint_realworld_webextensions"], "webextensions_zip": "webextensions.zip" + }, + "pdfpaint": { + "tests": ["pdfpaint"] } } } diff --git a/testing/talos/talos/cmdline.py b/testing/talos/talos/cmdline.py index dc7ecd9a09..697342e969 100644 --- a/testing/talos/talos/cmdline.py +++ b/testing/talos/talos/cmdline.py @@ -223,6 +223,26 @@ def create_parser(mach_interface=False): dest="pdfpaint", help="Wait for the first page of a PDF to be rendered", ) + add_arg( + "--pdfPaintChunk", + type=int, + default=None, + dest="pdfpaint_chunk", + help=( + "Chunk of the pdfpaint test to run (each chunk runs at most 100 pdfs). " + "Defaults to None to run all the pdfs at the same time." + ), + ) + add_arg( + "--pdfPaintName", + type=str, + default=None, + dest="pdfpaint_name", + help=( + "Name of a pdfpaint test to run (e.g. xfa_imm5257e.pdf). Chunking will be " + "ignored/disabled if this option is used." + ), + ) add_arg("--webServer", dest="webserver", help="DEPRECATED") if not mach_interface: add_arg( diff --git a/testing/talos/talos/config.py b/testing/talos/talos/config.py index e732e92d2f..31162d692c 100644 --- a/testing/talos/talos/config.py +++ b/testing/talos/talos/config.py @@ -2,7 +2,9 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this file, # You can obtain one at http://mozilla.org/MPL/2.0/. import copy +import json import os +import pathlib import re import sys @@ -50,7 +52,6 @@ DEFAULTS = dict( tploadnocache=False, tpscrolltest=False, win_counters=[], - w7_counters=[], linux_counters=[], mac_counters=[], xperf_counters=[], @@ -118,7 +119,6 @@ def fix_xperf(config): # BBB: remove doubly-quoted xperf values from command line # (needed for buildbot) # https://bugzilla.mozilla.org/show_bug.cgi?id=704654#c43 - win7_path = "c:/Program Files/Microsoft Windows Performance Toolkit/xperf.exe" if config["xperf_path"]: xperf_path = config["xperf_path"] quotes = ('"', "'") @@ -126,13 +126,6 @@ def fix_xperf(config): if xperf_path.startswith(quote) and xperf_path.endswith(quote): config["xperf_path"] = xperf_path[1:-1] break - if not os.path.exists(config["xperf_path"]): - # look for old win7 path - if not os.path.exists(win7_path): - raise ConfigurationError( - "xperf.exe cannot be found at the path specified" - ) - config["xperf_path"] = win7_path @validator @@ -182,11 +175,6 @@ def determine_local_symbols_path(config): ) -def get_counters(config): - counters = set() - return counters - - def get_active_tests(config): activeTests = config.pop("activeTests").strip().split(":") @@ -212,6 +200,77 @@ def get_global_overrides(config): return global_overrides +def setup_pdfpaint_test(config, test_instance): + # Get the root of the location of the PDFs artifact + if os.environ.get("MOZ_FETCHES_DIR", None): + pdfs_root = pathlib.Path(os.environ.get("MOZ_FETCHES_DIR"), "talos-pdfs") + else: + pdfs_root = pathlib.Path(os.environ.get("MOZBUILD_PATH"), "talos-pdfs") + if not pdfs_root.exists(): + raise Exception(f"Cannot find webserver root: {pdfs_root}") + + pdfpaint_manifest_path = pathlib.Path(pdfs_root, "pdfpaint.manifest") + test_manifest_path = pathlib.Path(pdfs_root, "test_manifest.json") + test_manifest = json.loads(test_manifest_path.read_text(encoding="utf8")) + + # If a pdfpaint test was specified, prevent any chunking + chunk_number = config.get("pdfpaint_chunk", None) + pdfpaint_test = None + if config.get("pdfpaint_name") is not None: + chunk_number = None + pdfpaint_test = config["pdfpaint_name"] + + # Gather all the pdf files that can be used in the test, and write + # all the pdfs to be tested to the manifest file + start_ind = 0 + end_ind = None + pdfs_per_chunk = 100 + if chunk_number is not None: + start_ind = pdfs_per_chunk * (chunk_number - 1) + end_ind = pdfs_per_chunk * chunk_number + + pdf_files = set() + for pdf_info in test_manifest: + if pdf_info.get("password", None) is not None: + # PDFs that require passwords cause timeouts + continue + + pdf_name = pathlib.Path(pdf_info["file"]).name + if ( + config.get("pdfpaint_name", None) is not None + and config["pdfpaint_name"] != pdf_name + ): + # If a user passed a name of a pdf, skip all the others + continue + + pdf_files.add(pdf_name) + + if start_ind > len(pdf_files): + raise ConfigurationError( + f"Chunk {chunk_number} contains no PDFs to test. " + f"For {len(pdf_files)} PDFs, the max chunk is " + f"{int((len(pdf_files)-1)/pdfs_per_chunk)+1}." + ) + + with pdfpaint_manifest_path.open("w") as f: + for pdf_file in sorted(list(pdf_files))[start_ind:end_ind]: + print(f"http://localhost/tests/pdfpaint/pdfs/{pdf_file}{os.linesep}") + f.write(f"http://localhost/tests/pdfpaint/pdfs/{pdf_file}{os.linesep}") + + # Make a symbolic link to the mozbuild pdf folder since the talos + # webserver depends on having the doc root as the talos folder for getInfo.html + symlink_dest = pathlib.Path(__file__).parent / "tests" / "pdfpaint" / "pdfs" + symlink_dest.unlink(missing_ok=True) + symlink_dest.symlink_to(pdfs_root, target_is_directory=True) + test_instance.tpmanifest = str(pdfpaint_manifest_path) + + # Increase the pagecycles for each pdf to 5 if we're running chunks, otherwise + # it can take a very long time to complete testing of all pdfs + if chunk_number is not None or pdfpaint_test is not None: + print("Setting pdfpaint tppagecycles to 5") + test_instance.tppagecycles = 5 + + def get_test_host(manifest_line): match = re.match(r"^http://localhost/page_load_test/tp5n/([^/]+)/", manifest_line) host = match.group(1) @@ -262,6 +321,9 @@ def get_test(config, global_overrides, counters, test_instance): if pdfPaint is not None: test_instance.pdfpaint = pdfPaint + if test_instance.pdfpaint: + setup_pdfpaint_test(config, test_instance) + # fix up url url = getattr(test_instance, "url", None) if url: @@ -281,7 +343,6 @@ def get_test(config, global_overrides, counters, test_instance): "linux_counters", "mac_counters", "win_counters", - "w7_counters", "xperf_counters", ) for key in keys: @@ -300,7 +361,7 @@ def get_test(config, global_overrides, counters, test_instance): @validator def tests(config): - counters = get_counters(config) + counters = set() global_overrides = get_global_overrides(config) activeTests = get_active_tests(config) test_dict = test.test_dict() @@ -374,6 +435,11 @@ def get_config(argv=None): elif not cli_opts.activeTests: raise ConfigurationError("--activeTests or --suite required!") + if cli_opts.pdfpaint_chunk is not None and cli_opts.pdfpaint_chunk < 1: + raise ConfigurationError( + "pdfpaint chunk must be a positive integer greater than or equal to 1" + ) + cli_opts = parse_args(argv=argv) setup_logging("talos", cli_opts, {"tbpl": sys.stdout}) config = copy.deepcopy(DEFAULTS) diff --git a/testing/talos/talos/pageloader/chrome/report.js b/testing/talos/talos/pageloader/chrome/report.js index 7f5d61922a..d1eea8f3c9 100644 --- a/testing/talos/talos/pageloader/chrome/report.js +++ b/testing/talos/talos/pageloader/chrome/report.js @@ -4,6 +4,11 @@ // given an array of strings, finds the longest common prefix function findCommonPrefixLength(strs) { + if (strs.every(str => str.includes("/pdfs/"))) { + // In all cases for pdfpaint PDFs, return the full file name + return strs[0].lastIndexOf("/") + 1; + } + if (strs.length < 2) { // only one page in the manifest // i.e. http://localhost/tests/perf-reftest/bloom-basic.html diff --git a/testing/talos/talos/run_tests.py b/testing/talos/talos/run_tests.py index 1d3333e3a8..b6ed4d2481 100755 --- a/testing/talos/talos/run_tests.py +++ b/testing/talos/talos/run_tests.py @@ -16,7 +16,6 @@ import six from mozgeckoprofiler import view_gecko_profile from mozlog import get_proxy_logger from wptserve import server -from wptserve.handlers import handler from talos import utils from talos.config import ConfigurationError, get_configs @@ -94,19 +93,9 @@ def setup_webserver(webserver): """Set up a new web server with wptserve.""" LOG.info("starting webserver on %r" % webserver) - @handler - def tracemonkey_pdf_handler(request, response): - """Handler for the talos pdfpaint test.""" - headers = [("Content-Type", "application/pdf")] - with open("%s/tests/pdfpaint/tracemonkey.pdf" % here, "rb") as file: - content = file.read() - return headers, content - host, port = webserver.split(":") httpd = server.WebTestHttpd(host=host, port=int(port), doc_root=here) - httpd.router.register( - "GET", "tests/pdfpaint/tracemonkey.pdf", tracemonkey_pdf_handler - ) + return httpd diff --git a/testing/talos/talos/startup_test/tspaint_test.html b/testing/talos/talos/startup_test/tspaint_test.html index 7282f1fcfa..2666b8ab43 100644 --- a/testing/talos/talos/startup_test/tspaint_test.html +++ b/testing/talos/talos/startup_test/tspaint_test.html @@ -25,7 +25,9 @@ async function painted() { let startupInfo = await TalosPowersContent.getStartupInfo(); - let startupTime = startupInfo.firstPaint - startupInfo.process; + // firstPaint2 is a more accurate measurement that checks that pixels are actually visible, + // not merely that the calls to paint them have been completed. + let startupTime = startupInfo.firstPaint2 - startupInfo.process; document.body.textContent = "Startup time = " + startupTime + " ms"; if (window.dump) { diff --git a/testing/talos/talos/talos-powers/api.js b/testing/talos/talos/talos-powers/api.js index c6b7611976..1021c63aab 100644 --- a/testing/talos/talos/talos-powers/api.js +++ b/testing/talos/talos/talos-powers/api.js @@ -332,9 +332,9 @@ TalosPowersService.prototype = { let mm = message.target.messageManager; let startupInfo = Services.startup.getStartupInfo(); - if (!startupInfo.firstPaint) { + if (!startupInfo.firstPaint2) { // It's possible that we were called early enough that - // the firstPaint measurement hasn't been set yet. In + // the firstPaint2 measurement hasn't been set yet. In // that case, we set up an observer for the next time // a window is painted and re-retrieve the startup info. let obs = function (subject, topic) { diff --git a/testing/talos/talos/test.py b/testing/talos/talos/test.py index 9f6bd78e58..f64d5004ef 100644 --- a/testing/talos/talos/test.py +++ b/testing/talos/talos/test.py @@ -345,7 +345,6 @@ class PageloaderTest(Test): "gecko_profile_extra_threads", "tptimeout", "win_counters", - "w7_counters", "linux_counters", "mac_counters", "tpscrolltest", @@ -420,8 +419,9 @@ class pdfpaint(PageloaderTest): """ tpmanifest = "${talos}/tests/pdfpaint/pdfpaint.manifest" - tppagecycles = 20 - timeout = 600 + tppagecycles = 1 + timeout = 1800 + tptimeout = 60000 pdfpaint = True unit = "ms" @@ -542,7 +542,7 @@ class tart(PageloaderTest): tploadnocache = True tpmozafterpaint = False gecko_profile_interval = 10 - win_counters = w7_counters = linux_counters = mac_counters = None + win_counters = linux_counters = mac_counters = None """ ASAP mode The recording API is broken with OMTC before ~2013-11-27 @@ -577,7 +577,7 @@ class damp(PageloaderTest): tpmozafterpaint = False gecko_profile_interval = 10 gecko_profile_extra_threads = "DOM Worker" - win_counters = w7_counters = linux_counters = mac_counters = None + win_counters = linux_counters = mac_counters = None filters = filter.ignore_first.prepare(1) + filter.median.prepare() preferences = {"devtools.memory.enabled": True} unit = "ms" @@ -604,7 +604,7 @@ class glterrain(PageloaderTest): tpchrome = False timeout = 600 gecko_profile_interval = 10 - win_counters = w7_counters = linux_counters = mac_counters = None + win_counters = linux_counters = mac_counters = None """ ASAP mode """ preferences = { "layout.frame_rate": 0, @@ -632,7 +632,7 @@ class glvideo(PageloaderTest): timeout = 600 gecko_profile_interval = 2 gecko_profile_extra_threads = "CanvasRenderer,CanvasWorker,MediaSupervisor" - win_counters = w7_counters = linux_counters = mac_counters = None + win_counters = linux_counters = mac_counters = None filters = filter.ignore_first.prepare(1) + filter.median.prepare() unit = "ms" @@ -654,7 +654,7 @@ class canvas2dvideo(PageloaderTest): timeout = 600 gecko_profile_interval = 2 gecko_profile_extra_threads = "CanvasRenderer,CanvasWorker,MediaSupervisor" - win_counters = w7_counters = linux_counters = mac_counters = None + win_counters = linux_counters = mac_counters = None filters = filter.ignore_first.prepare(1) + filter.median.prepare() unit = "ms" @@ -676,7 +676,7 @@ class offscreencanvas_webcodecs_main_webgl_h264(PageloaderTest): timeout = 600 gecko_profile_interval = 2 gecko_profile_extra_threads = "CanvasRenderer,MediaSupervisor" - win_counters = w7_counters = linux_counters = mac_counters = None + win_counters = linux_counters = mac_counters = None preferences = { "dom.media.webcodecs.enabled": True, "dom.media.webcodecs.force-osx-h264-enabled": True, @@ -702,7 +702,7 @@ class offscreencanvas_webcodecs_main_webgl_vp9(PageloaderTest): timeout = 600 gecko_profile_interval = 2 gecko_profile_extra_threads = "CanvasRenderer,MediaSupervisor" - win_counters = w7_counters = linux_counters = mac_counters = None + win_counters = linux_counters = mac_counters = None preferences = { "dom.media.webcodecs.enabled": True, "dom.media.webcodecs.force-osx-h264-enabled": True, @@ -728,7 +728,7 @@ class offscreencanvas_webcodecs_main_webgl_av1(PageloaderTest): timeout = 600 gecko_profile_interval = 2 gecko_profile_extra_threads = "CanvasRenderer,MediaSupervisor" - win_counters = w7_counters = linux_counters = mac_counters = None + win_counters = linux_counters = mac_counters = None preferences = { "dom.media.webcodecs.enabled": True, "dom.media.webcodecs.force-osx-h264-enabled": True, @@ -754,7 +754,7 @@ class offscreencanvas_webcodecs_worker_webgl_h264(PageloaderTest): timeout = 600 gecko_profile_interval = 2 gecko_profile_extra_threads = "DOM Worker,CanvasRenderer,MediaSupervisor" - win_counters = w7_counters = linux_counters = mac_counters = None + win_counters = linux_counters = mac_counters = None preferences = { "dom.media.webcodecs.enabled": True, "dom.media.webcodecs.force-osx-h264-enabled": True, @@ -780,7 +780,7 @@ class offscreencanvas_webcodecs_worker_webgl_vp9(PageloaderTest): timeout = 600 gecko_profile_interval = 2 gecko_profile_extra_threads = "DOM Worker,CanvasRenderer,MediaSupervisor" - win_counters = w7_counters = linux_counters = mac_counters = None + win_counters = linux_counters = mac_counters = None preferences = { "dom.media.webcodecs.enabled": True, "dom.media.webcodecs.force-osx-h264-enabled": True, @@ -806,7 +806,7 @@ class offscreencanvas_webcodecs_worker_webgl_av1(PageloaderTest): timeout = 600 gecko_profile_interval = 2 gecko_profile_extra_threads = "DOM Worker,CanvasRenderer,MediaSupervisor" - win_counters = w7_counters = linux_counters = mac_counters = None + win_counters = linux_counters = mac_counters = None preferences = { "dom.media.webcodecs.enabled": True, "dom.media.webcodecs.force-osx-h264-enabled": True, @@ -834,7 +834,7 @@ class offscreencanvas_webcodecs_main_2d_h264(PageloaderTest): timeout = 600 gecko_profile_interval = 2 gecko_profile_extra_threads = "CanvasRenderer,CanvasWorker,MediaSupervisor" - win_counters = w7_counters = linux_counters = mac_counters = None + win_counters = linux_counters = mac_counters = None preferences = { "dom.media.webcodecs.enabled": True, "dom.media.webcodecs.force-osx-h264-enabled": True, @@ -862,7 +862,7 @@ class offscreencanvas_webcodecs_main_2d_vp9(PageloaderTest): timeout = 600 gecko_profile_interval = 2 gecko_profile_extra_threads = "CanvasRenderer,CanvasWorker,MediaSupervisor" - win_counters = w7_counters = linux_counters = mac_counters = None + win_counters = linux_counters = mac_counters = None preferences = { "dom.media.webcodecs.enabled": True, "dom.media.webcodecs.force-osx-h264-enabled": True, @@ -890,7 +890,7 @@ class offscreencanvas_webcodecs_main_2d_av1(PageloaderTest): timeout = 600 gecko_profile_interval = 2 gecko_profile_extra_threads = "CanvasRenderer,CanvasWorker,MediaSupervisor" - win_counters = w7_counters = linux_counters = mac_counters = None + win_counters = linux_counters = mac_counters = None preferences = { "dom.media.webcodecs.enabled": True, "dom.media.webcodecs.force-osx-h264-enabled": True, @@ -918,7 +918,7 @@ class offscreencanvas_webcodecs_worker_2d_h264(PageloaderTest): gecko_profile_extra_threads = ( "DOM Worker,CanvasRenderer,CanvasWorker,MediaSupervisor" ) - win_counters = w7_counters = linux_counters = mac_counters = None + win_counters = linux_counters = mac_counters = None preferences = { "dom.media.webcodecs.enabled": True, "dom.media.webcodecs.force-osx-h264-enabled": True, @@ -946,7 +946,7 @@ class offscreencanvas_webcodecs_worker_2d_vp9(PageloaderTest): gecko_profile_extra_threads = ( "DOM Worker,CanvasRenderer,CanvasWorker,MediaSupervisor" ) - win_counters = w7_counters = linux_counters = mac_counters = None + win_counters = linux_counters = mac_counters = None preferences = { "dom.media.webcodecs.enabled": True, "dom.media.webcodecs.force-osx-h264-enabled": True, @@ -974,7 +974,7 @@ class offscreencanvas_webcodecs_worker_2d_av1(PageloaderTest): gecko_profile_extra_threads = ( "DOM Worker,CanvasRenderer,CanvasWorker,MediaSupervisor" ) - win_counters = w7_counters = linux_counters = mac_counters = None + win_counters = linux_counters = mac_counters = None preferences = { "dom.media.webcodecs.enabled": True, "dom.media.webcodecs.force-osx-h264-enabled": True, @@ -1003,7 +1003,6 @@ class tp5n(PageloaderTest): tpmozafterpaint = True tptimeout = 10000 mainthread = True - w7_counters = [] win_counters = [] linux_counters = [] mac_counters = [] @@ -1057,7 +1056,6 @@ class tp5o(PageloaderTest): multidomain = True tpmanifest = "${talos}/fis/tp5n/tp5o.manifest" win_counters = ["% Processor Time"] - w7_counters = ["% Processor Time"] linux_counters = ["XRes"] mac_counters = [] responsiveness = True @@ -1456,7 +1454,7 @@ class displaylist_mutate(PageloaderTest): tpchrome = False timeout = 600 gecko_profile_interval = 2 - win_counters = w7_counters = linux_counters = mac_counters = None + win_counters = linux_counters = mac_counters = None filters = filter.ignore_first.prepare(1) + filter.median.prepare() """ASAP mode""" preferences = { @@ -1482,7 +1480,7 @@ class rasterflood_svg(PageloaderTest): tpchrome = False timeout = 600 gecko_profile_interval = 2 - win_counters = w7_counters = linux_counters = mac_counters = None + win_counters = linux_counters = mac_counters = None filters = filter.ignore_first.prepare(1) + filter.median.prepare() """ASAP mode""" preferences = { @@ -1507,7 +1505,7 @@ class rasterflood_gradient(PageloaderTest): tpchrome = False timeout = 600 gecko_profile_interval = 2 - win_counters = w7_counters = linux_counters = mac_counters = None + win_counters = linux_counters = mac_counters = None filters = filter.ignore_first.prepare(1) + filter.median.prepare() """ASAP mode""" preferences = { diff --git a/testing/talos/talos/tests/devtools/addon/content/.eslintrc.js b/testing/talos/talos/tests/devtools/addon/content/.eslintrc.js index e2b9246fbb..32aa87888e 100644 --- a/testing/talos/talos/tests/devtools/addon/content/.eslintrc.js +++ b/testing/talos/talos/tests/devtools/addon/content/.eslintrc.js @@ -12,7 +12,7 @@ module.exports = { dampWindow: true, }, rules: { - "no-unused-vars": ["error", { args: "none", vars: "all" }], + "no-unused-vars": ["error", { argsIgnorePattern: "^_", vars: "all" }], // These are the rules that have been configured so far to match the // devtools coding style. diff --git a/testing/talos/talos/tests/devtools/addon/content/damp.js b/testing/talos/talos/tests/devtools/addon/content/damp.js index cad8c8433d..0465edec15 100644 --- a/testing/talos/talos/tests/devtools/addon/content/damp.js +++ b/testing/talos/talos/tests/devtools/addon/content/damp.js @@ -8,9 +8,9 @@ const { gBrowser, MozillaFileLogger, requestIdleCallback } = dampWindow; -const { AddonManager } = require("resource://gre/modules/AddonManager.jsm"); +const { AddonManager } = require("resource://gre/modules/AddonManager.sys.mjs"); -const DampLoadParentModule = require("damp-test/actors/DampLoadParent.jsm"); +const DampLoadParentModule = require("damp-test/actors/DampLoadParent.sys.mjs"); const DAMP_TESTS = require("damp-test/damp-tests.js"); // Record allocation count in new subtests if DEBUG_DEVTOOLS_ALLOCATIONS is set to @@ -66,7 +66,7 @@ Damp.prototype = { try { const { TalosParentProfiler, - } = require("resource://talos-powers/TalosParentProfiler.jsm"); + } = require("resource://talos-powers/TalosParentProfiler.sys.mjs"); return TalosParentProfiler; } catch (err) { await new Promise(resolve => setTimeout(resolve, 500)); diff --git a/testing/talos/talos/tests/devtools/addon/content/pages/custom/inspector/iframe.html b/testing/talos/talos/tests/devtools/addon/content/pages/custom/inspector/iframe.html index 894f647089..7926f65aac 100644 --- a/testing/talos/talos/tests/devtools/addon/content/pages/custom/inspector/iframe.html +++ b/testing/talos/talos/tests/devtools/addon/content/pages/custom/inspector/iframe.html @@ -58,6 +58,10 @@ font-family: Arial; margin: 20px; }`; + manyCssRules += ` + .many-css-rules::after { + content: "rule #${i}"; + }`; } let CSS_VARIABLES_COUNT = 300; diff --git a/testing/talos/talos/tests/devtools/addon/content/tests/styleeditor/complicated.js b/testing/talos/talos/tests/devtools/addon/content/tests/styleeditor/complicated.js index 29f3f2bdbb..648073bd36 100644 --- a/testing/talos/talos/tests/devtools/addon/content/tests/styleeditor/complicated.js +++ b/testing/talos/talos/tests/devtools/addon/content/tests/styleeditor/complicated.js @@ -7,19 +7,22 @@ const { openToolboxAndLog, closeToolboxAndLog, - reloadPageAndLog, testSetup, testTeardown, COMPLICATED_URL, } = require("damp-test/tests/head"); +const { + reloadStyleEditorAndLog, +} = require("damp-test/tests/styleeditor/styleeditor-helpers"); + module.exports = async function () { await testSetup(COMPLICATED_URL); const toolbox = await openToolboxAndLog( "complicated.styleeditor", "styleeditor" ); - await reloadPageAndLog("complicated.styleeditor", toolbox); + await reloadStyleEditorAndLog("complicated.styleeditor", toolbox); await closeToolboxAndLog("complicated.styleeditor", toolbox); await testTeardown(); }; diff --git a/testing/talos/talos/tests/devtools/addon/content/tests/styleeditor/custom.js b/testing/talos/talos/tests/devtools/addon/content/tests/styleeditor/custom.js index 2ea2af1b64..8664c1b606 100644 --- a/testing/talos/talos/tests/devtools/addon/content/tests/styleeditor/custom.js +++ b/testing/talos/talos/tests/devtools/addon/content/tests/styleeditor/custom.js @@ -7,18 +7,21 @@ const { openToolboxAndLog, closeToolboxAndLog, - reloadPageAndLog, testSetup, testTeardown, PAGES_BASE_URL, } = require("damp-test/tests/head"); +const { + reloadStyleEditorAndLog, +} = require("damp-test/tests/styleeditor/styleeditor-helpers"); + const TEST_URL = PAGES_BASE_URL + "custom/styleeditor/index.html"; module.exports = async function () { await testSetup(TEST_URL); const toolbox = await openToolboxAndLog("custom.styleeditor", "styleeditor"); - await reloadPageAndLog("custom.styleeditor", toolbox); + await reloadStyleEditorAndLog("custom.styleeditor", toolbox); await closeToolboxAndLog("custom.styleeditor", toolbox); await testTeardown(); diff --git a/testing/talos/talos/tests/devtools/addon/content/tests/styleeditor/styleeditor-helpers.js b/testing/talos/talos/tests/devtools/addon/content/tests/styleeditor/styleeditor-helpers.js new file mode 100644 index 0000000000..3e195c6481 --- /dev/null +++ b/testing/talos/talos/tests/devtools/addon/content/tests/styleeditor/styleeditor-helpers.js @@ -0,0 +1,15 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +"use strict"; + +const { reloadPageAndLog } = require("damp-test/tests/head"); + +exports.reloadStyleEditorAndLog = async function (testName, toolbox) { + const onReloaded = async function () { + let panel = await toolbox.getPanelWhenReady("styleeditor"); + await panel.once("reloaded"); + }; + await reloadPageAndLog(testName, toolbox, onReloaded); +}; diff --git a/testing/talos/talos/tests/devtools/addon/content/tests/toolbox/screenshot.js b/testing/talos/talos/tests/devtools/addon/content/tests/toolbox/screenshot.js index 215f7770c4..327d1e28e5 100644 --- a/testing/talos/talos/tests/devtools/addon/content/tests/toolbox/screenshot.js +++ b/testing/talos/talos/tests/devtools/addon/content/tests/toolbox/screenshot.js @@ -12,7 +12,7 @@ const { testSetup, testTeardown, } = require("damp-test/tests/head"); -const { Downloads } = require("resource://gre/modules/Downloads.jsm"); +const { Downloads } = require("resource://gre/modules/Downloads.sys.mjs"); module.exports = async function () { await testSetup(COMPLICATED_URL); diff --git a/testing/talos/talos/tests/tart/addon/content/tart.js b/testing/talos/talos/tests/tart/addon/content/tart.js index f3616cd4a9..dd98617cda 100644 --- a/testing/talos/talos/tests/tart/addon/content/tart.js +++ b/testing/talos/talos/tests/tart/addon/content/tart.js @@ -126,7 +126,7 @@ Tart.prototype = { clickNewTab() { this._endDetection = this.tabDetector; - this._win.BrowserOpenTab(); + this._win.BrowserCommands.openTab(); // Modifying the style for each tab right after opening seems like it could regress performance, // However, overlaying a global style over browser.xhtml actually ends up having greater ovrehead, // especially while closing the last of many tabs (a noticeable ~250ms delay before expanding the rest). @@ -146,7 +146,7 @@ Tart.prototype = { clickCloseCurrentTab() { this._endDetection = this.tabDetector; - this._win.BrowserCloseTabOrWindow(); + this._win.BrowserCommands.closeTabOrWindow(); return this._win.gBrowser.selectedTab; }, diff --git a/testing/talos/talos/ttest.py b/testing/talos/talos/ttest.py index 46cc53265a..96e5850c40 100644 --- a/testing/talos/talos/ttest.py +++ b/testing/talos/talos/ttest.py @@ -28,7 +28,7 @@ from talos import results, talosconfig, utils from talos.cmanager import CounterManagement from talos.ffsetup import FFSetup from talos.talos_process import run_browser -from talos.utils import TalosCrash, TalosError, TalosRegression, run_in_debug_mode +from talos.utils import TalosCrash, TalosRegression, run_in_debug_mode LOG = get_proxy_logger() @@ -67,18 +67,7 @@ class TTest(object): if platform.system() == "Linux": return "linux" elif platform.system() in ("Windows", "Microsoft"): - if "6.1" in platform.version(): # w7 - return "w7" - elif "6.2" in platform.version(): # w8 - return "w8" - # Bug 1264325 - FIXME: with python 2.7.11: reports win8 instead of 8.1 - elif "6.3" in platform.version(): - return "w8" - # Bug 1264325 - FIXME: with python 2.7.11: reports win8 instead of 10 - elif "10.0" in platform.version(): - return "w8" - else: - raise TalosError("unsupported windows version") + return "win" elif platform.system() == "Darwin": return "mac" diff --git a/testing/talos/talos/unittests/conftest.py b/testing/talos/talos/unittests/conftest.py index 96cc9ca9b9..ffa56cb4ff 100644 --- a/testing/talos/talos/unittests/conftest.py +++ b/testing/talos/talos/unittests/conftest.py @@ -2,7 +2,13 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. +import json import os +import pathlib +import shutil +import tempfile + +import pytest here = os.path.realpath(__file__) __TESTS_DIR = os.path.join(os.path.dirname(os.path.dirname(here)), "tests") @@ -45,3 +51,26 @@ def patched_build_manifest(config, manifestName): # return new manifest return newManifestName + + +@pytest.fixture(scope="module") +def pdfpaint_dir_info(): + with tempfile.TemporaryDirectory() as tmpdir: + try: + # Setup the temporary files + tmpdir_path = pathlib.Path(tmpdir) + talos_pdfs_dir = pathlib.Path(tmpdir_path, "talos-pdfs") + talos_pdfs_dir.mkdir(parents=True, exist_ok=True) + + pdf_count = 101 + pdf_manifest_json = [] + for i in range(pdf_count): + pdf_manifest_json.append({"file": str(i)}) + + pdf_manifest = talos_pdfs_dir / "test_manifest.json" + with pdf_manifest.open("w", encoding="utf-8") as f: + json.dump(pdf_manifest_json, f) + + yield tmpdir_path, pdf_count + finally: + shutil.rmtree(tmpdir) diff --git a/testing/talos/talos/unittests/test_config.py b/testing/talos/talos/unittests/test_config.py index ae58364b07..d5e1b5eebc 100644 --- a/testing/talos/talos/unittests/test_config.py +++ b/testing/talos/talos/unittests/test_config.py @@ -1,5 +1,6 @@ import copy import os +import pathlib from unittest import mock import conftest @@ -43,7 +44,6 @@ class mock_test(PageloaderTest): "gecko_profile_entries", "tptimeout", "win_counters", - "w7_counters", "linux_counters", "mac_counters", "tpscrolltest", @@ -143,7 +143,6 @@ class Test_get_test(object): linux_counters=None, mac_counters=[], win_counters=["counter_a"], - w7_counters=["counter_a", "counter_b"], xperf_counters=["counter_a", "counter_extra"], ) @@ -155,7 +154,6 @@ class Test_get_test(object): assert test_dict["linux_counters"] == counters assert test_dict["mac_counters"] == counters assert test_dict["win_counters"] == counters - assert test_dict["w7_counters"] == counters assert set(test_dict["xperf_counters"]) == set(counters + ["counter_extra"]) @@ -310,6 +308,7 @@ class Test_get_config(object): cls.argv_perf_reftest_singletons = ( "--activeTests perf_reftest_singletons -e /some/random/path".split() ) + cls.argv_pdfpaint = "--activeTests pdfpaint -e /some/random/path".split() @classmethod def teardown_class(cls): @@ -553,7 +552,6 @@ class Test_get_config(object): assert test_config["gecko_profile_interval"] == 10 assert test_config["gecko_profile_entries"] == 1000000 assert "win_counters" not in test_config - assert "w7_counters" not in test_config assert "linux_counters" not in test_config assert "mac_counters" not in test_config assert test_config["preferences"] == { @@ -578,7 +576,6 @@ class Test_get_config(object): assert test_config["gecko_profile_interval"] == 10 assert test_config["gecko_profile_entries"] == 1000000 assert "win_counters" not in test_config - assert "w7_counters" not in test_config assert "linux_counters" not in test_config assert "mac_counters" not in test_config assert test_config["filters"] is not None @@ -599,7 +596,6 @@ class Test_get_config(object): assert test_config["gecko_profile_interval"] == 10 assert test_config["gecko_profile_entries"] == 2000000 assert "win_counters" not in test_config - assert "w7_counters" not in test_config assert "linux_counters" not in test_config assert "mac_counters" not in test_config assert test_config["preferences"] == { @@ -624,7 +620,6 @@ class Test_get_config(object): assert test_config["gecko_profile_interval"] == 2 assert test_config["gecko_profile_entries"] == 2000000 assert "win_counters" not in test_config - assert "w7_counters" not in test_config assert "linux_counters" not in test_config assert "mac_counters" not in test_config assert test_config["filters"] is not None @@ -647,7 +642,6 @@ class Test_get_config(object): assert test_config["gecko_profile_interval"] == 2 assert test_config["gecko_profile_entries"] == 2000000 assert "win_counters" not in test_config - assert "w7_counters" not in test_config assert "linux_counters" not in test_config assert "mac_counters" not in test_config assert test_config["filters"] is not None @@ -670,7 +664,6 @@ class Test_get_config(object): assert test_config["gecko_profile_interval"] == 2 assert test_config["gecko_profile_entries"] == 2000000 assert "win_counters" not in test_config - assert "w7_counters" not in test_config assert "linux_counters" not in test_config assert "mac_counters" not in test_config assert test_config["filters"] is not None @@ -693,7 +686,6 @@ class Test_get_config(object): assert test_config["gecko_profile_interval"] == 2 assert test_config["gecko_profile_entries"] == 2000000 assert "win_counters" not in test_config - assert "w7_counters" not in test_config assert "linux_counters" not in test_config assert "mac_counters" not in test_config assert test_config["filters"] is not None @@ -716,7 +708,6 @@ class Test_get_config(object): assert test_config["gecko_profile_interval"] == 2 assert test_config["gecko_profile_entries"] == 2000000 assert "win_counters" not in test_config - assert "w7_counters" not in test_config assert "linux_counters" not in test_config assert "mac_counters" not in test_config assert test_config["filters"] is not None @@ -739,7 +730,6 @@ class Test_get_config(object): assert test_config["gecko_profile_interval"] == 2 assert test_config["gecko_profile_entries"] == 2000000 assert "win_counters" not in test_config - assert "w7_counters" not in test_config assert "linux_counters" not in test_config assert "mac_counters" not in test_config assert test_config["filters"] is not None @@ -762,7 +752,6 @@ class Test_get_config(object): assert test_config["gecko_profile_interval"] == 2 assert test_config["gecko_profile_entries"] == 2000000 assert "win_counters" not in test_config - assert "w7_counters" not in test_config assert "linux_counters" not in test_config assert "mac_counters" not in test_config assert test_config["filters"] is not None @@ -785,7 +774,6 @@ class Test_get_config(object): assert test_config["gecko_profile_interval"] == 2 assert test_config["gecko_profile_entries"] == 2000000 assert "win_counters" not in test_config - assert "w7_counters" not in test_config assert "linux_counters" not in test_config assert "mac_counters" not in test_config assert test_config["filters"] is not None @@ -808,7 +796,6 @@ class Test_get_config(object): assert test_config["gecko_profile_interval"] == 2 assert test_config["gecko_profile_entries"] == 2000000 assert "win_counters" not in test_config - assert "w7_counters" not in test_config assert "linux_counters" not in test_config assert "mac_counters" not in test_config assert test_config["filters"] is not None @@ -831,7 +818,6 @@ class Test_get_config(object): assert test_config["gecko_profile_interval"] == 2 assert test_config["gecko_profile_entries"] == 2000000 assert "win_counters" not in test_config - assert "w7_counters" not in test_config assert "linux_counters" not in test_config assert "mac_counters" not in test_config assert test_config["filters"] is not None @@ -854,7 +840,6 @@ class Test_get_config(object): assert test_config["gecko_profile_interval"] == 2 assert test_config["gecko_profile_entries"] == 2000000 assert "win_counters" not in test_config - assert "w7_counters" not in test_config assert "linux_counters" not in test_config assert "mac_counters" not in test_config assert test_config["filters"] is not None @@ -877,7 +862,6 @@ class Test_get_config(object): assert test_config["gecko_profile_interval"] == 2 assert test_config["gecko_profile_entries"] == 2000000 assert "win_counters" not in test_config - assert "w7_counters" not in test_config assert "linux_counters" not in test_config assert "mac_counters" not in test_config assert test_config["filters"] is not None @@ -900,7 +884,6 @@ class Test_get_config(object): assert test_config["gecko_profile_interval"] == 2 assert test_config["gecko_profile_entries"] == 2000000 assert "win_counters" not in test_config - assert "w7_counters" not in test_config assert "linux_counters" not in test_config assert "mac_counters" not in test_config assert test_config["filters"] is not None @@ -923,7 +906,6 @@ class Test_get_config(object): assert test_config["gecko_profile_interval"] == 2 assert test_config["gecko_profile_entries"] == 2000000 assert "win_counters" not in test_config - assert "w7_counters" not in test_config assert "linux_counters" not in test_config assert "mac_counters" not in test_config assert test_config["filters"] is not None @@ -943,7 +925,6 @@ class Test_get_config(object): assert test_config["tpmozafterpaint"] is True assert test_config["tptimeout"] == 5000 assert test_config["mainthread"] is True - assert test_config["w7_counters"] == [] assert test_config["win_counters"] == [] assert test_config["linux_counters"] == [] assert test_config["mac_counters"] == [] @@ -1001,7 +982,6 @@ class Test_get_config(object): assert test_config["mainthread"] is False assert test_config["tpmanifest"] != "${talos}/tests/tp5n/tp5o.manifest" assert test_config["win_counters"] == ["% Processor Time"] - assert test_config["w7_counters"] == ["% Processor Time"] assert test_config["linux_counters"] == ["XRes"] assert test_config["mac_counters"] == [] assert test_config["responsiveness"] is True @@ -1025,7 +1005,6 @@ class Test_get_config(object): assert test_config["mainthread"] is False assert test_config["tpmanifest"] != "${talos}/tests/tp5n/tp5o.manifest" assert test_config["win_counters"] == ["% Processor Time"] - assert test_config["w7_counters"] == ["% Processor Time"] assert test_config["linux_counters"] == ["XRes"] assert test_config["mac_counters"] == [] assert test_config["responsiveness"] is True @@ -1298,6 +1277,110 @@ class Test_get_config(object): assert test_config["alert_threshold"] == 5.0 +# The tests in the Test_get_config class don't currently run, so these +# pdfpaint tests live outside of it for now. See bug 1888132. +@mock.patch("pathlib.Path.unlink", new=mock.MagicMock()) +@mock.patch("pathlib.Path.symlink_to", new=mock.MagicMock()) +def test_pdfpaint_has_expected_attributes_no_chunk(pdfpaint_dir_info): + pdfpaint_dir, pdf_count = pdfpaint_dir_info + + Test_get_config.setup_class() + with mock.patch.dict( + os.environ, {"MOZ_FETCHES_DIR": "", "MOZBUILD_PATH": str(pdfpaint_dir)} + ): + config = get_config(Test_get_config.argv_pdfpaint) + + test_config = config["tests"][0] + + assert test_config["name"] == "pdfpaint" + assert test_config["tpmanifest"] != "${talos}/tests/pdfpaint/pdfpaint.manifest" + + manifest_content = pathlib.Path(test_config["tpmanifest"]).read_text() + manifest_lines = manifest_content.split("\n") + assert len([line for line in manifest_lines if line]) == pdf_count + + assert test_config["tpcycles"] == 1 + assert test_config["tppagecycles"] == 1 + assert test_config["tptimeout"] == 60000 + assert test_config["gecko_profile_entries"] == 16777216 + assert test_config["filters"] is not None + assert test_config["unit"] == "ms" + assert test_config["lower_is_better"] is True + assert test_config["alert_threshold"] == 2.0 + + +@mock.patch("pathlib.Path.unlink", new=mock.MagicMock()) +@mock.patch("pathlib.Path.symlink_to", new=mock.MagicMock()) +def test_pdfpaint_has_expected_attributes_with_chunk(pdfpaint_dir_info): + pdfpaint_dir, _ = pdfpaint_dir_info + + Test_get_config.setup_class() + args = Test_get_config.argv_pdfpaint + ["--pdfPaintChunk", "1"] + with mock.patch.dict( + os.environ, + {"MOZ_FETCHES_DIR": str(pdfpaint_dir), "MOZBUILD_PATH": ""}, + ): + config = get_config(args) + + test_config = config["tests"][0] + + assert test_config["name"] == "pdfpaint" + assert test_config["tpmanifest"] != "${talos}/tests/pdfpaint/pdfpaint.manifest" + + manifest_content = pathlib.Path(test_config["tpmanifest"]).read_text() + manifest_lines = manifest_content.split("\n") + assert len([line for line in manifest_lines if line]) == 100 + + assert test_config["tpcycles"] == 1 + assert test_config["tppagecycles"] == 5 + assert test_config["tptimeout"] == 60000 + assert test_config["gecko_profile_entries"] == 16777216 + assert test_config["filters"] is not None + assert test_config["unit"] == "ms" + assert test_config["lower_is_better"] is True + assert test_config["alert_threshold"] == 2.0 + + +def test_pdfpaint_fails_on_bad_chunk(pdfpaint_dir_info): + pdfpaint_dir, _ = pdfpaint_dir_info + + Test_get_config.setup_class() + args = Test_get_config.argv_pdfpaint + ["--pdfPaintChunk", "10"] + with pytest.raises(ConfigurationError): + with mock.patch.dict( + os.environ, + {"MOZ_FETCHES_DIR": str(pdfpaint_dir), "MOZBUILD_PATH": ""}, + ): + get_config(args) + + +@mock.patch("pathlib.Path.unlink", new=mock.MagicMock()) +@mock.patch("pathlib.Path.symlink_to", new=mock.MagicMock()) +def test_pdfpaint_with_pdf_name(pdfpaint_dir_info): + pdfpaint_dir, _ = pdfpaint_dir_info + + Test_get_config.setup_class() + args = Test_get_config.argv_pdfpaint + ["--pdfPaintName", "1"] + with mock.patch.dict( + os.environ, + {"MOZ_FETCHES_DIR": str(pdfpaint_dir), "MOZBUILD_PATH": ""}, + ): + config = get_config(args) + + test_config = config["tests"][0] + + assert test_config["name"] == "pdfpaint" + assert test_config["tpmanifest"] != "${talos}/tests/pdfpaint/pdfpaint.manifest" + + manifest_content = pathlib.Path(test_config["tpmanifest"]).read_text() + manifest_lines = manifest_content.split("\n") + assert len([line for line in manifest_lines if line]) == 1 + assert manifest_lines[0].split("/")[-1] == "1" + + assert test_config["tpcycles"] == 1 + assert test_config["tppagecycles"] == 5 + + @mock.patch("talos.config.get_browser_config") @mock.patch("talos.config.get_config") def test_get_configs(get_config_mock, get_browser_config_mock): |