diff options
Diffstat (limited to 'testing/raptor/browsertime')
20 files changed, 2389 insertions, 0 deletions
diff --git a/testing/raptor/browsertime/browsertime_benchmark.js b/testing/raptor/browsertime/browsertime_benchmark.js new file mode 100644 index 0000000000..c5ea55319f --- /dev/null +++ b/testing/raptor/browsertime/browsertime_benchmark.js @@ -0,0 +1,98 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* eslint-env node */ + +module.exports = async function (context, commands) { + context.log.info("Starting a browsertime benchamrk"); + let url = context.options.browsertime.url; + let page_cycles = context.options.browsertime.page_cycles; + let page_cycle_delay = context.options.browsertime.page_cycle_delay; + let post_startup_delay = context.options.browsertime.post_startup_delay; + let page_timeout = context.options.timeouts.pageLoad; + let ret = false; + let expose_profiler = context.options.browsertime.expose_profiler; + + context.log.info( + "Waiting for %d ms (post_startup_delay)", + post_startup_delay + ); + await commands.wait.byTime(post_startup_delay); + + for (let count = 0; count < page_cycles; count++) { + context.log.info("Navigating to about:blank"); + await commands.navigate("about:blank"); + + context.log.info("Cycle %d, waiting for %d ms", count, page_cycle_delay); + await commands.wait.byTime(page_cycle_delay); + + context.log.info("Cycle %d, starting the measure", count); + if (expose_profiler === "true") { + context.log.info("Custom profiler start!"); + if (context.options.browser === "firefox") { + await commands.profiler.start(); + } else if (context.options.browser === "chrome") { + await commands.trace.start(); + } + } + await commands.measure.start(url); + + context.log.info("Benchmark custom metric collection"); + + let data = null; + let starttime = await commands.js.run(`return performance.now();`); + while ( + data == null && + (await commands.js.run(`return performance.now();`)) - starttime < + page_timeout + ) { + let wait_time = 3000; + context.log.info("Waiting %d ms for data from benchmark...", wait_time); + await commands.wait.byTime(wait_time); + data = await commands.js.run( + "return window.sessionStorage.getItem('benchmark_results');" + ); + } + if (expose_profiler === "true") { + context.log.info("Custom profiler stop!"); + if (context.options.browser === "firefox") { + await commands.profiler.stop(); + } else if (context.options.browser === "chrome") { + await commands.trace.stop(); + } + } + if ( + data == null && + (await commands.js.run(`return performance.now();`)) - starttime >= + page_timeout + ) { + ret = false; + context.log.error("Benchmark timed out. Aborting..."); + } else if (data) { + // Reset benchmark results + await commands.js.run( + "return window.sessionStorage.removeItem('benchmark_results');" + ); + + context.log.info("Value of benchmark data: ", data); + data = JSON.parse(data); + + if (!Array.isArray(data)) { + commands.measure.addObject({ browsertime_benchmark: data }); + } else { + commands.measure.addObject({ + browsertime_benchmark: { + [data[1]]: data.slice(2), + }, + }); + } + ret = true; + } else { + context.log.error("No data collected from benchmark."); + ret = false; + } + } + context.log.info("Browsertime benchmark ended."); + return ret; +}; diff --git a/testing/raptor/browsertime/browsertime_interactive.js b/testing/raptor/browsertime/browsertime_interactive.js new file mode 100644 index 0000000000..20ce23ccf1 --- /dev/null +++ b/testing/raptor/browsertime/browsertime_interactive.js @@ -0,0 +1,103 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* eslint-env node */ + +async function get_command_function(cmd, commands) { + /* + Converts a string such as `measure.start` into the actual + function that is found in the `commands` module. + + XXX: Find a way to share this function between + perftest_record.js and browsertime_interactive.js + */ + if (cmd == "") { + throw new Error("A blank command was given."); + } else if (cmd.endsWith(".")) { + throw new Error( + "An extra `.` was found at the end of this command: " + cmd + ); + } + + // `func` will hold the actual method that needs to be called, + // and the `parent_mod` is the context required to run the `func` + // method. Without that context, `this` becomes undefined in the browsertime + // classes. + let func = null; + let parent_mod = null; + for (let func_part of cmd.split(".")) { + if (func_part == "") { + throw new Error( + "An empty function part was found in the command: " + cmd + ); + } + + if (func === null) { + parent_mod = commands; + func = commands[func_part]; + } else if (func !== undefined) { + parent_mod = func; + func = func[func_part]; + } else { + break; + } + } + + if (func == undefined) { + throw new Error( + "The given command could not be found as a function: " + cmd + ); + } + + return [func, parent_mod]; +} + +module.exports = async function (context, commands) { + context.log.info("Starting an interactive browsertime test"); + let page_cycles = context.options.browsertime.page_cycles; + let post_startup_delay = context.options.browsertime.post_startup_delay; + let input_cmds = context.options.browsertime.commands; + + context.log.info( + "Waiting for %d ms (post_startup_delay)", + post_startup_delay + ); + await commands.wait.byTime(post_startup_delay); + + // unpack commands from python + let cmds = input_cmds.split(";;;"); + + for (let count = 0; count < page_cycles; count++) { + context.log.info("Navigating to about:blank w/nav, count: " + count); + await commands.navigate("about:blank"); + + let pages_visited = []; + for (let cmdstr of cmds) { + let [cmd, ...args] = cmdstr.split(":::"); + + if (cmd == "measure.start") { + if (args[0] != "") { + pages_visited.push(args[0]); + } + } + + let [func, parent_mod] = await get_command_function(cmd, commands); + + try { + await func.call(parent_mod, ...args); + } catch (e) { + context.log.info( + `Exception found while running \`commands.${cmd}(${args})\`: ` + ); + context.log.info(e.stack); + } + } + + // Log the number of pages visited for results parsing + context.log.info("[] metrics: pages_visited: " + pages_visited); + } + + context.log.info("Browsertime pageload ended."); + return true; +}; diff --git a/testing/raptor/browsertime/browsertime_pageload.js b/testing/raptor/browsertime/browsertime_pageload.js new file mode 100644 index 0000000000..869efcf3a0 --- /dev/null +++ b/testing/raptor/browsertime/browsertime_pageload.js @@ -0,0 +1,412 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* eslint-env node */ + +const fs = require("fs"); +const http = require("http"); + +const URL = "/secrets/v1/secret/project/perftest/gecko/level-"; +const SECRET = "/perftest-login"; +const DEFAULT_SERVER = "https://firefox-ci-tc.services.mozilla.com"; + +const SCM_LOGIN_SITES = ["facebook", "netflix"]; + +/** + * This function obtains the perftest secret from Taskcluster. + * + * It will NOT work locally. Please see the get_logins function, you + * will need to define a JSON file and set the RAPTOR_LOGINS + * env variable to its path. + */ +async function get_tc_secrets(context) { + const MOZ_AUTOMATION = process.env.MOZ_AUTOMATION; + if (!MOZ_AUTOMATION) { + throw Error( + "Not running in CI. Set RAPTOR_LOGINS to a JSON file containing the logins." + ); + } + + let TASKCLUSTER_PROXY_URL = process.env.TASKCLUSTER_PROXY_URL + ? process.env.TASKCLUSTER_PROXY_URL + : DEFAULT_SERVER; + + let MOZ_SCM_LEVEL = process.env.MOZ_SCM_LEVEL ? process.env.MOZ_SCM_LEVEL : 1; + + const url = TASKCLUSTER_PROXY_URL + URL + MOZ_SCM_LEVEL + SECRET; + + const data = await new Promise((resolve, reject) => { + context.log.info("Obtaining secrets for login..."); + + http.get( + url, + { + headers: { + "Content-Type": "application/json", + Accept: "application/json", + }, + }, + res => { + let data = ""; + context.log.info(`Secret status code: ${res.statusCode}`); + + res.on("data", d => { + data += d.toString(); + }); + + res.on("end", () => { + resolve(data); + }); + + res.on("error", error => { + context.log.error(error); + reject(error); + }); + } + ); + }); + + return JSON.parse(data); +} + +/** + * This function gets the login information required. + * + * It starts by looking for a local file whose path is defined + * within RAPTOR_LOGINS. If we don't find this file, then we'll + * attempt to get the login information from our Taskcluster secret. + * If MOZ_AUTOMATION is undefined, then the test will fail, Taskcluster + * secrets can only be obtained in CI. + */ +async function get_logins(context) { + let logins; + + let RAPTOR_LOGINS = process.env.RAPTOR_LOGINS; + if (RAPTOR_LOGINS) { + // Get logins from a local file + if (!RAPTOR_LOGINS.endsWith(".json")) { + throw Error( + `File given for logins does not end in '.json': ${RAPTOR_LOGINS}` + ); + } + + let logins_file = null; + try { + logins_file = await fs.readFileSync(RAPTOR_LOGINS, "utf8"); + } catch (err) { + throw Error(`Failed to read the file ${RAPTOR_LOGINS}: ${err}`); + } + + logins = await JSON.parse(logins_file); + } else { + // Get logins from a perftest Taskcluster secret + logins = await get_tc_secrets(context); + } + + return logins; +} + +/** + * This function returns the type of login to do. + * + * This function returns "single-form" when we find a single form. If we only + * find a single input field, we assume that there is one page per input + * and return "multi-page". Otherwise, we return null. + */ +async function get_login_type(context, commands) { + /* + Determine if there's a password field visible with this + query selector. Some sites use `tabIndex` to hide the password + field behind other elements. In this case, we are searching + for any password-type field that has a tabIndex of 0 or undefined and + is not hidden. + */ + let input_length = await commands.js.run(` + return document.querySelectorAll( + "input[type=password][tabIndex='0']:not([type=hidden])," + + "input[type=password]:not([tabIndex]):not([type=hidden])" + ).length; + `); + + if (input_length == 0) { + context.log.info("Found a multi-page login"); + return multi_page_login; + } else if (input_length == 1) { + context.log.info("Found a single-page login"); + return single_page_login; + } + + if ( + (await commands.js.run( + `return document.querySelectorAll("form").length;` + )) >= 1 + ) { + context.log.info("Found a single-form login"); + return single_form_login; + } + + return null; +} + +/** + * This function sets up the login for a single form. + * + * The username field is defined as the field which immediately precedes + * the password field. We have to do this in two steps because we need + * to make sure that the event we emit from the change has the `isTrusted` + * field set to `true`. Otherwise, some websites will ignore the input and + * the form submission. + */ +async function single_page_login(login_info, context, commands, prefix = "") { + // Get the first input field in the form that is not hidden and add the + // username. Assumes that email/username is always the first input field. + await commands.addText.bySelector( + login_info.username, + `${prefix}input:not([type=hidden]):not([type=password])` + ); + + // Get the password field and ensure it's not hidden. + await commands.addText.bySelector( + login_info.password, + `${prefix}input[type=password]:not([type=hidden])` + ); + + return undefined; +} + +/** + * See single_page_login. + */ +async function single_form_login(login_info, context, commands) { + return single_page_login(login_info, context, commands, "form "); +} + +/** + * Login to a website that uses multiple pages for the login. + * + * WARNING: Assumes that the first page is for the username. + */ +// TODO cleanup comments +async function multi_page_login(login_info, context, commands) { + const driver = context.selenium.driver; + const webdriver = context.selenium.webdriver; + //TODO fails here in netflix for Try... + const username_field = await driver.findElement( + webdriver.By.css(`input:not([type=hidden]):not([type=password])`) + ); + await username_field.sendKeys(login_info.username); + await username_field.sendKeys(webdriver.Key.ENTER); + await commands.wait.byTime(5000); + let password_field; + try { + password_field = await driver.findElement( + webdriver.By.css(`input[type=password]:not([type=hidden])`) + ); + } catch (err) { + if (err.toString().includes("NoSuchElementError")) { + // Sometimes we're suspicious (i.e. they think we're a bot/bad-actor) + let name_field = await driver.findElement( + webdriver.By.css(`input:not([type=hidden]):not([type=password])`) + ); + await name_field.sendKeys(login_info.suspicious_answer); + await name_field.sendKeys(webdriver.Key.ENTER); + await commands.wait.byTime(5000); + + // Try getting the password field again + password_field = await driver.findElement( + webdriver.By.css(`input[type=password]:not([type=hidden])`) + ); + } else { + throw err; + } + } + + await password_field.sendKeys(login_info.password); + + return async function () { + password_field.sendKeys(webdriver.Key.ENTER); + await commands.wait.byTime(5000); + }; +} + +/** + * This function sets up the login. + * + * This is done by first the login type, and then performing the + * actual login setup. The return is a possible button to click + * to perform the login. + */ +async function setup_login(login_info, context, commands) { + let login_func = await get_login_type(context, commands); + if (!login_func) { + throw Error("Could not determine the type of login page."); + } + + try { + return await login_func(login_info, context, commands); + } catch (err) { + throw Error(`Could not setup login information: ${err}`); + } +} + +/** + * This function performs the login. + * + * It does this by either clicking on a button with a type + * of "sumbit", or running a final_button function that was + * obtained from the setup_login function. Some pages also ask + * questions about setting up 2FA or other information. Generally, + * these contain the "skip" text. + */ +async function login(context, commands, final_button) { + try { + if (!final_button) { + // The mouse double click emits an event with `evt.isTrusted=true` + await commands.mouse.doubleClick.bySelector("button[type=submit]"); + await commands.wait.byTime(10000); + } else { + // In some cases, it's preferable to be given a function for the final button + await final_button(); + } + + // Some pages ask to setup 2FA, skip this based on the text + const XPATHS = [ + "//a[contains(text(), 'skip')]", + "//button[contains(text(), 'skip')]", + "//input[contains(text(), 'skip')]", + "//div[contains(text(), 'skip')]", + ]; + + for (let xpath of XPATHS) { + try { + await commands.mouse.doubleClick.byXpath(xpath); + } catch (err) { + if (err.toString().includes("not double click")) { + context.log.info(`Can't find a button with the text: ${xpath}`); + } else { + throw err; + } + } + } + } catch (err) { + throw Error( + `Could not login to website as we could not find the submit button/input: ${err}` + ); + } +} + +/** + * Grab the base URL from the browsertime url. + * + * This is a necessary step for getting the login values from the Taskcluster + * secrets, which are hashed by the base URL. + * + * The first entry is the protocal, third is the top-level domain (or host) + */ +function get_base_URL(fullUrl) { + let pathAsArray = fullUrl.split("/"); + return pathAsArray[0] + "//" + pathAsArray[2]; +} + +/** + * This function attempts the login-login sequence for a live pageload test + */ +async function perform_live_login(context, commands) { + let testUrl = context.options.browsertime.url; + + let logins = await get_logins(context); + const baseUrl = get_base_URL(testUrl); + + await commands.navigate("about:blank"); + + let login_info = logins.secret[baseUrl]; + try { + await commands.navigate(login_info.login_url); + } catch (err) { + context.log.info("Unable to acquire login information"); + throw err; + } + await commands.wait.byTime(10000); + + let final_button = await setup_login(login_info, context, commands); + await login(context, commands, final_button); +} + +module.exports = async function (context, commands) { + context.log.info("Starting a browsertime pageload"); + let test_url = context.options.browsertime.url; + let secondary_url = context.options.browsertime.secondary_url; + let page_cycles = context.options.browsertime.page_cycles; + let page_cycle_delay = context.options.browsertime.page_cycle_delay; + let post_startup_delay = context.options.browsertime.post_startup_delay; + let chimera_mode = context.options.browsertime.chimera; + let test_bytecode_cache = context.options.browsertime.test_bytecode_cache; + let login_required = context.options.browsertime.loginRequired; + let live_site = context.options.browsertime.liveSite; + let test_name = context.options.browsertime.testName; + + context.log.info( + "Waiting for %d ms (post_startup_delay)", + post_startup_delay + ); + await commands.wait.byTime(post_startup_delay); + let cached = false; + + // Login once before testing the test_url/secondary_url cycles + // If the user has RAPTOR_LOGINS configured correctly, a local login pageload + // test can be attempted. Otherwise if attempting it in CI, only sites with the + // associated MOZ_SCM_LEVEL will be attempted (e.g. Try = 1, autoland = 3). + // In addition, ensure login sequence is only attempted on live sites and for sites + // that we have Taskcluster secrets for. + if ( + login_required == "True" && + live_site == "True" && + SCM_LOGIN_SITES.includes(test_name) + ) { + await perform_live_login(context, commands); + } + + for (let count = 0; count < page_cycles; count++) { + if (count !== 0 && secondary_url !== undefined) { + context.log.info("Navigating to secondary url:" + secondary_url); + await commands.navigate(secondary_url); + await commands.wait.byTime(1000); + await commands.js.runAndWait(` + (function() { + const white = document.createElement('div'); + white.id = 'raptor-white'; + white.style.position = 'absolute'; + white.style.top = '0'; + white.style.left = '0'; + white.style.width = Math.max(document.documentElement.clientWidth, document.body.clientWidth) + 'px'; + white.style.height = Math.max(document.documentElement.clientHeight,document.body.clientHeight) + 'px'; + white.style.backgroundColor = 'white'; + white.style.zIndex = '2147483647'; + document.body.appendChild(white); + document.body.style.display = ''; + })();`); + await commands.wait.byTime(1000); + } else { + context.log.info("Navigating to about:blank, count: " + count); + await commands.navigate("about:blank"); + } + + context.log.info("Navigating to primary url:" + test_url); + context.log.info("Cycle %d, waiting for %d ms", count, page_cycle_delay); + await commands.wait.byTime(page_cycle_delay); + + context.log.info("Cycle %d, starting the measure", count); + await commands.measure.start(test_url); + + // Wait 20 seconds to populate bytecode cache + if (test_bytecode_cache == "true" && chimera_mode == "true" && !cached) { + context.log.info("Waiting 20s to populate bytecode cache..."); + await commands.wait.byTime(20000); + cached = true; + } + } + + context.log.info("Browsertime pageload ended."); + return true; +}; diff --git a/testing/raptor/browsertime/browsertime_scenario.js b/testing/raptor/browsertime/browsertime_scenario.js new file mode 100644 index 0000000000..3abf0d142b --- /dev/null +++ b/testing/raptor/browsertime/browsertime_scenario.js @@ -0,0 +1,67 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* eslint-env node */ + +module.exports = async function (context, commands) { + context.log.info("Starting a browsertime scenario test"); + let page_cycles = context.options.browsertime.page_cycles; + let page_cycle_delay = context.options.browsertime.page_cycle_delay; + let post_startup_delay = context.options.browsertime.post_startup_delay; + let scenario_time = context.options.browsertime.scenario_time; + let background_app = context.options.browsertime.background_app == "true"; + let app = null; + + if (background_app) { + if (!context.options.android) { + throw new Error("Cannot background an application on desktop"); + } + app = context.options.firefox.android.package; + } + + context.log.info( + "Waiting for %d ms (post_startup_delay)", + post_startup_delay + ); + await commands.wait.byTime(post_startup_delay); + + for (let count = 0; count < page_cycles; count++) { + context.log.info("Navigating to about:blank"); + await commands.navigate("about:blank"); + + context.log.info("Cycle %d, waiting for %d ms", count, page_cycle_delay); + await commands.wait.byTime(page_cycle_delay); + + context.log.info("Cycle %d, starting the measure", count); + + if (background_app) { + // Background the application and disable doze for it + await commands.android.shell(`dumpsys deviceidle whitelist +${app}`); + await commands.android.shell(`input keyevent 3`); + await commands.wait.byTime(1000); + const foreground = await commands.android.shell( + "dumpsys window windows | grep mCurrentFocus" + ); + if (foreground.includes(app)) { + throw new Error("Application was not backgrounded successfully"); + } else { + context.log.info("Application was backgrounded successfully"); + } + } + + // Run the test + await commands.measure.start("about:blank test"); + context.log.info("Waiting for %d ms for this scenario", scenario_time); + await commands.wait.byTime(scenario_time); + await commands.measure.stop(); + + if (background_app) { + // Foreground the application and enable doze again + await commands.android.shell(`am start --activity-single-top ${app}`); + await commands.android.shell(`dumpsys deviceidle enable`); + } + } + context.log.info("Browsertime scenario test ended."); + return true; +}; diff --git a/testing/raptor/browsertime/browsertime_tp6_bench.js b/testing/raptor/browsertime/browsertime_tp6_bench.js new file mode 100644 index 0000000000..670ee7bcf3 --- /dev/null +++ b/testing/raptor/browsertime/browsertime_tp6_bench.js @@ -0,0 +1,29 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* eslint-env node */ + +module.exports = async function (context, commands) { + let urlstr = context.options.browsertime.url; + const parsedUrls = urlstr.split(","); + + let startTime = await commands.js.run( + `return performance.timeOrigin + performance.now();` + ); + for (let count = 0; count < parsedUrls.length; count++) { + context.log.info("Navigating to url:" + parsedUrls[count]); + context.log.info("Cycle %d, starting the measure", count); + await commands.measure.start(parsedUrls[count]); + } + + let endTime = await commands.js.run( + `return performance.timeOrigin + performance.now();` + ); + + context.log.info("Browsertime pageload benchmark ended."); + await commands.measure.add("pageload-benchmark", { + totalTime: endTime - startTime, + }); + return true; +}; diff --git a/testing/raptor/browsertime/constant_regression_test.js b/testing/raptor/browsertime/constant_regression_test.js new file mode 100644 index 0000000000..5373936208 --- /dev/null +++ b/testing/raptor/browsertime/constant_regression_test.js @@ -0,0 +1,14 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* eslint-env node */ + +module.exports = async function (context, commands) { + context.log.info("Starting constant value regression test"); + await commands.measure.start("regression-test"); + await commands.measure.stop(); + await commands.measure.addObject({ + data: { metric: context.options.browsertime.constant_value }, + }); +}; diff --git a/testing/raptor/browsertime/grandprix.js b/testing/raptor/browsertime/grandprix.js new file mode 100644 index 0000000000..530eb203e3 --- /dev/null +++ b/testing/raptor/browsertime/grandprix.js @@ -0,0 +1,80 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* eslint-env node */ + +module.exports = async function (context, commands) { + const benchmark_url = "https://grandprixbench.netlify.app/"; + const iterations = `${context.options.browsertime.grandprix_iterations}`; + const tests = [ + "Vanilla-HTML-TodoMVC", + "Vanilla-TodoMVC", + "Preact-TodoMVC", + "Preact-TodoMVC-Modern", + "React-TodoMVC", + //"Elm-TodoMVC", // removed from suite + //"Rust-Yew-TodoMVC", // removed from suite + //"Hydration-Preact", // removed from suite + //"Scroll-Windowing-React", // removed from suite + "Monaco-Editor", + "Monaco-Syntax-Highlight", + "React-Stockcharts", + "React-Stockcharts-SVG", + "Leaflet-Fractal", + //"SVG-UI", // removed from suite + "Proxx-Tables", + "Proxx-Tables-Lit", + "Proxx-Tables-Canvas", + ]; + + // Build test url + let url = benchmark_url + "?suites="; + tests.forEach(test => { + url = url + test + ","; + }); + url = url + "&iterations=" + iterations; + + console.log("Using url=" + url); + await commands.measure.start(url); + + await commands.wait.byTime("3000"); + await commands.click.byXpath("/html/body/main/div/div[1]/div/div/button"); + + let finished = 0; + do { + await commands.wait.byTime("30000"); + finished = await commands.js.run('return typeof results != "undefined"'); + } while (!finished); + + if (context.options.browser === "firefox") { + let buildId = await commands.js.runPrivileged( + "return Services.appinfo.appBuildID;" + ); + console.log(buildId); + } + + let output = {}; + output.score = {}; + output.score.total = await commands.js.run("return results.Score"); + output.iterations = await commands.js.run("return results.iterations"); + + let subtests = {}; + for (let i = 0; i < output.iterations.length; i++) { + for (const [key, value] of Object.entries(output.iterations[i])) { + if (!subtests.hasOwnProperty(key)) { + subtests[key] = []; + } + subtests[key].push(value.total); + } + } + + console.log("score, ", output.score.total.mean); + for (const [key, value] of Object.entries(subtests)) { + const average = value.reduce((a, b) => a + b, 0) / value.length; + output.score[key] = average; + console.log(key, ",", average); + } + + await commands.measure.add("grandprix-s3", output); +}; diff --git a/testing/raptor/browsertime/indexeddb_getkeyrange.js b/testing/raptor/browsertime/indexeddb_getkeyrange.js new file mode 100644 index 0000000000..100ac20e2a --- /dev/null +++ b/testing/raptor/browsertime/indexeddb_getkeyrange.js @@ -0,0 +1,69 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* eslint-env node */ + +module.exports = async function (context, commands) { + context.log.info("Starting a indexedDB getkeyrange"); + + const test_url = context.options.browsertime.url; + let page_cycles = context.options.browsertime.page_cycles; + let page_cycle_delay = context.options.browsertime.page_cycle_delay; + let post_startup_delay = context.options.browsertime.post_startup_delay; + + context.log.info( + "Waiting for %d ms (post_startup_delay)", + post_startup_delay + ); + await commands.wait.byTime(post_startup_delay); + + await commands.navigate(test_url); + + for (let count = 0; count < page_cycles; count++) { + context.log.info("Cycle %d, waiting for %d ms", count, page_cycle_delay); + await commands.wait.byTime(page_cycle_delay); + + context.log.info("Cycle %d, starting the measure", count); + await await commands.measure.start(); + await commands.js.run(` + const notifyDone = arguments[arguments.length - 1]; + async function resPromise() { + return new Promise((resolve, reject) => { + const results = {}; + const request = indexedDB.open('get-keyrange', 1); + + request.onsuccess = () => { + const db = request.result; + const start = Date.now(); + const keyRange = IDBKeyRange.bound(0, 99); + const transaction = db.transaction('entries', 'readonly'); + const store = transaction.objectStore('entries'); + const index = store.index('index'); + const getAllRequest = index.getAll(keyRange); + getAllRequest.onsuccess = () => { + const items = getAllRequest.result; + items.forEach((item) => { + results[item.key] = item; + }); + const end = Date.now(); + db.close(); + resolve(end - start); + }; + getAllRequest.onerror = () => { + reject(getAllRequest.error); + }; + }; + + }); + } + resPromise().then(() => { + notifyDone(); + }); + `); + await commands.measure.stop(); + } + + context.log.info("IndexedDB getkeyrange ended."); + return true; +}; diff --git a/testing/raptor/browsertime/indexeddb_write.js b/testing/raptor/browsertime/indexeddb_write.js new file mode 100644 index 0000000000..30c5c84f8e --- /dev/null +++ b/testing/raptor/browsertime/indexeddb_write.js @@ -0,0 +1,153 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* eslint-env node */ + +module.exports = async function (context, commands) { + context.log.info("Starting a indexedDB write"); + const post_startup_delay = context.options.browsertime.post_startup_delay; + console.log("context options", context.options); + + const test_url = context.options.browsertime.url; + const chunk_size = context.options.browsertime.chunk_size; + const iterations = context.options.browsertime.iterations; + const buffer_type = context.options.browsertime.buffer_type; + const atomic_value = context.options.browsertime.atomic; + if (atomic_value * atomic_value != atomic_value) { + throw Error("Value of atomic shall be 0 for falsehood, 1 for truth."); + } + const atomic = 0 != atomic_value; + + const accepted_buffers = ["Array", "ArrayBuffer", "Blob"]; + if (!accepted_buffers.includes(buffer_type)) { + throw Error("Buffer type " + buffer_type + " is unknown."); + } + + context.log.info("IndexedDB write URL = " + test_url); + context.log.info("IndexedDB write chunk size = " + chunk_size); + context.log.info("IndexedDB write iterations = " + iterations); + context.log.info( + "IndexedDB writes " + + (atomic ? "all in one big transaction" : "in separate transactions") + ); + context.log.info("IndexedDB write data format " + buffer_type); + + context.log.info( + "Waiting for %d ms (post_startup_delay)", + post_startup_delay + ); + + await commands.navigate(test_url); + + const seleniumDriver = context.selenium.driver; + + await commands.wait.byTime(post_startup_delay); + + await commands.measure.start(); + const time_duration = await seleniumDriver.executeAsyncScript(` + const notifyDone = arguments[arguments.length - 1]; + + const iterations = ${iterations}; + const sizeInBytes = ${chunk_size}; + const bufferType = "${buffer_type}"; + const atomic = ${atomic}; + + const makeData = (() => { + if (bufferType === "ArrayBuffer") { + return () => { + const valueBuffer = new ArrayBuffer(sizeInBytes); + + const charCodeView = new Uint16Array(valueBuffer); + const sizeInUint16 = sizeInBytes / 2; + for (let i=0; i < sizeInUint16; ++i) { + charCodeView[i] = "qwerty".charCodeAt(i % 6); + } + + return valueBuffer; + }; + } + + if (bufferType === "Array") { + return () => { + return Array.from({length: sizeInBytes}, (_, i) => "qwerty"[i % 6]); + }; + } + + if (bufferType !== "Blob") { + throw Error("Unknown buffer type " + bufferType); + } + + return () => { + return new Blob([Array.from({length: sizeInBytes}, (_, i) => "qwerty"[i % 6])]); + }; + })(); + + function addData(txSource, txProvider, i) { + try { + const keyName = "doc_" + i; + const valueData = makeData(); + const record = { key: keyName, property: valueData }; + + const rq = txProvider(txSource).add(record); + + return new Promise((res_ad, rej_ad) => { + rq.onsuccess = () => { res_ad(); }; + rq.onerror = e => { rej_ad(e); }; + }); + } catch (e) { + return new Promise((_, rej_ad) => rej_ad(e)); + } + } + + function waitForData(txSource) { + try { + if (!atomic) { + const txProvider = src => src.transaction("store", "readwrite").objectStore("store"); + return Promise.all( + Array.from({ length: iterations }, (_, i) => { + return addData(txSource, txProvider, i); + })); + } + + const currentTx = txSource.transaction("store", "readwrite").objectStore("store"); + return Promise.all( + Array.from({ length: iterations }, (_, i) => { + return addData(currentTx, src => src, i); + })); + } catch (e) { + return new Promise((_, rej_tx) => rej_tx(e)); + } + } + + function upgradePromise() { + try { + const open_db = indexedDB.open("rootsdb"); + return new Promise((res_upgrade, rej_upgrade) => { + open_db.onupgradeneeded = e => { + e.target.result.createObjectStore("store", { keyPath: "key" }); + }; + open_db.onsuccess = e => { res_upgrade(e.target.result); }; + open_db.onerror = e => { rej_upgrade(e); }; + }); + } catch (e) { + return new Promise((_, rej_upgrade) => rej_upgrade(e)); + } + } + + const startTime = performance.now(); + upgradePromise().then(waitForData).then(() => { + notifyDone(performance.now() - startTime); + }); + `); + await commands.measure.stop(); + + console.log("Time duration was ", time_duration); + + await commands.measure.addObject({ + custom_data: { time_duration }, + }); + + context.log.info("IndexedDB write ended."); + return true; +}; diff --git a/testing/raptor/browsertime/pageload_sites.json b/testing/raptor/browsertime/pageload_sites.json new file mode 100644 index 0000000000..475470d459 --- /dev/null +++ b/testing/raptor/browsertime/pageload_sites.json @@ -0,0 +1,611 @@ +{ + "mobile": [ + { + "login": false, + "name": "allrecipes", + "test_url": "https://www.allrecipes.com/" + }, + { + "login": false, + "name": "amazon", + "test_url": "https://www.amazon.com" + }, + { + "login": false, + "name": "amazon-search", + "test_url": "https://www.amazon.com/s/ref=nb_sb_noss_2/139-6317191-5622045?url=search-alias%3Daps&field-keywords=mobile+phone" + }, + { + "login": false, + "name": "bing", + "test_url": "https://www.bing.com/" + }, + { + "login": false, + "name": "bing-search-restaurants", + "test_url": "https://www.bing.com/search?q=restaurants+in+exton+pa+19341" + }, + { + "login": false, + "name": "booking", + "test_url": "https://www.booking.com/" + }, + { + "login": false, + "name": "cnn", + "test_url": "https://cnn.com", + "secondary_url": "https://www.cnn.com/weather" + }, + { + "login": false, + "name": "cnn-ampstories", + "test_url": "https://cnn.com/ampstories/us/why-hurricane-michael-is-a-monster-unlike-any-other" + }, + { + "login": false, + "name": "dailymail", + "test_url": "https://www.dailymail.co.uk/sciencetech/article-9749081/Experts-say-Hubble-repair-despite-NASA-insisting-multiple-options-fix.html" + }, + { + "login": false, + "name": "ebay-kleinanzeigen", + "test_url": "https://m.ebay-kleinanzeigen.de" + }, + { + "login": false, + "name": "ebay-kleinanzeigen-search", + "test_url": "https://m.ebay-kleinanzeigen.de/s-anzeigen/auf-zeit-wg-berlin/zimmer/c199-l3331" + }, + { + "login": false, + "name": "espn", + "test_url": "http://www.espn.com/nba/story/_/page/allstarweekend25788027/the-comparison-lebron-james-michael-jordan-their-own-words" + }, + { + "login": true, + "name": "facebook", + "test_url": "https://m.facebook.com" + }, + { + "login": false, + "name": "facebook-cristiano", + "test_url": "https://m.facebook.com/Cristiano" + }, + { + "login": true, + "name": "google", + "test_url": "https://www.google.com" + }, + { + "login": false, + "name": "google-maps", + "test_url": "https://www.google.com/maps?force=pwa" + }, + { + "login": true, + "name": "google-search-restaurants", + "test_url": "https://www.google.com/search?q=restaurants+near+me" + }, + { + "login": false, + "name": "imdb", + "test_url": "https://m.imdb.com/" + }, + { + "login": true, + "name": "instagram", + "test_url": "https://www.instagram.com" + }, + { + "login": false, + "name": "sina", + "test_url": "https://www.sina.com.cn/" + }, + { + "login": false, + "name": "microsoft-support", + "test_url": "https://support.microsoft.com/en-us" + }, + { + "login": false, + "name": "reddit", + "test_url": "https://www.reddit.com" + }, + { + "login": false, + "name": "stackoverflow", + "test_url": "https://stackoverflow.com/" + }, + { + "login": false, + "name": "web-de", + "test_url": "https://web.de/magazine/politik/politologe-glaubt-grossen-koalition-herbst-knallen-33563566" + }, + { + "login": false, + "name": "wikipedia", + "test_url": "https://en.m.wikipedia.org/wiki/Main_Page" + }, + { + "login": false, + "name": "youtube", + "test_url": "https://m.youtube.com" + }, + { + "login": false, + "name": "youtube-watch", + "test_url": "https://www.youtube.com/watch?v=COU5T-Wafa4" + } + ], + "desktop": [ + { + "login": false, + "name": "amazon", + "test_url": "https://www.amazon.com/s?k=laptop&ref=nb_sb_noss_1", + "secondary_url": "https://www.amazon.com/Acer-A515-46-R14K-Quad-Core-Processor-Backlit/dp/B08VKNVDDR/ref=sr_1_3?dchild=1&keywords=laptop&qid=1627047187&sr=8-3" + }, + { + "login": false, + "name": "bing-search", + "test_url": "https://www.bing.com/search?q=barack+obama" + }, + { + "login": false, + "name": "buzzfeed", + "test_url": "https://www.buzzfeed.com/", + "secondary_url": "https://www.buzzfeed.com/quizzes", + "test_cmds": [ + [ + "click.byXpathAndWait", + "/html/body/div[1]/div/div/div/div[2]/div/button[2]" + ], + [ + "click.byXpathAndWait", + "/html/body/div[1]/div/div/div/div[3]/div[2]/button" + ] + ] + }, + { + "login": false, + "name": "cnn", + "test_url": "https://www.cnn.com/2021/03/22/weather/climate-change-warm-waters-lake-michigan/index.html", + "secondary_url": "https://www.cnn.com/weather", + "preferences": { + "media.autoplay.default": 5, + "media.autoplay.ask-permission": true, + "media.autoplay.blocking_policy": 1, + "media.allowed-to-play.enabled": false, + "media.block-autoplay-until-in-foreground": true + }, + "test_cmds": [ + ["click.byXpathAndWait", "//*[@id='onetrust-pc-btn-handler']"], + [ + "click.byXpathAndWait", + "/html/body/div[13]/div[2]/div/section/div[17]/button[2]" + ] + ] + }, + { + "login": false, + "name": "ebay", + "test_url": "https://www.ebay.com/", + "secondary_url": "https://www.ebay.com/deals", + "test_cmds": [ + [ + "click.byXpathAndWait", + "/html/body/div[5]/div[1]/div[2]/div[2]/div[2]/a" + ], + [ + "click.byXpathAndWait", + "/html/body/div[4]/div/div[2]/section[3]/section/button" + ] + ] + }, + { + "login": false, + "name": "espn", + "test_url": "http://www.espn.com/nba/story/_/page/allstarweekend25788027/the-comparison-lebron-james-michael-jordan-their-own-words", + "test_cmds": [ + ["click.byXpathAndWait", "//*[@id='onetrust-pc-btn-handler']"], + [ + "click.byXpathAndWait", + "/html/body/div[10]/div[2]/div[3]/div[1]/button" + ] + ] + }, + { + "login": false, + "name": "expedia", + "test_url": "https://expedia.com/Hotel-Search?destination=New+York%2C+New+York&latLong=40.756680%2C-73.986470®ionId=178293&startDate=&endDate=&rooms=1&_xpid=11905%7C1&adults=2", + "secondary_url": "https://groups.expedia.com/Group-Rate/?locale=en_US&ol=1" + }, + { + "login": true, + "name": "facebook", + "test_url": "https://www.facebook.com", + "secondary_url": "https://www.facebook.com/marketplace/?ref=bookmark", + "test_cmds": [ + [ + "click.byXpathAndWait", + "/html/body/div[3]/div[2]/div/div/div/div/div[3]/button[1]" + ], + [ + "click.byXpathAndWait", + "/html/body/div[3]/div[2]/div/div/div/div/div[3]/button[1]" + ] + ] + }, + { + "login": true, + "login-test": true, + "name": "facebook-login", + "test_url": "https://www.facebook.com", + "type": "interactive", + "test_cmds": [ + ["setup_login", "https://www.facebook.com"], + ["wait.byTime", 1000], + ["login", ""], + ["measure.start", "marketplace"], + ["navigate", "https://www.facebook.com/marketplace"], + ["measure.stop", ""] + ] + }, + { + "login": false, + "name": "fandom", + "test_url": "https://www.fandom.com/articles/fallout-76-will-live-and-die-on-the-creativity-of-its-playerbase", + "test_cmds": [ + ["click.byXpathAndWait", "/html/body/div[9]/div/div/div[2]/div[1]"], + ["click.byXpathAndWait", "/html/body/div[9]/div/div/div[2]/div[2]"] + ] + }, + { + "login": true, + "name": "google", + "test_url": "https://www.google.com/search?hl=en&q=barack+obama&cad=h", + "test_cmds": [ + [ + "click.byXpathAndWait", + "/html/body/div[3]/div[3]/span/div/div/div/div[3]/button[1]/div" + ], + [ + "click.byXpathAndWait", + "/html/body/c-wiz/div/div/div/div[2]/form/div/button/div[2]" + ] + ] + }, + { + "login": false, + "name": "google-docs", + "test_url": "https://docs.google.com/document/d/1US-07msg12slQtI_xchzYxcKlTs6Fp7WqIc6W5GK5M8/edit?usp=sharing", + "secondary_url": "https://docs.google.com/document/d/1vUnn0ePU-ynArE1OdxyEHXR2G0sl74ja_st_4OOzlgE/preview" + }, + { + "login": true, + "name": "google-mail", + "test_url": "https://mail.google.com/", + "test_cmds": [["click.byXpathAndWait", "/html/body/div/div/span[2]/a[2]"]] + }, + { + "login": false, + "name": "google-slides", + "test_url": "https://docs.google.com/presentation/d/1Ici0ceWwpFvmIb3EmKeWSq_vAQdmmdFcWqaiLqUkJng/edit?usp=sharing", + "secondary_url": "https://docs.google.com/document/d/1vUnn0ePU-ynArE1OdxyEHXR2G0sl74ja_st_4OOzlgE/preview" + }, + { + "login": false, + "name": "imdb", + "test_url": "https://www.imdb.com/title/tt0084967/?ref_=nv_sr_2", + "secondary_url": "https://www.imdb.com/title/tt0084967/episodes/?ref_=tt_ov_epl" + }, + { + "login": false, + "name": "imgur", + "test_url": "https://imgur.com/gallery/m5tYJL6", + "inject_deterministic": false, + "test_cmds": [ + [ + "click.byXpathAndWait", + "/html/body/div[1]/div/div/div/div[2]/div/button[2]" + ], + [ + "click.byXpathAndWait", + "/html/body/div[1]/div/div/div/div[3]/div[2]/button" + ] + ], + "secondary_url": "https://imgur.com/gallery/rCXZUil", + "preferences": { + "media.autoplay.default": 5, + "media.autoplay.ask-permission": true, + "media.autoplay.blocking_policy": 1, + "media.allowed-to-play.enabled": false, + "media.block-autoplay-until-in-foreground": true + } + }, + { + "login": true, + "name": "instagram", + "test_url": "https://www.instagram.com/" + }, + { + "login": true, + "name": "linkedin", + "test_url": "https://www.linkedin.com/in/thommy-harris-hk-385723106/", + "secondary_url": "https://www.linkedin.com/company/github?trk=affiliated-pages" + }, + { + "login": false, + "name": "microsoft", + "test_url": "https://www.microsoft.com/en-us/", + "secondary_url": "https://support.microsoft.com/en-us", + "test_cmds": [ + [ + "click.byXpathAndWait", + "/html/body/div[1]/div/div/div[2]/div/div/div/div[2]/button[2]" + ], + [ + "click.byXpath", + "/html/body/div[1]/div/div/div[2]/div/div/div[2]/div[2]/div/form/dl/dt[2]/div/div/div[2]/label" + ], + [ + "click.byXpath", + "/html/body/div[1]/div/div/div[2]/div/div/div[2]/div[2]/div/form/dl/dt[3]/div/div/div[2]/label" + ], + [ + "click.byXpath", + "/html/body/div[1]/div/div/div[2]/div/div/div[2]/div[2]/div/form/dl/dt[4]/div/div/div[2]/label" + ], + [ + "click.byXpathAndWait", + "/html/body/div[1]/div/div/div[2]/div/div/div[2]/div[2]/div/div[2]/button[1]" + ] + ] + }, + { + "login": true, + "name": "netflix", + "test_url": "https://www.netflix.com/title/80117263", + "secondary_url": "https://www.netflix.com/title/699257" + }, + { + "login": false, + "name": "nytimes", + "test_url": "https://www.nytimes.com/2020/02/19/opinion/surprise-medical-bill.html", + "secondary_url": "https://www.nytimes.com/section/opinion/columnists", + "test_cmds": [ + [ + "click.byXpathAndWait", + "/html/body/div/div/div[2]/div/div[2]/div/div/div[1]/div[3]/div[3]/a" + ], + [ + "click.byXpathAndWait", + "//*[@id='opt-out-of-new-york-times-nonessential-trackers']" + ] + ] + }, + { + "login": true, + "name": "office", + "test_url": "https://www.office.com/launch/powerpoint/", + "secondary_url": "https://www.office.com/" + }, + { + "login": true, + "name": "outlook", + "test_url": "https://outlook.live.com/mail/inbox" + }, + { + "login": true, + "name": "paypal", + "test_url": "https://www.paypal.com/myaccount/summary/" + }, + { + "login": true, + "name": "pinterest", + "test_url": "https://pinterest.com/", + "secondary_url": "https://www.pinterest.com/today/best/halloween-costumes-for-your-furry-friends/75787/" + }, + { + "login": false, + "name": "reddit", + "test_url": "https://www.reddit.com/r/technology/comments/9sqwyh/we_posed_as_100_senators_to_run_ads_on_facebook/", + "secondary_url": "https://www.reddit.com/r/technology/", + "test_cmds": [ + [ + "click.byXpathAndWait", + "/html/body/div[1]/div/div[2]/div[3]/div[1]/section/div/section/section/form[1]/button" + ] + ] + }, + { + "login": true, + "name": "tumblr", + "test_url": "https://www.tumblr.com/dashboard", + "secondary_url": "https://www.tumblr.com/tagged/funny+cats?sort=top" + }, + { + "login": false, + "name": "twitch", + "test_url": "https://www.twitch.tv/videos/894226211", + "secondary_url": "https://www.twitch.tv/gmashley", + "preferences": { + "media.autoplay.default": 5, + "media.autoplay.ask-permission": true, + "media.autoplay.blocking_policy": 1, + "media.allowed-to-play.enabled": false, + "media.block-autoplay-until-in-foreground": true + }, + "test_cmds": [ + [ + "click.byXpathAndWait", + "/html/body/div[1]/div/div[2]/div[1]/div/div/div[3]/div/button/div/div/div" + ], + [ + "click.byXpathAndWait", + "/html/body/div[3]/div/div/div/div/div/div[1]/div[2]/div/button/div/div" + ] + ] + }, + { + "login": true, + "name": "twitter", + "test_url": "https://twitter.com/BarackObama" + }, + { + "login": false, + "name": "wikia", + "test_url": "https://marvel.fandom.com/wiki/Black_Panther", + "secondary_url": "https://marvel.fandom.com/wiki/Celestials", + "test_cmds": [ + ["click.byXpathAndWait", "/html/body/div[6]/div/div/div[2]/div[1]"], + ["click.byXpathAndWait", "/html/body/div[7]/div/div/div[2]/div[2]"] + ] + }, + { + "login": false, + "name": "wikipedia", + "test_url": "https://en.wikipedia.org/wiki/Barack_Obama", + "secondary_url": "https://en.wikipedia.org/wiki/Joe_Biden" + }, + { + "login": true, + "name": "yahoo-mail", + "test_url": "https://mail.yahoo.com/" + }, + { + "login": false, + "name": "youtube", + "test_url": "https://www.youtube.com", + "secondary_url": "https://www.youtube.com/watch?v=JrdEMERq8MA", + "test_cmds ": [ + [ + "click.byXpathAndWait", + "/html/body/ytd-app/ytd-consent-bump-v2-lightbox/tp-yt-paper-dialog/div[4]/div[2]/div[5]/div[2]/ytd-button-renderer[1]/a" + ], + [ + "click.byXpath", + "/html/body/c-wiz/div/div/div/div[2]/div[2]/div[2]/div/div[2]/div[1]/div/button" + ], + [ + "click.byXpath", + "/html/body/c-wiz/div/div/div/div[2]/div[3]/div[2]/div/div[2]/div[1]/div/button" + ], + [ + "click.byXpath", + "/html/body/c-wiz/div/div/div/div[2]/div[4]/div[2]/div[2]/div/div[2]/div[1]/div/button" + ], + [ + "click.byXpathAndWait", + "/html/body/c-wiz/div/div/div/div[2]/form/div/button/div[2]" + ] + ] + }, + { + "login": false, + "name": "cnn-nav", + "test_url": "https://www.cnn.com/", + "type": "interactive", + "test_cmds": [ + ["measure.start", "landing"], + ["navigate", "https://www.cnn.com"], + ["wait.byTime", 4000], + ["measure.stop", ""], + ["measure.start", "world"], + [ + "click.byXpathAndWait", + "/html/body/div[5]/div/div/header/div/div[1]/div/div[2]/nav/ul/li[2]/a" + ], + ["wait.byTime", 1000], + ["measure.stop", ""] + ] + }, + { + "login": false, + "name": "reddit-billgates-ama", + "test_url": "https://www.reddit.com/", + "type": "interactive", + "test_cmds": [ + ["measure.start", "billg-ama"], + [ + "navigate", + "https://www.reddit.com/r/IAmA/comments/m8n4vt/im_bill_gates_cochair_of_the_bill_and_melinda/" + ], + ["wait.byTime", 5000], + ["measure.stop", ""], + ["measure.start", "members"], + [ + "click.byXpathAndWait", + "/html/body/div[1]/div/div[2]/div[2]/div/div[3]/div[2]/div/div[1]/div/div[4]/div[1]/div" + ], + ["wait.byTime", 1000], + ["measure.stop", ""] + ] + }, + { + "login": false, + "name": "reddit-billgates-post", + "test_url": "https://www.reddit.com/user/thisisbillgates/", + "type": "interactive", + "test_cmds": [ + ["measure.start", "billg"], + ["navigate", "https://www.reddit.com/user/thisisbillgates/"], + ["wait.byTime", 5000], + ["measure.stop", ""], + ["measure.start", "posts"], + [ + "click.byXpathAndWait", + "/html/body/div[1]/div/div[2]/div[2]/div/div/div/div[2]/div[2]/div/div/div/a[2]" + ], + ["wait.byTime", 15000], + ["measure.stop", ""], + ["measure.start", "comments"], + [ + "click.byXpathAndWait", + "/html/body/div[1]/div/div[2]/div[2]/div/div/div/div[2]/div[2]/div/div/div/a[3]" + ], + ["wait.byTime", 15000], + ["measure.stop", ""], + ["measure.start", "hot"], + [ + "click.byXpathAndWait", + "/html/body/div[1]/div/div[2]/div[2]/div/div/div/div[2]/div[4]/div[1]/div[1]/div[2]/a[2]" + ], + ["wait.byTime", 15000], + ["measure.stop", ""], + ["measure.start", "top"], + [ + "click.byXpathAndWait", + "/html/body/div[1]/div/div[2]/div[2]/div/div/div/div[2]/div[4]/div[1]/div[1]/div[2]/a[3]" + ], + ["wait.byTime", 15000], + ["measure.stop", ""] + ] + }, + { + "login": false, + "name": "facebook-nav", + "test_url": "https://www.facebook.com/", + "type": "interactive", + "test_cmds": [ + ["navigate", "https://www.facebook.com/login"], + ["wait.byTime", 30000], + ["measure.start", "landing"], + ["navigate", "https://www.facebook.com/"], + ["wait.byTime", 5000], + ["measure.stop", ""], + ["measure.start", "marketplace"], + ["navigate", "https://www.facebook.com/marketplace"], + ["wait.byTime", 5000], + ["measure.stop", ""], + ["measure.start", "groups"], + ["navigate", "https://www.facebook.com/groups/discover/"], + ["wait.byTime", 5000], + ["measure.stop", ""], + ["measure.start", "friends"], + ["navigate", "https://www.facebook.com/friends/"], + ["wait.byTime", 5000], + ["measure.stop", ""] + ] + } + ] +} diff --git a/testing/raptor/browsertime/process_switch.js b/testing/raptor/browsertime/process_switch.js new file mode 100644 index 0000000000..a13094c6c5 --- /dev/null +++ b/testing/raptor/browsertime/process_switch.js @@ -0,0 +1,45 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* eslint-env node */ + +module.exports = async function (context, commands) { + context.log.info("Starting a process switch test"); + let urlstr = context.options.browsertime.url; + let page_cycles = context.options.browsertime.page_cycles; + let page_cycle_delay = context.options.browsertime.page_cycle_delay; + let post_startup_delay = context.options.browsertime.post_startup_delay; + + // Get the two urls to use in the test (the second one will be measured) + let urls = urlstr.split(","); + if (urls.length != 2) { + context.log.error( + `Wrong number of urls given. Expecting: 2, Given: ${urls.length}` + ); + return false; + } + + context.log.info( + "Waiting for %d ms (post_startup_delay)", + post_startup_delay + ); + await commands.wait.byTime(post_startup_delay); + + for (let count = 0; count < page_cycles; count++) { + context.log.info("Navigating to about:blank"); + await commands.navigate("about:blank"); + + context.log.info("Cycle %d, waiting for %d ms", count, page_cycle_delay); + await commands.wait.byTime(page_cycle_delay); + context.log.info("Cycle %d, starting the measure", count); + + await commands.navigate(urls[0]); + await commands.wait.byTime(3000); + await commands.measure.start(urls[1]); + await commands.wait.byTime(2000); + } + + context.log.info("Process switch test ended."); + return true; +}; diff --git a/testing/raptor/browsertime/speculative-connect.js b/testing/raptor/browsertime/speculative-connect.js new file mode 100644 index 0000000000..e984b21d92 --- /dev/null +++ b/testing/raptor/browsertime/speculative-connect.js @@ -0,0 +1,44 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* eslint-env node */ + +module.exports = async function (context, commands) { + context.log.info( + "Starting a pageload for which we will first make a speculative connection to the host" + ); + + const url = "https://en.wikipedia.org/wiki/Barack_Obama"; + + await commands.navigate("about:blank"); + await commands.wait.byTime(1000); + + context.log.debug("Make privileged call to speculativeConnect"); + const script = ` + var URI = Services.io.newURI("${url}"); + var principal = Services.scriptSecurityManager.createContentPrincipal(URI, {}); + Services.io.QueryInterface(Ci.nsISpeculativeConnect).speculativeConnect(URI, principal, null, false); + `; + + commands.js.runPrivileged(script); + + // More than enough time for the connection to be made + await commands.wait.byTime(1000); + + await commands.measure.start(); + await commands.navigate(url); + await commands.measure.stop(); + + let connect_time = await commands.js.run( + `return (window.performance.timing.connectEnd - window.performance.timing.navigationStart);` + ); + context.log.info("connect_time: " + connect_time); + + await commands.measure.addObject({ + custom_data: { connect_time }, + }); + + context.log.info("Speculative connect test finished."); + return true; +}; diff --git a/testing/raptor/browsertime/speedometer3.js b/testing/raptor/browsertime/speedometer3.js new file mode 100644 index 0000000000..d588fd4418 --- /dev/null +++ b/testing/raptor/browsertime/speedometer3.js @@ -0,0 +1,93 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* eslint-env node */ + +module.exports = async function (context, commands) { + context.log.info("Starting Speedometer 3 test"); + let url = context.options.browsertime.url; + let page_cycles = context.options.browsertime.page_cycles; + let page_cycle_delay = context.options.browsertime.page_cycle_delay; + let post_startup_delay = context.options.browsertime.post_startup_delay; + let page_timeout = context.options.timeouts.pageLoad; + let expose_profiler = context.options.browsertime.expose_profiler; + + context.log.info( + "Waiting for %d ms (post_startup_delay)", + post_startup_delay + ); + await commands.wait.byTime(post_startup_delay); + + for (let count = 0; count < page_cycles; count++) { + context.log.info("Navigating to about:blank"); + await commands.navigate("about:blank"); + + context.log.info("Cycle %d, waiting for %d ms", count, page_cycle_delay); + await commands.wait.byTime(page_cycle_delay); + + context.log.info("Cycle %d, starting the measure", count); + if (expose_profiler === "true") { + context.log.info("Custom profiler start!"); + if (context.options.browser === "firefox") { + await commands.profiler.start(); + } else if (context.options.browser === "chrome") { + await commands.trace.start(); + } + } + await commands.measure.start(url); + + await commands.js.runAndWait(` + this.benchmarkClient.start() + `); + + let data_exists = false; + let starttime = await commands.js.run(`return performance.now();`); + while ( + !data_exists && + (await commands.js.run(`return performance.now();`)) - starttime < + page_timeout + ) { + let wait_time = 3000; + context.log.info("Waiting %d ms for data from speedometer...", wait_time); + await commands.wait.byTime(wait_time); + data_exists = await commands.js.run( + "return !(this.benchmarkClient._isRunning)" + ); + } + if (expose_profiler === "true") { + context.log.info("Custom profiler stop!"); + if (context.options.browser === "firefox") { + await commands.profiler.stop(); + } else if (context.options.browser === "chrome") { + await commands.trace.stop(); + } + } + if ( + !data_exists && + (await commands.js.run(`return performance.now();`)) - starttime >= + page_timeout + ) { + context.log.error("Benchmark timed out. Aborting..."); + return false; + } + let internal_data = await commands.js.run( + `return this.benchmarkClient._measuredValuesList;` + ); + context.log.info("Value of internal benchmark iterations: ", internal_data); + + let data = await commands.js.run(` + const values = this.benchmarkClient._computeResults(this.benchmarkClient._measuredValuesList, "ms"); + const score = this.benchmarkClient._computeResults(this.benchmarkClient._measuredValuesList, "score"); + return { + score, + values: values.formattedMean, + }; + `); + context.log.info("Value of summarized benchmark data: ", data); + + commands.measure.addObject({ s3: data, s3_internal: internal_data }); + } + + return true; +}; diff --git a/testing/raptor/browsertime/support-scripts/browsertime_tp6_bench.py b/testing/raptor/browsertime/support-scripts/browsertime_tp6_bench.py new file mode 100644 index 0000000000..f1a514ccd0 --- /dev/null +++ b/testing/raptor/browsertime/support-scripts/browsertime_tp6_bench.py @@ -0,0 +1,217 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +import copy +import re + +import filters +from base_python_support import BasePythonSupport +from utils import bool_from_str + +DOMAIN_MATCHER = re.compile(r"(?:https?:\/\/)?(?:[^@\n]+@)?(?:www\.)?([^:\/\n]+)") +VARIANCE_THRESHOLD = 0.2 + + +def extract_domain(link): + match = DOMAIN_MATCHER.search(link) + if match: + return match.group(1) + raise Exception(f"Could not find domain for {link}") + + +class TP6BenchSupport(BasePythonSupport): + def __init__(self, **kwargs): + self._load_times = [] + self._total_times = [] + self._geomean_load_times = [] + self._sites_tested = 0 + self._test_pages = {} + + def setup_test(self, test, args): + from cmdline import DESKTOP_APPS + from manifest import get_browser_test_list + from utils import transform_subtest + + all_tests = get_browser_test_list(args.app, args.run_local) + + manifest_to_find = "browsertime-tp6.toml" + if args.app not in DESKTOP_APPS: + manifest_to_find = "browsertime-tp6m.toml" + + test_urls = [] + playback_pageset_manifests = [] + for parsed_test in all_tests: + if manifest_to_find in parsed_test["manifest"]: + if not bool_from_str(parsed_test.get("benchmark_page", "false")): + continue + test_url = parsed_test["test_url"] + test_urls.append(test_url) + playback_pageset_manifests.append( + transform_subtest( + parsed_test["playback_pageset_manifest"], parsed_test["name"] + ) + ) + self._test_pages[test_url] = parsed_test + + if len(playback_pageset_manifests) == 0: + raise Exception("Could not find any manifests for testing.") + + test["test_url"] = ",".join(test_urls) + test["playback_pageset_manifest"] = ",".join(playback_pageset_manifests) + + def handle_result(self, bt_result, raw_result, last_result=False, **kwargs): + measurements = {"totalTime": []} + + # Find new results to add + for res in raw_result["extras"]: + if "pageload-benchmark" in res: + total_time = int( + round(res["pageload-benchmark"].get("totalTime", 0), 0) + ) + measurements["totalTime"].append(total_time) + self._total_times.append(total_time) + + # Gather the load times of each page/cycle to help with diagnosing issues + load_times = [] + result_name = None + for cycle in raw_result["browserScripts"]: + if not result_name: + # When the test name is unknown, we use the url TLD combined + # with the page title to differentiate pages with similar domains, and + # limit it to 35 characters + page_url = cycle["pageinfo"].get("url", "") + if self._test_pages.get(page_url, None) is not None: + result_name = self._test_pages[page_url]["name"] + else: + page_name = extract_domain(page_url) + page_title = ( + cycle["pageinfo"].get("documentTitle", "")[:35].replace(" ", "") + ) + result_name = f"{page_name} - {page_title}" + + load_time = cycle["timings"]["loadEventEnd"] + fcp = cycle["timings"]["paintTiming"]["first-contentful-paint"] + lcp = ( + cycle["timings"] + .get("largestContentfulPaint", {}) + .get("renderTime", None) + ) + measurements.setdefault(f"{result_name} - loadTime", []).append(load_time) + + measurements.setdefault(f"{result_name} - fcp", []).append(fcp) + + if lcp is not None: + measurements.setdefault(f"{result_name} - lcp", []).append(lcp) + + load_times.append(load_time) + + cur_load_times = self._load_times or [0] * len(load_times) + self._load_times = list(map(sum, zip(cur_load_times, load_times))) + self._sites_tested += 1 + + for measurement, values in measurements.items(): + bt_result["measurements"].setdefault(measurement, []).extend(values) + if last_result: + bt_result["measurements"]["totalLoadTime"] = self._load_times + bt_result["measurements"]["totalLoadTimePerSite"] = [ + round(total_load_time / self._sites_tested, 2) + for total_load_time in self._load_times + ] + bt_result["measurements"]["totalTimePerSite"] = [ + round(total_time / self._sites_tested, 2) + for total_time in self._total_times + ] + + def _build_subtest(self, measurement_name, replicates, test): + unit = test.get("unit", "ms") + if test.get("subtest_unit"): + unit = test.get("subtest_unit") + + return { + "name": measurement_name, + "lowerIsBetter": test.get("lower_is_better", True), + "alertThreshold": float(test.get("alert_threshold", 2.0)), + "unit": unit, + "replicates": replicates, + "value": round(filters.geometric_mean(replicates), 3), + } + + def summarize_test(self, test, suite, **kwargs): + suite["type"] = "pageload" + if suite["subtests"] == {}: + suite["subtests"] = [] + for measurement_name, replicates in test["measurements"].items(): + if not replicates: + continue + suite["subtests"].append( + self._build_subtest(measurement_name, replicates, test) + ) + suite["subtests"].sort(key=lambda subtest: subtest["name"]) + + def _produce_suite_alts(self, suite_base, subtests, suite_name_prefix): + geomean_suite = copy.deepcopy(suite_base) + geomean_suite["subtests"] = copy.deepcopy(subtests) + median_suite = copy.deepcopy(geomean_suite) + median_suite["subtests"] = copy.deepcopy(subtests) + + subtest_values = [] + for subtest in subtests: + subtest_values.extend(subtest["replicates"]) + + geomean_suite["name"] = suite_name_prefix + "-geomean" + geomean_suite["value"] = filters.geometric_mean(subtest_values) + for subtest in geomean_suite["subtests"]: + subtest["value"] = filters.geometric_mean(subtest["replicates"]) + + median_suite["name"] = suite_name_prefix + "-median" + median_suite["value"] = filters.median(subtest_values) + for subtest in median_suite["subtests"]: + subtest["value"] = filters.median(subtest["replicates"]) + + return [ + geomean_suite, + median_suite, + ] + + def summarize_suites(self, suites): + fcp_subtests = [] + lcp_subtests = [] + load_time_subtests = [] + + for suite in suites: + for subtest in suite["subtests"]: + if "- fcp" in subtest["name"]: + fcp_subtests.append(subtest) + elif "- lcp" in subtest["name"]: + lcp_subtests.append(subtest) + elif "- loadTime" in subtest["name"]: + load_time_subtests.append(subtest) + + fcp_bench_suites = self._produce_suite_alts( + suites[0], fcp_subtests, "fcp-bench" + ) + + lcp_bench_suites = self._produce_suite_alts( + suites[0], lcp_subtests, "lcp-bench" + ) + + load_time_bench_suites = self._produce_suite_alts( + suites[0], load_time_subtests, "loadtime-bench" + ) + + overall_suite = copy.deepcopy(suites[0]) + new_subtests = [ + subtest + for subtest in suites[0]["subtests"] + if subtest["name"].startswith("total") + ] + overall_suite["subtests"] = new_subtests + + new_suites = [ + overall_suite, + *fcp_bench_suites, + *lcp_bench_suites, + *load_time_bench_suites, + ] + suites.pop() + suites.extend(new_suites) diff --git a/testing/raptor/browsertime/support-scripts/sample_python_support.py b/testing/raptor/browsertime/support-scripts/sample_python_support.py new file mode 100644 index 0000000000..a1ec0069a5 --- /dev/null +++ b/testing/raptor/browsertime/support-scripts/sample_python_support.py @@ -0,0 +1,12 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from base_python_support import BasePythonSupport + + +class SamplePythonSupport(BasePythonSupport): + def modify_command(self, cmd): + for i, entry in enumerate(cmd): + if "{replace-with-constant-value}" in entry: + cmd[i] = "25" diff --git a/testing/raptor/browsertime/support-scripts/speedometer3.py b/testing/raptor/browsertime/support-scripts/speedometer3.py new file mode 100644 index 0000000000..ed2c26df51 --- /dev/null +++ b/testing/raptor/browsertime/support-scripts/speedometer3.py @@ -0,0 +1,85 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import filters +from base_python_support import BasePythonSupport +from utils import flatten + + +class Speedometer3Support(BasePythonSupport): + def handle_result(self, bt_result, raw_result, **kwargs): + """Parse a result for the required results. + + See base_python_support.py for what's expected from this method. + """ + for res in raw_result["extras"]: + sp3_mean_score = round(res["s3"]["score"]["mean"], 3) + flattened_metrics_s3_internal = flatten(res["s3_internal"], ()) + + clean_flat_internal_metrics = {} + for k, vals in flattened_metrics_s3_internal.items(): + if k in ("mean", "geomean"): + # Skip these for parity with what was being + # returned in the results.py/output.py + continue + clean_flat_internal_metrics[k.replace("tests/", "")] = [ + round(val, 3) for val in vals + ] + + clean_flat_internal_metrics["score-internal"] = clean_flat_internal_metrics[ + "score" + ] + clean_flat_internal_metrics["score"] = [sp3_mean_score] + + for k, v in clean_flat_internal_metrics.items(): + bt_result["measurements"].setdefault(k, []).extend(v) + + def _build_subtest(self, measurement_name, replicates, test): + unit = test.get("unit", "ms") + if test.get("subtest_unit"): + unit = test.get("subtest_unit") + + lower_is_better = test.get( + "subtest_lower_is_better", test.get("lower_is_better", True) + ) + if "score" in measurement_name: + lower_is_better = False + unit = "score" + + subtest = { + "unit": unit, + "alertThreshold": float(test.get("alert_threshold", 2.0)), + "lowerIsBetter": lower_is_better, + "name": measurement_name, + "replicates": replicates, + "value": round(filters.mean(replicates), 3), + } + + if "score-internal" in measurement_name: + subtest["shouldAlert"] = False + + return subtest + + def summarize_test(self, test, suite, **kwargs): + """Summarize the measurements found in the test as a suite with subtests. + + See base_python_support.py for what's expected from this method. + """ + suite["type"] = "benchmark" + if suite["subtests"] == {}: + suite["subtests"] = [] + for measurement_name, replicates in test["measurements"].items(): + if not replicates: + continue + suite["subtests"].append( + self._build_subtest(measurement_name, replicates, test) + ) + suite["subtests"].sort(key=lambda subtest: subtest["name"]) + + score = 0 + for subtest in suite["subtests"]: + if subtest["name"] == "score": + score = subtest["value"] + break + suite["value"] = score diff --git a/testing/raptor/browsertime/throttled_pageload.js b/testing/raptor/browsertime/throttled_pageload.js new file mode 100644 index 0000000000..46701b9b59 --- /dev/null +++ b/testing/raptor/browsertime/throttled_pageload.js @@ -0,0 +1,56 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* eslint-env node */ + +const fs = require("fs"); +const path = require("path"); + +module.exports = async function (context, commands) { + context.log.info( + "Starting a pageload for which we will first enable the dev tools network throttler" + ); + + const url = "https://en.wikipedia.org/wiki/Barack_Obama"; + + await commands.navigate("about:blank"); + await commands.wait.byTime(1000); + + // Load the throttling script + let throttler_script = null; + let throttlerScriptPath = path.join( + `${path.resolve(__dirname)}`, + "utils", + "NetworkThrottlingUtils.js" + ); + try { + throttler_script = fs.readFileSync(throttlerScriptPath, "utf8"); + } catch (error) { + throw Error( + `Failed to load network throttler script ${throttlerScriptPath}` + ); + } + + // Create a throttler and configure the network + let usage = ` + let networkThrottler = new NetworkThrottler(); + let throttleData = { + latencyMean: 50, + latencyMax: 50, + downloadBPSMean: 250000, + downloadBPSMax: 250000, + uploadBPSMean: 250000, + uploadBPSMax: 250000 + }; + networkThrottler.start(throttleData); + `; + + throttler_script += usage; + commands.js.runPrivileged(throttler_script); + + await commands.measure.start(url); + + context.log.info("Throttled pageload test finished."); + return true; +}; diff --git a/testing/raptor/browsertime/upload.js b/testing/raptor/browsertime/upload.js new file mode 100644 index 0000000000..4f46ba0f50 --- /dev/null +++ b/testing/raptor/browsertime/upload.js @@ -0,0 +1,86 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* eslint-env node */ + +const path = require("path"); + +async function waitForUpload(timeout, commands, context) { + let starttime = await commands.js.run(`return performance.now();`); + let status = ""; + + while ( + (await commands.js.run(`return performance.now();`)) - starttime < + timeout && + status != "error" && + status != "success" + ) { + await commands.wait.byTime(10); + + status = await commands.js.run( + `return document.getElementById('upload_status').innerHTML;` + ); + + context.log.info("context.log test: " + status); + console.log("test: " + status); + } + + let endtime = await commands.js.run(`return performance.now();`); + + return { + start: starttime, + end: endtime, + upload_status: status, + }; +} + +module.exports = async function (context, commands) { + let uploadSiteUrl = "https://uploadtest-381620.uc.r.appspot.com"; + let iterations = `${context.options.browsertime.upload_iterations}`; + + await commands.measure.start(uploadSiteUrl); + let accumulatedResults = []; + for (let iteration = 0; iteration < iterations; iteration++) { + await commands.navigate(uploadSiteUrl); + + const driver = context.selenium.driver; + const webdriver = context.selenium.webdriver; + + const uploadItem = await driver.findElement(webdriver.By.id("fileUpload")); + + if (context.options.browsertime.moz_fetch_dir == "None") { + context.log.error( + "This test depends on the fetch task. Download the file, 'https://github.com/mozilla/perf-automation/raw/master/test_files/upload-test-32MB.dat' and set the os environment variable MOZ_FETCHES_DIR to that directory." + ); + } + + let localFilePath = path.join( + `${context.options.browsertime.moz_fetch_dir}`, + "upload-test-32MB.dat" + ); + + context.log.info("Sending file path: " + localFilePath); + await uploadItem.sendKeys(localFilePath); + + // Start the test and wait for the upload to complete + let results = await waitForUpload(120000, commands, context); + let uploadTime = results.end - results.start; + + // Store result in megabit/seconds, (Upload is a 50 MB file) + let uploadBandwidth = (50 * 8) / (uploadTime / 1000.0); + context.log.info( + "upload results: " + + results.upload_status + + " duration: " + + uploadTime + + " uploadBandwidth: " + + uploadBandwidth + ); + accumulatedResults.push(uploadBandwidth); + } + + commands.measure.addObject({ + custom_data: { "upload-bandwidth": accumulatedResults }, + }); +}; diff --git a/testing/raptor/browsertime/utils/NetworkThrottlingUtils.js b/testing/raptor/browsertime/utils/NetworkThrottlingUtils.js new file mode 100644 index 0000000000..8a0f78150e --- /dev/null +++ b/testing/raptor/browsertime/utils/NetworkThrottlingUtils.js @@ -0,0 +1,85 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +const lazy = {}; + +ChromeUtils.defineESModuleGetters(lazy, { + NetworkObserver: + "resource://devtools/shared/network-observer/NetworkObserver.sys.mjs", +}); + +/** + * The NetworkThrottler uses the dev tools NetworkObserver to provide api to throttle all network activity. + * This can be used to fix network conditions in browsertime pageload tests. + * + */ + +// A minimal struct for onNetworkEvent handling +class NetworkEventRecord { + addRequestPostData() {} + addResponseStart() {} + addSecurityInfo() {} + addEventTimings() {} + addResponseCache() {} + addResponseContent() {} + addServerTimings() {} + addServiceWorkerTimings() {} +} + +class NetworkThrottler { + #devtoolsNetworkObserver; + #throttling; + + constructor() { + this.#throttling = false; + } + + destroy() { + this.stop(); + } + + start(throttleData) { + if (this.#throttling) { + console.error("NetworkThrottler already started"); + return; + } + + this.#devtoolsNetworkObserver = new lazy.NetworkObserver({ + ignoreChannelFunction: this.#ignoreChannelFunction, + onNetworkEvent: this.#onNetworkEvent, + }); + + this.#devtoolsNetworkObserver.setThrottleData(throttleData); + + this.#throttling = true; + } + + stop() { + if (!this.#throttling) { + return; + } + + this.#devtoolsNetworkObserver.destroy(); + this.#devtoolsNetworkObserver = null; + + this.#throttling = false; + } + + #ignoreChannelFunction = channel => { + // Ignore chrome-privileged or DevTools-initiated requests + if ( + channel.loadInfo?.loadingDocument === null && + (channel.loadInfo.loadingPrincipal === + Services.scriptSecurityManager.getSystemPrincipal() || + channel.loadInfo.isInDevToolsContext) + ) { + return true; + } + return false; + }; + + #onNetworkEvent = (networkEvent, channel) => { + return new NetworkEventRecord(networkEvent, channel, this); + }; +} diff --git a/testing/raptor/browsertime/welcome.js b/testing/raptor/browsertime/welcome.js new file mode 100644 index 0000000000..42861c8d03 --- /dev/null +++ b/testing/raptor/browsertime/welcome.js @@ -0,0 +1,30 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* eslint-env node */ + +module.exports = async function (context, commands) { + context.log.info("Starting a first-install test"); + let page_cycles = context.options.browsertime.page_cycles; + + for (let count = 0; count < page_cycles; count++) { + context.log.info("Navigating to about:blank"); + + // See bug 1717754 + context.log.info(await commands.js.run(`return document.documentURI;`)); + + await commands.navigate("about:blank"); + + await commands.measure.start(); + await commands.wait.byTime(1000); + await commands.navigate("about:welcome"); + await commands.wait.byTime(2000); + await commands.measure.stop(); + + await commands.wait.byTime(2000); + } + + context.log.info("First-install test ended."); + return true; +}; |