summaryrefslogtreecommitdiffstats
path: root/testing/raptor/browsertime
diff options
context:
space:
mode:
Diffstat (limited to 'testing/raptor/browsertime')
-rw-r--r--testing/raptor/browsertime/browsertime_benchmark.js90
-rw-r--r--testing/raptor/browsertime/browsertime_interactive.js103
-rw-r--r--testing/raptor/browsertime/browsertime_pageload.js412
-rw-r--r--testing/raptor/browsertime/browsertime_scenario.js67
-rw-r--r--testing/raptor/browsertime/constant_regression_test.js14
-rw-r--r--testing/raptor/browsertime/grandprix.js80
-rw-r--r--testing/raptor/browsertime/pageload_sites.json614
-rw-r--r--testing/raptor/browsertime/process_switch.js45
-rw-r--r--testing/raptor/browsertime/speedometer3.js90
-rw-r--r--testing/raptor/browsertime/support-scripts/sample_python_support.py13
-rw-r--r--testing/raptor/browsertime/upload.js86
-rw-r--r--testing/raptor/browsertime/welcome.js30
12 files changed, 1644 insertions, 0 deletions
diff --git a/testing/raptor/browsertime/browsertime_benchmark.js b/testing/raptor/browsertime/browsertime_benchmark.js
new file mode 100644
index 0000000000..35bc2c1844
--- /dev/null
+++ b/testing/raptor/browsertime/browsertime_benchmark.js
@@ -0,0 +1,90 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/* eslint-env node */
+
+module.exports = async function (context, commands) {
+ context.log.info("Starting a browsertime benchamrk");
+ let url = context.options.browsertime.url;
+ let page_cycles = context.options.browsertime.page_cycles;
+ let page_cycle_delay = context.options.browsertime.page_cycle_delay;
+ let post_startup_delay = context.options.browsertime.post_startup_delay;
+ let page_timeout = context.options.timeouts.pageLoad;
+ let ret = false;
+ let expose_profiler = context.options.browsertime.expose_profiler;
+
+ context.log.info(
+ "Waiting for %d ms (post_startup_delay)",
+ post_startup_delay
+ );
+ await commands.wait.byTime(post_startup_delay);
+
+ for (let count = 0; count < page_cycles; count++) {
+ context.log.info("Navigating to about:blank");
+ await commands.navigate("about:blank");
+
+ context.log.info("Cycle %d, waiting for %d ms", count, page_cycle_delay);
+ await commands.wait.byTime(page_cycle_delay);
+
+ context.log.info("Cycle %d, starting the measure", count);
+ if (expose_profiler === "true") {
+ context.log.info("Custom profiler start!");
+ await commands.profiler.start();
+ }
+ await commands.measure.start(url);
+
+ context.log.info("Benchmark custom metric collection");
+
+ let data = null;
+ let starttime = await commands.js.run(`return performance.now();`);
+ while (
+ data == null &&
+ (await commands.js.run(`return performance.now();`)) - starttime <
+ page_timeout
+ ) {
+ let wait_time = 3000;
+ context.log.info("Waiting %d ms for data from benchmark...", wait_time);
+ await commands.wait.byTime(wait_time);
+ data = await commands.js.run(
+ "return window.sessionStorage.getItem('benchmark_results');"
+ );
+ }
+ if (expose_profiler === "true") {
+ context.log.info("Custom profiler stop!");
+ await commands.profiler.stop();
+ }
+ if (
+ data == null &&
+ (await commands.js.run(`return performance.now();`)) - starttime >=
+ page_timeout
+ ) {
+ ret = false;
+ context.log.error("Benchmark timed out. Aborting...");
+ } else if (data) {
+ // Reset benchmark results
+ await commands.js.run(
+ "return window.sessionStorage.removeItem('benchmark_results');"
+ );
+
+ context.log.info("Value of benchmark data: ", data);
+ data = JSON.parse(data);
+
+ if (!Array.isArray(data)) {
+ commands.measure.addObject({ browsertime_benchmark: data });
+ } else {
+ commands.measure.addObject({
+ browsertime_benchmark: {
+ [data[1]]: data.slice(2),
+ },
+ });
+ }
+ ret = true;
+ } else {
+ context.log.error("No data collected from benchmark.");
+ ret = false;
+ }
+ }
+ context.log.info("Browsertime benchmark ended.");
+ return ret;
+};
diff --git a/testing/raptor/browsertime/browsertime_interactive.js b/testing/raptor/browsertime/browsertime_interactive.js
new file mode 100644
index 0000000000..20ce23ccf1
--- /dev/null
+++ b/testing/raptor/browsertime/browsertime_interactive.js
@@ -0,0 +1,103 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/* eslint-env node */
+
+async function get_command_function(cmd, commands) {
+ /*
+ Converts a string such as `measure.start` into the actual
+ function that is found in the `commands` module.
+
+ XXX: Find a way to share this function between
+ perftest_record.js and browsertime_interactive.js
+ */
+ if (cmd == "") {
+ throw new Error("A blank command was given.");
+ } else if (cmd.endsWith(".")) {
+ throw new Error(
+ "An extra `.` was found at the end of this command: " + cmd
+ );
+ }
+
+ // `func` will hold the actual method that needs to be called,
+ // and the `parent_mod` is the context required to run the `func`
+ // method. Without that context, `this` becomes undefined in the browsertime
+ // classes.
+ let func = null;
+ let parent_mod = null;
+ for (let func_part of cmd.split(".")) {
+ if (func_part == "") {
+ throw new Error(
+ "An empty function part was found in the command: " + cmd
+ );
+ }
+
+ if (func === null) {
+ parent_mod = commands;
+ func = commands[func_part];
+ } else if (func !== undefined) {
+ parent_mod = func;
+ func = func[func_part];
+ } else {
+ break;
+ }
+ }
+
+ if (func == undefined) {
+ throw new Error(
+ "The given command could not be found as a function: " + cmd
+ );
+ }
+
+ return [func, parent_mod];
+}
+
+module.exports = async function (context, commands) {
+ context.log.info("Starting an interactive browsertime test");
+ let page_cycles = context.options.browsertime.page_cycles;
+ let post_startup_delay = context.options.browsertime.post_startup_delay;
+ let input_cmds = context.options.browsertime.commands;
+
+ context.log.info(
+ "Waiting for %d ms (post_startup_delay)",
+ post_startup_delay
+ );
+ await commands.wait.byTime(post_startup_delay);
+
+ // unpack commands from python
+ let cmds = input_cmds.split(";;;");
+
+ for (let count = 0; count < page_cycles; count++) {
+ context.log.info("Navigating to about:blank w/nav, count: " + count);
+ await commands.navigate("about:blank");
+
+ let pages_visited = [];
+ for (let cmdstr of cmds) {
+ let [cmd, ...args] = cmdstr.split(":::");
+
+ if (cmd == "measure.start") {
+ if (args[0] != "") {
+ pages_visited.push(args[0]);
+ }
+ }
+
+ let [func, parent_mod] = await get_command_function(cmd, commands);
+
+ try {
+ await func.call(parent_mod, ...args);
+ } catch (e) {
+ context.log.info(
+ `Exception found while running \`commands.${cmd}(${args})\`: `
+ );
+ context.log.info(e.stack);
+ }
+ }
+
+ // Log the number of pages visited for results parsing
+ context.log.info("[] metrics: pages_visited: " + pages_visited);
+ }
+
+ context.log.info("Browsertime pageload ended.");
+ return true;
+};
diff --git a/testing/raptor/browsertime/browsertime_pageload.js b/testing/raptor/browsertime/browsertime_pageload.js
new file mode 100644
index 0000000000..869efcf3a0
--- /dev/null
+++ b/testing/raptor/browsertime/browsertime_pageload.js
@@ -0,0 +1,412 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/* eslint-env node */
+
+const fs = require("fs");
+const http = require("http");
+
+const URL = "/secrets/v1/secret/project/perftest/gecko/level-";
+const SECRET = "/perftest-login";
+const DEFAULT_SERVER = "https://firefox-ci-tc.services.mozilla.com";
+
+const SCM_LOGIN_SITES = ["facebook", "netflix"];
+
+/**
+ * This function obtains the perftest secret from Taskcluster.
+ *
+ * It will NOT work locally. Please see the get_logins function, you
+ * will need to define a JSON file and set the RAPTOR_LOGINS
+ * env variable to its path.
+ */
+async function get_tc_secrets(context) {
+ const MOZ_AUTOMATION = process.env.MOZ_AUTOMATION;
+ if (!MOZ_AUTOMATION) {
+ throw Error(
+ "Not running in CI. Set RAPTOR_LOGINS to a JSON file containing the logins."
+ );
+ }
+
+ let TASKCLUSTER_PROXY_URL = process.env.TASKCLUSTER_PROXY_URL
+ ? process.env.TASKCLUSTER_PROXY_URL
+ : DEFAULT_SERVER;
+
+ let MOZ_SCM_LEVEL = process.env.MOZ_SCM_LEVEL ? process.env.MOZ_SCM_LEVEL : 1;
+
+ const url = TASKCLUSTER_PROXY_URL + URL + MOZ_SCM_LEVEL + SECRET;
+
+ const data = await new Promise((resolve, reject) => {
+ context.log.info("Obtaining secrets for login...");
+
+ http.get(
+ url,
+ {
+ headers: {
+ "Content-Type": "application/json",
+ Accept: "application/json",
+ },
+ },
+ res => {
+ let data = "";
+ context.log.info(`Secret status code: ${res.statusCode}`);
+
+ res.on("data", d => {
+ data += d.toString();
+ });
+
+ res.on("end", () => {
+ resolve(data);
+ });
+
+ res.on("error", error => {
+ context.log.error(error);
+ reject(error);
+ });
+ }
+ );
+ });
+
+ return JSON.parse(data);
+}
+
+/**
+ * This function gets the login information required.
+ *
+ * It starts by looking for a local file whose path is defined
+ * within RAPTOR_LOGINS. If we don't find this file, then we'll
+ * attempt to get the login information from our Taskcluster secret.
+ * If MOZ_AUTOMATION is undefined, then the test will fail, Taskcluster
+ * secrets can only be obtained in CI.
+ */
+async function get_logins(context) {
+ let logins;
+
+ let RAPTOR_LOGINS = process.env.RAPTOR_LOGINS;
+ if (RAPTOR_LOGINS) {
+ // Get logins from a local file
+ if (!RAPTOR_LOGINS.endsWith(".json")) {
+ throw Error(
+ `File given for logins does not end in '.json': ${RAPTOR_LOGINS}`
+ );
+ }
+
+ let logins_file = null;
+ try {
+ logins_file = await fs.readFileSync(RAPTOR_LOGINS, "utf8");
+ } catch (err) {
+ throw Error(`Failed to read the file ${RAPTOR_LOGINS}: ${err}`);
+ }
+
+ logins = await JSON.parse(logins_file);
+ } else {
+ // Get logins from a perftest Taskcluster secret
+ logins = await get_tc_secrets(context);
+ }
+
+ return logins;
+}
+
+/**
+ * This function returns the type of login to do.
+ *
+ * This function returns "single-form" when we find a single form. If we only
+ * find a single input field, we assume that there is one page per input
+ * and return "multi-page". Otherwise, we return null.
+ */
+async function get_login_type(context, commands) {
+ /*
+ Determine if there's a password field visible with this
+ query selector. Some sites use `tabIndex` to hide the password
+ field behind other elements. In this case, we are searching
+ for any password-type field that has a tabIndex of 0 or undefined and
+ is not hidden.
+ */
+ let input_length = await commands.js.run(`
+ return document.querySelectorAll(
+ "input[type=password][tabIndex='0']:not([type=hidden])," +
+ "input[type=password]:not([tabIndex]):not([type=hidden])"
+ ).length;
+ `);
+
+ if (input_length == 0) {
+ context.log.info("Found a multi-page login");
+ return multi_page_login;
+ } else if (input_length == 1) {
+ context.log.info("Found a single-page login");
+ return single_page_login;
+ }
+
+ if (
+ (await commands.js.run(
+ `return document.querySelectorAll("form").length;`
+ )) >= 1
+ ) {
+ context.log.info("Found a single-form login");
+ return single_form_login;
+ }
+
+ return null;
+}
+
+/**
+ * This function sets up the login for a single form.
+ *
+ * The username field is defined as the field which immediately precedes
+ * the password field. We have to do this in two steps because we need
+ * to make sure that the event we emit from the change has the `isTrusted`
+ * field set to `true`. Otherwise, some websites will ignore the input and
+ * the form submission.
+ */
+async function single_page_login(login_info, context, commands, prefix = "") {
+ // Get the first input field in the form that is not hidden and add the
+ // username. Assumes that email/username is always the first input field.
+ await commands.addText.bySelector(
+ login_info.username,
+ `${prefix}input:not([type=hidden]):not([type=password])`
+ );
+
+ // Get the password field and ensure it's not hidden.
+ await commands.addText.bySelector(
+ login_info.password,
+ `${prefix}input[type=password]:not([type=hidden])`
+ );
+
+ return undefined;
+}
+
+/**
+ * See single_page_login.
+ */
+async function single_form_login(login_info, context, commands) {
+ return single_page_login(login_info, context, commands, "form ");
+}
+
+/**
+ * Login to a website that uses multiple pages for the login.
+ *
+ * WARNING: Assumes that the first page is for the username.
+ */
+// TODO cleanup comments
+async function multi_page_login(login_info, context, commands) {
+ const driver = context.selenium.driver;
+ const webdriver = context.selenium.webdriver;
+ //TODO fails here in netflix for Try...
+ const username_field = await driver.findElement(
+ webdriver.By.css(`input:not([type=hidden]):not([type=password])`)
+ );
+ await username_field.sendKeys(login_info.username);
+ await username_field.sendKeys(webdriver.Key.ENTER);
+ await commands.wait.byTime(5000);
+ let password_field;
+ try {
+ password_field = await driver.findElement(
+ webdriver.By.css(`input[type=password]:not([type=hidden])`)
+ );
+ } catch (err) {
+ if (err.toString().includes("NoSuchElementError")) {
+ // Sometimes we're suspicious (i.e. they think we're a bot/bad-actor)
+ let name_field = await driver.findElement(
+ webdriver.By.css(`input:not([type=hidden]):not([type=password])`)
+ );
+ await name_field.sendKeys(login_info.suspicious_answer);
+ await name_field.sendKeys(webdriver.Key.ENTER);
+ await commands.wait.byTime(5000);
+
+ // Try getting the password field again
+ password_field = await driver.findElement(
+ webdriver.By.css(`input[type=password]:not([type=hidden])`)
+ );
+ } else {
+ throw err;
+ }
+ }
+
+ await password_field.sendKeys(login_info.password);
+
+ return async function () {
+ password_field.sendKeys(webdriver.Key.ENTER);
+ await commands.wait.byTime(5000);
+ };
+}
+
+/**
+ * This function sets up the login.
+ *
+ * This is done by first the login type, and then performing the
+ * actual login setup. The return is a possible button to click
+ * to perform the login.
+ */
+async function setup_login(login_info, context, commands) {
+ let login_func = await get_login_type(context, commands);
+ if (!login_func) {
+ throw Error("Could not determine the type of login page.");
+ }
+
+ try {
+ return await login_func(login_info, context, commands);
+ } catch (err) {
+ throw Error(`Could not setup login information: ${err}`);
+ }
+}
+
+/**
+ * This function performs the login.
+ *
+ * It does this by either clicking on a button with a type
+ * of "sumbit", or running a final_button function that was
+ * obtained from the setup_login function. Some pages also ask
+ * questions about setting up 2FA or other information. Generally,
+ * these contain the "skip" text.
+ */
+async function login(context, commands, final_button) {
+ try {
+ if (!final_button) {
+ // The mouse double click emits an event with `evt.isTrusted=true`
+ await commands.mouse.doubleClick.bySelector("button[type=submit]");
+ await commands.wait.byTime(10000);
+ } else {
+ // In some cases, it's preferable to be given a function for the final button
+ await final_button();
+ }
+
+ // Some pages ask to setup 2FA, skip this based on the text
+ const XPATHS = [
+ "//a[contains(text(), 'skip')]",
+ "//button[contains(text(), 'skip')]",
+ "//input[contains(text(), 'skip')]",
+ "//div[contains(text(), 'skip')]",
+ ];
+
+ for (let xpath of XPATHS) {
+ try {
+ await commands.mouse.doubleClick.byXpath(xpath);
+ } catch (err) {
+ if (err.toString().includes("not double click")) {
+ context.log.info(`Can't find a button with the text: ${xpath}`);
+ } else {
+ throw err;
+ }
+ }
+ }
+ } catch (err) {
+ throw Error(
+ `Could not login to website as we could not find the submit button/input: ${err}`
+ );
+ }
+}
+
+/**
+ * Grab the base URL from the browsertime url.
+ *
+ * This is a necessary step for getting the login values from the Taskcluster
+ * secrets, which are hashed by the base URL.
+ *
+ * The first entry is the protocal, third is the top-level domain (or host)
+ */
+function get_base_URL(fullUrl) {
+ let pathAsArray = fullUrl.split("/");
+ return pathAsArray[0] + "//" + pathAsArray[2];
+}
+
+/**
+ * This function attempts the login-login sequence for a live pageload test
+ */
+async function perform_live_login(context, commands) {
+ let testUrl = context.options.browsertime.url;
+
+ let logins = await get_logins(context);
+ const baseUrl = get_base_URL(testUrl);
+
+ await commands.navigate("about:blank");
+
+ let login_info = logins.secret[baseUrl];
+ try {
+ await commands.navigate(login_info.login_url);
+ } catch (err) {
+ context.log.info("Unable to acquire login information");
+ throw err;
+ }
+ await commands.wait.byTime(10000);
+
+ let final_button = await setup_login(login_info, context, commands);
+ await login(context, commands, final_button);
+}
+
+module.exports = async function (context, commands) {
+ context.log.info("Starting a browsertime pageload");
+ let test_url = context.options.browsertime.url;
+ let secondary_url = context.options.browsertime.secondary_url;
+ let page_cycles = context.options.browsertime.page_cycles;
+ let page_cycle_delay = context.options.browsertime.page_cycle_delay;
+ let post_startup_delay = context.options.browsertime.post_startup_delay;
+ let chimera_mode = context.options.browsertime.chimera;
+ let test_bytecode_cache = context.options.browsertime.test_bytecode_cache;
+ let login_required = context.options.browsertime.loginRequired;
+ let live_site = context.options.browsertime.liveSite;
+ let test_name = context.options.browsertime.testName;
+
+ context.log.info(
+ "Waiting for %d ms (post_startup_delay)",
+ post_startup_delay
+ );
+ await commands.wait.byTime(post_startup_delay);
+ let cached = false;
+
+ // Login once before testing the test_url/secondary_url cycles
+ // If the user has RAPTOR_LOGINS configured correctly, a local login pageload
+ // test can be attempted. Otherwise if attempting it in CI, only sites with the
+ // associated MOZ_SCM_LEVEL will be attempted (e.g. Try = 1, autoland = 3).
+ // In addition, ensure login sequence is only attempted on live sites and for sites
+ // that we have Taskcluster secrets for.
+ if (
+ login_required == "True" &&
+ live_site == "True" &&
+ SCM_LOGIN_SITES.includes(test_name)
+ ) {
+ await perform_live_login(context, commands);
+ }
+
+ for (let count = 0; count < page_cycles; count++) {
+ if (count !== 0 && secondary_url !== undefined) {
+ context.log.info("Navigating to secondary url:" + secondary_url);
+ await commands.navigate(secondary_url);
+ await commands.wait.byTime(1000);
+ await commands.js.runAndWait(`
+ (function() {
+ const white = document.createElement('div');
+ white.id = 'raptor-white';
+ white.style.position = 'absolute';
+ white.style.top = '0';
+ white.style.left = '0';
+ white.style.width = Math.max(document.documentElement.clientWidth, document.body.clientWidth) + 'px';
+ white.style.height = Math.max(document.documentElement.clientHeight,document.body.clientHeight) + 'px';
+ white.style.backgroundColor = 'white';
+ white.style.zIndex = '2147483647';
+ document.body.appendChild(white);
+ document.body.style.display = '';
+ })();`);
+ await commands.wait.byTime(1000);
+ } else {
+ context.log.info("Navigating to about:blank, count: " + count);
+ await commands.navigate("about:blank");
+ }
+
+ context.log.info("Navigating to primary url:" + test_url);
+ context.log.info("Cycle %d, waiting for %d ms", count, page_cycle_delay);
+ await commands.wait.byTime(page_cycle_delay);
+
+ context.log.info("Cycle %d, starting the measure", count);
+ await commands.measure.start(test_url);
+
+ // Wait 20 seconds to populate bytecode cache
+ if (test_bytecode_cache == "true" && chimera_mode == "true" && !cached) {
+ context.log.info("Waiting 20s to populate bytecode cache...");
+ await commands.wait.byTime(20000);
+ cached = true;
+ }
+ }
+
+ context.log.info("Browsertime pageload ended.");
+ return true;
+};
diff --git a/testing/raptor/browsertime/browsertime_scenario.js b/testing/raptor/browsertime/browsertime_scenario.js
new file mode 100644
index 0000000000..3abf0d142b
--- /dev/null
+++ b/testing/raptor/browsertime/browsertime_scenario.js
@@ -0,0 +1,67 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/* eslint-env node */
+
+module.exports = async function (context, commands) {
+ context.log.info("Starting a browsertime scenario test");
+ let page_cycles = context.options.browsertime.page_cycles;
+ let page_cycle_delay = context.options.browsertime.page_cycle_delay;
+ let post_startup_delay = context.options.browsertime.post_startup_delay;
+ let scenario_time = context.options.browsertime.scenario_time;
+ let background_app = context.options.browsertime.background_app == "true";
+ let app = null;
+
+ if (background_app) {
+ if (!context.options.android) {
+ throw new Error("Cannot background an application on desktop");
+ }
+ app = context.options.firefox.android.package;
+ }
+
+ context.log.info(
+ "Waiting for %d ms (post_startup_delay)",
+ post_startup_delay
+ );
+ await commands.wait.byTime(post_startup_delay);
+
+ for (let count = 0; count < page_cycles; count++) {
+ context.log.info("Navigating to about:blank");
+ await commands.navigate("about:blank");
+
+ context.log.info("Cycle %d, waiting for %d ms", count, page_cycle_delay);
+ await commands.wait.byTime(page_cycle_delay);
+
+ context.log.info("Cycle %d, starting the measure", count);
+
+ if (background_app) {
+ // Background the application and disable doze for it
+ await commands.android.shell(`dumpsys deviceidle whitelist +${app}`);
+ await commands.android.shell(`input keyevent 3`);
+ await commands.wait.byTime(1000);
+ const foreground = await commands.android.shell(
+ "dumpsys window windows | grep mCurrentFocus"
+ );
+ if (foreground.includes(app)) {
+ throw new Error("Application was not backgrounded successfully");
+ } else {
+ context.log.info("Application was backgrounded successfully");
+ }
+ }
+
+ // Run the test
+ await commands.measure.start("about:blank test");
+ context.log.info("Waiting for %d ms for this scenario", scenario_time);
+ await commands.wait.byTime(scenario_time);
+ await commands.measure.stop();
+
+ if (background_app) {
+ // Foreground the application and enable doze again
+ await commands.android.shell(`am start --activity-single-top ${app}`);
+ await commands.android.shell(`dumpsys deviceidle enable`);
+ }
+ }
+ context.log.info("Browsertime scenario test ended.");
+ return true;
+};
diff --git a/testing/raptor/browsertime/constant_regression_test.js b/testing/raptor/browsertime/constant_regression_test.js
new file mode 100644
index 0000000000..5373936208
--- /dev/null
+++ b/testing/raptor/browsertime/constant_regression_test.js
@@ -0,0 +1,14 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/* eslint-env node */
+
+module.exports = async function (context, commands) {
+ context.log.info("Starting constant value regression test");
+ await commands.measure.start("regression-test");
+ await commands.measure.stop();
+ await commands.measure.addObject({
+ data: { metric: context.options.browsertime.constant_value },
+ });
+};
diff --git a/testing/raptor/browsertime/grandprix.js b/testing/raptor/browsertime/grandprix.js
new file mode 100644
index 0000000000..530eb203e3
--- /dev/null
+++ b/testing/raptor/browsertime/grandprix.js
@@ -0,0 +1,80 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/* eslint-env node */
+
+module.exports = async function (context, commands) {
+ const benchmark_url = "https://grandprixbench.netlify.app/";
+ const iterations = `${context.options.browsertime.grandprix_iterations}`;
+ const tests = [
+ "Vanilla-HTML-TodoMVC",
+ "Vanilla-TodoMVC",
+ "Preact-TodoMVC",
+ "Preact-TodoMVC-Modern",
+ "React-TodoMVC",
+ //"Elm-TodoMVC", // removed from suite
+ //"Rust-Yew-TodoMVC", // removed from suite
+ //"Hydration-Preact", // removed from suite
+ //"Scroll-Windowing-React", // removed from suite
+ "Monaco-Editor",
+ "Monaco-Syntax-Highlight",
+ "React-Stockcharts",
+ "React-Stockcharts-SVG",
+ "Leaflet-Fractal",
+ //"SVG-UI", // removed from suite
+ "Proxx-Tables",
+ "Proxx-Tables-Lit",
+ "Proxx-Tables-Canvas",
+ ];
+
+ // Build test url
+ let url = benchmark_url + "?suites=";
+ tests.forEach(test => {
+ url = url + test + ",";
+ });
+ url = url + "&iterations=" + iterations;
+
+ console.log("Using url=" + url);
+ await commands.measure.start(url);
+
+ await commands.wait.byTime("3000");
+ await commands.click.byXpath("/html/body/main/div/div[1]/div/div/button");
+
+ let finished = 0;
+ do {
+ await commands.wait.byTime("30000");
+ finished = await commands.js.run('return typeof results != "undefined"');
+ } while (!finished);
+
+ if (context.options.browser === "firefox") {
+ let buildId = await commands.js.runPrivileged(
+ "return Services.appinfo.appBuildID;"
+ );
+ console.log(buildId);
+ }
+
+ let output = {};
+ output.score = {};
+ output.score.total = await commands.js.run("return results.Score");
+ output.iterations = await commands.js.run("return results.iterations");
+
+ let subtests = {};
+ for (let i = 0; i < output.iterations.length; i++) {
+ for (const [key, value] of Object.entries(output.iterations[i])) {
+ if (!subtests.hasOwnProperty(key)) {
+ subtests[key] = [];
+ }
+ subtests[key].push(value.total);
+ }
+ }
+
+ console.log("score, ", output.score.total.mean);
+ for (const [key, value] of Object.entries(subtests)) {
+ const average = value.reduce((a, b) => a + b, 0) / value.length;
+ output.score[key] = average;
+ console.log(key, ",", average);
+ }
+
+ await commands.measure.add("grandprix-s3", output);
+};
diff --git a/testing/raptor/browsertime/pageload_sites.json b/testing/raptor/browsertime/pageload_sites.json
new file mode 100644
index 0000000000..7f2c6185f3
--- /dev/null
+++ b/testing/raptor/browsertime/pageload_sites.json
@@ -0,0 +1,614 @@
+{
+ "mobile": [
+ {
+ "login": false,
+ "name": "allrecipes",
+ "test_url": "https://www.allrecipes.com/"
+ },
+ {
+ "login": false,
+ "name": "amazon",
+ "test_url": "https://www.amazon.com"
+ },
+ {
+ "login": false,
+ "name": "amazon-search",
+ "test_url": "https://www.amazon.com/s/ref=nb_sb_noss_2/139-6317191-5622045?url=search-alias%3Daps&field-keywords=mobile+phone"
+ },
+ {
+ "login": false,
+ "name": "bing",
+ "test_url": "https://www.bing.com/"
+ },
+ {
+ "login": false,
+ "name": "bing-search-restaurants",
+ "test_url": "https://www.bing.com/search?q=restaurants+in+exton+pa+19341"
+ },
+ {
+ "login": false,
+ "name": "booking",
+ "test_url": "https://www.booking.com/"
+ },
+ {
+ "login": false,
+ "name": "cnn",
+ "test_url": "https://cnn.com",
+ "secondary_url": "https://www.cnn.com/weather"
+ },
+ {
+ "login": false,
+ "name": "cnn-ampstories",
+ "test_url": "https://cnn.com/ampstories/us/why-hurricane-michael-is-a-monster-unlike-any-other"
+ },
+ {
+ "login": false,
+ "name": "dailymail",
+ "test_url": "https://www.dailymail.co.uk/sciencetech/article-9749081/Experts-say-Hubble-repair-despite-NASA-insisting-multiple-options-fix.html"
+ },
+ {
+ "login": false,
+ "name": "ebay-kleinanzeigen",
+ "test_url": "https://m.ebay-kleinanzeigen.de"
+ },
+ {
+ "login": false,
+ "name": "ebay-kleinanzeigen-search",
+ "test_url": "https://m.ebay-kleinanzeigen.de/s-anzeigen/auf-zeit-wg-berlin/zimmer/c199-l3331"
+ },
+ {
+ "login": false,
+ "name": "espn",
+ "test_url": "http://www.espn.com/nba/story/_/page/allstarweekend25788027/the-comparison-lebron-james-michael-jordan-their-own-words"
+ },
+ {
+ "login": true,
+ "name": "facebook",
+ "test_url": "https://m.facebook.com"
+ },
+ {
+ "login": false,
+ "name": "facebook-cristiano",
+ "test_url": "https://m.facebook.com/Cristiano"
+ },
+ {
+ "login": true,
+ "name": "google",
+ "test_url": "https://www.google.com"
+ },
+ {
+ "login": false,
+ "name": "google-maps",
+ "test_url": "https://www.google.com/maps?force=pwa"
+ },
+ {
+ "login": true,
+ "name": "google-search-restaurants",
+ "test_url": "https://www.google.com/search?q=restaurants+near+me"
+ },
+ {
+ "login": false,
+ "name": "imdb",
+ "test_url": "https://m.imdb.com/"
+ },
+ {
+ "login": true,
+ "name": "instagram",
+ "test_url": "https://www.instagram.com"
+ },
+ {
+ "login": false,
+ "name": "sina",
+ "test_url": "https://www.sina.com.cn/"
+ },
+ {
+ "login": false,
+ "name": "microsoft-support",
+ "test_url": "https://support.microsoft.com/en-us"
+ },
+ {
+ "login": false,
+ "name": "reddit",
+ "test_url": "https://www.reddit.com"
+ },
+ {
+ "login": false,
+ "name": "stackoverflow",
+ "test_url": "https://stackoverflow.com/"
+ },
+ {
+ "login": false,
+ "name": "web-de",
+ "test_url": "https://web.de/magazine/politik/politologe-glaubt-grossen-koalition-herbst-knallen-33563566"
+ },
+ {
+ "login": false,
+ "name": "wikipedia",
+ "test_url": "https://en.m.wikipedia.org/wiki/Main_Page"
+ },
+ {
+ "login": false,
+ "name": "youtube",
+ "test_url": "https://m.youtube.com"
+ },
+ {
+ "login": false,
+ "name": "youtube-watch",
+ "test_url": "https://www.youtube.com/watch?v=COU5T-Wafa4"
+ }
+ ],
+ "desktop": [
+ {
+ "login": false,
+ "name": "amazon",
+ "test_url": "https://www.amazon.com/s?k=laptop&ref=nb_sb_noss_1",
+ "secondary_url": "https://www.amazon.com/Acer-A515-46-R14K-Quad-Core-Processor-Backlit/dp/B08VKNVDDR/ref=sr_1_3?dchild=1&keywords=laptop&qid=1627047187&sr=8-3"
+ },
+ {
+ "login": false,
+ "name": "bing-search",
+ "test_url": "https://www.bing.com/search?q=barack+obama"
+ },
+ {
+ "login": false,
+ "name": "buzzfeed",
+ "test_url": "https://www.buzzfeed.com/",
+ "secondary_url": "https://www.buzzfeed.com/quizzes",
+ "test_cmds": [
+ [
+ "click.byXpathAndWait",
+ "/html/body/div[1]/div/div/div/div[2]/div/button[2]"
+ ],
+ [
+ "click.byXpathAndWait",
+ "/html/body/div[1]/div/div/div/div[3]/div[2]/button"
+ ]
+ ]
+ },
+ {
+ "login": false,
+ "name": "cnn",
+ "test_url": "https://www.cnn.com/2021/03/22/weather/climate-change-warm-waters-lake-michigan/index.html",
+ "secondary_url": "https://www.cnn.com/weather",
+ "preferences": {
+ "media.autoplay.default": 5,
+ "media.autoplay.ask-permission": true,
+ "media.autoplay.blocking_policy": 1,
+ "media.autoplay.block-webaudio": true,
+ "media.allowed-to-play.enabled": false,
+ "media.block-autoplay-until-in-foreground": true
+ },
+ "test_cmds": [
+ ["click.byXpathAndWait", "//*[@id='onetrust-pc-btn-handler']"],
+ [
+ "click.byXpathAndWait",
+ "/html/body/div[13]/div[2]/div/section/div[17]/button[2]"
+ ]
+ ]
+ },
+ {
+ "login": false,
+ "name": "ebay",
+ "test_url": "https://www.ebay.com/",
+ "secondary_url": "https://www.ebay.com/deals",
+ "test_cmds": [
+ [
+ "click.byXpathAndWait",
+ "/html/body/div[5]/div[1]/div[2]/div[2]/div[2]/a"
+ ],
+ [
+ "click.byXpathAndWait",
+ "/html/body/div[4]/div/div[2]/section[3]/section/button"
+ ]
+ ]
+ },
+ {
+ "login": false,
+ "name": "espn",
+ "test_url": "http://www.espn.com/nba/story/_/page/allstarweekend25788027/the-comparison-lebron-james-michael-jordan-their-own-words",
+ "test_cmds": [
+ ["click.byXpathAndWait", "//*[@id='onetrust-pc-btn-handler']"],
+ [
+ "click.byXpathAndWait",
+ "/html/body/div[10]/div[2]/div[3]/div[1]/button"
+ ]
+ ]
+ },
+ {
+ "login": false,
+ "name": "expedia",
+ "test_url": "https://expedia.com/Hotel-Search?destination=New+York%2C+New+York&latLong=40.756680%2C-73.986470&regionId=178293&startDate=&endDate=&rooms=1&_xpid=11905%7C1&adults=2",
+ "secondary_url": "https://groups.expedia.com/Group-Rate/?locale=en_US&ol=1"
+ },
+ {
+ "login": true,
+ "name": "facebook",
+ "test_url": "https://www.facebook.com",
+ "secondary_url": "https://www.facebook.com/marketplace/?ref=bookmark",
+ "test_cmds": [
+ [
+ "click.byXpathAndWait",
+ "/html/body/div[3]/div[2]/div/div/div/div/div[3]/button[1]"
+ ],
+ [
+ "click.byXpathAndWait",
+ "/html/body/div[3]/div[2]/div/div/div/div/div[3]/button[1]"
+ ]
+ ]
+ },
+ {
+ "login": true,
+ "login-test": true,
+ "name": "facebook-login",
+ "test_url": "https://www.facebook.com",
+ "type": "interactive",
+ "test_cmds": [
+ ["setup_login", "https://www.facebook.com"],
+ ["wait.byTime", 1000],
+ ["login", ""],
+ ["measure.start", "marketplace"],
+ ["navigate", "https://www.facebook.com/marketplace"],
+ ["measure.stop", ""]
+ ]
+ },
+ {
+ "login": false,
+ "name": "fandom",
+ "test_url": "https://www.fandom.com/articles/fallout-76-will-live-and-die-on-the-creativity-of-its-playerbase",
+ "test_cmds": [
+ ["click.byXpathAndWait", "/html/body/div[9]/div/div/div[2]/div[1]"],
+ ["click.byXpathAndWait", "/html/body/div[9]/div/div/div[2]/div[2]"]
+ ]
+ },
+ {
+ "login": true,
+ "name": "google",
+ "test_url": "https://www.google.com/search?hl=en&q=barack+obama&cad=h",
+ "test_cmds": [
+ [
+ "click.byXpathAndWait",
+ "/html/body/div[3]/div[3]/span/div/div/div/div[3]/button[1]/div"
+ ],
+ [
+ "click.byXpathAndWait",
+ "/html/body/c-wiz/div/div/div/div[2]/form/div/button/div[2]"
+ ]
+ ]
+ },
+ {
+ "login": false,
+ "name": "google-docs",
+ "test_url": "https://docs.google.com/document/d/1US-07msg12slQtI_xchzYxcKlTs6Fp7WqIc6W5GK5M8/edit?usp=sharing",
+ "secondary_url": "https://docs.google.com/document/d/1vUnn0ePU-ynArE1OdxyEHXR2G0sl74ja_st_4OOzlgE/preview"
+ },
+ {
+ "login": true,
+ "name": "google-mail",
+ "test_url": "https://mail.google.com/",
+ "test_cmds": [["click.byXpathAndWait", "/html/body/div/div/span[2]/a[2]"]]
+ },
+ {
+ "login": false,
+ "name": "google-slides",
+ "test_url": "https://docs.google.com/presentation/d/1Ici0ceWwpFvmIb3EmKeWSq_vAQdmmdFcWqaiLqUkJng/edit?usp=sharing",
+ "secondary_url": "https://docs.google.com/document/d/1vUnn0ePU-ynArE1OdxyEHXR2G0sl74ja_st_4OOzlgE/preview"
+ },
+ {
+ "login": false,
+ "name": "imdb",
+ "test_url": "https://www.imdb.com/title/tt0084967/?ref_=nv_sr_2",
+ "secondary_url": "https://www.imdb.com/title/tt0084967/episodes/?ref_=tt_ov_epl"
+ },
+ {
+ "login": false,
+ "name": "imgur",
+ "test_url": "https://imgur.com/gallery/m5tYJL6",
+ "inject_deterministic": false,
+ "test_cmds": [
+ [
+ "click.byXpathAndWait",
+ "/html/body/div[1]/div/div/div/div[2]/div/button[2]"
+ ],
+ [
+ "click.byXpathAndWait",
+ "/html/body/div[1]/div/div/div/div[3]/div[2]/button"
+ ]
+ ],
+ "secondary_url": "https://imgur.com/gallery/rCXZUil",
+ "preferences": {
+ "media.autoplay.default": 5,
+ "media.autoplay.ask-permission": true,
+ "media.autoplay.blocking_policy": 1,
+ "media.autoplay.block-webaudio": true,
+ "media.allowed-to-play.enabled": false,
+ "media.block-autoplay-until-in-foreground": true
+ }
+ },
+ {
+ "login": true,
+ "name": "instagram",
+ "test_url": "https://www.instagram.com/"
+ },
+ {
+ "login": true,
+ "name": "linkedin",
+ "test_url": "https://www.linkedin.com/in/thommy-harris-hk-385723106/",
+ "secondary_url": "https://www.linkedin.com/company/github?trk=affiliated-pages"
+ },
+ {
+ "login": false,
+ "name": "microsoft",
+ "test_url": "https://www.microsoft.com/en-us/",
+ "secondary_url": "https://support.microsoft.com/en-us",
+ "test_cmds": [
+ [
+ "click.byXpathAndWait",
+ "/html/body/div[1]/div/div/div[2]/div/div/div/div[2]/button[2]"
+ ],
+ [
+ "click.byXpath",
+ "/html/body/div[1]/div/div/div[2]/div/div/div[2]/div[2]/div/form/dl/dt[2]/div/div/div[2]/label"
+ ],
+ [
+ "click.byXpath",
+ "/html/body/div[1]/div/div/div[2]/div/div/div[2]/div[2]/div/form/dl/dt[3]/div/div/div[2]/label"
+ ],
+ [
+ "click.byXpath",
+ "/html/body/div[1]/div/div/div[2]/div/div/div[2]/div[2]/div/form/dl/dt[4]/div/div/div[2]/label"
+ ],
+ [
+ "click.byXpathAndWait",
+ "/html/body/div[1]/div/div/div[2]/div/div/div[2]/div[2]/div/div[2]/button[1]"
+ ]
+ ]
+ },
+ {
+ "login": true,
+ "name": "netflix",
+ "test_url": "https://www.netflix.com/title/80117263",
+ "secondary_url": "https://www.netflix.com/title/699257"
+ },
+ {
+ "login": false,
+ "name": "nytimes",
+ "test_url": "https://www.nytimes.com/2020/02/19/opinion/surprise-medical-bill.html",
+ "secondary_url": "https://www.nytimes.com/section/opinion/columnists",
+ "test_cmds": [
+ [
+ "click.byXpathAndWait",
+ "/html/body/div/div/div[2]/div/div[2]/div/div/div[1]/div[3]/div[3]/a"
+ ],
+ [
+ "click.byXpathAndWait",
+ "//*[@id='opt-out-of-new-york-times-nonessential-trackers']"
+ ]
+ ]
+ },
+ {
+ "login": true,
+ "name": "office",
+ "test_url": "https://www.office.com/launch/powerpoint/",
+ "secondary_url": "https://www.office.com/"
+ },
+ {
+ "login": true,
+ "name": "outlook",
+ "test_url": "https://outlook.live.com/mail/inbox"
+ },
+ {
+ "login": true,
+ "name": "paypal",
+ "test_url": "https://www.paypal.com/myaccount/summary/"
+ },
+ {
+ "login": true,
+ "name": "pinterest",
+ "test_url": "https://pinterest.com/",
+ "secondary_url": "https://www.pinterest.com/today/best/halloween-costumes-for-your-furry-friends/75787/"
+ },
+ {
+ "login": false,
+ "name": "reddit",
+ "test_url": "https://www.reddit.com/r/technology/comments/9sqwyh/we_posed_as_100_senators_to_run_ads_on_facebook/",
+ "secondary_url": "https://www.reddit.com/r/technology/",
+ "test_cmds": [
+ [
+ "click.byXpathAndWait",
+ "/html/body/div[1]/div/div[2]/div[3]/div[1]/section/div/section/section/form[1]/button"
+ ]
+ ]
+ },
+ {
+ "login": true,
+ "name": "tumblr",
+ "test_url": "https://www.tumblr.com/dashboard",
+ "secondary_url": "https://www.tumblr.com/tagged/funny+cats?sort=top"
+ },
+ {
+ "login": false,
+ "name": "twitch",
+ "test_url": "https://www.twitch.tv/videos/894226211",
+ "secondary_url": "https://www.twitch.tv/gmashley",
+ "preferences": {
+ "media.autoplay.default": 5,
+ "media.autoplay.ask-permission": true,
+ "media.autoplay.blocking_policy": 1,
+ "media.autoplay.block-webaudio": true,
+ "media.allowed-to-play.enabled": false,
+ "media.block-autoplay-until-in-foreground": true
+ },
+ "test_cmds": [
+ [
+ "click.byXpathAndWait",
+ "/html/body/div[1]/div/div[2]/div[1]/div/div/div[3]/div/button/div/div/div"
+ ],
+ [
+ "click.byXpathAndWait",
+ "/html/body/div[3]/div/div/div/div/div/div[1]/div[2]/div/button/div/div"
+ ]
+ ]
+ },
+ {
+ "login": true,
+ "name": "twitter",
+ "test_url": "https://twitter.com/BarackObama"
+ },
+ {
+ "login": false,
+ "name": "wikia",
+ "test_url": "https://marvel.fandom.com/wiki/Black_Panther",
+ "secondary_url": "https://marvel.fandom.com/wiki/Celestials",
+ "test_cmds": [
+ ["click.byXpathAndWait", "/html/body/div[6]/div/div/div[2]/div[1]"],
+ ["click.byXpathAndWait", "/html/body/div[7]/div/div/div[2]/div[2]"]
+ ]
+ },
+ {
+ "login": false,
+ "name": "wikipedia",
+ "test_url": "https://en.wikipedia.org/wiki/Barack_Obama",
+ "secondary_url": "https://en.wikipedia.org/wiki/Joe_Biden"
+ },
+ {
+ "login": true,
+ "name": "yahoo-mail",
+ "test_url": "https://mail.yahoo.com/"
+ },
+ {
+ "login": false,
+ "name": "youtube",
+ "test_url": "https://www.youtube.com",
+ "secondary_url": "https://www.youtube.com/watch?v=JrdEMERq8MA",
+ "test_cmds ": [
+ [
+ "click.byXpathAndWait",
+ "/html/body/ytd-app/ytd-consent-bump-v2-lightbox/tp-yt-paper-dialog/div[4]/div[2]/div[5]/div[2]/ytd-button-renderer[1]/a"
+ ],
+ [
+ "click.byXpath",
+ "/html/body/c-wiz/div/div/div/div[2]/div[2]/div[2]/div/div[2]/div[1]/div/button"
+ ],
+ [
+ "click.byXpath",
+ "/html/body/c-wiz/div/div/div/div[2]/div[3]/div[2]/div/div[2]/div[1]/div/button"
+ ],
+ [
+ "click.byXpath",
+ "/html/body/c-wiz/div/div/div/div[2]/div[4]/div[2]/div[2]/div/div[2]/div[1]/div/button"
+ ],
+ [
+ "click.byXpathAndWait",
+ "/html/body/c-wiz/div/div/div/div[2]/form/div/button/div[2]"
+ ]
+ ]
+ },
+ {
+ "login": false,
+ "name": "cnn-nav",
+ "test_url": "https://www.cnn.com/",
+ "type": "interactive",
+ "test_cmds": [
+ ["measure.start", "landing"],
+ ["navigate", "https://www.cnn.com"],
+ ["wait.byTime", 4000],
+ ["measure.stop", ""],
+ ["measure.start", "world"],
+ [
+ "click.byXpathAndWait",
+ "/html/body/div[5]/div/div/header/div/div[1]/div/div[2]/nav/ul/li[2]/a"
+ ],
+ ["wait.byTime", 1000],
+ ["measure.stop", ""]
+ ]
+ },
+ {
+ "login": false,
+ "name": "reddit-billgates-ama",
+ "test_url": "https://www.reddit.com/",
+ "type": "interactive",
+ "test_cmds": [
+ ["measure.start", "billg-ama"],
+ [
+ "navigate",
+ "https://www.reddit.com/r/IAmA/comments/m8n4vt/im_bill_gates_cochair_of_the_bill_and_melinda/"
+ ],
+ ["wait.byTime", 5000],
+ ["measure.stop", ""],
+ ["measure.start", "members"],
+ [
+ "click.byXpathAndWait",
+ "/html/body/div[1]/div/div[2]/div[2]/div/div[3]/div[2]/div/div[1]/div/div[4]/div[1]/div"
+ ],
+ ["wait.byTime", 1000],
+ ["measure.stop", ""]
+ ]
+ },
+ {
+ "login": false,
+ "name": "reddit-billgates-post",
+ "test_url": "https://www.reddit.com/user/thisisbillgates/",
+ "type": "interactive",
+ "test_cmds": [
+ ["measure.start", "billg"],
+ ["navigate", "https://www.reddit.com/user/thisisbillgates/"],
+ ["wait.byTime", 5000],
+ ["measure.stop", ""],
+ ["measure.start", "posts"],
+ [
+ "click.byXpathAndWait",
+ "/html/body/div[1]/div/div[2]/div[2]/div/div/div/div[2]/div[2]/div/div/div/a[2]"
+ ],
+ ["wait.byTime", 15000],
+ ["measure.stop", ""],
+ ["measure.start", "comments"],
+ [
+ "click.byXpathAndWait",
+ "/html/body/div[1]/div/div[2]/div[2]/div/div/div/div[2]/div[2]/div/div/div/a[3]"
+ ],
+ ["wait.byTime", 15000],
+ ["measure.stop", ""],
+ ["measure.start", "hot"],
+ [
+ "click.byXpathAndWait",
+ "/html/body/div[1]/div/div[2]/div[2]/div/div/div/div[2]/div[4]/div[1]/div[1]/div[2]/a[2]"
+ ],
+ ["wait.byTime", 15000],
+ ["measure.stop", ""],
+ ["measure.start", "top"],
+ [
+ "click.byXpathAndWait",
+ "/html/body/div[1]/div/div[2]/div[2]/div/div/div/div[2]/div[4]/div[1]/div[1]/div[2]/a[3]"
+ ],
+ ["wait.byTime", 15000],
+ ["measure.stop", ""]
+ ]
+ },
+ {
+ "login": false,
+ "name": "facebook-nav",
+ "test_url": "https://www.facebook.com/",
+ "type": "interactive",
+ "test_cmds": [
+ ["navigate", "https://www.facebook.com/login"],
+ ["wait.byTime", 30000],
+ ["measure.start", "landing"],
+ ["navigate", "https://www.facebook.com/"],
+ ["wait.byTime", 5000],
+ ["measure.stop", ""],
+ ["measure.start", "marketplace"],
+ ["navigate", "https://www.facebook.com/marketplace"],
+ ["wait.byTime", 5000],
+ ["measure.stop", ""],
+ ["measure.start", "groups"],
+ ["navigate", "https://www.facebook.com/groups/discover/"],
+ ["wait.byTime", 5000],
+ ["measure.stop", ""],
+ ["measure.start", "friends"],
+ ["navigate", "https://www.facebook.com/friends/"],
+ ["wait.byTime", 5000],
+ ["measure.stop", ""]
+ ]
+ }
+ ]
+}
diff --git a/testing/raptor/browsertime/process_switch.js b/testing/raptor/browsertime/process_switch.js
new file mode 100644
index 0000000000..a13094c6c5
--- /dev/null
+++ b/testing/raptor/browsertime/process_switch.js
@@ -0,0 +1,45 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/* eslint-env node */
+
+module.exports = async function (context, commands) {
+ context.log.info("Starting a process switch test");
+ let urlstr = context.options.browsertime.url;
+ let page_cycles = context.options.browsertime.page_cycles;
+ let page_cycle_delay = context.options.browsertime.page_cycle_delay;
+ let post_startup_delay = context.options.browsertime.post_startup_delay;
+
+ // Get the two urls to use in the test (the second one will be measured)
+ let urls = urlstr.split(",");
+ if (urls.length != 2) {
+ context.log.error(
+ `Wrong number of urls given. Expecting: 2, Given: ${urls.length}`
+ );
+ return false;
+ }
+
+ context.log.info(
+ "Waiting for %d ms (post_startup_delay)",
+ post_startup_delay
+ );
+ await commands.wait.byTime(post_startup_delay);
+
+ for (let count = 0; count < page_cycles; count++) {
+ context.log.info("Navigating to about:blank");
+ await commands.navigate("about:blank");
+
+ context.log.info("Cycle %d, waiting for %d ms", count, page_cycle_delay);
+ await commands.wait.byTime(page_cycle_delay);
+ context.log.info("Cycle %d, starting the measure", count);
+
+ await commands.navigate(urls[0]);
+ await commands.wait.byTime(3000);
+ await commands.measure.start(urls[1]);
+ await commands.wait.byTime(2000);
+ }
+
+ context.log.info("Process switch test ended.");
+ return true;
+};
diff --git a/testing/raptor/browsertime/speedometer3.js b/testing/raptor/browsertime/speedometer3.js
new file mode 100644
index 0000000000..48a06fcd1c
--- /dev/null
+++ b/testing/raptor/browsertime/speedometer3.js
@@ -0,0 +1,90 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/* eslint-env node */
+
+module.exports = async function (context, commands) {
+ context.log.info("Starting Speedometer 3 test");
+ let url = context.options.browsertime.url;
+ let page_cycles = context.options.browsertime.page_cycles;
+ let speedometer_iterations =
+ context.options.browsertime.speedometer_iterations;
+ let page_cycle_delay = context.options.browsertime.page_cycle_delay;
+ let post_startup_delay = context.options.browsertime.post_startup_delay;
+ let page_timeout = context.options.timeouts.pageLoad;
+ let expose_profiler = context.options.browsertime.expose_profiler;
+
+ context.log.info(
+ "Waiting for %d ms (post_startup_delay)",
+ post_startup_delay
+ );
+ await commands.wait.byTime(post_startup_delay);
+
+ for (let count = 0; count < page_cycles; count++) {
+ context.log.info("Navigating to about:blank");
+ await commands.navigate("about:blank");
+
+ context.log.info("Cycle %d, waiting for %d ms", count, page_cycle_delay);
+ await commands.wait.byTime(page_cycle_delay);
+
+ context.log.info("Cycle %d, starting the measure", count);
+ if (expose_profiler === "true") {
+ context.log.info("Custom profiler start!");
+ await commands.profiler.start();
+ }
+ await commands.measure.start(url);
+
+ await commands.js.runAndWait(`
+ window.testDone = false;
+ window.suiteValues = [];
+ const benchmarkClient = {
+ didRunSuites(measuredValues) {
+ window.suiteValues.push(measuredValues);
+ },
+ didFinishLastIteration() {
+ window.testDone = true;
+ }
+ };
+ window.Suites.forEach((el) => {
+ el.disabled = false;
+ });
+ // BenchmarkRunner is overriden as the InteractiveBenchmarkRunner
+ const runner = new BenchmarkRunner(window.Suites, ${speedometer_iterations});
+ runner._client = benchmarkClient;
+
+ runner.runSuites();
+ `);
+
+ let data_exists = false;
+ let starttime = await commands.js.run(`return performance.now();`);
+ while (
+ !data_exists &&
+ (await commands.js.run(`return performance.now();`)) - starttime <
+ page_timeout
+ ) {
+ let wait_time = 3000;
+ context.log.info("Waiting %d ms for data from speedometer...", wait_time);
+ await commands.wait.byTime(wait_time);
+ data_exists = await commands.js.run("return window.testDone;");
+ }
+ if (expose_profiler === "true") {
+ context.log.info("Custom profiler stop!");
+ await commands.profiler.stop();
+ }
+ if (
+ !data_exists &&
+ (await commands.js.run(`return performance.now();`)) - starttime >=
+ page_timeout
+ ) {
+ context.log.error("Benchmark timed out. Aborting...");
+ return false;
+ }
+ let data = await commands.js.run(`return window.suiteValues;`);
+ context.log.info("Value of benchmark data: ", data);
+
+ commands.measure.addObject({ browsertime_benchmark: { s3: data } });
+ }
+
+ return true;
+};
diff --git a/testing/raptor/browsertime/support-scripts/sample_python_support.py b/testing/raptor/browsertime/support-scripts/sample_python_support.py
new file mode 100644
index 0000000000..529c7c2cb0
--- /dev/null
+++ b/testing/raptor/browsertime/support-scripts/sample_python_support.py
@@ -0,0 +1,13 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+
+class SamplePythonSupport:
+ def __init__(self, **kwargs):
+ pass
+
+ def modify_command(self, cmd):
+ for i, entry in enumerate(cmd):
+ if "{replace-with-constant-value}" in entry:
+ cmd[i] = "25"
diff --git a/testing/raptor/browsertime/upload.js b/testing/raptor/browsertime/upload.js
new file mode 100644
index 0000000000..4f46ba0f50
--- /dev/null
+++ b/testing/raptor/browsertime/upload.js
@@ -0,0 +1,86 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/* eslint-env node */
+
+const path = require("path");
+
+async function waitForUpload(timeout, commands, context) {
+ let starttime = await commands.js.run(`return performance.now();`);
+ let status = "";
+
+ while (
+ (await commands.js.run(`return performance.now();`)) - starttime <
+ timeout &&
+ status != "error" &&
+ status != "success"
+ ) {
+ await commands.wait.byTime(10);
+
+ status = await commands.js.run(
+ `return document.getElementById('upload_status').innerHTML;`
+ );
+
+ context.log.info("context.log test: " + status);
+ console.log("test: " + status);
+ }
+
+ let endtime = await commands.js.run(`return performance.now();`);
+
+ return {
+ start: starttime,
+ end: endtime,
+ upload_status: status,
+ };
+}
+
+module.exports = async function (context, commands) {
+ let uploadSiteUrl = "https://uploadtest-381620.uc.r.appspot.com";
+ let iterations = `${context.options.browsertime.upload_iterations}`;
+
+ await commands.measure.start(uploadSiteUrl);
+ let accumulatedResults = [];
+ for (let iteration = 0; iteration < iterations; iteration++) {
+ await commands.navigate(uploadSiteUrl);
+
+ const driver = context.selenium.driver;
+ const webdriver = context.selenium.webdriver;
+
+ const uploadItem = await driver.findElement(webdriver.By.id("fileUpload"));
+
+ if (context.options.browsertime.moz_fetch_dir == "None") {
+ context.log.error(
+ "This test depends on the fetch task. Download the file, 'https://github.com/mozilla/perf-automation/raw/master/test_files/upload-test-32MB.dat' and set the os environment variable MOZ_FETCHES_DIR to that directory."
+ );
+ }
+
+ let localFilePath = path.join(
+ `${context.options.browsertime.moz_fetch_dir}`,
+ "upload-test-32MB.dat"
+ );
+
+ context.log.info("Sending file path: " + localFilePath);
+ await uploadItem.sendKeys(localFilePath);
+
+ // Start the test and wait for the upload to complete
+ let results = await waitForUpload(120000, commands, context);
+ let uploadTime = results.end - results.start;
+
+ // Store result in megabit/seconds, (Upload is a 50 MB file)
+ let uploadBandwidth = (50 * 8) / (uploadTime / 1000.0);
+ context.log.info(
+ "upload results: " +
+ results.upload_status +
+ " duration: " +
+ uploadTime +
+ " uploadBandwidth: " +
+ uploadBandwidth
+ );
+ accumulatedResults.push(uploadBandwidth);
+ }
+
+ commands.measure.addObject({
+ custom_data: { "upload-bandwidth": accumulatedResults },
+ });
+};
diff --git a/testing/raptor/browsertime/welcome.js b/testing/raptor/browsertime/welcome.js
new file mode 100644
index 0000000000..42861c8d03
--- /dev/null
+++ b/testing/raptor/browsertime/welcome.js
@@ -0,0 +1,30 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/* eslint-env node */
+
+module.exports = async function (context, commands) {
+ context.log.info("Starting a first-install test");
+ let page_cycles = context.options.browsertime.page_cycles;
+
+ for (let count = 0; count < page_cycles; count++) {
+ context.log.info("Navigating to about:blank");
+
+ // See bug 1717754
+ context.log.info(await commands.js.run(`return document.documentURI;`));
+
+ await commands.navigate("about:blank");
+
+ await commands.measure.start();
+ await commands.wait.byTime(1000);
+ await commands.navigate("about:welcome");
+ await commands.wait.byTime(2000);
+ await commands.measure.stop();
+
+ await commands.wait.byTime(2000);
+ }
+
+ context.log.info("First-install test ended.");
+ return true;
+};
generated by cgit v1.2.3 (git 2.39.1) at 2024-12-30 21:19:46 +0000