diff options
Diffstat (limited to '')
197 files changed, 23654 insertions, 0 deletions
diff --git a/testing/raptor/.eslintrc.js b/testing/raptor/.eslintrc.js new file mode 100644 index 0000000000..aaf8ae940c --- /dev/null +++ b/testing/raptor/.eslintrc.js @@ -0,0 +1,24 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +"use strict"; + +module.exports = { + env: { + webextensions: true, + }, + + globals: { + getTestConfig: false, + startMark: true, + endMark: true, + name: true, + }, + + plugins: ["mozilla"], + + rules: { + "mozilla/avoid-Date-timing": "error", + }, +}; diff --git a/testing/raptor/MANIFEST.in b/testing/raptor/MANIFEST.in new file mode 100644 index 0000000000..2de9af8012 --- /dev/null +++ b/testing/raptor/MANIFEST.in @@ -0,0 +1 @@ +recursive-include raptor *
\ No newline at end of file diff --git a/testing/raptor/browsertime/browsertime_benchmark.js b/testing/raptor/browsertime/browsertime_benchmark.js new file mode 100644 index 0000000000..1b42005695 --- /dev/null +++ b/testing/raptor/browsertime/browsertime_benchmark.js @@ -0,0 +1,81 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* eslint-env node */ + +module.exports = async function(context, commands) { + context.log.info("Starting a browsertime benchamrk"); + let url = context.options.browsertime.url; + let page_cycles = context.options.browsertime.page_cycles; + let page_cycle_delay = context.options.browsertime.page_cycle_delay; + let post_startup_delay = context.options.browsertime.post_startup_delay; + let page_timeout = context.options.timeouts.pageLoad; + let ret = false; + + context.log.info( + "Waiting for %d ms (post_startup_delay)", + post_startup_delay + ); + await commands.wait.byTime(post_startup_delay); + + for (let count = 0; count < page_cycles; count++) { + context.log.info("Navigating to about:blank"); + await commands.navigate("about:blank"); + + context.log.info("Cycle %d, waiting for %d ms", count, page_cycle_delay); + await commands.wait.byTime(page_cycle_delay); + + context.log.info("Cycle %d, starting the measure", count); + await commands.measure.start(url); + + context.log.info("Benchmark custom metric collection"); + + let data = null; + let starttime = await commands.js.run(`return performance.now();`); + while ( + data == null && + (await commands.js.run(`return performance.now();`)) - starttime < + page_timeout + ) { + let wait_time = 3000; + context.log.info("Waiting %d ms for data from benchmark...", wait_time); + await commands.wait.byTime(wait_time); + data = await commands.js.run( + "return window.sessionStorage.getItem('benchmark_results');" + ); + } + if ( + data == null && + (await commands.js.run(`return performance.now();`)) - starttime >= + page_timeout + ) { + ret = false; + context.log.error("Benchmark timed out. Aborting..."); + } else if (data) { + // Reset benchmark results + await commands.js.run( + "return window.sessionStorage.removeItem('benchmark_results');" + ); + + context.log.info("Value of benchmark data: ", data); + data = JSON.parse(data); + + if (!Array.isArray(data)) { + commands.measure.addObject({ browsertime_benchmark: data }); + } else { + commands.measure.addObject({ + browsertime_benchmark: { + [data[1]]: data.slice(2), + }, + }); + } + ret = true; + } else { + context.log.error("No data collected from benchmark."); + ret = false; + } + } + context.log.info("Browsertime benchmark ended."); + return ret; +}; diff --git a/testing/raptor/browsertime/browsertime_interactive.js b/testing/raptor/browsertime/browsertime_interactive.js new file mode 100644 index 0000000000..bffa5be082 --- /dev/null +++ b/testing/raptor/browsertime/browsertime_interactive.js @@ -0,0 +1,103 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* eslint-env node */ + +async function get_command_function(cmd, commands) { + /* + Converts a string such as `measure.start` into the actual + function that is found in the `commands` module. + + XXX: Find a way to share this function between + perftest_record.js and browsertime_interactive.js + */ + if (cmd == "") { + throw new Error("A blank command was given."); + } else if (cmd.endsWith(".")) { + throw new Error( + "An extra `.` was found at the end of this command: " + cmd + ); + } + + // `func` will hold the actual method that needs to be called, + // and the `parent_mod` is the context required to run the `func` + // method. Without that context, `this` becomes undefined in the browsertime + // classes. + let func = null; + let parent_mod = null; + for (let func_part of cmd.split(".")) { + if (func_part == "") { + throw new Error( + "An empty function part was found in the command: " + cmd + ); + } + + if (func === null) { + parent_mod = commands; + func = commands[func_part]; + } else if (func !== undefined) { + parent_mod = func; + func = func[func_part]; + } else { + break; + } + } + + if (func == undefined) { + throw new Error( + "The given command could not be found as a function: " + cmd + ); + } + + return [func, parent_mod]; +} + +module.exports = async function(context, commands) { + context.log.info("Starting an interactive browsertime test"); + let page_cycles = context.options.browsertime.page_cycles; + let post_startup_delay = context.options.browsertime.post_startup_delay; + let input_cmds = context.options.browsertime.commands; + + context.log.info( + "Waiting for %d ms (post_startup_delay)", + post_startup_delay + ); + await commands.wait.byTime(post_startup_delay); + + // unpack commands from python + let cmds = input_cmds.split(";;;"); + + for (let count = 0; count < page_cycles; count++) { + context.log.info("Navigating to about:blank w/nav, count: " + count); + await commands.navigate("about:blank"); + + let pages_visited = []; + for (let cmdstr of cmds) { + let [cmd, ...args] = cmdstr.split(":::"); + + if (cmd == "measure.start") { + if (args[0] != "") { + pages_visited.push(args[0]); + } + } + + let [func, parent_mod] = await get_command_function(cmd, commands); + + try { + await func.call(parent_mod, ...args); + } catch (e) { + context.log.info( + `Exception found while running \`commands.${cmd}(${args})\`: ` + ); + context.log.info(e.stack); + } + } + + // Log the number of pages visited for results parsing + context.log.info("[] metrics: pages_visited: " + pages_visited); + } + + context.log.info("Browsertime pageload ended."); + return true; +}; diff --git a/testing/raptor/browsertime/browsertime_pageload.js b/testing/raptor/browsertime/browsertime_pageload.js new file mode 100644 index 0000000000..e9c6a52f70 --- /dev/null +++ b/testing/raptor/browsertime/browsertime_pageload.js @@ -0,0 +1,412 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* eslint-env node */ + +const fs = require("fs"); +const http = require("http"); + +const URL = "/secrets/v1/secret/project/perftest/gecko/level-"; +const SECRET = "/perftest-login"; +const DEFAULT_SERVER = "https://firefox-ci-tc.services.mozilla.com"; + +const SCM_LOGIN_SITES = ["facebook", "netflix"]; + +/** + * This function obtains the perftest secret from Taskcluster. + * + * It will NOT work locally. Please see the get_logins function, you + * will need to define a JSON file and set the RAPTOR_LOGINS + * env variable to its path. + */ +async function get_tc_secrets(context) { + const MOZ_AUTOMATION = process.env.MOZ_AUTOMATION; + if (!MOZ_AUTOMATION) { + throw Error( + "Not running in CI. Set RAPTOR_LOGINS to a JSON file containing the logins." + ); + } + + let TASKCLUSTER_PROXY_URL = process.env.TASKCLUSTER_PROXY_URL + ? process.env.TASKCLUSTER_PROXY_URL + : DEFAULT_SERVER; + + let MOZ_SCM_LEVEL = process.env.MOZ_SCM_LEVEL ? process.env.MOZ_SCM_LEVEL : 1; + + const url = TASKCLUSTER_PROXY_URL + URL + MOZ_SCM_LEVEL + SECRET; + + const data = await new Promise((resolve, reject) => { + context.log.info("Obtaining secrets for login..."); + + http.get( + url, + { + headers: { + "Content-Type": "application/json", + Accept: "application/json", + }, + }, + res => { + let data = ""; + context.log.info(`Secret status code: ${res.statusCode}`); + + res.on("data", d => { + data += d.toString(); + }); + + res.on("end", () => { + resolve(data); + }); + + res.on("error", error => { + context.log.error(error); + reject(error); + }); + } + ); + }); + + return JSON.parse(data); +} + +/** + * This function gets the login information required. + * + * It starts by looking for a local file whose path is defined + * within RAPTOR_LOGINS. If we don't find this file, then we'll + * attempt to get the login information from our Taskcluster secret. + * If MOZ_AUTOMATION is undefined, then the test will fail, Taskcluster + * secrets can only be obtained in CI. + */ +async function get_logins(context) { + let logins; + + let RAPTOR_LOGINS = process.env.RAPTOR_LOGINS; + if (RAPTOR_LOGINS) { + // Get logins from a local file + if (!RAPTOR_LOGINS.endsWith(".json")) { + throw Error( + `File given for logins does not end in '.json': ${RAPTOR_LOGINS}` + ); + } + + let logins_file = null; + try { + logins_file = await fs.readFileSync(RAPTOR_LOGINS, "utf8"); + } catch (err) { + throw Error(`Failed to read the file ${RAPTOR_LOGINS}: ${err}`); + } + + logins = await JSON.parse(logins_file); + } else { + // Get logins from a perftest Taskcluster secret + logins = await get_tc_secrets(context); + } + + return logins; +} + +/** + * This function returns the type of login to do. + * + * This function returns "single-form" when we find a single form. If we only + * find a single input field, we assume that there is one page per input + * and return "multi-page". Otherwise, we return null. + */ +async function get_login_type(context, commands) { + /* + Determine if there's a password field visible with this + query selector. Some sites use `tabIndex` to hide the password + field behind other elements. In this case, we are searching + for any password-type field that has a tabIndex of 0 or undefined and + is not hidden. + */ + let input_length = await commands.js.run(` + return document.querySelectorAll( + "input[type=password][tabIndex='0']:not([type=hidden])," + + "input[type=password]:not([tabIndex]):not([type=hidden])" + ).length; + `); + + if (input_length == 0) { + context.log.info("Found a multi-page login"); + return multi_page_login; + } else if (input_length == 1) { + context.log.info("Found a single-page login"); + return single_page_login; + } + + if ( + (await commands.js.run( + `return document.querySelectorAll("form").length;` + )) >= 1 + ) { + context.log.info("Found a single-form login"); + return single_form_login; + } + + return null; +} + +/** + * This function sets up the login for a single form. + * + * The username field is defined as the field which immediately precedes + * the password field. We have to do this in two steps because we need + * to make sure that the event we emit from the change has the `isTrusted` + * field set to `true`. Otherwise, some websites will ignore the input and + * the form submission. + */ +async function single_page_login(login_info, context, commands, prefix = "") { + // Get the first input field in the form that is not hidden and add the + // username. Assumes that email/username is always the first input field. + await commands.addText.bySelector( + login_info.username, + `${prefix}input:not([type=hidden]):not([type=password])` + ); + + // Get the password field and ensure it's not hidden. + await commands.addText.bySelector( + login_info.password, + `${prefix}input[type=password]:not([type=hidden])` + ); + + return undefined; +} + +/** + * See single_page_login. + */ +async function single_form_login(login_info, context, commands) { + return single_page_login(login_info, context, commands, "form "); +} + +/** + * Login to a website that uses multiple pages for the login. + * + * WARNING: Assumes that the first page is for the username. + */ +// TODO cleanup comments +async function multi_page_login(login_info, context, commands) { + const driver = context.selenium.driver; + const webdriver = context.selenium.webdriver; + //TODO fails here in netflix for Try... + const username_field = await driver.findElement( + webdriver.By.css(`input:not([type=hidden]):not([type=password])`) + ); + await username_field.sendKeys(login_info.username); + await username_field.sendKeys(webdriver.Key.ENTER); + await commands.wait.byTime(5000); + let password_field; + try { + password_field = await driver.findElement( + webdriver.By.css(`input[type=password]:not([type=hidden])`) + ); + } catch (err) { + if (err.toString().includes("NoSuchElementError")) { + // Sometimes we're suspicious (i.e. they think we're a bot/bad-actor) + let name_field = await driver.findElement( + webdriver.By.css(`input:not([type=hidden]):not([type=password])`) + ); + await name_field.sendKeys(login_info.suspicious_answer); + await name_field.sendKeys(webdriver.Key.ENTER); + await commands.wait.byTime(5000); + + // Try getting the password field again + password_field = await driver.findElement( + webdriver.By.css(`input[type=password]:not([type=hidden])`) + ); + } else { + throw err; + } + } + + await password_field.sendKeys(login_info.password); + + return async function() { + password_field.sendKeys(webdriver.Key.ENTER); + await commands.wait.byTime(5000); + }; +} + +/** + * This function sets up the login. + * + * This is done by first the login type, and then performing the + * actual login setup. The return is a possible button to click + * to perform the login. + */ +async function setup_login(login_info, context, commands) { + let login_func = await get_login_type(context, commands); + if (!login_func) { + throw Error("Could not determine the type of login page."); + } + + try { + return await login_func(login_info, context, commands); + } catch (err) { + throw Error(`Could not setup login information: ${err}`); + } +} + +/** + * This function performs the login. + * + * It does this by either clicking on a button with a type + * of "sumbit", or running a final_button function that was + * obtained from the setup_login function. Some pages also ask + * questions about setting up 2FA or other information. Generally, + * these contain the "skip" text. + */ +async function login(context, commands, final_button) { + try { + if (!final_button) { + // The mouse double click emits an event with `evt.isTrusted=true` + await commands.mouse.doubleClick.bySelector("button[type=submit]"); + await commands.wait.byTime(10000); + } else { + // In some cases, it's preferable to be given a function for the final button + await final_button(); + } + + // Some pages ask to setup 2FA, skip this based on the text + const XPATHS = [ + "//a[contains(text(), 'skip')]", + "//button[contains(text(), 'skip')]", + "//input[contains(text(), 'skip')]", + "//div[contains(text(), 'skip')]", + ]; + + for (let xpath of XPATHS) { + try { + await commands.mouse.doubleClick.byXpath(xpath); + } catch (err) { + if (err.toString().includes("not double click")) { + context.log.info(`Can't find a button with the text: ${xpath}`); + } else { + throw err; + } + } + } + } catch (err) { + throw Error( + `Could not login to website as we could not find the submit button/input: ${err}` + ); + } +} + +/** + * Grab the base URL from the browsertime url. + * + * This is a necessary step for getting the login values from the Taskcluster + * secrets, which are hashed by the base URL. + * + * The first entry is the protocal, third is the top-level domain (or host) + */ +function get_base_URL(fullUrl) { + let pathAsArray = fullUrl.split("/"); + return pathAsArray[0] + "//" + pathAsArray[2]; +} + +/** + * This function attempts the login-login sequence for a live pageload test + */ +async function perform_live_login(context, commands) { + let testUrl = context.options.browsertime.url; + + let logins = await get_logins(context); + const baseUrl = get_base_URL(testUrl); + + await commands.navigate("about:blank"); + + let login_info = logins.secret[baseUrl]; + try { + await commands.navigate(login_info.login_url); + } catch (err) { + context.log.info("Unable to acquire login information"); + throw err; + } + await commands.wait.byTime(10000); + + let final_button = await setup_login(login_info, context, commands); + await login(context, commands, final_button); +} + +module.exports = async function(context, commands) { + context.log.info("Starting a browsertime pageload"); + let test_url = context.options.browsertime.url; + let secondary_url = context.options.browsertime.secondary_url; + let page_cycles = context.options.browsertime.page_cycles; + let page_cycle_delay = context.options.browsertime.page_cycle_delay; + let post_startup_delay = context.options.browsertime.post_startup_delay; + let chimera_mode = context.options.browsertime.chimera; + let test_bytecode_cache = context.options.browsertime.test_bytecode_cache; + let login_required = context.options.browsertime.loginRequired; + let live_site = context.options.browsertime.liveSite; + let test_name = context.options.browsertime.testName; + + context.log.info( + "Waiting for %d ms (post_startup_delay)", + post_startup_delay + ); + await commands.wait.byTime(post_startup_delay); + let cached = false; + + // Login once before testing the test_url/secondary_url cycles + // If the user has RAPTOR_LOGINS configured correctly, a local login pageload + // test can be attempted. Otherwise if attempting it in CI, only sites with the + // associated MOZ_SCM_LEVEL will be attempted (e.g. Try = 1, autoland = 3). + // In addition, ensure login sequence is only attempted on live sites and for sites + // that we have Taskcluster secrets for. + if ( + login_required == "True" && + live_site == "True" && + SCM_LOGIN_SITES.includes(test_name) + ) { + await perform_live_login(context, commands); + } + + for (let count = 0; count < page_cycles; count++) { + if (count !== 0 && secondary_url !== undefined) { + context.log.info("Navigating to secondary url:" + secondary_url); + await commands.navigate(secondary_url); + await commands.wait.byTime(1000); + await commands.js.runAndWait(` + (function() { + const white = document.createElement('div'); + white.id = 'raptor-white'; + white.style.position = 'absolute'; + white.style.top = '0'; + white.style.left = '0'; + white.style.width = Math.max(document.documentElement.clientWidth, document.body.clientWidth) + 'px'; + white.style.height = Math.max(document.documentElement.clientHeight,document.body.clientHeight) + 'px'; + white.style.backgroundColor = 'white'; + white.style.zIndex = '2147483647'; + document.body.appendChild(white); + document.body.style.display = ''; + })();`); + await commands.wait.byTime(1000); + } else { + context.log.info("Navigating to about:blank, count: " + count); + await commands.navigate("about:blank"); + } + + context.log.info("Navigating to primary url:" + test_url); + context.log.info("Cycle %d, waiting for %d ms", count, page_cycle_delay); + await commands.wait.byTime(page_cycle_delay); + + context.log.info("Cycle %d, starting the measure", count); + await commands.measure.start(test_url); + + // Wait 20 seconds to populate bytecode cache + if (test_bytecode_cache == "true" && chimera_mode == "true" && !cached) { + context.log.info("Waiting 20s to populate bytecode cache..."); + await commands.wait.byTime(20000); + cached = true; + } + } + + context.log.info("Browsertime pageload ended."); + return true; +}; diff --git a/testing/raptor/browsertime/browsertime_scenario.js b/testing/raptor/browsertime/browsertime_scenario.js new file mode 100644 index 0000000000..15b199e1a8 --- /dev/null +++ b/testing/raptor/browsertime/browsertime_scenario.js @@ -0,0 +1,67 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* eslint-env node */ + +module.exports = async function(context, commands) { + context.log.info("Starting a browsertime scenario test"); + let page_cycles = context.options.browsertime.page_cycles; + let page_cycle_delay = context.options.browsertime.page_cycle_delay; + let post_startup_delay = context.options.browsertime.post_startup_delay; + let scenario_time = context.options.browsertime.scenario_time; + let background_app = context.options.browsertime.background_app == "true"; + let app = null; + + if (background_app) { + if (!context.options.android) { + throw new Error("Cannot background an application on desktop"); + } + app = context.options.firefox.android.package; + } + + context.log.info( + "Waiting for %d ms (post_startup_delay)", + post_startup_delay + ); + await commands.wait.byTime(post_startup_delay); + + for (let count = 0; count < page_cycles; count++) { + context.log.info("Navigating to about:blank"); + await commands.navigate("about:blank"); + + context.log.info("Cycle %d, waiting for %d ms", count, page_cycle_delay); + await commands.wait.byTime(page_cycle_delay); + + context.log.info("Cycle %d, starting the measure", count); + + if (background_app) { + // Background the application and disable doze for it + await commands.android.shell(`dumpsys deviceidle whitelist +${app}`); + await commands.android.shell(`input keyevent 3`); + await commands.wait.byTime(1000); + const foreground = await commands.android.shell( + "dumpsys window windows | grep mCurrentFocus" + ); + if (foreground.includes(app)) { + throw new Error("Application was not backgrounded successfully"); + } else { + context.log.info("Application was backgrounded successfully"); + } + } + + // Run the test + await commands.measure.start("about:blank test"); + context.log.info("Waiting for %d ms for this scenario", scenario_time); + await commands.wait.byTime(scenario_time); + await commands.measure.stop(); + + if (background_app) { + // Foreground the application and enable doze again + await commands.android.shell(`am start --activity-single-top ${app}`); + await commands.android.shell(`dumpsys deviceidle enable`); + } + } + context.log.info("Browsertime scenario test ended."); + return true; +}; diff --git a/testing/raptor/browsertime/grandprix.js b/testing/raptor/browsertime/grandprix.js new file mode 100644 index 0000000000..99bb31935d --- /dev/null +++ b/testing/raptor/browsertime/grandprix.js @@ -0,0 +1,80 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* eslint-env node */ + +module.exports = async function(context, commands) { + const benchmark_url = "https://grandprixbench.netlify.app/"; + const iterations = `${context.options.browsertime.grandprix_iterations}`; + const tests = [ + "Vanilla-HTML-TodoMVC", + "Vanilla-TodoMVC", + "Preact-TodoMVC", + "Preact-TodoMVC-Modern", + "React-TodoMVC", + //"Elm-TodoMVC", // removed from suite + //"Rust-Yew-TodoMVC", // removed from suite + //"Hydration-Preact", // removed from suite + //"Scroll-Windowing-React", // removed from suite + "Monaco-Editor", + "Monaco-Syntax-Highlight", + "React-Stockcharts", + "React-Stockcharts-SVG", + "Leaflet-Fractal", + //"SVG-UI", // removed from suite + "Proxx-Tables", + "Proxx-Tables-Lit", + "Proxx-Tables-Canvas", + ]; + + // Build test url + let url = benchmark_url + "?suites="; + tests.forEach(test => { + url = url + test + ","; + }); + url = url + "&iterations=" + iterations; + + console.log("Using url=" + url); + await commands.measure.start(url); + + await commands.wait.byTime("3000"); + await commands.click.byXpath("/html/body/main/div/div[1]/div/div/button"); + + let finished = 0; + do { + await commands.wait.byTime("30000"); + finished = await commands.js.run('return typeof results != "undefined"'); + } while (!finished); + + if (context.options.browser === "firefox") { + let buildId = await commands.js.runPrivileged( + "return Services.appinfo.appBuildID;" + ); + console.log(buildId); + } + + let output = {}; + output.score = {}; + output.score.total = await commands.js.run("return results.Score"); + output.iterations = await commands.js.run("return results.iterations"); + + let subtests = {}; + for (let i = 0; i < output.iterations.length; i++) { + for (const [key, value] of Object.entries(output.iterations[i])) { + if (!subtests.hasOwnProperty(key)) { + subtests[key] = []; + } + subtests[key].push(value.total); + } + } + + console.log("score, ", output.score.total.mean); + for (const [key, value] of Object.entries(subtests)) { + const average = value.reduce((a, b) => a + b, 0) / value.length; + output.score[key] = average; + console.log(key, ",", average); + } + + await commands.measure.add("grandprix-s3", output); +}; diff --git a/testing/raptor/browsertime/pageload_sites.json b/testing/raptor/browsertime/pageload_sites.json new file mode 100644 index 0000000000..5872b401e1 --- /dev/null +++ b/testing/raptor/browsertime/pageload_sites.json @@ -0,0 +1,514 @@ +{ + "mobile": [ + { + "login": false, + "name": "allrecipes", + "test_url": "https://www.allrecipes.com/" + }, + { + "login": false, + "name": "amazon", + "test_url": "https://www.amazon.com" + }, + { + "login": false, + "name": "amazon-search", + "test_url": "https://www.amazon.com/s/ref=nb_sb_noss_2/139-6317191-5622045?url=search-alias%3Daps&field-keywords=mobile+phone" + }, + { + "login": false, + "name": "bing", + "test_url": "https://www.bing.com/" + }, + { + "login": false, + "name": "bing-search-restaurants", + "test_url": "https://www.bing.com/search?q=restaurants+in+exton+pa+19341" + }, + { + "login": false, + "name": "booking", + "test_url": "https://www.booking.com/" + }, + { + "login": false, + "name": "cnn", + "test_url": "https://cnn.com", + "secondary_url": "https://www.cnn.com/weather" + }, + { + "login": false, + "name": "cnn-ampstories", + "test_url": "https://cnn.com/ampstories/us/why-hurricane-michael-is-a-monster-unlike-any-other" + }, + { + "login": false, + "name": "dailymail", + "test_url": "https://www.dailymail.co.uk/sciencetech/article-9749081/Experts-say-Hubble-repair-despite-NASA-insisting-multiple-options-fix.html" + }, + { + "login": false, + "name": "ebay-kleinanzeigen", + "test_url": "https://m.ebay-kleinanzeigen.de" + }, + { + "login": false, + "name": "ebay-kleinanzeigen-search", + "test_url": "https://m.ebay-kleinanzeigen.de/s-anzeigen/auf-zeit-wg-berlin/zimmer/c199-l3331" + }, + { + "login": false, + "name": "espn", + "test_url": "http://www.espn.com/nba/story/_/page/allstarweekend25788027/the-comparison-lebron-james-michael-jordan-their-own-words" + }, + { + "login": true, + "name": "facebook", + "test_url": "https://m.facebook.com" + }, + { + "login": false, + "name": "facebook-cristiano", + "test_url": "https://m.facebook.com/Cristiano" + }, + { + "login": true, + "name": "google", + "test_url": "https://www.google.com" + }, + { + "login": false, + "name": "google-maps", + "test_url": "https://www.google.com/maps?force=pwa" + }, + { + "login": true, + "name": "google-search-restaurants", + "test_url": "https://www.google.com/search?q=restaurants+near+me" + }, + { + "login": false, + "name": "imdb", + "test_url": "https://m.imdb.com/" + }, + { + "login": true, + "name": "instagram", + "test_url": "https://www.instagram.com" + }, + { + "login": false, + "name": "sina", + "test_url": "https://www.sina.com.cn/" + }, + { + "login": false, + "name": "microsoft-support", + "test_url": "https://support.microsoft.com/en-us" + }, + { + "login": false, + "name": "reddit", + "test_url": "https://www.reddit.com" + }, + { + "login": false, + "name": "stackoverflow", + "test_url": "https://stackoverflow.com/" + }, + { + "login": false, + "name": "web-de", + "test_url": "https://web.de/magazine/politik/politologe-glaubt-grossen-koalition-herbst-knallen-33563566" + }, + { + "login": false, + "name": "wikipedia", + "test_url": "https://en.m.wikipedia.org/wiki/Main_Page" + }, + { + "login": false, + "name": "youtube", + "test_url": "https://m.youtube.com" + }, + { + "login": false, + "name": "youtube-watch", + "test_url": "https://www.youtube.com/watch?v=COU5T-Wafa4" + } + ], + "desktop": [ + { + "login": false, + "name": "amazon", + "test_url": "https://www.amazon.com/s?k=laptop&ref=nb_sb_noss_1", + "secondary_url": "https://www.amazon.com/Acer-A515-46-R14K-Quad-Core-Processor-Backlit/dp/B08VKNVDDR/ref=sr_1_3?dchild=1&keywords=laptop&qid=1627047187&sr=8-3" + }, + { + "login": false, + "name": "bing-search", + "test_url": "https://www.bing.com/search?q=barack+obama" + }, + { + "login": false, + "name": "buzzfeed", + "test_url": "https://www.buzzfeed.com/", + "secondary_url": "https://www.buzzfeed.com/quizzes", + "test_cmds": [ + ["click.byXpathAndWait", "/html/body/div[1]/div/div/div/div[2]/div/button[2]"], + ["click.byXpathAndWait", "/html/body/div[1]/div/div/div/div[3]/div[2]/button"] + ] + }, + { + "login": false, + "name": "cnn", + "test_url": "https://www.cnn.com/2021/03/22/weather/climate-change-warm-waters-lake-michigan/index.html", + "secondary_url": "https://www.cnn.com/weather", + "preferences": { + "media.autoplay.default": 5, + "media.autoplay.ask-permission": true, + "media.autoplay.blocking_policy": 1, + "media.autoplay.block-webaudio": true, + "media.allowed-to-play.enabled": false, + "media.block-autoplay-until-in-foreground": true + }, + "test_cmds": [ + ["click.byXpathAndWait", "//*[@id='onetrust-pc-btn-handler']"], + ["click.byXpathAndWait", "/html/body/div[13]/div[2]/div/section/div[17]/button[2]"] + ] + }, + { + "login": false, + "name": "ebay", + "test_url": "https://www.ebay.com/", + "secondary_url": "https://www.ebay.com/deals", + "test_cmds": [ + ["click.byXpathAndWait", "/html/body/div[5]/div[1]/div[2]/div[2]/div[2]/a"], + ["click.byXpathAndWait", "/html/body/div[4]/div/div[2]/section[3]/section/button"] + ] + }, + { + "login": false, + "name": "espn", + "test_url": "http://www.espn.com/nba/story/_/page/allstarweekend25788027/the-comparison-lebron-james-michael-jordan-their-own-words", + "test_cmds": [ + ["click.byXpathAndWait", "//*[@id='onetrust-pc-btn-handler']"], + ["click.byXpathAndWait", "/html/body/div[10]/div[2]/div[3]/div[1]/button"] + ] + }, + { + "login": false, + "name": "expedia", + "test_url": "https://expedia.com/Hotel-Search?destination=New+York%2C+New+York&latLong=40.756680%2C-73.986470®ionId=178293&startDate=&endDate=&rooms=1&_xpid=11905%7C1&adults=2", + "secondary_url": "https://groups.expedia.com/Group-Rate/?locale=en_US&ol=1" + }, + { + "login": true, + "name": "facebook", + "test_url": "https://www.facebook.com", + "secondary_url": "https://www.facebook.com/marketplace/?ref=bookmark", + "test_cmds": [ + ["click.byXpathAndWait", "/html/body/div[3]/div[2]/div/div/div/div/div[3]/button[1]"], + ["click.byXpathAndWait", "/html/body/div[3]/div[2]/div/div/div/div/div[3]/button[1]"] + ] + }, + { + "login": true, + "login-test": true, + "name": "facebook-login", + "test_url": "https://www.facebook.com", + "type": "interactive", + "test_cmds": [ + ["setup_login", "https://www.facebook.com"], + ["wait.byTime", 1000], + ["login", ""], + ["measure.start", "marketplace"], + ["navigate", "https://www.facebook.com/marketplace"], + ["measure.stop", ""] + ] + }, + { + "login": false, + "name": "fandom", + "test_url": "https://www.fandom.com/articles/fallout-76-will-live-and-die-on-the-creativity-of-its-playerbase", + "test_cmds": [ + ["click.byXpathAndWait", "/html/body/div[9]/div/div/div[2]/div[1]"], + ["click.byXpathAndWait", "/html/body/div[9]/div/div/div[2]/div[2]"] + ] + }, + { + "login": true, + "name": "google", + "test_url": "https://www.google.com/search?hl=en&q=barack+obama&cad=h", + "test_cmds": [ + ["click.byXpathAndWait", "/html/body/div[3]/div[3]/span/div/div/div/div[3]/button[1]/div"], + ["click.byXpathAndWait", "/html/body/c-wiz/div/div/div/div[2]/form/div/button/div[2]"] + ] + }, + { + "login": false, + "name": "google-docs", + "test_url": "https://docs.google.com/document/d/1US-07msg12slQtI_xchzYxcKlTs6Fp7WqIc6W5GK5M8/edit?usp=sharing", + "secondary_url": "https://docs.google.com/document/d/1vUnn0ePU-ynArE1OdxyEHXR2G0sl74ja_st_4OOzlgE/preview" + }, + { + "login": true, + "name": "google-mail", + "test_url": "https://mail.google.com/", + "test_cmds": [ + ["click.byXpathAndWait", "/html/body/div/div/span[2]/a[2]"] + ] + }, + { + "login": false, + "name": "google-slides", + "test_url": "https://docs.google.com/presentation/d/1Ici0ceWwpFvmIb3EmKeWSq_vAQdmmdFcWqaiLqUkJng/edit?usp=sharing", + "secondary_url": "https://docs.google.com/document/d/1vUnn0ePU-ynArE1OdxyEHXR2G0sl74ja_st_4OOzlgE/preview" + }, + { + "login": false, + "name": "imdb", + "test_url": "https://www.imdb.com/title/tt0084967/?ref_=nv_sr_2", + "secondary_url": "https://www.imdb.com/title/tt0084967/episodes/?ref_=tt_ov_epl" + }, + { + "login": false, + "name": "imgur", + "test_url": "https://imgur.com/gallery/m5tYJL6", + "inject_deterministic": false, + "test_cmds": [ + ["click.byXpathAndWait", "/html/body/div[1]/div/div/div/div[2]/div/button[2]"], + ["click.byXpathAndWait", "/html/body/div[1]/div/div/div/div[3]/div[2]/button"] + ], + "secondary_url": "https://imgur.com/gallery/L13Ci", + "preferences": { + "media.autoplay.default": 5, + "media.autoplay.ask-permission": true, + "media.autoplay.blocking_policy": 1, + "media.autoplay.block-webaudio": true, + "media.allowed-to-play.enabled": false, + "media.block-autoplay-until-in-foreground": true + } + }, + { + "login": true, + "name": "instagram", + "test_url": "https://www.instagram.com/" + }, + { + "login": true, + "name": "linkedin", + "test_url": "https://www.linkedin.com/in/thommy-harris-hk-385723106/", + "secondary_url": "https://www.linkedin.com/company/github?trk=affiliated-pages" + }, + { + "login": false, + "name": "microsoft", + "test_url": "https://www.microsoft.com/en-us/", + "secondary_url": "https://support.microsoft.com/en-us", + "test_cmds": [ + ["click.byXpathAndWait", "/html/body/div[1]/div/div/div[2]/div/div/div/div[2]/button[2]"], + ["click.byXpath", "/html/body/div[1]/div/div/div[2]/div/div/div[2]/div[2]/div/form/dl/dt[2]/div/div/div[2]/label"], + ["click.byXpath", "/html/body/div[1]/div/div/div[2]/div/div/div[2]/div[2]/div/form/dl/dt[3]/div/div/div[2]/label"], + ["click.byXpath", "/html/body/div[1]/div/div/div[2]/div/div/div[2]/div[2]/div/form/dl/dt[4]/div/div/div[2]/label"], + ["click.byXpathAndWait", "/html/body/div[1]/div/div/div[2]/div/div/div[2]/div[2]/div/div[2]/button[1]"] + ] + }, + { + "login": true, + "name": "netflix", + "test_url": "https://www.netflix.com/title/80117263", + "secondary_url": "https://www.netflix.com/title/699257" + }, + { + "login": false, + "name": "nytimes", + "test_url": "https://www.nytimes.com/2020/02/19/opinion/surprise-medical-bill.html", + "secondary_url": "https://www.nytimes.com/section/opinion/columnists", + "test_cmds": [ + ["click.byXpathAndWait", "/html/body/div/div/div[2]/div/div[2]/div/div/div[1]/div[3]/div[3]/a"], + ["click.byXpathAndWait", "//*[@id='opt-out-of-new-york-times-nonessential-trackers']"] + ] + }, + { + "login": true, + "name": "office", + "test_url": "https://www.office.com/launch/powerpoint/", + "secondary_url": "https://www.office.com/" + }, + { + "login": true, + "name": "outlook", + "test_url": "https://outlook.live.com/mail/inbox" + }, + { + "login": true, + "name": "paypal", + "test_url": "https://www.paypal.com/myaccount/summary/" + }, + { + "login": true, + "name": "pinterest", + "test_url": "https://pinterest.com/", + "secondary_url": "https://www.pinterest.com/today/best/halloween-costumes-for-your-furry-friends/75787/" + }, + { + "login": false, + "name": "reddit", + "test_url": "https://www.reddit.com/r/technology/comments/9sqwyh/we_posed_as_100_senators_to_run_ads_on_facebook/", + "secondary_url": "https://www.reddit.com/r/technology/", + "test_cmds": [ + ["click.byXpathAndWait", "/html/body/div[1]/div/div[2]/div[3]/div[1]/section/div/section/section/form[1]/button"] + ] + }, + { + "login": true, + "name": "tumblr", + "test_url": "https://www.tumblr.com/dashboard", + "secondary_url": "https://www.tumblr.com/tagged/funny+cats?sort=top" + }, + { + "login": false, + "name": "twitch", + "test_url": "https://www.twitch.tv/videos/894226211", + "secondary_url": "https://www.twitch.tv/gmashley", + "preferences": { + "media.autoplay.default": 5, + "media.autoplay.ask-permission": true, + "media.autoplay.blocking_policy": 1, + "media.autoplay.block-webaudio": true, + "media.allowed-to-play.enabled": false, + "media.block-autoplay-until-in-foreground": true + }, + "test_cmds": [ + ["click.byXpathAndWait", "/html/body/div[1]/div/div[2]/div[1]/div/div/div[3]/div/button/div/div/div"], + ["click.byXpathAndWait", "/html/body/div[3]/div/div/div/div/div/div[1]/div[2]/div/button/div/div"] + ] + }, + { + "login": true, + "name": "twitter", + "test_url": "https://twitter.com/BarackObama" + }, + { + "login": false, + "name": "wikia", + "test_url": "https://marvel.fandom.com/wiki/Black_Panther", + "secondary_url": "https://marvel.fandom.com/wiki/Celestials", + "test_cmds": [ + ["click.byXpathAndWait", "/html/body/div[6]/div/div/div[2]/div[1]"], + ["click.byXpathAndWait", "/html/body/div[7]/div/div/div[2]/div[2]"] + ] + }, + { + "login": false, + "name": "wikipedia", + "test_url": "https://en.wikipedia.org/wiki/Barack_Obama", + "secondary_url": "https://en.wikipedia.org/wiki/Joe_Biden" + }, + { + "login": true, + "name": "yahoo-mail", + "test_url": "https://mail.yahoo.com/" + }, + { + "login": false, + "name": "youtube", + "test_url": "https://www.youtube.com", + "secondary_url": "https://www.youtube.com/watch?v=JrdEMERq8MA", + "test_cmds ": [ + ["click.byXpathAndWait", "/html/body/ytd-app/ytd-consent-bump-v2-lightbox/tp-yt-paper-dialog/div[4]/div[2]/div[5]/div[2]/ytd-button-renderer[1]/a"], + ["click.byXpath", "/html/body/c-wiz/div/div/div/div[2]/div[2]/div[2]/div/div[2]/div[1]/div/button"], + ["click.byXpath", "/html/body/c-wiz/div/div/div/div[2]/div[3]/div[2]/div/div[2]/div[1]/div/button"], + ["click.byXpath", "/html/body/c-wiz/div/div/div/div[2]/div[4]/div[2]/div[2]/div/div[2]/div[1]/div/button"], + ["click.byXpathAndWait", "/html/body/c-wiz/div/div/div/div[2]/form/div/button/div[2]"] + ] + }, + { + "login": false, + "name": "cnn-nav", + "test_url": "https://www.cnn.com/", + "type": "interactive", + "test_cmds": [ + ["measure.start", "landing"], + ["navigate", "https://www.cnn.com"], + ["wait.byTime", 4000], + ["measure.stop", ""], + ["measure.start", "world"], + ["click.byXpathAndWait", "/html/body/div[5]/div/div/header/div/div[1]/div/div[2]/nav/ul/li[2]/a"], + ["wait.byTime", 1000], + ["measure.stop", ""] + ] + }, + { + "login": false, + "name": "reddit-billgates-ama", + "test_url": "https://www.reddit.com/", + "type": "interactive", + "test_cmds": [ + ["measure.start", "billg-ama"], + ["navigate", "https://www.reddit.com/r/IAmA/comments/m8n4vt/im_bill_gates_cochair_of_the_bill_and_melinda/"], + ["wait.byTime", 5000], + ["measure.stop", ""], + ["measure.start", "members"], + ["click.byXpathAndWait", "/html/body/div[1]/div/div[2]/div[2]/div/div[3]/div[2]/div/div[1]/div/div[4]/div[1]/div"], + ["wait.byTime", 1000], + ["measure.stop", ""] + ] + }, + { + "login": false, + "name": "reddit-billgates-post", + "test_url": "https://www.reddit.com/user/thisisbillgates/", + "type": "interactive", + "test_cmds": [ + ["measure.start", "billg"], + ["navigate", "https://www.reddit.com/user/thisisbillgates/"], + ["wait.byTime", 5000], + ["measure.stop", ""], + ["measure.start", "posts"], + ["click.byXpathAndWait", "/html/body/div[1]/div/div[2]/div[2]/div/div/div/div[2]/div[2]/div/div/div/a[2]"], + ["wait.byTime", 15000], + ["measure.stop", ""], + ["measure.start", "comments"], + ["click.byXpathAndWait", "/html/body/div[1]/div/div[2]/div[2]/div/div/div/div[2]/div[2]/div/div/div/a[3]"], + ["wait.byTime", 15000], + ["measure.stop", ""], + ["measure.start", "hot"], + ["click.byXpathAndWait", "/html/body/div[1]/div/div[2]/div[2]/div/div/div/div[2]/div[4]/div[1]/div[1]/div[2]/a[2]"], + ["wait.byTime", 15000], + ["measure.stop", ""], + ["measure.start", "top"], + ["click.byXpathAndWait", "/html/body/div[1]/div/div[2]/div[2]/div/div/div/div[2]/div[4]/div[1]/div[1]/div[2]/a[3]"], + ["wait.byTime", 15000], + ["measure.stop", ""] + ] + }, + { + "login": false, + "name": "facebook-nav", + "test_url": "https://www.facebook.com/", + "type": "interactive", + "test_cmds": [ + ["navigate", "https://www.facebook.com/login"], + ["wait.byTime", 30000], + ["measure.start", "landing"], + ["navigate", "https://www.facebook.com/"], + ["wait.byTime", 5000], + ["measure.stop", ""], + ["measure.start", "marketplace"], + ["navigate", "https://www.facebook.com/marketplace"], + ["wait.byTime", 5000], + ["measure.stop", ""], + ["measure.start", "groups"], + ["navigate", "https://www.facebook.com/groups/discover/"], + ["wait.byTime", 5000], + ["measure.stop", ""], + ["measure.start", "friends"], + ["navigate", "https://www.facebook.com/friends/"], + ["wait.byTime", 5000], + ["measure.stop", ""] + ] + } + ] +} diff --git a/testing/raptor/browsertime/process_switch.js b/testing/raptor/browsertime/process_switch.js new file mode 100644 index 0000000000..a904074a5d --- /dev/null +++ b/testing/raptor/browsertime/process_switch.js @@ -0,0 +1,45 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* eslint-env node */ + +module.exports = async function(context, commands) { + context.log.info("Starting a process switch test"); + let urlstr = context.options.browsertime.url; + let page_cycles = context.options.browsertime.page_cycles; + let page_cycle_delay = context.options.browsertime.page_cycle_delay; + let post_startup_delay = context.options.browsertime.post_startup_delay; + + // Get the two urls to use in the test (the second one will be measured) + let urls = urlstr.split(","); + if (urls.length != 2) { + context.log.error( + `Wrong number of urls given. Expecting: 2, Given: ${urls.length}` + ); + return false; + } + + context.log.info( + "Waiting for %d ms (post_startup_delay)", + post_startup_delay + ); + await commands.wait.byTime(post_startup_delay); + + for (let count = 0; count < page_cycles; count++) { + context.log.info("Navigating to about:blank"); + await commands.navigate("about:blank"); + + context.log.info("Cycle %d, waiting for %d ms", count, page_cycle_delay); + await commands.wait.byTime(page_cycle_delay); + context.log.info("Cycle %d, starting the measure", count); + + await commands.navigate(urls[0]); + await commands.wait.byTime(3000); + await commands.measure.start(urls[1]); + await commands.wait.byTime(2000); + } + + context.log.info("Process switch test ended."); + return true; +}; diff --git a/testing/raptor/browsertime/speedometer3.js b/testing/raptor/browsertime/speedometer3.js new file mode 100644 index 0000000000..dbda22117f --- /dev/null +++ b/testing/raptor/browsertime/speedometer3.js @@ -0,0 +1,81 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* eslint-env node */ + +module.exports = async function(context, commands) { + context.log.info("Starting Speedometer 3 test"); + let url = context.options.browsertime.url; + let page_cycles = context.options.browsertime.page_cycles; + let speedometer_iterations = + context.options.browsertime.speedometer_iterations; + let page_cycle_delay = context.options.browsertime.page_cycle_delay; + let post_startup_delay = context.options.browsertime.post_startup_delay; + let page_timeout = context.options.timeouts.pageLoad; + + context.log.info( + "Waiting for %d ms (post_startup_delay)", + post_startup_delay + ); + await commands.wait.byTime(post_startup_delay); + + for (let count = 0; count < page_cycles; count++) { + context.log.info("Navigating to about:blank"); + await commands.navigate("about:blank"); + + context.log.info("Cycle %d, waiting for %d ms", count, page_cycle_delay); + await commands.wait.byTime(page_cycle_delay); + + context.log.info("Cycle %d, starting the measure", count); + await commands.measure.start(url); + + await commands.js.runAndWait(` + window.testDone = false; + window.suiteValues = []; + const benchmarkClient = { + didRunSuites(measuredValues) { + window.suiteValues.push(measuredValues); + }, + didFinishLastIteration() { + window.testDone = true; + } + }; + window.Suites.forEach((el) => { + el.disabled = false; + }); + // BenchmarkRunner is overriden as the InteractiveBenchmarkRunner + const runner = new BenchmarkRunner(window.Suites, ${speedometer_iterations}); + runner._client = benchmarkClient; + + runner.runSuites(); + `); + + let data_exists = false; + let starttime = await commands.js.run(`return performance.now();`); + while ( + !data_exists && + (await commands.js.run(`return performance.now();`)) - starttime < + page_timeout + ) { + let wait_time = 3000; + context.log.info("Waiting %d ms for data from speedometer...", wait_time); + await commands.wait.byTime(wait_time); + data_exists = await commands.js.run("return window.testDone;"); + } + if ( + !data_exists && + (await commands.js.run(`return performance.now();`)) - starttime >= + page_timeout + ) { + context.log.error("Benchmark timed out. Aborting..."); + return false; + } + let data = await commands.js.run(`return window.suiteValues;`); + context.log.info("Value of benchmark data: ", data); + + commands.measure.addObject({ browsertime_benchmark: { s3: data } }); + } + + return true; +}; diff --git a/testing/raptor/browsertime/welcome.js b/testing/raptor/browsertime/welcome.js new file mode 100644 index 0000000000..67150c4947 --- /dev/null +++ b/testing/raptor/browsertime/welcome.js @@ -0,0 +1,30 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* eslint-env node */ + +module.exports = async function(context, commands) { + context.log.info("Starting a first-install test"); + let page_cycles = context.options.browsertime.page_cycles; + + for (let count = 0; count < page_cycles; count++) { + context.log.info("Navigating to about:blank"); + + // See bug 1717754 + context.log.info(await commands.js.run(`return document.documentURI;`)); + + await commands.navigate("about:blank"); + + await commands.measure.start(); + await commands.wait.byTime(1000); + await commands.navigate("about:welcome"); + await commands.wait.byTime(2000); + await commands.measure.stop(); + + await commands.wait.byTime(2000); + } + + context.log.info("First-install test ended."); + return true; +}; diff --git a/testing/raptor/constants/__init__.py b/testing/raptor/constants/__init__.py new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/testing/raptor/constants/__init__.py diff --git a/testing/raptor/constants/raptor_tests_constants.py b/testing/raptor/constants/raptor_tests_constants.py new file mode 100644 index 0000000000..a071c8cb29 --- /dev/null +++ b/testing/raptor/constants/raptor_tests_constants.py @@ -0,0 +1,1439 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# raptor-youtube-playback constant for measure field +# if the alert_on is set but measure is not specified, +# then we will use the list below + +# main ytp tests - HFR +HFR = [ + "AV11080p60fps@0.25X_%_dropped_frames", + "AV11080p60fps@0.25X_decoded_frames", + "AV11080p60fps@0.25X_dropped_frames", + "AV11080p60fps@0.5X_%_dropped_frames", + "AV11080p60fps@0.5X_decoded_frames", + "AV11080p60fps@0.5X_dropped_frames", + "AV11080p60fps@0.75X_%_dropped_frames", + "AV11080p60fps@0.75X_decoded_frames", + "AV11080p60fps@0.75X_dropped_frames", + "AV11080p60fps@1.25X_%_dropped_frames", + "AV11080p60fps@1.25X_decoded_frames", + "AV11080p60fps@1.25X_dropped_frames", + "AV11080p60fps@1.5X_%_dropped_frames", + "AV11080p60fps@1.5X_decoded_frames", + "AV11080p60fps@1.5X_dropped_frames", + "AV11080p60fps@1.75X_%_dropped_frames", + "AV11080p60fps@1.75X_decoded_frames", + "AV11080p60fps@1.75X_dropped_frames", + "AV11080p60fps@1X_%_dropped_frames", + "AV11080p60fps@1X_decoded_frames", + "AV11080p60fps@1X_dropped_frames", + "AV11080p60fps@2X_%_dropped_frames", + "AV11080p60fps@2X_decoded_frames", + "AV11080p60fps@2X_dropped_frames", + "AV11440p60fps@0.25X_%_dropped_frames", + "AV11440p60fps@0.25X_decoded_frames", + "AV11440p60fps@0.25X_dropped_frames", + "AV11440p60fps@0.5X_%_dropped_frames", + "AV11440p60fps@0.5X_decoded_frames", + "AV11440p60fps@0.5X_dropped_frames", + "AV11440p60fps@0.75X_%_dropped_frames", + "AV11440p60fps@0.75X_decoded_frames", + "AV11440p60fps@0.75X_dropped_frames", + "AV11440p60fps@1.25X_%_dropped_frames", + "AV11440p60fps@1.25X_decoded_frames", + "AV11440p60fps@1.25X_dropped_frames", + "AV11440p60fps@1.5X_%_dropped_frames", + "AV11440p60fps@1.5X_decoded_frames", + "AV11440p60fps@1.5X_dropped_frames", + "AV11440p60fps@1.75X_%_dropped_frames", + "AV11440p60fps@1.75X_decoded_frames", + "AV11440p60fps@1.75X_dropped_frames", + "AV11440p60fps@1X_%_dropped_frames", + "AV11440p60fps@1X_decoded_frames", + "AV11440p60fps@1X_dropped_frames", + "AV11440p60fps@2X_%_dropped_frames", + "AV11440p60fps@2X_decoded_frames", + "AV11440p60fps@2X_dropped_frames", + "AV1720p60fps@0.25X_%_dropped_frames", + "AV1720p60fps@0.25X_decoded_frames", + "AV1720p60fps@0.25X_dropped_frames", + "AV1720p60fps@0.5X_%_dropped_frames", + "AV1720p60fps@0.5X_decoded_frames", + "AV1720p60fps@0.5X_dropped_frames", + "AV1720p60fps@0.75X_%_dropped_frames", + "AV1720p60fps@0.75X_decoded_frames", + "AV1720p60fps@0.75X_dropped_frames", + "AV1720p60fps@1.25X_%_dropped_frames", + "AV1720p60fps@1.25X_decoded_frames", + "AV1720p60fps@1.25X_dropped_frames", + "AV1720p60fps@1.5X_%_dropped_frames", + "AV1720p60fps@1.5X_decoded_frames", + "AV1720p60fps@1.5X_dropped_frames", + "AV1720p60fps@1.75X_%_dropped_frames", + "AV1720p60fps@1.75X_decoded_frames", + "AV1720p60fps@1.75X_dropped_frames", + "AV1720p60fps@1X_%_dropped_frames", + "AV1720p60fps@1X_decoded_frames", + "AV1720p60fps@1X_dropped_frames", + "AV1720p60fps@2X_%_dropped_frames", + "AV1720p60fps@2X_decoded_frames", + "AV1720p60fps@2X_dropped_frames", + "H2641080p60fps@0.25X_%_dropped_frames", + "H2641080p60fps@0.25X_decoded_frames", + "H2641080p60fps@0.25X_dropped_frames", + "H2641080p60fps@0.5X_%_dropped_frames", + "H2641080p60fps@0.5X_decoded_frames", + "H2641080p60fps@0.5X_dropped_frames", + "H2641080p60fps@0.75X_%_dropped_frames", + "H2641080p60fps@0.75X_decoded_frames", + "H2641080p60fps@0.75X_dropped_frames", + "H2641080p60fps@1.25X_%_dropped_frames", + "H2641080p60fps@1.25X_decoded_frames", + "H2641080p60fps@1.25X_dropped_frames", + "H2641080p60fps@1.5X_%_dropped_frames", + "H2641080p60fps@1.5X_decoded_frames", + "H2641080p60fps@1.5X_dropped_frames", + "H2641080p60fps@1.75X_%_dropped_frames", + "H2641080p60fps@1.75X_decoded_frames", + "H2641080p60fps@1.75X_dropped_frames", + "H2641080p60fps@1X_%_dropped_frames", + "H2641080p60fps@1X_decoded_frames", + "H2641080p60fps@1X_dropped_frames", + "H2641080p60fps@2X_%_dropped_frames", + "H2641080p60fps@2X_decoded_frames", + "H2641080p60fps@2X_dropped_frames", + "H264720p60fps@0.25X_%_dropped_frames", + "H264720p60fps@0.25X_decoded_frames", + "H264720p60fps@0.25X_dropped_frames", + "H264720p60fps@0.5X_%_dropped_frames", + "H264720p60fps@0.5X_decoded_frames", + "H264720p60fps@0.5X_dropped_frames", + "H264720p60fps@0.75X_%_dropped_frames", + "H264720p60fps@0.75X_decoded_frames", + "H264720p60fps@0.75X_dropped_frames", + "H264720p60fps@1.25X_%_dropped_frames", + "H264720p60fps@1.25X_decoded_frames", + "H264720p60fps@1.25X_dropped_frames", + "H264720p60fps@1.5X_%_dropped_frames", + "H264720p60fps@1.5X_decoded_frames", + "H264720p60fps@1.5X_dropped_frames", + "H264720p60fps@1.75X_%_dropped_frames", + "H264720p60fps@1.75X_decoded_frames", + "H264720p60fps@1.75X_dropped_frames", + "H264720p60fps@1X_%_dropped_frames", + "H264720p60fps@1X_decoded_frames", + "H264720p60fps@1X_dropped_frames", + "H264720p60fps@2X_%_dropped_frames", + "H264720p60fps@2X_decoded_frames", + "H264720p60fps@2X_dropped_frames", + "VP91080p60fps@0.25X_%_dropped_frames", + "VP91080p60fps@0.25X_decoded_frames", + "VP91080p60fps@0.25X_dropped_frames", + "VP91080p60fps@0.5X_%_dropped_frames", + "VP91080p60fps@0.5X_decoded_frames", + "VP91080p60fps@0.5X_dropped_frames", + "VP91080p60fps@0.75X_%_dropped_frames", + "VP91080p60fps@0.75X_decoded_frames", + "VP91080p60fps@0.75X_dropped_frames", + "VP91080p60fps@1.25X_%_dropped_frames", + "VP91080p60fps@1.25X_decoded_frames", + "VP91080p60fps@1.25X_dropped_frames", + "VP91080p60fps@1.5X_%_dropped_frames", + "VP91080p60fps@1.5X_decoded_frames", + "VP91080p60fps@1.5X_dropped_frames", + "VP91080p60fps@1.75X_%_dropped_frames", + "VP91080p60fps@1.75X_decoded_frames", + "VP91080p60fps@1.75X_dropped_frames", + "VP91080p60fps@1X_%_dropped_frames", + "VP91080p60fps@1X_decoded_frames", + "VP91080p60fps@1X_dropped_frames", + "VP91080p60fps@2X_%_dropped_frames", + "VP91080p60fps@2X_decoded_frames", + "VP91080p60fps@2X_dropped_frames", + "VP91440p60fps@0.25X_%_dropped_frames", + "VP91440p60fps@0.25X_decoded_frames", + "VP91440p60fps@0.25X_dropped_frames", + "VP91440p60fps@0.5X_%_dropped_frames", + "VP91440p60fps@0.5X_decoded_frames", + "VP91440p60fps@0.5X_dropped_frames", + "VP91440p60fps@0.75X_%_dropped_frames", + "VP91440p60fps@0.75X_decoded_frames", + "VP91440p60fps@0.75X_dropped_frames", + "VP91440p60fps@1.25X_%_dropped_frames", + "VP91440p60fps@1.25X_decoded_frames", + "VP91440p60fps@1.25X_dropped_frames", + "VP91440p60fps@1.5X_%_dropped_frames", + "VP91440p60fps@1.5X_decoded_frames", + "VP91440p60fps@1.5X_dropped_frames", + "VP91440p60fps@1.75X_%_dropped_frames", + "VP91440p60fps@1.75X_decoded_frames", + "VP91440p60fps@1.75X_dropped_frames", + "VP91440p60fps@1X_%_dropped_frames", + "VP91440p60fps@1X_decoded_frames", + "VP91440p60fps@1X_dropped_frames", + "VP91440p60fps@2X_%_dropped_frames", + "VP91440p60fps@2X_decoded_frames", + "VP91440p60fps@2X_dropped_frames", + "VP92160p60fps@0.25X_%_dropped_frames", + "VP92160p60fps@0.25X_decoded_frames", + "VP92160p60fps@0.25X_dropped_frames", + "VP92160p60fps@0.5X_%_dropped_frames", + "VP92160p60fps@0.5X_decoded_frames", + "VP92160p60fps@0.5X_dropped_frames", + "VP92160p60fps@0.75X_%_dropped_frames", + "VP92160p60fps@0.75X_decoded_frames", + "VP92160p60fps@0.75X_dropped_frames", + "VP92160p60fps@1.25X_%_dropped_frames", + "VP92160p60fps@1.25X_decoded_frames", + "VP92160p60fps@1.25X_dropped_frames", + "VP92160p60fps@1.5X_%_dropped_frames", + "VP92160p60fps@1.5X_decoded_frames", + "VP92160p60fps@1.5X_dropped_frames", + "VP92160p60fps@1.75X_%_dropped_frames", + "VP92160p60fps@1.75X_decoded_frames", + "VP92160p60fps@1.75X_dropped_frames", + "VP92160p60fps@1X_%_dropped_frames", + "VP92160p60fps@1X_decoded_frames", + "VP92160p60fps@1X_dropped_frames", + "VP92160p60fps@2X_%_dropped_frames", + "VP92160p60fps@2X_decoded_frames", + "VP92160p60fps@2X_dropped_frames", + "VP9720p60fps@0.25X_%_dropped_frames", + "VP9720p60fps@0.25X_decoded_frames", + "VP9720p60fps@0.25X_dropped_frames", + "VP9720p60fps@0.5X_%_dropped_frames", + "VP9720p60fps@0.5X_decoded_frames", + "VP9720p60fps@0.5X_dropped_frames", + "VP9720p60fps@0.75X_%_dropped_frames", + "VP9720p60fps@0.75X_decoded_frames", + "VP9720p60fps@0.75X_dropped_frames", + "VP9720p60fps@1.25X_%_dropped_frames", + "VP9720p60fps@1.25X_decoded_frames", + "VP9720p60fps@1.25X_dropped_frames", + "VP9720p60fps@1.5X_%_dropped_frames", + "VP9720p60fps@1.5X_decoded_frames", + "VP9720p60fps@1.5X_dropped_frames", + "VP9720p60fps@1.75X_%_dropped_frames", + "VP9720p60fps@1.75X_decoded_frames", + "VP9720p60fps@1.75X_dropped_frames", + "VP9720p60fps@1X_%_dropped_frames", + "VP9720p60fps@1X_decoded_frames", + "VP9720p60fps@1X_dropped_frames", + "VP9720p60fps@2X_%_dropped_frames", + "VP9720p60fps@2X_decoded_frames", + "VP9720p60fps@2X_dropped_frames", +] + + +# sfr tests +SFR_AV1 = [ + "AV11080p30fps@0.25X_%_dropped_frames", + "AV11080p30fps@0.25X_decoded_frames", + "AV11080p30fps@0.25X_dropped_frames", + "AV11080p30fps@0.5X_%_dropped_frames", + "AV11080p30fps@0.5X_decoded_frames", + "AV11080p30fps@0.5X_dropped_frames", + "AV11080p30fps@0.75X_%_dropped_frames", + "AV11080p30fps@0.75X_decoded_frames", + "AV11080p30fps@0.75X_dropped_frames", + "AV11080p30fps@1.25X_%_dropped_frames", + "AV11080p30fps@1.25X_decoded_frames", + "AV11080p30fps@1.25X_dropped_frames", + "AV11080p30fps@1.5X_%_dropped_frames", + "AV11080p30fps@1.5X_decoded_frames", + "AV11080p30fps@1.5X_dropped_frames", + "AV11080p30fps@1.75X_%_dropped_frames", + "AV11080p30fps@1.75X_decoded_frames", + "AV11080p30fps@1.75X_dropped_frames", + "AV11080p30fps@1X_%_dropped_frames", + "AV11080p30fps@1X_decoded_frames", + "AV11080p30fps@1X_dropped_frames", + "AV11080p30fps@2X_%_dropped_frames", + "AV11080p30fps@2X_decoded_frames", + "AV11080p30fps@2X_dropped_frames", + "AV11440p30fps@0.25X_%_dropped_frames", + "AV11440p30fps@0.25X_decoded_frames", + "AV11440p30fps@0.25X_dropped_frames", + "AV11440p30fps@0.5X_%_dropped_frames", + "AV11440p30fps@0.5X_decoded_frames", + "AV11440p30fps@0.5X_dropped_frames", + "AV11440p30fps@0.75X_%_dropped_frames", + "AV11440p30fps@0.75X_decoded_frames", + "AV11440p30fps@0.75X_dropped_frames", + "AV11440p30fps@1.25X_%_dropped_frames", + "AV11440p30fps@1.25X_decoded_frames", + "AV11440p30fps@1.25X_dropped_frames", + "AV11440p30fps@1.5X_%_dropped_frames", + "AV11440p30fps@1.5X_decoded_frames", + "AV11440p30fps@1.5X_dropped_frames", + "AV11440p30fps@1.75X_%_dropped_frames", + "AV11440p30fps@1.75X_decoded_frames", + "AV11440p30fps@1.75X_dropped_frames", + "AV11440p30fps@1X_%_dropped_frames", + "AV11440p30fps@1X_decoded_frames", + "AV11440p30fps@1X_dropped_frames", + "AV11440p30fps@2X_%_dropped_frames", + "AV11440p30fps@2X_decoded_frames", + "AV11440p30fps@2X_dropped_frames", + "AV1144p30fps@1X_%_dropped_frames", + "AV1144p30fps@1X_decoded_frames", + "AV1144p30fps@1X_dropped_frames", + "AV12160p30fps@0.25X_%_dropped_frames", + "AV12160p30fps@0.25X_decoded_frames", + "AV12160p30fps@0.25X_dropped_frames", + "AV12160p30fps@0.5X_%_dropped_frames", + "AV12160p30fps@0.5X_decoded_frames", + "AV12160p30fps@0.5X_dropped_frames", + "AV12160p30fps@0.75X_%_dropped_frames", + "AV12160p30fps@0.75X_decoded_frames", + "AV12160p30fps@0.75X_dropped_frames", + "AV12160p30fps@1.25X_%_dropped_frames", + "AV12160p30fps@1.25X_decoded_frames", + "AV12160p30fps@1.25X_dropped_frames", + "AV12160p30fps@1.5X_%_dropped_frames", + "AV12160p30fps@1.5X_decoded_frames", + "AV12160p30fps@1.5X_dropped_frames", + "AV12160p30fps@1.75X_%_dropped_frames", + "AV12160p30fps@1.75X_decoded_frames", + "AV12160p30fps@1.75X_dropped_frames", + "AV12160p30fps@1X_%_dropped_frames", + "AV12160p30fps@1X_decoded_frames", + "AV12160p30fps@1X_dropped_frames", + "AV12160p30fps@2X_%_dropped_frames", + "AV12160p30fps@2X_decoded_frames", + "AV12160p30fps@2X_dropped_frames", + "AV1240p30fps@1X_%_dropped_frames", + "AV1240p30fps@1X_decoded_frames", + "AV1240p30fps@1X_dropped_frames", + "AV1360p30fps@1X_%_dropped_frames", + "AV1360p30fps@1X_decoded_frames", + "AV1360p30fps@1X_dropped_frames", + "AV1480p30fps@1X_%_dropped_frames", + "AV1480p30fps@1X_decoded_frames", + "AV1480p30fps@1X_dropped_frames", + "AV1720p30fps@0.25X_%_dropped_frames", + "AV1720p30fps@0.25X_decoded_frames", + "AV1720p30fps@0.25X_dropped_frames", + "AV1720p30fps@0.5X_%_dropped_frames", + "AV1720p30fps@0.5X_decoded_frames", + "AV1720p30fps@0.5X_dropped_frames", + "AV1720p30fps@0.75X_%_dropped_frames", + "AV1720p30fps@0.75X_decoded_frames", + "AV1720p30fps@0.75X_dropped_frames", + "AV1720p30fps@1.25X_%_dropped_frames", + "AV1720p30fps@1.25X_decoded_frames", + "AV1720p30fps@1.25X_dropped_frames", + "AV1720p30fps@1.5X_%_dropped_frames", + "AV1720p30fps@1.5X_decoded_frames", + "AV1720p30fps@1.5X_dropped_frames", + "AV1720p30fps@1.75X_%_dropped_frames", + "AV1720p30fps@1.75X_decoded_frames", + "AV1720p30fps@1.75X_dropped_frames", + "AV1720p30fps@1X_%_dropped_frames", + "AV1720p30fps@1X_decoded_frames", + "AV1720p30fps@1X_dropped_frames", + "AV1720p30fps@2X_%_dropped_frames", + "AV1720p30fps@2X_decoded_frames", + "AV1720p30fps@2X_dropped_frames", +] + +SFR_H264 = [ + "H2641080p30fps@0.25X_%_dropped_frames", + "H2641080p30fps@0.25X_decoded_frames", + "H2641080p30fps@0.25X_dropped_frames", + "H2641080p30fps@0.5X_%_dropped_frames", + "H2641080p30fps@0.5X_decoded_frames", + "H2641080p30fps@0.5X_dropped_frames", + "H2641080p30fps@0.75X_%_dropped_frames", + "H2641080p30fps@0.75X_decoded_frames", + "H2641080p30fps@0.75X_dropped_frames", + "H2641080p30fps@1.25X_%_dropped_frames", + "H2641080p30fps@1.25X_decoded_frames", + "H2641080p30fps@1.25X_dropped_frames", + "H2641080p30fps@1.5X_%_dropped_frames", + "H2641080p30fps@1.5X_decoded_frames", + "H2641080p30fps@1.5X_dropped_frames", + "H2641080p30fps@1.75X_%_dropped_frames", + "H2641080p30fps@1.75X_decoded_frames", + "H2641080p30fps@1.75X_dropped_frames", + "H2641080p30fps@1X_%_dropped_frames", + "H2641080p30fps@1X_decoded_frames", + "H2641080p30fps@1X_dropped_frames", + "H2641080p30fps@2X_%_dropped_frames", + "H2641080p30fps@2X_decoded_frames", + "H2641080p30fps@2X_dropped_frames", + "H2641440p30fps@0.25X_%_dropped_frames", + "H2641440p30fps@0.25X_decoded_frames", + "H2641440p30fps@0.25X_dropped_frames", + "H2641440p30fps@0.5X_%_dropped_frames", + "H2641440p30fps@0.5X_decoded_frames", + "H2641440p30fps@0.5X_dropped_frames", + "H2641440p30fps@0.75X_%_dropped_frames", + "H2641440p30fps@0.75X_decoded_frames", + "H2641440p30fps@0.75X_dropped_frames", + "H2641440p30fps@1.25X_%_dropped_frames", + "H2641440p30fps@1.25X_decoded_frames", + "H2641440p30fps@1.25X_dropped_frames", + "H2641440p30fps@1.5X_%_dropped_frames", + "H2641440p30fps@1.5X_decoded_frames", + "H2641440p30fps@1.5X_dropped_frames", + "H2641440p30fps@1.75X_%_dropped_frames", + "H2641440p30fps@1.75X_decoded_frames", + "H2641440p30fps@1.75X_dropped_frames", + "H2641440p30fps@1X_%_dropped_frames", + "H2641440p30fps@1X_decoded_frames", + "H2641440p30fps@1X_dropped_frames", + "H2641440p30fps@2X_%_dropped_frames", + "H2641440p30fps@2X_decoded_frames", + "H2641440p30fps@2X_dropped_frames", + "H264144p15fps@1X_%_dropped_frames", + "H264144p15fps@1X_decoded_frames", + "H264144p15fps@1X_dropped_frames", + "H2642160p30fps@0.25X_%_dropped_frames", + "H2642160p30fps@0.25X_decoded_frames", + "H2642160p30fps@0.25X_dropped_frames", + "H2642160p30fps@0.5X_%_dropped_frames", + "H2642160p30fps@0.5X_decoded_frames", + "H2642160p30fps@0.5X_dropped_frames", + "H2642160p30fps@0.75X_%_dropped_frames", + "H2642160p30fps@0.75X_decoded_frames", + "H2642160p30fps@0.75X_dropped_frames", + "H2642160p30fps@1.25X_%_dropped_frames", + "H2642160p30fps@1.25X_decoded_frames", + "H2642160p30fps@1.25X_dropped_frames", + "H2642160p30fps@1.5X_%_dropped_frames", + "H2642160p30fps@1.5X_decoded_frames", + "H2642160p30fps@1.5X_dropped_frames", + "H2642160p30fps@1.75X_%_dropped_frames", + "H2642160p30fps@1.75X_decoded_frames", + "H2642160p30fps@1.75X_dropped_frames", + "H2642160p30fps@1X_%_dropped_frames", + "H2642160p30fps@1X_decoded_frames", + "H2642160p30fps@1X_dropped_frames", + "H2642160p30fps@2X_%_dropped_frames", + "H2642160p30fps@2X_decoded_frames", + "H2642160p30fps@2X_dropped_frames", + "H264240p30fps@1X_%_dropped_frames", + "H264240p30fps@1X_decoded_frames", + "H264240p30fps@1X_dropped_frames", + "H264360p30fps@1X_%_dropped_frames", + "H264360p30fps@1X_decoded_frames", + "H264360p30fps@1X_dropped_frames", + "H264480p30fps@1X_%_dropped_frames", + "H264480p30fps@1X_decoded_frames", + "H264480p30fps@1X_dropped_frames", + "H264720p30fps@0.25X_%_dropped_frames", + "H264720p30fps@0.25X_decoded_frames", + "H264720p30fps@0.25X_dropped_frames", + "H264720p30fps@0.5X_%_dropped_frames", + "H264720p30fps@0.5X_decoded_frames", + "H264720p30fps@0.5X_dropped_frames", + "H264720p30fps@0.75X_%_dropped_frames", + "H264720p30fps@0.75X_decoded_frames", + "H264720p30fps@0.75X_dropped_frames", + "H264720p30fps@1.25X_%_dropped_frames", + "H264720p30fps@1.25X_decoded_frames", + "H264720p30fps@1.25X_dropped_frames", + "H264720p30fps@1.5X_%_dropped_frames", + "H264720p30fps@1.5X_decoded_frames", + "H264720p30fps@1.5X_dropped_frames", + "H264720p30fps@1.75X_%_dropped_frames", + "H264720p30fps@1.75X_decoded_frames", + "H264720p30fps@1.75X_dropped_frames", + "H264720p30fps@1X_%_dropped_frames", + "H264720p30fps@1X_decoded_frames", + "H264720p30fps@1X_dropped_frames", + "H264720p30fps@2X_%_dropped_frames", + "H264720p30fps@2X_decoded_frames", + "H264720p30fps@2X_dropped_frames", +] + +SFR_VP9 = [ + "VP91080p30fps@0.25X_%_dropped_frames", + "VP91080p30fps@0.25X_decoded_frames", + "VP91080p30fps@0.25X_dropped_frames", + "VP91080p30fps@0.5X_%_dropped_frames", + "VP91080p30fps@0.5X_decoded_frames", + "VP91080p30fps@0.5X_dropped_frames", + "VP91080p30fps@0.75X_%_dropped_frames", + "VP91080p30fps@0.75X_decoded_frames", + "VP91080p30fps@0.75X_dropped_frames", + "VP91080p30fps@1.25X_%_dropped_frames", + "VP91080p30fps@1.25X_decoded_frames", + "VP91080p30fps@1.25X_dropped_frames", + "VP91080p30fps@1.5X_%_dropped_frames", + "VP91080p30fps@1.5X_decoded_frames", + "VP91080p30fps@1.5X_dropped_frames", + "VP91080p30fps@1.75X_%_dropped_frames", + "VP91080p30fps@1.75X_decoded_frames", + "VP91080p30fps@1.75X_dropped_frames", + "VP91080p30fps@1X_%_dropped_frames", + "VP91080p30fps@1X_decoded_frames", + "VP91080p30fps@1X_dropped_frames", + "VP91080p30fps@2X_%_dropped_frames", + "VP91080p30fps@2X_decoded_frames", + "VP91080p30fps@2X_dropped_frames", + "VP91440p30fps@0.25X_%_dropped_frames", + "VP91440p30fps@0.25X_decoded_frames", + "VP91440p30fps@0.25X_dropped_frames", + "VP91440p30fps@0.5X_%_dropped_frames", + "VP91440p30fps@0.5X_decoded_frames", + "VP91440p30fps@0.5X_dropped_frames", + "VP91440p30fps@0.75X_%_dropped_frames", + "VP91440p30fps@0.75X_decoded_frames", + "VP91440p30fps@0.75X_dropped_frames", + "VP91440p30fps@1.25X_%_dropped_frames", + "VP91440p30fps@1.25X_decoded_frames", + "VP91440p30fps@1.25X_dropped_frames", + "VP91440p30fps@1.5X_%_dropped_frames", + "VP91440p30fps@1.5X_decoded_frames", + "VP91440p30fps@1.5X_dropped_frames", + "VP91440p30fps@1.75X_%_dropped_frames", + "VP91440p30fps@1.75X_decoded_frames", + "VP91440p30fps@1.75X_dropped_frames", + "VP91440p30fps@1X_%_dropped_frames", + "VP91440p30fps@1X_decoded_frames", + "VP91440p30fps@1X_dropped_frames", + "VP91440p30fps@2X_%_dropped_frames", + "VP91440p30fps@2X_decoded_frames", + "VP91440p30fps@2X_dropped_frames", + "VP9144p30fps@1X_%_dropped_frames", + "VP9144p30fps@1X_decoded_frames", + "VP9144p30fps@1X_dropped_frames", + "VP92160p30fps@0.25X_%_dropped_frames", + "VP92160p30fps@0.25X_decoded_frames", + "VP92160p30fps@0.25X_dropped_frames", + "VP92160p30fps@0.5X_%_dropped_frames", + "VP92160p30fps@0.5X_decoded_frames", + "VP92160p30fps@0.5X_dropped_frames", + "VP92160p30fps@0.75X_%_dropped_frames", + "VP92160p30fps@0.75X_decoded_frames", + "VP92160p30fps@0.75X_dropped_frames", + "VP92160p30fps@1.25X_%_dropped_frames", + "VP92160p30fps@1.25X_decoded_frames", + "VP92160p30fps@1.25X_dropped_frames", + "VP92160p30fps@1.5X_%_dropped_frames", + "VP92160p30fps@1.5X_decoded_frames", + "VP92160p30fps@1.5X_dropped_frames", + "VP92160p30fps@1.75X_%_dropped_frames", + "VP92160p30fps@1.75X_decoded_frames", + "VP92160p30fps@1.75X_dropped_frames", + "VP92160p30fps@1X_%_dropped_frames", + "VP92160p30fps@1X_decoded_frames", + "VP92160p30fps@1X_dropped_frames", + "VP92160p30fps@2X_%_dropped_frames", + "VP92160p30fps@2X_decoded_frames", + "VP92160p30fps@2X_dropped_frames", + "VP9240p30fps@1X_%_dropped_frames", + "VP9240p30fps@1X_decoded_frames", + "VP9240p30fps@1X_dropped_frames", + "VP9360p30fps@1X_%_dropped_frames", + "VP9360p30fps@1X_decoded_frames", + "VP9360p30fps@1X_dropped_frames", + "VP9480p30fps@1X_%_dropped_frames", + "VP9480p30fps@1X_decoded_frames", + "VP9480p30fps@1X_dropped_frames", + "VP9720p30fps@0.25X_%_dropped_frames", + "VP9720p30fps@0.25X_decoded_frames", + "VP9720p30fps@0.25X_dropped_frames", + "VP9720p30fps@0.5X_%_dropped_frames", + "VP9720p30fps@0.5X_decoded_frames", + "VP9720p30fps@0.5X_dropped_frames", + "VP9720p30fps@0.75X_%_dropped_frames", + "VP9720p30fps@0.75X_decoded_frames", + "VP9720p30fps@0.75X_dropped_frames", + "VP9720p30fps@1.25X_%_dropped_frames", + "VP9720p30fps@1.25X_decoded_frames", + "VP9720p30fps@1.25X_dropped_frames", + "VP9720p30fps@1.5X_%_dropped_frames", + "VP9720p30fps@1.5X_decoded_frames", + "VP9720p30fps@1.5X_dropped_frames", + "VP9720p30fps@1.75X_%_dropped_frames", + "VP9720p30fps@1.75X_decoded_frames", + "VP9720p30fps@1.75X_dropped_frames", + "VP9720p30fps@1X_%_dropped_frames", + "VP9720p30fps@1X_decoded_frames", + "VP9720p30fps@1X_dropped_frames", + "VP9720p30fps@2X_%_dropped_frames", + "VP9720p30fps@2X_decoded_frames", + "VP9720p30fps@2X_dropped_frames", +] + + +# widevine tests +WIDEVINE_HFR = [ + "H2641080p60fps@0.25X_%_dropped_frames", + "H2641080p60fps@0.25X_decoded_frames", + "H2641080p60fps@0.25X_dropped_frames", + "H2641080p60fps@0.5X_%_dropped_frames", + "H2641080p60fps@0.5X_decoded_frames", + "H2641080p60fps@0.5X_dropped_frames", + "H2641080p60fps@0.75X_%_dropped_frames", + "H2641080p60fps@0.75X_decoded_frames", + "H2641080p60fps@0.75X_dropped_frames", + "H2641080p60fps@1.25X_%_dropped_frames", + "H2641080p60fps@1.25X_decoded_frames", + "H2641080p60fps@1.25X_dropped_frames", + "H2641080p60fps@1.5X_%_dropped_frames", + "H2641080p60fps@1.5X_decoded_frames", + "H2641080p60fps@1.5X_dropped_frames", + "H2641080p60fps@1.75X_%_dropped_frames", + "H2641080p60fps@1.75X_decoded_frames", + "H2641080p60fps@1.75X_dropped_frames", + "H2641080p60fps@1X_%_dropped_frames", + "H2641080p60fps@1X_decoded_frames", + "H2641080p60fps@1X_dropped_frames", + "H2641080p60fps@2X_%_dropped_frames", + "H2641080p60fps@2X_decoded_frames", + "H2641080p60fps@2X_dropped_frames", + "H2641080pMQ60fps@0.25X_%_dropped_frames", + "H2641080pMQ60fps@0.25X_decoded_frames", + "H2641080pMQ60fps@0.25X_dropped_frames", + "H2641080pMQ60fps@0.5X_%_dropped_frames", + "H2641080pMQ60fps@0.5X_decoded_frames", + "H2641080pMQ60fps@0.5X_dropped_frames", + "H2641080pMQ60fps@0.75X_%_dropped_frames", + "H2641080pMQ60fps@0.75X_decoded_frames", + "H2641080pMQ60fps@0.75X_dropped_frames", + "H2641080pMQ60fps@1.25X_%_dropped_frames", + "H2641080pMQ60fps@1.25X_decoded_frames", + "H2641080pMQ60fps@1.25X_dropped_frames", + "H2641080pMQ60fps@1.5X_%_dropped_frames", + "H2641080pMQ60fps@1.5X_decoded_frames", + "H2641080pMQ60fps@1.5X_dropped_frames", + "H2641080pMQ60fps@1.75X_%_dropped_frames", + "H2641080pMQ60fps@1.75X_decoded_frames", + "H2641080pMQ60fps@1.75X_dropped_frames", + "H2641080pMQ60fps@1X_%_dropped_frames", + "H2641080pMQ60fps@1X_decoded_frames", + "H2641080pMQ60fps@1X_dropped_frames", + "H2641080pMQ60fps@2X_%_dropped_frames", + "H2641080pMQ60fps@2X_decoded_frames", + "H2641080pMQ60fps@2X_dropped_frames", + "H264720p60fps@0.25X_%_dropped_frames", + "H264720p60fps@0.25X_decoded_frames", + "H264720p60fps@0.25X_dropped_frames", + "H264720p60fps@0.5X_%_dropped_frames", + "H264720p60fps@0.5X_decoded_frames", + "H264720p60fps@0.5X_dropped_frames", + "H264720p60fps@0.75X_%_dropped_frames", + "H264720p60fps@0.75X_decoded_frames", + "H264720p60fps@0.75X_dropped_frames", + "H264720p60fps@1.25X_%_dropped_frames", + "H264720p60fps@1.25X_decoded_frames", + "H264720p60fps@1.25X_dropped_frames", + "H264720p60fps@1.5X_%_dropped_frames", + "H264720p60fps@1.5X_decoded_frames", + "H264720p60fps@1.5X_dropped_frames", + "H264720p60fps@1.75X_%_dropped_frames", + "H264720p60fps@1.75X_decoded_frames", + "H264720p60fps@1.75X_dropped_frames", + "H264720p60fps@1X_%_dropped_frames", + "H264720p60fps@1X_decoded_frames", + "H264720p60fps@1X_dropped_frames", + "H264720p60fps@2X_%_dropped_frames", + "H264720p60fps@2X_decoded_frames", + "H264720p60fps@2X_dropped_frames", + "H264720pMQ60fps@0.25X_%_dropped_frames", + "H264720pMQ60fps@0.25X_decoded_frames", + "H264720pMQ60fps@0.25X_dropped_frames", + "H264720pMQ60fps@0.5X_%_dropped_frames", + "H264720pMQ60fps@0.5X_decoded_frames", + "H264720pMQ60fps@0.5X_dropped_frames", + "H264720pMQ60fps@0.75X_%_dropped_frames", + "H264720pMQ60fps@0.75X_decoded_frames", + "H264720pMQ60fps@0.75X_dropped_frames", + "H264720pMQ60fps@1.25X_%_dropped_frames", + "H264720pMQ60fps@1.25X_decoded_frames", + "H264720pMQ60fps@1.25X_dropped_frames", + "H264720pMQ60fps@1.5X_%_dropped_frames", + "H264720pMQ60fps@1.5X_decoded_frames", + "H264720pMQ60fps@1.5X_dropped_frames", + "H264720pMQ60fps@1.75X_%_dropped_frames", + "H264720pMQ60fps@1.75X_decoded_frames", + "H264720pMQ60fps@1.75X_dropped_frames", + "H264720pMQ60fps@1X_%_dropped_frames", + "H264720pMQ60fps@1X_decoded_frames", + "H264720pMQ60fps@1X_dropped_frames", + "H264720pMQ60fps@2X_%_dropped_frames", + "H264720pMQ60fps@2X_decoded_frames", + "H264720pMQ60fps@2X_dropped_frames", + "VP91080p60fps@0.25X_%_dropped_frames", + "VP91080p60fps@0.25X_decoded_frames", + "VP91080p60fps@0.25X_dropped_frames", + "VP91080p60fps@0.5X_%_dropped_frames", + "VP91080p60fps@0.5X_decoded_frames", + "VP91080p60fps@0.5X_dropped_frames", + "VP91080p60fps@0.75X_%_dropped_frames", + "VP91080p60fps@0.75X_decoded_frames", + "VP91080p60fps@0.75X_dropped_frames", + "VP91080p60fps@1.25X_%_dropped_frames", + "VP91080p60fps@1.25X_decoded_frames", + "VP91080p60fps@1.25X_dropped_frames", + "VP91080p60fps@1.5X_%_dropped_frames", + "VP91080p60fps@1.5X_decoded_frames", + "VP91080p60fps@1.5X_dropped_frames", + "VP91080p60fps@1.75X_%_dropped_frames", + "VP91080p60fps@1.75X_decoded_frames", + "VP91080p60fps@1.75X_dropped_frames", + "VP91080p60fps@1X_%_dropped_frames", + "VP91080p60fps@1X_decoded_frames", + "VP91080p60fps@1X_dropped_frames", + "VP91080p60fps@2X_%_dropped_frames", + "VP91080p60fps@2X_decoded_frames", + "VP91080p60fps@2X_dropped_frames", + "VP91080pMQ60fps@0.25X_%_dropped_frames", + "VP91080pMQ60fps@0.25X_decoded_frames", + "VP91080pMQ60fps@0.25X_dropped_frames", + "VP91080pMQ60fps@0.5X_%_dropped_frames", + "VP91080pMQ60fps@0.5X_decoded_frames", + "VP91080pMQ60fps@0.5X_dropped_frames", + "VP91080pMQ60fps@0.75X_%_dropped_frames", + "VP91080pMQ60fps@0.75X_decoded_frames", + "VP91080pMQ60fps@0.75X_dropped_frames", + "VP91080pMQ60fps@1.25X_%_dropped_frames", + "VP91080pMQ60fps@1.25X_decoded_frames", + "VP91080pMQ60fps@1.25X_dropped_frames", + "VP91080pMQ60fps@1.5X_%_dropped_frames", + "VP91080pMQ60fps@1.5X_decoded_frames", + "VP91080pMQ60fps@1.5X_dropped_frames", + "VP91080pMQ60fps@1.75X_%_dropped_frames", + "VP91080pMQ60fps@1.75X_decoded_frames", + "VP91080pMQ60fps@1.75X_dropped_frames", + "VP91080pMQ60fps@1X_%_dropped_frames", + "VP91080pMQ60fps@1X_decoded_frames", + "VP91080pMQ60fps@1X_dropped_frames", + "VP91080pMQ60fps@2X_%_dropped_frames", + "VP91080pMQ60fps@2X_decoded_frames", + "VP91080pMQ60fps@2X_dropped_frames", + "VP9720p60fps@0.25X_%_dropped_frames", + "VP9720p60fps@0.25X_decoded_frames", + "VP9720p60fps@0.25X_dropped_frames", + "VP9720p60fps@0.5X_%_dropped_frames", + "VP9720p60fps@0.5X_decoded_frames", + "VP9720p60fps@0.5X_dropped_frames", + "VP9720p60fps@0.75X_%_dropped_frames", + "VP9720p60fps@0.75X_decoded_frames", + "VP9720p60fps@0.75X_dropped_frames", + "VP9720p60fps@1.25X_%_dropped_frames", + "VP9720p60fps@1.25X_decoded_frames", + "VP9720p60fps@1.25X_dropped_frames", + "VP9720p60fps@1.5X_%_dropped_frames", + "VP9720p60fps@1.5X_decoded_frames", + "VP9720p60fps@1.5X_dropped_frames", + "VP9720p60fps@1.75X_%_dropped_frames", + "VP9720p60fps@1.75X_decoded_frames", + "VP9720p60fps@1.75X_dropped_frames", + "VP9720p60fps@1X_%_dropped_frames", + "VP9720p60fps@1X_decoded_frames", + "VP9720p60fps@1X_dropped_frames", + "VP9720p60fps@2X_%_dropped_frames", + "VP9720p60fps@2X_decoded_frames", + "VP9720p60fps@2X_dropped_frames", + "VP9720pMQ60fps@0.25X_%_dropped_frames", + "VP9720pMQ60fps@0.25X_decoded_frames", + "VP9720pMQ60fps@0.25X_dropped_frames", + "VP9720pMQ60fps@0.5X_%_dropped_frames", + "VP9720pMQ60fps@0.5X_decoded_frames", + "VP9720pMQ60fps@0.5X_dropped_frames", + "VP9720pMQ60fps@0.75X_%_dropped_frames", + "VP9720pMQ60fps@0.75X_decoded_frames", + "VP9720pMQ60fps@0.75X_dropped_frames", + "VP9720pMQ60fps@1.25X_%_dropped_frames", + "VP9720pMQ60fps@1.25X_decoded_frames", + "VP9720pMQ60fps@1.25X_dropped_frames", + "VP9720pMQ60fps@1.5X_%_dropped_frames", + "VP9720pMQ60fps@1.5X_decoded_frames", + "VP9720pMQ60fps@1.5X_dropped_frames", + "VP9720pMQ60fps@1.75X_%_dropped_frames", + "VP9720pMQ60fps@1.75X_decoded_frames", + "VP9720pMQ60fps@1.75X_dropped_frames", + "VP9720pMQ60fps@1X_%_dropped_frames", + "VP9720pMQ60fps@1X_decoded_frames", + "VP9720pMQ60fps@1X_dropped_frames", + "VP9720pMQ60fps@2X_%_dropped_frames", + "VP9720pMQ60fps@2X_decoded_frames", + "VP9720pMQ60fps@2X_dropped_frames", +] + +WIDEVINE_SFR_H264 = [ + "H2641080p30fps@0.25X_%_dropped_frames", + "H2641080p30fps@0.25X_decoded_frames", + "H2641080p30fps@0.25X_dropped_frames", + "H2641080p30fps@0.5X_%_dropped_frames", + "H2641080p30fps@0.5X_decoded_frames", + "H2641080p30fps@0.5X_dropped_frames", + "H2641080p30fps@0.75X_%_dropped_frames", + "H2641080p30fps@0.75X_decoded_frames", + "H2641080p30fps@0.75X_dropped_frames", + "H2641080p30fps@1.25X_%_dropped_frames", + "H2641080p30fps@1.25X_decoded_frames", + "H2641080p30fps@1.25X_dropped_frames", + "H2641080p30fps@1.5X_%_dropped_frames", + "H2641080p30fps@1.5X_decoded_frames", + "H2641080p30fps@1.5X_dropped_frames", + "H2641080p30fps@1.75X_%_dropped_frames", + "H2641080p30fps@1.75X_decoded_frames", + "H2641080p30fps@1.75X_dropped_frames", + "H2641080p30fps@1X_%_dropped_frames", + "H2641080p30fps@1X_decoded_frames", + "H2641080p30fps@1X_dropped_frames", + "H2641080p30fps@2X_%_dropped_frames", + "H2641080p30fps@2X_decoded_frames", + "H2641080p30fps@2X_dropped_frames", + "H2641080pHQ30fps@0.25X_%_dropped_frames", + "H2641080pHQ30fps@0.25X_decoded_frames", + "H2641080pHQ30fps@0.25X_dropped_frames", + "H2641080pHQ30fps@0.5X_%_dropped_frames", + "H2641080pHQ30fps@0.5X_decoded_frames", + "H2641080pHQ30fps@0.5X_dropped_frames", + "H2641080pHQ30fps@0.75X_%_dropped_frames", + "H2641080pHQ30fps@0.75X_decoded_frames", + "H2641080pHQ30fps@0.75X_dropped_frames", + "H2641080pHQ30fps@1.25X_%_dropped_frames", + "H2641080pHQ30fps@1.25X_decoded_frames", + "H2641080pHQ30fps@1.25X_dropped_frames", + "H2641080pHQ30fps@1.5X_%_dropped_frames", + "H2641080pHQ30fps@1.5X_decoded_frames", + "H2641080pHQ30fps@1.5X_dropped_frames", + "H2641080pHQ30fps@1.75X_%_dropped_frames", + "H2641080pHQ30fps@1.75X_decoded_frames", + "H2641080pHQ30fps@1.75X_dropped_frames", + "H2641080pHQ30fps@1X_%_dropped_frames", + "H2641080pHQ30fps@1X_decoded_frames", + "H2641080pHQ30fps@1X_dropped_frames", + "H2641080pHQ30fps@2X_%_dropped_frames", + "H2641080pHQ30fps@2X_decoded_frames", + "H2641080pHQ30fps@2X_dropped_frames", + "H2641080pMQ30fps@0.25X_%_dropped_frames", + "H2641080pMQ30fps@0.25X_decoded_frames", + "H2641080pMQ30fps@0.25X_dropped_frames", + "H2641080pMQ30fps@0.5X_%_dropped_frames", + "H2641080pMQ30fps@0.5X_decoded_frames", + "H2641080pMQ30fps@0.5X_dropped_frames", + "H2641080pMQ30fps@0.75X_%_dropped_frames", + "H2641080pMQ30fps@0.75X_decoded_frames", + "H2641080pMQ30fps@0.75X_dropped_frames", + "H2641080pMQ30fps@1.25X_%_dropped_frames", + "H2641080pMQ30fps@1.25X_decoded_frames", + "H2641080pMQ30fps@1.25X_dropped_frames", + "H2641080pMQ30fps@1.5X_%_dropped_frames", + "H2641080pMQ30fps@1.5X_decoded_frames", + "H2641080pMQ30fps@1.5X_dropped_frames", + "H2641080pMQ30fps@1.75X_%_dropped_frames", + "H2641080pMQ30fps@1.75X_decoded_frames", + "H2641080pMQ30fps@1.75X_dropped_frames", + "H2641080pMQ30fps@1X_%_dropped_frames", + "H2641080pMQ30fps@1X_decoded_frames", + "H2641080pMQ30fps@1X_dropped_frames", + "H2641080pMQ30fps@2X_%_dropped_frames", + "H2641080pMQ30fps@2X_decoded_frames", + "H2641080pMQ30fps@2X_dropped_frames", + "H264144p30fps@1X_%_dropped_frames", + "H264144p30fps@1X_decoded_frames", + "H264144p30fps@1X_dropped_frames", + "H264240p30fps@1X_%_dropped_frames", + "H264240p30fps@1X_decoded_frames", + "H264240p30fps@1X_dropped_frames", + "H264360p30fps@1X_%_dropped_frames", + "H264360p30fps@1X_decoded_frames", + "H264360p30fps@1X_dropped_frames", + "H264480p30fps@1X_%_dropped_frames", + "H264480p30fps@1X_decoded_frames", + "H264480p30fps@1X_dropped_frames", + "H264480pHQ30fps@1X_%_dropped_frames", + "H264480pHQ30fps@1X_decoded_frames", + "H264480pHQ30fps@1X_dropped_frames", + "H264480pMQ30fps@1X_%_dropped_frames", + "H264480pMQ30fps@1X_decoded_frames", + "H264480pMQ30fps@1X_dropped_frames", + "H264720p30fps@0.25X_%_dropped_frames", + "H264720p30fps@0.25X_decoded_frames", + "H264720p30fps@0.25X_dropped_frames", + "H264720p30fps@0.5X_%_dropped_frames", + "H264720p30fps@0.5X_decoded_frames", + "H264720p30fps@0.5X_dropped_frames", + "H264720p30fps@0.75X_%_dropped_frames", + "H264720p30fps@0.75X_decoded_frames", + "H264720p30fps@0.75X_dropped_frames", + "H264720p30fps@1.25X_%_dropped_frames", + "H264720p30fps@1.25X_decoded_frames", + "H264720p30fps@1.25X_dropped_frames", + "H264720p30fps@1.5X_%_dropped_frames", + "H264720p30fps@1.5X_decoded_frames", + "H264720p30fps@1.5X_dropped_frames", + "H264720p30fps@1.75X_%_dropped_frames", + "H264720p30fps@1.75X_decoded_frames", + "H264720p30fps@1.75X_dropped_frames", + "H264720p30fps@1X_%_dropped_frames", + "H264720p30fps@1X_decoded_frames", + "H264720p30fps@1X_dropped_frames", + "H264720p30fps@2X_%_dropped_frames", + "H264720p30fps@2X_decoded_frames", + "H264720p30fps@2X_dropped_frames", + "H264720pHQ30fps@0.25X_%_dropped_frames", + "H264720pHQ30fps@0.25X_decoded_frames", + "H264720pHQ30fps@0.25X_dropped_frames", + "H264720pHQ30fps@0.5X_%_dropped_frames", + "H264720pHQ30fps@0.5X_decoded_frames", + "H264720pHQ30fps@0.5X_dropped_frames", + "H264720pHQ30fps@0.75X_%_dropped_frames", + "H264720pHQ30fps@0.75X_decoded_frames", + "H264720pHQ30fps@0.75X_dropped_frames", + "H264720pHQ30fps@1.25X_%_dropped_frames", + "H264720pHQ30fps@1.25X_decoded_frames", + "H264720pHQ30fps@1.25X_dropped_frames", + "H264720pHQ30fps@1.5X_%_dropped_frames", + "H264720pHQ30fps@1.5X_decoded_frames", + "H264720pHQ30fps@1.5X_dropped_frames", + "H264720pHQ30fps@1.75X_%_dropped_frames", + "H264720pHQ30fps@1.75X_decoded_frames", + "H264720pHQ30fps@1.75X_dropped_frames", + "H264720pHQ30fps@1X_%_dropped_frames", + "H264720pHQ30fps@1X_decoded_frames", + "H264720pHQ30fps@1X_dropped_frames", + "H264720pHQ30fps@2X_%_dropped_frames", + "H264720pHQ30fps@2X_decoded_frames", + "H264720pHQ30fps@2X_dropped_frames", + "H264720pMQ30fps@0.25X_%_dropped_frames", + "H264720pMQ30fps@0.25X_decoded_frames", + "H264720pMQ30fps@0.25X_dropped_frames", + "H264720pMQ30fps@0.5X_%_dropped_frames", + "H264720pMQ30fps@0.5X_decoded_frames", + "H264720pMQ30fps@0.5X_dropped_frames", + "H264720pMQ30fps@0.75X_%_dropped_frames", + "H264720pMQ30fps@0.75X_decoded_frames", + "H264720pMQ30fps@0.75X_dropped_frames", + "H264720pMQ30fps@1.25X_%_dropped_frames", + "H264720pMQ30fps@1.25X_decoded_frames", + "H264720pMQ30fps@1.25X_dropped_frames", + "H264720pMQ30fps@1.5X_%_dropped_frames", + "H264720pMQ30fps@1.5X_decoded_frames", + "H264720pMQ30fps@1.5X_dropped_frames", + "H264720pMQ30fps@1.75X_%_dropped_frames", + "H264720pMQ30fps@1.75X_decoded_frames", + "H264720pMQ30fps@1.75X_dropped_frames", + "H264720pMQ30fps@1X_%_dropped_frames", + "H264720pMQ30fps@1X_decoded_frames", + "H264720pMQ30fps@1X_dropped_frames", + "H264720pMQ30fps@2X_%_dropped_frames", + "H264720pMQ30fps@2X_decoded_frames", + "H264720pMQ30fps@2X_dropped_frames", +] + +WIDEVINE_SFR_VP9 = [ + "VP91080p30fps@0.25X_%_dropped_frames", + "VP91080p30fps@0.25X_decoded_frames", + "VP91080p30fps@0.25X_dropped_frames", + "VP91080p30fps@0.5X_%_dropped_frames", + "VP91080p30fps@0.5X_decoded_frames", + "VP91080p30fps@0.5X_dropped_frames", + "VP91080p30fps@0.75X_%_dropped_frames", + "VP91080p30fps@0.75X_decoded_frames", + "VP91080p30fps@0.75X_dropped_frames", + "VP91080p30fps@1.25X_%_dropped_frames", + "VP91080p30fps@1.25X_decoded_frames", + "VP91080p30fps@1.25X_dropped_frames", + "VP91080p30fps@1.5X_%_dropped_frames", + "VP91080p30fps@1.5X_decoded_frames", + "VP91080p30fps@1.5X_dropped_frames", + "VP91080p30fps@1.75X_%_dropped_frames", + "VP91080p30fps@1.75X_decoded_frames", + "VP91080p30fps@1.75X_dropped_frames", + "VP91080p30fps@1X_%_dropped_frames", + "VP91080p30fps@1X_decoded_frames", + "VP91080p30fps@1X_dropped_frames", + "VP91080p30fps@2X_%_dropped_frames", + "VP91080p30fps@2X_decoded_frames", + "VP91080p30fps@2X_dropped_frames", + "VP91080pHQ30fps@0.25X_%_dropped_frames", + "VP91080pHQ30fps@0.25X_decoded_frames", + "VP91080pHQ30fps@0.25X_dropped_frames", + "VP91080pHQ30fps@0.5X_%_dropped_frames", + "VP91080pHQ30fps@0.5X_decoded_frames", + "VP91080pHQ30fps@0.5X_dropped_frames", + "VP91080pHQ30fps@0.75X_%_dropped_frames", + "VP91080pHQ30fps@0.75X_decoded_frames", + "VP91080pHQ30fps@0.75X_dropped_frames", + "VP91080pHQ30fps@1.25X_%_dropped_frames", + "VP91080pHQ30fps@1.25X_decoded_frames", + "VP91080pHQ30fps@1.25X_dropped_frames", + "VP91080pHQ30fps@1.5X_%_dropped_frames", + "VP91080pHQ30fps@1.5X_decoded_frames", + "VP91080pHQ30fps@1.5X_dropped_frames", + "VP91080pHQ30fps@1.75X_%_dropped_frames", + "VP91080pHQ30fps@1.75X_decoded_frames", + "VP91080pHQ30fps@1.75X_dropped_frames", + "VP91080pHQ30fps@1X_%_dropped_frames", + "VP91080pHQ30fps@1X_decoded_frames", + "VP91080pHQ30fps@1X_dropped_frames", + "VP91080pHQ30fps@2X_%_dropped_frames", + "VP91080pHQ30fps@2X_decoded_frames", + "VP91080pHQ30fps@2X_dropped_frames", + "VP91080pMQ30fps@0.25X_%_dropped_frames", + "VP91080pMQ30fps@0.25X_decoded_frames", + "VP91080pMQ30fps@0.25X_dropped_frames", + "VP91080pMQ30fps@0.5X_%_dropped_frames", + "VP91080pMQ30fps@0.5X_decoded_frames", + "VP91080pMQ30fps@0.5X_dropped_frames", + "VP91080pMQ30fps@0.75X_%_dropped_frames", + "VP91080pMQ30fps@0.75X_decoded_frames", + "VP91080pMQ30fps@0.75X_dropped_frames", + "VP91080pMQ30fps@1.25X_%_dropped_frames", + "VP91080pMQ30fps@1.25X_decoded_frames", + "VP91080pMQ30fps@1.25X_dropped_frames", + "VP91080pMQ30fps@1.5X_%_dropped_frames", + "VP91080pMQ30fps@1.5X_decoded_frames", + "VP91080pMQ30fps@1.5X_dropped_frames", + "VP91080pMQ30fps@1.75X_%_dropped_frames", + "VP91080pMQ30fps@1.75X_decoded_frames", + "VP91080pMQ30fps@1.75X_dropped_frames", + "VP91080pMQ30fps@1X_%_dropped_frames", + "VP91080pMQ30fps@1X_decoded_frames", + "VP91080pMQ30fps@1X_dropped_frames", + "VP91080pMQ30fps@2X_%_dropped_frames", + "VP91080pMQ30fps@2X_decoded_frames", + "VP91080pMQ30fps@2X_dropped_frames", + "VP91440p24fps@0.25X_%_dropped_frames", + "VP91440p24fps@0.25X_decoded_frames", + "VP91440p24fps@0.25X_dropped_frames", + "VP91440p24fps@0.5X_%_dropped_frames", + "VP91440p24fps@0.5X_decoded_frames", + "VP91440p24fps@0.5X_dropped_frames", + "VP91440p24fps@0.75X_%_dropped_frames", + "VP91440p24fps@0.75X_decoded_frames", + "VP91440p24fps@0.75X_dropped_frames", + "VP91440p24fps@1.25X_%_dropped_frames", + "VP91440p24fps@1.25X_decoded_frames", + "VP91440p24fps@1.25X_dropped_frames", + "VP91440p24fps@1.5X_%_dropped_frames", + "VP91440p24fps@1.5X_decoded_frames", + "VP91440p24fps@1.5X_dropped_frames", + "VP91440p24fps@1.75X_%_dropped_frames", + "VP91440p24fps@1.75X_decoded_frames", + "VP91440p24fps@1.75X_dropped_frames", + "VP91440p24fps@1X_%_dropped_frames", + "VP91440p24fps@1X_decoded_frames", + "VP91440p24fps@1X_dropped_frames", + "VP91440p24fps@2X_%_dropped_frames", + "VP91440p24fps@2X_decoded_frames", + "VP91440p24fps@2X_dropped_frames", + "VP92160p24fps@0.25X_%_dropped_frames", + "VP92160p24fps@0.25X_decoded_frames", + "VP92160p24fps@0.25X_dropped_frames", + "VP92160p24fps@0.5X_%_dropped_frames", + "VP92160p24fps@0.5X_decoded_frames", + "VP92160p24fps@0.5X_dropped_frames", + "VP92160p24fps@0.75X_%_dropped_frames", + "VP92160p24fps@0.75X_decoded_frames", + "VP92160p24fps@0.75X_dropped_frames", + "VP92160p24fps@1.25X_%_dropped_frames", + "VP92160p24fps@1.25X_decoded_frames", + "VP92160p24fps@1.25X_dropped_frames", + "VP92160p24fps@1.5X_%_dropped_frames", + "VP92160p24fps@1.5X_decoded_frames", + "VP92160p24fps@1.5X_dropped_frames", + "VP92160p24fps@1.75X_%_dropped_frames", + "VP92160p24fps@1.75X_decoded_frames", + "VP92160p24fps@1.75X_dropped_frames", + "VP92160p24fps@1X_%_dropped_frames", + "VP92160p24fps@1X_decoded_frames", + "VP92160p24fps@1X_dropped_frames", + "VP92160p24fps@2X_%_dropped_frames", + "VP92160p24fps@2X_decoded_frames", + "VP92160p24fps@2X_dropped_frames", + "VP9240p30fps@1X_%_dropped_frames", + "VP9240p30fps@1X_decoded_frames", + "VP9240p30fps@1X_dropped_frames", + "VP9360p30fps@1X_%_dropped_frames", + "VP9360p30fps@1X_decoded_frames", + "VP9360p30fps@1X_dropped_frames", + "VP9480p30fps@1X_%_dropped_frames", + "VP9480p30fps@1X_decoded_frames", + "VP9480p30fps@1X_dropped_frames", + "VP9480pHQ30fps@1X_%_dropped_frames", + "VP9480pHQ30fps@1X_decoded_frames", + "VP9480pHQ30fps@1X_dropped_frames", + "VP9480pMQ30fps@1X_%_dropped_frames", + "VP9480pMQ30fps@1X_decoded_frames", + "VP9480pMQ30fps@1X_dropped_frames", + "VP9720p30fps@0.25X_%_dropped_frames", + "VP9720p30fps@0.25X_decoded_frames", + "VP9720p30fps@0.25X_dropped_frames", + "VP9720p30fps@0.5X_%_dropped_frames", + "VP9720p30fps@0.5X_decoded_frames", + "VP9720p30fps@0.5X_dropped_frames", + "VP9720p30fps@0.75X_%_dropped_frames", + "VP9720p30fps@0.75X_decoded_frames", + "VP9720p30fps@0.75X_dropped_frames", + "VP9720p30fps@1.25X_%_dropped_frames", + "VP9720p30fps@1.25X_decoded_frames", + "VP9720p30fps@1.25X_dropped_frames", + "VP9720p30fps@1.5X_%_dropped_frames", + "VP9720p30fps@1.5X_decoded_frames", + "VP9720p30fps@1.5X_dropped_frames", + "VP9720p30fps@1.75X_%_dropped_frames", + "VP9720p30fps@1.75X_decoded_frames", + "VP9720p30fps@1.75X_dropped_frames", + "VP9720p30fps@1X_%_dropped_frames", + "VP9720p30fps@1X_decoded_frames", + "VP9720p30fps@1X_dropped_frames", + "VP9720p30fps@2X_%_dropped_frames", + "VP9720p30fps@2X_decoded_frames", + "VP9720p30fps@2X_dropped_frames", + "VP9720pHQ30fps@0.25X_%_dropped_frames", + "VP9720pHQ30fps@0.25X_decoded_frames", + "VP9720pHQ30fps@0.25X_dropped_frames", + "VP9720pHQ30fps@0.5X_%_dropped_frames", + "VP9720pHQ30fps@0.5X_decoded_frames", + "VP9720pHQ30fps@0.5X_dropped_frames", + "VP9720pHQ30fps@0.75X_%_dropped_frames", + "VP9720pHQ30fps@0.75X_decoded_frames", + "VP9720pHQ30fps@0.75X_dropped_frames", + "VP9720pHQ30fps@1.25X_%_dropped_frames", + "VP9720pHQ30fps@1.25X_decoded_frames", + "VP9720pHQ30fps@1.25X_dropped_frames", + "VP9720pHQ30fps@1.5X_%_dropped_frames", + "VP9720pHQ30fps@1.5X_decoded_frames", + "VP9720pHQ30fps@1.5X_dropped_frames", + "VP9720pHQ30fps@1.75X_%_dropped_frames", + "VP9720pHQ30fps@1.75X_decoded_frames", + "VP9720pHQ30fps@1.75X_dropped_frames", + "VP9720pHQ30fps@1X_%_dropped_frames", + "VP9720pHQ30fps@1X_decoded_frames", + "VP9720pHQ30fps@1X_dropped_frames", + "VP9720pHQ30fps@2X_%_dropped_frames", + "VP9720pHQ30fps@2X_decoded_frames", + "VP9720pHQ30fps@2X_dropped_frames", + "VP9720pMQ30fps@0.25X_%_dropped_frames", + "VP9720pMQ30fps@0.25X_decoded_frames", + "VP9720pMQ30fps@0.25X_dropped_frames", + "VP9720pMQ30fps@0.5X_%_dropped_frames", + "VP9720pMQ30fps@0.5X_decoded_frames", + "VP9720pMQ30fps@0.5X_dropped_frames", + "VP9720pMQ30fps@0.75X_%_dropped_frames", + "VP9720pMQ30fps@0.75X_decoded_frames", + "VP9720pMQ30fps@0.75X_dropped_frames", + "VP9720pMQ30fps@1.25X_%_dropped_frames", + "VP9720pMQ30fps@1.25X_decoded_frames", + "VP9720pMQ30fps@1.25X_dropped_frames", + "VP9720pMQ30fps@1.5X_%_dropped_frames", + "VP9720pMQ30fps@1.5X_decoded_frames", + "VP9720pMQ30fps@1.5X_dropped_frames", + "VP9720pMQ30fps@1.75X_%_dropped_frames", + "VP9720pMQ30fps@1.75X_decoded_frames", + "VP9720pMQ30fps@1.75X_dropped_frames", + "VP9720pMQ30fps@1X_%_dropped_frames", + "VP9720pMQ30fps@1X_decoded_frames", + "VP9720pMQ30fps@1X_dropped_frames", + "VP9720pMQ30fps@2X_%_dropped_frames", + "VP9720pMQ30fps@2X_decoded_frames", + "VP9720pMQ30fps@2X_dropped_frames", +] + +YOUTUBE_PLAYBACK_2019 = [ + "VP9.720p60@2X_dropped_frames", + "VP9.720p60@2X_decoded_frames", + "VP9.720p60@2X_%_dropped_frames", + "VP9.720p60@1X_dropped_frames", + "VP9.720p60@1X_decoded_frames", + "VP9.720p60@1X_%_dropped_frames", + "VP9.720p60@1.5X_dropped_frames", + "VP9.720p60@1.5X_decoded_frames", + "VP9.720p60@1.5X_%_dropped_frames", + "VP9.720p60@1.25X_dropped_frames", + "VP9.720p60@1.25X_decoded_frames", + "VP9.720p60@1.25X_%_dropped_frames", + "VP9.720p60@0.5X_dropped_frames", + "VP9.720p60@0.5X_decoded_frames", + "VP9.720p60@0.5X_%_dropped_frames", + "VP9.720p60@0.25X_dropped_frames", + "VP9.720p60@0.25X_decoded_frames", + "VP9.720p60@0.25X_%_dropped_frames", + "VP9.720p30@2X_dropped_frames", + "VP9.720p30@2X_decoded_frames", + "VP9.720p30@2X_%_dropped_frames", + "VP9.720p30@1X_dropped_frames", + "VP9.720p30@1X_decoded_frames", + "VP9.720p30@1X_%_dropped_frames", + "VP9.720p30@1.5X_dropped_frames", + "VP9.720p30@1.5X_decoded_frames", + "VP9.720p30@1.5X_%_dropped_frames", + "VP9.720p30@1.25X_dropped_frames", + "VP9.720p30@1.25X_decoded_frames", + "VP9.720p30@1.25X_%_dropped_frames", + "VP9.480p30@2X_dropped_frames", + "VP9.480p30@2X_decoded_frames", + "VP9.480p30@2X_%_dropped_frames", + "VP9.480p30@1X_dropped_frames", + "VP9.480p30@1X_decoded_frames", + "VP9.480p30@1X_%_dropped_frames", + "VP9.480p30@1.5X_dropped_frames", + "VP9.480p30@1.5X_decoded_frames", + "VP9.480p30@1.5X_%_dropped_frames", + "VP9.480p30@1.25X_dropped_frames", + "VP9.480p30@1.25X_decoded_frames", + "VP9.480p30@1.25X_%_dropped_frames", + "VP9.360p30@2X_dropped_frames", + "VP9.360p30@2X_decoded_frames", + "VP9.360p30@2X_%_dropped_frames", + "VP9.360p30@1X_dropped_frames", + "VP9.360p30@1X_decoded_frames", + "VP9.360p30@1X_%_dropped_frames", + "VP9.360p30@1.5X_dropped_frames", + "VP9.360p30@1.5X_decoded_frames", + "VP9.360p30@1.5X_%_dropped_frames", + "VP9.360p30@1.25X_dropped_frames", + "VP9.360p30@1.25X_decoded_frames", + "VP9.360p30@1.25X_%_dropped_frames", + "VP9.240p30@2X_dropped_frames", + "VP9.240p30@2X_decoded_frames", + "VP9.240p30@2X_%_dropped_frames", + "VP9.240p30@1X_dropped_frames", + "VP9.240p30@1X_decoded_frames", + "VP9.240p30@1X_%_dropped_frames", + "VP9.240p30@1.5X_dropped_frames", + "VP9.240p30@1.5X_decoded_frames", + "VP9.240p30@1.5X_%_dropped_frames", + "VP9.240p30@1.25X_dropped_frames", + "VP9.240p30@1.25X_decoded_frames", + "VP9.240p30@1.25X_%_dropped_frames", + "VP9.2160p60@2X_dropped_frames", + "VP9.2160p60@2X_decoded_frames", + "VP9.2160p60@2X_%_dropped_frames", + "VP9.2160p60@1X_dropped_frames", + "VP9.2160p60@1X_decoded_frames", + "VP9.2160p60@1X_%_dropped_frames", + "VP9.2160p60@1.5X_dropped_frames", + "VP9.2160p60@1.5X_decoded_frames", + "VP9.2160p60@1.5X_%_dropped_frames", + "VP9.2160p60@1.25X_dropped_frames", + "VP9.2160p60@1.25X_decoded_frames", + "VP9.2160p60@1.25X_%_dropped_frames", + "VP9.2160p60@0.5X_dropped_frames", + "VP9.2160p60@0.5X_decoded_frames", + "VP9.2160p60@0.5X_%_dropped_frames", + "VP9.2160p60@0.25X_dropped_frames", + "VP9.2160p60@0.25X_decoded_frames", + "VP9.2160p60@0.25X_%_dropped_frames", + "VP9.2160p30@2X_dropped_frames", + "VP9.2160p30@2X_decoded_frames", + "VP9.2160p30@2X_%_dropped_frames", + "VP9.2160p30@1X_dropped_frames", + "VP9.2160p30@1X_decoded_frames", + "VP9.2160p30@1X_%_dropped_frames", + "VP9.2160p30@1.5X_dropped_frames", + "VP9.2160p30@1.5X_decoded_frames", + "VP9.2160p30@1.5X_%_dropped_frames", + "VP9.2160p30@1.25X_dropped_frames", + "VP9.2160p30@1.25X_decoded_frames", + "VP9.2160p30@1.25X_%_dropped_frames", + "VP9.144p30@2X_dropped_frames", + "VP9.144p30@2X_decoded_frames", + "VP9.144p30@2X_%_dropped_frames", + "VP9.144p30@1X_dropped_frames", + "VP9.144p30@1X_decoded_frames", + "VP9.144p30@1X_%_dropped_frames", + "VP9.144p30@1.5X_dropped_frames", + "VP9.144p30@1.5X_decoded_frames", + "VP9.144p30@1.5X_%_dropped_frames", + "VP9.144p30@1.25X_dropped_frames", + "VP9.144p30@1.25X_decoded_frames", + "VP9.144p30@1.25X_%_dropped_frames", + "VP9.1440p60@2X_dropped_frames", + "VP9.1440p60@2X_decoded_frames", + "VP9.1440p60@2X_%_dropped_frames", + "VP9.1440p60@1X_dropped_frames", + "VP9.1440p60@1X_decoded_frames", + "VP9.1440p60@1X_%_dropped_frames", + "VP9.1440p60@1.5X_dropped_frames", + "VP9.1440p60@1.5X_decoded_frames", + "VP9.1440p60@1.5X_%_dropped_frames", + "VP9.1440p60@1.25X_dropped_frames", + "VP9.1440p60@1.25X_decoded_frames", + "VP9.1440p60@1.25X_%_dropped_frames", + "VP9.1440p30@2X_dropped_frames", + "VP9.1440p30@2X_decoded_frames", + "VP9.1440p30@2X_%_dropped_frames", + "VP9.1440p30@1X_dropped_frames", + "VP9.1440p30@1X_decoded_frames", + "VP9.1440p30@1X_%_dropped_frames", + "VP9.1440p30@1.5X_dropped_frames", + "VP9.1440p30@1.5X_decoded_frames", + "VP9.1440p30@1.5X_%_dropped_frames", + "VP9.1440p30@1.25X_dropped_frames", + "VP9.1440p30@1.25X_decoded_frames", + "VP9.1440p30@1.25X_%_dropped_frames", + "VP9.1080p60@2X_dropped_frames", + "VP9.1080p60@2X_decoded_frames", + "VP9.1080p60@2X_%_dropped_frames", + "VP9.1080p60@1X_dropped_frames", + "VP9.1080p60@1X_decoded_frames", + "VP9.1080p60@1X_%_dropped_frames", + "VP9.1080p60@1.5X_dropped_frames", + "VP9.1080p60@1.5X_decoded_frames", + "VP9.1080p60@1.5X_%_dropped_frames", + "VP9.1080p60@1.25X_dropped_frames", + "VP9.1080p60@1.25X_decoded_frames", + "VP9.1080p60@1.25X_%_dropped_frames", + "VP9.1080p30@2X_dropped_frames", + "VP9.1080p30@2X_decoded_frames", + "VP9.1080p30@2X_%_dropped_frames", + "VP9.1080p30@1X_dropped_frames", + "VP9.1080p30@1X_decoded_frames", + "VP9.1080p30@1X_%_dropped_frames", + "VP9.1080p30@1.5X_dropped_frames", + "VP9.1080p30@1.5X_decoded_frames", + "VP9.1080p30@1.5X_%_dropped_frames", + "VP9.1080p30@1.25X_dropped_frames", + "VP9.1080p30@1.25X_decoded_frames", + "VP9.1080p30@1.25X_%_dropped_frames", + "H264.720p60@2X_dropped_frames", + "H264.720p60@2X_decoded_frames", + "H264.720p60@2X_%_dropped_frames", + "H264.720p60@1X_dropped_frames", + "H264.720p60@1X_decoded_frames", + "H264.720p60@1X_%_dropped_frames", + "H264.720p60@1.5X_dropped_frames", + "H264.720p60@1.5X_decoded_frames", + "H264.720p60@1.5X_%_dropped_frames", + "H264.720p60@1.25X_dropped_frames", + "H264.720p60@1.25X_decoded_frames", + "H264.720p60@1.25X_%_dropped_frames", + "H264.720p30@2X_dropped_frames", + "H264.720p30@2X_decoded_frames", + "H264.720p30@2X_%_dropped_frames", + "H264.720p30@1X_dropped_frames", + "H264.720p30@1X_decoded_frames", + "H264.720p30@1X_%_dropped_frames", + "H264.720p30@1.5X_dropped_frames", + "H264.720p30@1.5X_decoded_frames", + "H264.720p30@1.5X_%_dropped_frames", + "H264.720p30@1.25X_dropped_frames", + "H264.720p30@1.25X_decoded_frames", + "H264.720p30@1.25X_%_dropped_frames", + "H264.720p30@0.5X_dropped_frames", + "H264.720p30@0.5X_decoded_frames", + "H264.720p30@0.5X_%_dropped_frames", + "H264.720p30@0.25X_dropped_frames", + "H264.720p30@0.25X_decoded_frames", + "H264.720p30@0.25X_%_dropped_frames", + "H264.480p30@2X_dropped_frames", + "H264.480p30@2X_decoded_frames", + "H264.480p30@2X_%_dropped_frames", + "H264.480p30@1X_dropped_frames", + "H264.480p30@1X_decoded_frames", + "H264.480p30@1X_%_dropped_frames", + "H264.480p30@1.5X_dropped_frames", + "H264.480p30@1.5X_decoded_frames", + "H264.480p30@1.5X_%_dropped_frames", + "H264.480p30@1.25X_dropped_frames", + "H264.480p30@1.25X_decoded_frames", + "H264.480p30@1.25X_%_dropped_frames", + "H264.360p30@2X_dropped_frames", + "H264.360p30@2X_decoded_frames", + "H264.360p30@2X_%_dropped_frames", + "H264.360p30@1X_dropped_frames", + "H264.360p30@1X_decoded_frames", + "H264.360p30@1X_%_dropped_frames", + "H264.360p30@1.5X_dropped_frames", + "H264.360p30@1.5X_decoded_frames", + "H264.360p30@1.5X_%_dropped_frames", + "H264.360p30@1.25X_dropped_frames", + "H264.360p30@1.25X_decoded_frames", + "H264.360p30@1.25X_%_dropped_frames", + "H264.240p30@2X_dropped_frames", + "H264.240p30@2X_decoded_frames", + "H264.240p30@2X_%_dropped_frames", + "H264.240p30@1X_dropped_frames", + "H264.240p30@1X_decoded_frames", + "H264.240p30@1X_%_dropped_frames", + "H264.240p30@1.5X_dropped_frames", + "H264.240p30@1.5X_decoded_frames", + "H264.240p30@1.5X_%_dropped_frames", + "H264.240p30@1.25X_dropped_frames", + "H264.240p30@1.25X_decoded_frames", + "H264.240p30@1.25X_%_dropped_frames", + "H264.2160p30@2X_dropped_frames", + "H264.2160p30@2X_decoded_frames", + "H264.2160p30@2X_%_dropped_frames", + "H264.2160p30@1X_dropped_frames", + "H264.2160p30@1X_decoded_frames", + "H264.2160p30@1X_%_dropped_frames", + "H264.2160p30@1.5X_dropped_frames", + "H264.2160p30@1.5X_decoded_frames", + "H264.2160p30@1.5X_%_dropped_frames", + "H264.2160p30@1.25X_dropped_frames", + "H264.2160p30@1.25X_decoded_frames", + "H264.2160p30@1.25X_%_dropped_frames", + "H264.2160p30@0.5X_dropped_frames", + "H264.2160p30@0.5X_decoded_frames", + "H264.2160p30@0.5X_%_dropped_frames", + "H264.2160p30@0.25X_dropped_frames", + "H264.2160p30@0.25X_decoded_frames", + "H264.2160p30@0.25X_%_dropped_frames", + "H264.144p15@2X_dropped_frames", + "H264.144p15@2X_decoded_frames", + "H264.144p15@2X_%_dropped_frames", + "H264.144p15@1X_dropped_frames", + "H264.144p15@1X_decoded_frames", + "H264.144p15@1X_%_dropped_frames", + "H264.144p15@1.5X_dropped_frames", + "H264.144p15@1.5X_decoded_frames", + "H264.144p15@1.5X_%_dropped_frames", + "H264.144p15@1.25X_dropped_frames", + "H264.144p15@1.25X_decoded_frames", + "H264.144p15@1.25X_%_dropped_frames", + "H264.1440p30@2X_dropped_frames", + "H264.1440p30@2X_decoded_frames", + "H264.1440p30@2X_%_dropped_frames", + "H264.1440p30@1X_dropped_frames", + "H264.1440p30@1X_decoded_frames", + "H264.1440p30@1X_%_dropped_frames", + "H264.1440p30@1.5X_dropped_frames", + "H264.1440p30@1.5X_decoded_frames", + "H264.1440p30@1.5X_%_dropped_frames", + "H264.1440p30@1.25X_dropped_frames", + "H264.1440p30@1.25X_decoded_frames", + "H264.1440p30@1.25X_%_dropped_frames", + "H264.1080p60@2X_dropped_frames", + "H264.1080p60@2X_decoded_frames", + "H264.1080p60@2X_%_dropped_frames", + "H264.1080p60@1X_dropped_frames", + "H264.1080p60@1X_decoded_frames", + "H264.1080p60@1X_%_dropped_frames", + "H264.1080p60@1.5X_dropped_frames", + "H264.1080p60@1.5X_decoded_frames", + "H264.1080p60@1.5X_%_dropped_frames", + "H264.1080p60@1.25X_dropped_frames", + "H264.1080p60@1.25X_decoded_frames", + "H264.1080p60@1.25X_%_dropped_frames", + "H264.1080p30@2X_dropped_frames", + "H264.1080p30@2X_decoded_frames", + "H264.1080p30@2X_%_dropped_frames", + "H264.1080p30@1X_dropped_frames", + "H264.1080p30@1X_decoded_frames", + "H264.1080p30@1X_%_dropped_frames", + "H264.1080p30@1.5X_dropped_frames", + "H264.1080p30@1.5X_decoded_frames", + "H264.1080p30@1.5X_%_dropped_frames", + "H264.1080p30@1.25X_dropped_frames", + "H264.1080p30@1.25X_decoded_frames", + "H264.1080p30@1.25X_%_dropped_frames", +] + +r = ( + HFR + + SFR_AV1 + + SFR_H264 + + SFR_VP9 + + WIDEVINE_HFR + + WIDEVINE_SFR_H264 + + WIDEVINE_SFR_VP9 + + YOUTUBE_PLAYBACK_2019 +) + +YOUTUBE_PLAYBACK_MEASURE = list(set(r)) +YOUTUBE_PLAYBACK_MEASURE.sort() diff --git a/testing/raptor/logger/__init__.py b/testing/raptor/logger/__init__.py new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/testing/raptor/logger/__init__.py diff --git a/testing/raptor/logger/logger.py b/testing/raptor/logger/logger.py new file mode 100644 index 0000000000..7e5f3b6f8d --- /dev/null +++ b/testing/raptor/logger/logger.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python + +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from mozlog.proxy import ProxyLogger + + +class RaptorLogger: + app = "" + + def __init__(self, component=None): + self.logger = ProxyLogger(component) + + def set_app(self, app_name): + """Sets the app name to prefix messages with.""" + RaptorLogger.app = " [" + app_name + "]" + + def exception(self, message, **kwargs): + self.critical(message, **kwargs) + + def debug(self, message, **kwargs): + return self.logger.debug("Debug: {}".format(message), **kwargs) + + def info(self, message, **kwargs): + return self.logger.info("Info: {}".format(message), **kwargs) + + def warning(self, message, **kwargs): + return self.logger.warning( + "Warning:{} {}".format(RaptorLogger.app, message), **kwargs + ) + + def error(self, message, **kwargs): + return self.logger.error( + "Error:{} {}".format(RaptorLogger.app, message), **kwargs + ) + + def critical(self, message, **kwargs): + return self.logger.critical( + "Critical:{} {}".format(RaptorLogger.app, message), **kwargs + ) + + def log_raw(self, message, **kwargs): + return self.logger.log_raw(message, **kwargs) + + def process_output(self, *args, **kwargs): + return self.logger.process_output(*args, **kwargs) + + def crash(self, *args, **kwargs): + return self.logger.crash(*args, **kwargs) diff --git a/testing/raptor/mach_commands.py b/testing/raptor/mach_commands.py new file mode 100644 index 0000000000..54a25a75e4 --- /dev/null +++ b/testing/raptor/mach_commands.py @@ -0,0 +1,447 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# Originally taken from /talos/mach_commands.py + +# Integrates raptor mozharness with mach + +import json +import logging +import os +import socket +import subprocess +import sys + +from mach.decorators import Command +from mach.util import get_state_dir +from mozbuild.base import BinaryNotFoundException +from mozbuild.base import MachCommandConditions as Conditions +from mozbuild.base import MozbuildObject + +HERE = os.path.dirname(os.path.realpath(__file__)) + +ANDROID_BROWSERS = ["geckoview", "refbrow", "fenix", "chrome-m"] + + +class RaptorRunner(MozbuildObject): + def run_test(self, raptor_args, kwargs): + """Setup and run mozharness. + + We want to do a few things before running Raptor: + + 1. Clone mozharness + 2. Make the config for Raptor mozharness + 3. Run mozharness + """ + self.init_variables(raptor_args, kwargs) + self.make_config() + self.write_config() + self.make_args() + return self.run_mozharness() + + def init_variables(self, raptor_args, kwargs): + self.raptor_args = raptor_args + + if kwargs.get("host") == "HOST_IP": + kwargs["host"] = os.environ["HOST_IP"] + self.host = kwargs["host"] + self.is_release_build = kwargs["is_release_build"] + self.memory_test = kwargs["memory_test"] + self.power_test = kwargs["power_test"] + self.cpu_test = kwargs["cpu_test"] + self.live_sites = kwargs["live_sites"] + self.disable_perf_tuning = kwargs["disable_perf_tuning"] + self.conditioned_profile = kwargs["conditioned_profile"] + self.device_name = kwargs["device_name"] + self.enable_marionette_trace = kwargs["enable_marionette_trace"] + self.browsertime_visualmetrics = kwargs["browsertime_visualmetrics"] + self.browsertime_node = kwargs["browsertime_node"] + self.clean = kwargs["clean"] + + if Conditions.is_android(self) or kwargs["app"] in ANDROID_BROWSERS: + self.binary_path = None + else: + self.binary_path = kwargs.get("binary") or self.get_binary_path() + + self.python = sys.executable + + self.raptor_dir = os.path.join(self.topsrcdir, "testing", "raptor") + self.mozharness_dir = os.path.join(self.topsrcdir, "testing", "mozharness") + self.config_file_path = os.path.join( + self._topobjdir, "testing", "raptor-in_tree_conf.json" + ) + + self.virtualenv_script = os.path.join( + self.topsrcdir, "third_party", "python", "virtualenv", "virtualenv.py" + ) + self.virtualenv_path = os.path.join(self._topobjdir, "testing", "raptor-venv") + + def make_config(self): + default_actions = [ + "populate-webroot", + "create-virtualenv", + "install-chromium-distribution", + "run-tests", + ] + self.config = { + "run_local": True, + "binary_path": self.binary_path, + "repo_path": self.topsrcdir, + "raptor_path": self.raptor_dir, + "obj_path": self.topobjdir, + "log_name": "raptor", + "virtualenv_path": self.virtualenv_path, + "pypi_url": "http://pypi.org/simple", + "base_work_dir": self.mozharness_dir, + "exes": { + "python": self.python, + "virtualenv": [self.python, self.virtualenv_script], + }, + "title": socket.gethostname(), + "default_actions": default_actions, + "raptor_cmd_line_args": self.raptor_args, + "host": self.host, + "power_test": self.power_test, + "memory_test": self.memory_test, + "cpu_test": self.cpu_test, + "live_sites": self.live_sites, + "disable_perf_tuning": self.disable_perf_tuning, + "conditioned_profile": self.conditioned_profile, + "is_release_build": self.is_release_build, + "device_name": self.device_name, + "enable_marionette_trace": self.enable_marionette_trace, + "browsertime_visualmetrics": self.browsertime_visualmetrics, + "browsertime_node": self.browsertime_node, + "mozbuild_path": get_state_dir(), + "clean": self.clean, + } + + sys.path.insert(0, os.path.join(self.topsrcdir, "tools", "browsertime")) + try: + import platform + + import mach_commands as browsertime + + # We don't set `browsertime_{chromedriver,geckodriver} -- those will be found by + # browsertime in its `node_modules` directory, which is appropriate for local builds. + # We don't set `browsertime_ffmpeg` yet: it will need to be on the path. There is code + # to configure the environment including the path in + # `tools/browsertime/mach_commands.py` but integrating it here will take more effort. + self.config.update( + { + "browsertime_browsertimejs": browsertime.browsertime_path(), + "browsertime_vismet_script": browsertime.visualmetrics_path(), + } + ) + + def _get_browsertime_package(): + with open( + os.path.join( + self.topsrcdir, + "tools", + "browsertime", + "node_modules", + "browsertime", + "package.json", + ) + ) as package: + return json.load(package) + + def _get_browsertime_resolved(): + try: + with open( + os.path.join( + self.topsrcdir, + "tools", + "browsertime", + "node_modules", + ".package-lock.json", + ) + ) as package_lock: + return json.load(package_lock)["packages"][ + "node_modules/browsertime" + ]["resolved"] + except FileNotFoundError: + # Older versions of node/npm add this metadata to package.json + return _get_browsertime_package()["_from"] + + def _should_install(): + # If ffmpeg doesn't exist in the .mozbuild directory, + # then we should install + btime_cache = os.path.join(self.config["mozbuild_path"], "browsertime") + if not os.path.exists(btime_cache) or not any( + ["ffmpeg" in cache_dir for cache_dir in os.listdir(btime_cache)] + ): + return True + + # If browsertime doesn't exist, install it + if not os.path.exists( + self.config["browsertime_browsertimejs"] + ) or not os.path.exists(self.config["browsertime_vismet_script"]): + return True + + # Browsertime exists, check if it's outdated + with open( + os.path.join(self.topsrcdir, "tools", "browsertime", "package.json") + ) as new: + new_pkg = json.load(new) + + return not _get_browsertime_resolved().endswith( + new_pkg["devDependencies"]["browsertime"] + ) + + def _get_browsertime_version(): + # Returns the (version number, current commit) used + return ( + _get_browsertime_package()["version"], + _get_browsertime_resolved(), + ) + + # Check if browsertime scripts exist and try to install them if + # they aren't + if _should_install(): + # TODO: Make this "integration" nicer in the near future + print("Missing browsertime files...attempting to install") + subprocess.check_output( + [ + os.path.join(self.topsrcdir, "mach"), + "browsertime", + "--setup", + "--clobber", + ], + shell="windows" in platform.system().lower(), + ) + if _should_install(): + raise Exception( + "Failed installation attempt. Cannot find browsertime scripts. " + "Run `./mach browsertime --setup --clobber` to set it up." + ) + + # Bug 1766112 - For the time being, we need to trigger a + # clean build to upgrade browsertime. This should be disabled + # after some time. + print( + "Setting --clean to True to rebuild Python " + "environment for Browsertime upgrade..." + ) + self.config["clean"] = True + + print("Using browsertime version %s from %s" % _get_browsertime_version()) + + finally: + sys.path = sys.path[1:] + + def make_args(self): + self.args = { + "config": {}, + "initial_config_file": self.config_file_path, + } + + def write_config(self): + try: + config_file = open(self.config_file_path, "w") + config_file.write(json.dumps(self.config)) + config_file.close() + except IOError as e: + err_str = "Error writing to Raptor Mozharness config file {0}:{1}" + print(err_str.format(self.config_file_path, str(e))) + raise e + + def run_mozharness(self): + sys.path.insert(0, self.mozharness_dir) + from mozharness.mozilla.testing.raptor import Raptor + + raptor_mh = Raptor( + config=self.args["config"], + initial_config_file=self.args["initial_config_file"], + ) + return raptor_mh.run() + + +def setup_node(command_context): + """Fetch the latest node-16 binary and install it into the .mozbuild directory.""" + import platform + from distutils.version import StrictVersion + + from mozbuild.artifact_commands import artifact_toolchain + from mozbuild.nodeutil import find_node_executable + + print("Setting up node for browsertime...") + state_dir = get_state_dir() + cache_path = os.path.join(state_dir, "browsertime", "node-16") + + def __check_for_node(): + # Check standard locations first + node_exe = find_node_executable(min_version=StrictVersion("16.0.0")) + if node_exe and (node_exe[0] is not None): + return node_exe[0] + if not os.path.exists(cache_path): + return None + + # Check the browsertime-specific node location next + node_name = "node" + if platform.system() == "Windows": + node_name = "node.exe" + node_exe_path = os.path.join( + state_dir, + "browsertime", + "node-16", + "node", + ) + else: + node_exe_path = os.path.join( + state_dir, + "browsertime", + "node-16", + "node", + "bin", + ) + + node_exe = os.path.join(node_exe_path, node_name) + if not os.path.exists(node_exe): + return None + + return node_exe + + node_exe = __check_for_node() + if node_exe is None: + toolchain_job = "{}-node-16" + plat = platform.system() + if plat == "Windows": + toolchain_job = toolchain_job.format("win64") + elif plat == "Darwin": + toolchain_job = toolchain_job.format("macosx64") + else: + toolchain_job = toolchain_job.format("linux64") + + print( + "Downloading Node v16 from Taskcluster toolchain {}...".format( + toolchain_job + ) + ) + + if not os.path.exists(cache_path): + os.makedirs(cache_path, exist_ok=True) + + # Change directories to where node should be installed + # before installing. Otherwise, it gets installed in the + # top level of the repo (or the current working directory). + cur_dir = os.getcwd() + os.chdir(cache_path) + artifact_toolchain( + command_context, + verbose=False, + from_build=[toolchain_job], + no_unpack=False, + retry=0, + cache_dir=cache_path, + ) + os.chdir(cur_dir) + + node_exe = __check_for_node() + if node_exe is None: + raise Exception("Could not find Node v16 binary for Raptor-Browsertime") + + print("Finished downloading Node v16 from Taskcluster") + + print("Node v16+ found at: %s" % node_exe) + return node_exe + + +def create_parser(): + sys.path.insert(0, HERE) # allow to import the raptor package + from raptor.cmdline import create_parser + + return create_parser(mach_interface=True) + + +@Command( + "raptor", + category="testing", + description="Run Raptor performance tests.", + parser=create_parser, +) +def run_raptor(command_context, **kwargs): + # Defers this import so that a transitive dependency doesn't + # stop |mach bootstrap| from running + from raptor.power import disable_charging, enable_charging + + build_obj = command_context + + # Setup node for browsertime + kwargs["browsertime_node"] = setup_node(command_context) + + is_android = Conditions.is_android(build_obj) or kwargs["app"] in ANDROID_BROWSERS + + if is_android: + from mozdevice import ADBDeviceFactory + from mozrunner.devices.android_device import ( + InstallIntent, + verify_android_device, + ) + + install = ( + InstallIntent.NO if kwargs.pop("noinstall", False) else InstallIntent.YES + ) + verbose = False + if ( + kwargs.get("log_mach_verbose") + or kwargs.get("log_tbpl_level") == "debug" + or kwargs.get("log_mach_level") == "debug" + or kwargs.get("log_raw_level") == "debug" + ): + verbose = True + if not verify_android_device( + build_obj, + install=install, + app=kwargs["binary"], + verbose=verbose, + xre=True, + ): # Equivalent to 'run_local' = True. + return 1 + # Disable fission until geckoview supports fission by default. + # Need fission on Android? Use '--setpref fission.autostart=true' + kwargs["fission"] = False + + # Remove mach global arguments from sys.argv to prevent them + # from being consumed by raptor. Treat any item in sys.argv + # occuring before "raptor" as a mach global argument. + argv = [] + in_mach = True + for arg in sys.argv: + if not in_mach: + argv.append(arg) + if arg.startswith("raptor"): + in_mach = False + + raptor = command_context._spawn(RaptorRunner) + device = None + + try: + if kwargs["power_test"] and is_android: + device = ADBDeviceFactory(verbose=True) + disable_charging(device) + return raptor.run_test(argv, kwargs) + except BinaryNotFoundException as e: + command_context.log( + logging.ERROR, "raptor", {"error": str(e)}, "ERROR: {error}" + ) + command_context.log(logging.INFO, "raptor", {"help": e.help()}, "{help}") + return 1 + except Exception as e: + print(repr(e)) + return 1 + finally: + if kwargs["power_test"] and device: + enable_charging(device) + + +@Command( + "raptor-test", + category="testing", + description="Run Raptor performance tests.", + parser=create_parser, +) +def run_raptor_test(command_context, **kwargs): + return run_raptor(command_context, **kwargs) diff --git a/testing/raptor/moz.build b/testing/raptor/moz.build new file mode 100644 index 0000000000..7d2d5695f3 --- /dev/null +++ b/testing/raptor/moz.build @@ -0,0 +1,7 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +with Files("**"): + BUG_COMPONENT = ("Testing", "Raptor") + SCHEDULES.exclusive = ["raptor"] diff --git a/testing/raptor/raptor/__init__.py b/testing/raptor/raptor/__init__.py new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/testing/raptor/raptor/__init__.py diff --git a/testing/raptor/raptor/benchmark.py b/testing/raptor/raptor/benchmark.py new file mode 100644 index 0000000000..311728eff6 --- /dev/null +++ b/testing/raptor/raptor/benchmark.py @@ -0,0 +1,312 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import os +import pathlib +import shutil +import socket +import subprocess +import tempfile + +import mozfile +from logger.logger import RaptorLogger +from wptserve import handlers, server + +LOG = RaptorLogger(component="raptor-benchmark") +here = pathlib.Path(__file__).parent.resolve() + + +class Benchmark(object): + """utility class for running benchmarks in raptor""" + + def __init__(self, config, test): + self.config = config + self.test = test + + self.setup_benchmarks( + os.getenv("MOZ_DEVELOPER_REPO_DIR"), + os.getenv("MOZ_MOZBUILD_DIR"), + run_local=self.config.get("run_local", False), + ) + + LOG.info(f"bench_dir: {self.bench_dir}") + LOG.info("bench_dir contains:") + LOG.info(list(self.bench_dir.iterdir())) + + # now have the benchmark source ready, go ahead and serve it up! + self.start_http_server() + + def start_http_server(self): + self.write_server_headers() + + # pick a free port + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.bind(("", 0)) + self.host = self.config["host"] + self.port = sock.getsockname()[1] + sock.close() + _webserver = "%s:%d" % (self.host, self.port) + + self.httpd = self.setup_webserver(_webserver) + self.httpd.start() + + def write_server_headers(self): + # to add specific headers for serving files via wptserve, write out a headers dir file + # see http://wptserve.readthedocs.io/en/latest/handlers.html#file-handlers + LOG.info("writing wptserve headers file") + headers_file = pathlib.Path(self.bench_dir, "__dir__.headers") + file = headers_file.open("w") + file.write("Access-Control-Allow-Origin: *") + file.close() + LOG.info("wrote wpt headers file: %s" % headers_file) + + def setup_webserver(self, webserver): + LOG.info("starting webserver on %r" % webserver) + LOG.info("serving benchmarks from here: %s" % self.bench_dir) + self.host, self.port = webserver.split(":") + + return server.WebTestHttpd( + host=self.host, + port=int(self.port), + doc_root=str(self.bench_dir), + routes=[("GET", "*", handlers.file_handler)], + ) + + def stop_serve(self): + LOG.info("TODO: stop serving benchmark source") + pass + + def _full_clone(self, benchmark_repository, dest): + subprocess.check_call( + [ + "git", + "clone", + "-c", + "http.postBuffer=2147483648", + "-c", + "core.autocrlf=false", + benchmark_repository, + str(dest.resolve()), + ] + ) + + def _get_benchmark_folder(self, benchmark_dest, run_local): + if not run_local: + # If the test didn't specify a repo and we're in CI + # then we'll find them here and we don't need to do anything else + return pathlib.Path(benchmark_dest, "tests", "webkit", "PerformanceTests") + return pathlib.Path(benchmark_dest, "testing", "raptor", "benchmarks") + + def _sparse_clone(self, benchmark_repository, dest): + """Get a partial clone of the repo. + + This need git version 2.30+ so it's currently unused but it works. + See bug 1804694. This method should only be used in CI, locally we + can simply pull the whole repo. + """ + subprocess.check_call( + [ + "git", + "clone", + "--depth", + "1", + "--filter", + "blob:none", + "--sparse", + benchmark_repository, + str(dest.resolve()), + ] + ) + subprocess.check_call( + [ + "git", + "sparse-checkout", + "set", + self.test.get("repository_path", "benchmarks"), + ], + cwd=dest, + ) + + def _copy_or_link_files( + self, benchmark_path, benchmark_dest, skip_files_and_hidden=True + ): + if not benchmark_dest.exists(): + benchmark_dest.mkdir(parents=True, exist_ok=True) + + dest = pathlib.Path(benchmark_dest, benchmark_path.name) + if hasattr(os, "symlink") and os.name != "nt": + if not dest.exists(): + os.symlink(benchmark_path, dest) + else: + # Clobber the benchmark in case a recent update removed any files. + mozfile.remove(str(dest.resolve())) + shutil.copytree(benchmark_path, dest) + + if any(path.is_file() for path in benchmark_path.iterdir()): + # Host the parent of this directory to prevent hosting issues + # (e.g. linked files ending up with different routes) + host_folder = dest.parent + self.test["test_url"] = self.test["test_url"].replace( + "<port>/", f"<port>/{benchmark_path.name}/" + ) + dest = host_folder + + return dest + + def _verify_benchmark_revision(self, benchmark_revision, external_repo_path): + try: + # Check if the given revision is valid + subprocess.check_call( + ["git", "rev-parse", "--verify", f"{benchmark_revision}^{{commit}}"], + cwd=external_repo_path, + ) + LOG.info("Given benchmark repository revision verified") + except Exception: + LOG.error( + f"Given revision doesn't exist in this repository: {benchmark_revision}" + ) + raise + + def _update_benchmark_repo(self, external_repo_path): + default_branch = self.test.get("repository_branch", None) + if default_branch is None: + try: + # Get the default branch name, and check it if's been updated + default_branch = ( + subprocess.check_output( + ["git", "rev-parse", "--abbrev-ref", "origin/HEAD"], + cwd=external_repo_path, + ) + .decode("utf-8") + .strip() + .split("/")[-1] + ) + remote_default_branch = ( + subprocess.check_output( + ["git", "remote", "set-head", "origin", "-a"], + cwd=external_repo_path, + ) + .decode("utf-8") + .strip() + ) + if default_branch not in remote_default_branch: + default_branch = remote_default_branch.split()[-1] + except Exception: + LOG.critical("Failed to find the default branch of the repository!") + raise + else: + LOG.info(f"Using non-default branch {default_branch}") + try: + subprocess.check_call(["git", "pull", "--all"], cwd=external_repo_path) + except subprocess.CalledProcessError: + LOG.info("Failed to pull new branches from remote") + + LOG.info(external_repo_path) + subprocess.check_call( + ["git", "checkout", default_branch], cwd=external_repo_path + ) + subprocess.check_call(["git", "pull"], cwd=external_repo_path) + + def _setup_git_benchmarks(self, mozbuild_path, benchmark_dest, run_local=True): + """Setup a benchmark from a github repository.""" + benchmark_repository = self.test["repository"] + benchmark_revision = self.test["repository_revision"] + + # Specifies where we can find the benchmark within the cloned repo, this is the + # folder that will be hosted to run the test. If it isn't given, we'll host the + # root of the repository. + benchmark_repo_path = self.test.get("repository_path", "") + + # Get the performance-tests cache (if it exists), otherwise create a temp folder + if mozbuild_path is None: + mozbuild_path = tempfile.mkdtemp() + + external_repo_path = pathlib.Path( + mozbuild_path, "performance-tests", benchmark_repository.split("/")[-1] + ) + + try: + subprocess.check_output(["git", "--version"]) + except Exception as ex: + LOG.info( + "Git is not available! Please install git and " + "ensure it is included in the terminal path" + ) + raise ex + + if not external_repo_path.is_dir(): + LOG.info("Cloning the benchmarks to {}".format(external_repo_path)) + # Bug 1804694 - Use sparse checkouts instead of full clones + # Locally, we should always do a full clone + self._full_clone(benchmark_repository, external_repo_path) + else: + self._update_benchmark_repo(external_repo_path) + + self._verify_benchmark_revision(benchmark_revision, external_repo_path) + subprocess.check_call( + ["git", "checkout", benchmark_revision], cwd=external_repo_path + ) + + benchmark_dest = pathlib.Path( + self._get_benchmark_folder(benchmark_dest, run_local), self.test["name"] + ) + benchmark_dest = self._copy_or_link_files( + pathlib.Path(external_repo_path, benchmark_repo_path), + benchmark_dest, + skip_files_and_hidden=False, + ) + + return benchmark_dest + + def _setup_in_tree_benchmarks(self, topsrc_path, benchmark_dest, run_local=True): + """Setup a benchmakr that is found in-tree. + + This method will be deprecated once bug 1804578 is resolved (copying our + in-tree benchmarks into a repo) to have a standard way of running benchmarks. + """ + benchmark_dest = self._get_benchmark_folder(benchmark_dest, run_local) + if not run_local: + # If the test didn't specify a repo and we're in CI + # then we'll find them here and we don't need to do anything else + return benchmark_dest + + benchmark_dest = self._copy_or_link_files( + pathlib.Path(topsrc_path, "third_party", "webkit", "PerformanceTests"), + benchmark_dest, + ) + + return benchmark_dest + + def setup_benchmarks( + self, + topsrc_path, + mozbuild_path, + run_local=True, + ): + """Make sure benchmarks are linked to the proper location in the objdir. + + Benchmarks can either live in-tree or in an external repository. In the latter + case also clone/update the repository if necessary. + """ + # bench_dir is where we will download all mitmproxy required files + # when running locally it comes from obj_path via mozharness/mach + if self.config.get("obj_path", None) is not None: + bench_dir = pathlib.Path(self.config.get("obj_path")) + else: + # in production it is ../tasks/task_N/build/tests/raptor/raptor/... + # 'here' is that path, we can start with that + bench_dir = pathlib.Path(here) + + if self.test.get("repository", None) is not None: + # Setup benchmarks that are found on Github + bench_dir = self._setup_git_benchmarks( + mozbuild_path, bench_dir, run_local=run_local + ) + else: + # Setup the benchmarks that are available in-tree + bench_dir = self._setup_in_tree_benchmarks( + topsrc_path, bench_dir, run_local=run_local + ) + + self.bench_dir = bench_dir diff --git a/testing/raptor/raptor/browsertime/__init__.py b/testing/raptor/raptor/browsertime/__init__.py new file mode 100644 index 0000000000..2f20774686 --- /dev/null +++ b/testing/raptor/raptor/browsertime/__init__.py @@ -0,0 +1,8 @@ +# flake8: noqa +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +from .android import BrowsertimeAndroid +from .base import Browsertime +from .desktop import BrowsertimeDesktop diff --git a/testing/raptor/raptor/browsertime/android.py b/testing/raptor/raptor/browsertime/android.py new file mode 100644 index 0000000000..5d3e852a27 --- /dev/null +++ b/testing/raptor/raptor/browsertime/android.py @@ -0,0 +1,260 @@ +#!/usr/bin/env python + +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import os +import shutil +import tempfile + +import mozcrash +from logger.logger import RaptorLogger +from mozdevice import ADBDeviceFactory +from performance_tuning import tune_performance +from perftest import PerftestAndroid +from power import disable_charging, enable_charging + +from .base import Browsertime + +LOG = RaptorLogger(component="raptor-browsertime-android") + + +class BrowsertimeAndroid(PerftestAndroid, Browsertime): + """Android setup and configuration for browsertime + + When running raptor-browsertime tests on android, we create the profile (and set the proxy + prefs in the profile that is using playback) but we don't need to copy it onto the device + because geckodriver takes care of that. + We tell browsertime to use our profile (we pass it in with the firefox.profileTemplate arg); + browsertime creates a copy of that and passes that into geckodriver. Geckodriver then takes + the profile and copies it onto the mobile device's test root for us; and then it even writes + the geckoview app config.yaml file onto the device, which points the app to the profile on + the device's test root. + Therefore, raptor doesn't have to copy the profile onto the scard (and create the config.yaml) + file ourselves. Also note when using playback, the nss certificate db is created as usual when + mitmproxy is started (and saved in the profile) so it is already included in the profile that + browsertime/geckodriver copies onto the device. + XXX: bc: This doesn't work with scoped storage in Android 10 since the shell owns the profile + directory that is pushed to the device and the profile can no longer be on the sdcard. But when + geckodriver's android.rs defines the profile to be located on internal storage, it will be + owned by shell but if we are attempting to eliminate root, then when we run shell commands + as the app, they will fail due to the app being unable to write to the shell owned profile + directory. + """ + + def __init__(self, app, binary, activity=None, intent=None, **kwargs): + super(BrowsertimeAndroid, self).__init__( + app, binary, profile_class="firefox", **kwargs + ) + + self.config.update({"activity": activity, "intent": intent}) + self.remote_profile = None + + def _initialize_device(self): + if self.device is None: + self.device = ADBDeviceFactory(verbose=True) + if not self.config.get("disable_perf_tuning", False): + tune_performance(self.device, log=LOG) + + @property + def android_external_storage(self): + if self._remote_test_root is None: + self._initialize_device() + + external_storage = self.device.shell_output("echo $EXTERNAL_STORAGE") + self._remote_test_root = os.path.join( + external_storage, + "Android", + "data", + self.config["binary"], + "files", + "test_root", + ) + + return self._remote_test_root + + @property + def browsertime_args(self): + args_list = ["--viewPort", "1366x695"] + + if self.config["app"] == "chrome-m": + args_list.extend( + [ + "--browser", + "chrome", + "--android", + ] + ) + else: + activity = self.config["activity"] + if self.config["app"] == "fenix": + LOG.info( + "Changing initial activity to " + "`mozilla.telemetry.glean.debug.GleanDebugActivity`" + ) + activity = "mozilla.telemetry.glean.debug.GleanDebugActivity" + + args_list.extend( + [ + "--browser", + "firefox", + "--android", + "--firefox.android.package", + self.config["binary"], + "--firefox.android.activity", + activity, + ] + ) + + # Setup power testing + if self.config["power_test"]: + args_list.extend(["--androidPower", "true"]) + + if self.config["app"] == "fenix": + # See bug 1768889 + args_list.extend(["--ignoreShutdownFailures", "true"]) + + # If running on Fenix we must add the intent as we use a + # special non-default one there + if self.config.get("intent") is not None: + args_list.extend(["--firefox.android.intentArgument=-a"]) + args_list.extend( + ["--firefox.android.intentArgument", self.config["intent"]] + ) + + # Change glean ping names in all cases on Fenix + args_list.extend( + [ + "--firefox.android.intentArgument=--es", + "--firefox.android.intentArgument=startNext", + "--firefox.android.intentArgument=" + self.config["activity"], + "--firefox.android.intentArgument=--esa", + "--firefox.android.intentArgument=sourceTags", + "--firefox.android.intentArgument=automation", + "--firefox.android.intentArgument=--ez", + "--firefox.android.intentArgument=performancetest", + "--firefox.android.intentArgument=true", + ] + ) + + args_list.extend(["--firefox.android.intentArgument=-d"]) + args_list.extend( + ["--firefox.android.intentArgument", str("about:blank")] + ) + + return args_list + + def setup_chrome_args(self, test): + chrome_args = [ + "--use-mock-keychain", + "--no-default-browser-check", + "--no-first-run", + ] + + if test.get("playback", False): + pb_args = [ + "--proxy-server=%s:%d" % (self.playback.host, self.playback.port), + "--proxy-bypass-list=localhost;127.0.0.1", + "--ignore-certificate-errors", + ] + + if not self.is_localhost: + pb_args[0] = pb_args[0].replace("127.0.0.1", self.config["host"]) + + chrome_args.extend(pb_args) + + if self.debug_mode: + chrome_args.extend(["--auto-open-devtools-for-tabs"]) + + args_list = [] + for arg in chrome_args: + args_list.extend(["--chrome.args=" + str(arg.replace("'", '"'))]) + + return args_list + + def build_browser_profile(self): + super(BrowsertimeAndroid, self).build_browser_profile() + + # Merge in the Android profile. + path = os.path.join(self.profile_data_dir, "raptor-android") + LOG.info("Merging profile: {}".format(path)) + self.profile.merge(path) + self.profile.set_preferences( + {"browser.tabs.remote.autostart": self.config["e10s"]} + ) + + # There's no great way to have "after" advice in Python, so we do this + # in super and then again here since the profile merging re-introduces + # the "#MozRunner" delimiters. + self.remove_mozprofile_delimiters_from_profile() + + def setup_adb_device(self): + self._initialize_device() + + self.clear_app_data() + self.set_debug_app_flag() + self.device.run_as_package = self.config["binary"] + + self.geckodriver_profile = os.path.join( + self.android_external_storage, + "%s-geckodriver-profile" % self.config["binary"], + ) + + # make sure no remote profile exists + if self.device.exists(self.geckodriver_profile): + self.device.rm(self.geckodriver_profile, force=True, recursive=True) + + def check_for_crashes(self): + super(BrowsertimeAndroid, self).check_for_crashes() + + try: + dump_dir = tempfile.mkdtemp() + remote_dir = os.path.join(self.geckodriver_profile, "minidumps") + if not self.device.is_dir(remote_dir): + return + self.device.pull(remote_dir, dump_dir) + self.crashes += mozcrash.log_crashes( + LOG, dump_dir, self.config["symbols_path"] + ) + except Exception as e: + LOG.error( + "Could not pull the crash data!", + exc_info=True, + ) + raise e + finally: + try: + shutil.rmtree(dump_dir) + except Exception: + LOG.warning("unable to remove directory: %s" % dump_dir) + + def run_test_setup(self, test): + super(BrowsertimeAndroid, self).run_test_setup(test) + + self.set_reverse_ports() + + if self.playback: + self.turn_on_android_app_proxy() + self.remove_mozprofile_delimiters_from_profile() + + def run_tests(self, tests, test_names): + self.setup_adb_device() + + if self.config["app"] == "chrome-m": + # Make sure that chrome is enabled on the device + self.device.shell_output("pm enable com.android.chrome") + + try: + if self.config["power_test"]: + disable_charging(self.device) + return super(BrowsertimeAndroid, self).run_tests(tests, test_names) + finally: + if self.config["power_test"]: + enable_charging(self.device) + + def run_test_teardown(self, test): + LOG.info("removing reverse socket connections") + self.device.remove_socket_connections("reverse") + + super(BrowsertimeAndroid, self).run_test_teardown(test) diff --git a/testing/raptor/raptor/browsertime/base.py b/testing/raptor/raptor/browsertime/base.py new file mode 100644 index 0000000000..2f8b1f0861 --- /dev/null +++ b/testing/raptor/raptor/browsertime/base.py @@ -0,0 +1,831 @@ +#!/usr/bin/env python + +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import json +import os +import re +import sys +from abc import ABCMeta, abstractmethod +from copy import deepcopy + +import mozprocess +import six +from benchmark import Benchmark +from logger.logger import RaptorLogger +from manifestparser.util import evaluate_list_from_string +from perftest import Perftest +from results import BrowsertimeResultsHandler + +LOG = RaptorLogger(component="raptor-browsertime") + +DEFAULT_CHROMEVERSION = "77" +BROWSERTIME_PAGELOAD_OUTPUT_TIMEOUT = 120 # 2 minutes +BROWSERTIME_BENCHMARK_OUTPUT_TIMEOUT = ( + None # Disable output timeout for benchmark tests +) + + +@six.add_metaclass(ABCMeta) +class Browsertime(Perftest): + """Abstract base class for Browsertime""" + + @property + @abstractmethod + def browsertime_args(self): + pass + + def __init__(self, app, binary, process_handler=None, **kwargs): + self.browsertime = True + self.browsertime_failure = "" + self.browsertime_user_args = [] + + self.process_handler = process_handler or mozprocess.ProcessHandler + for key in list(kwargs): + if key.startswith("browsertime_"): + value = kwargs.pop(key) + setattr(self, key, value) + + def klass(**config): + root_results_dir = os.path.join( + os.environ.get("MOZ_UPLOAD_DIR", os.getcwd()), "browsertime-results" + ) + return BrowsertimeResultsHandler(config, root_results_dir=root_results_dir) + + super(Browsertime, self).__init__( + app, binary, results_handler_class=klass, **kwargs + ) + LOG.info("cwd: '{}'".format(os.getcwd())) + self.config["browsertime"] = True + + # Setup browsertime-specific settings for result parsing + self.results_handler.browsertime_visualmetrics = self.browsertime_visualmetrics + + # For debugging. + for k in ( + "browsertime_node", + "browsertime_browsertimejs", + "browsertime_ffmpeg", + "browsertime_geckodriver", + "browsertime_chromedriver", + "browsertime_user_args", + ): + try: + if not self.browsertime_video and k == "browsertime_ffmpeg": + continue + LOG.info("{}: {}".format(k, getattr(self, k))) + LOG.info("{}: {}".format(k, os.stat(getattr(self, k)))) + except Exception as e: + LOG.info("{}: {}".format(k, e)) + + def build_browser_profile(self): + super(Browsertime, self).build_browser_profile() + if self.profile is not None: + self.remove_mozprofile_delimiters_from_profile() + + def remove_mozprofile_delimiters_from_profile(self): + # Perftest.build_browser_profile uses mozprofile to create the profile and merge in prefs; + # while merging, mozprofile adds in special delimiters; these delimiters (along with blank + # lines) are not recognized by selenium-webdriver ultimately causing Firefox launch to + # fail. So we must remove these delimiters from the browser profile before passing into + # btime via firefox.profileTemplate. + + LOG.info("Removing mozprofile delimiters from browser profile") + userjspath = os.path.join(self.profile.profile, "user.js") + try: + with open(userjspath) as userjsfile: + lines = userjsfile.readlines() + lines = [ + line + for line in lines + if not line.startswith("#MozRunner") and line.strip() + ] + with open(userjspath, "w") as userjsfile: + userjsfile.writelines(lines) + except Exception as e: + LOG.critical("Exception {} while removing mozprofile delimiters".format(e)) + + def set_browser_test_prefs(self, raw_prefs): + # add test specific preferences + LOG.info("setting test-specific Firefox preferences") + self.profile.set_preferences(json.loads(raw_prefs)) + self.remove_mozprofile_delimiters_from_profile() + + def run_test_setup(self, test): + super(Browsertime, self).run_test_setup(test) + + if test.get("type") == "benchmark": + # benchmark-type tests require the benchmark test to be served out + self.benchmark = Benchmark(self.config, test) + test["test_url"] = test["test_url"].replace("<host>", self.benchmark.host) + test["test_url"] = test["test_url"].replace("<port>", self.benchmark.port) + + if test.get("playback") is not None and self.playback is None: + self.start_playback(test) + + # TODO: geckodriver/chromedriver from tasks. + self.driver_paths = [] + if self.browsertime_geckodriver: + self.driver_paths.extend( + ["--firefox.geckodriverPath", self.browsertime_geckodriver] + ) + if self.browsertime_chromedriver and self.config["app"] in ( + "chrome", + "chrome-m", + "chromium", + ): + if ( + not self.config.get("run_local", None) + or "{}" in self.browsertime_chromedriver + ): + if self.browser_version: + bvers = str(self.browser_version) + chromedriver_version = bvers.split(".")[0] + else: + chromedriver_version = DEFAULT_CHROMEVERSION + + self.browsertime_chromedriver = self.browsertime_chromedriver.format( + chromedriver_version + ) + + if not os.path.exists(self.browsertime_chromedriver): + raise Exception( + "Cannot find the chromedriver for the chrome version " + "being tested: %s" % self.browsertime_chromedriver + ) + + self.driver_paths.extend( + ["--chrome.chromedriverPath", self.browsertime_chromedriver] + ) + + # YTP tests fail in mozilla-release due to the `MOZ_DISABLE_NONLOCAL_CONNECTIONS` + # environment variable. This logic changes this variable for the browsertime test + # subprocess call in `run_test` + if "youtube-playback" in test["name"] and self.config["is_release_build"]: + os.environ["MOZ_DISABLE_NONLOCAL_CONNECTIONS"] = "0" + + LOG.info("test: {}".format(test)) + + def run_test_teardown(self, test): + super(Browsertime, self).run_test_teardown(test) + + # if we were using a playback tool, stop it + if self.playback is not None: + self.playback.stop() + self.playback = None + + def check_for_crashes(self): + super(Browsertime, self).check_for_crashes() + + def clean_up(self): + super(Browsertime, self).clean_up() + + def _compose_cmd(self, test, timeout, extra_profiler_run=False): + """Browsertime has the following overwrite priorities(in order of highest-lowest) + (1) User - input commandline flag. + (2) Browsertime args mentioned for a test + (3) Test-manifest settings + (4) Default settings + """ + + browsertime_path = os.path.join( + os.path.dirname(__file__), "..", "..", "browsertime" + ) + + if test.get("type", "") == "scenario": + browsertime_script = os.path.join( + browsertime_path, + "browsertime_scenario.js", + ) + elif ( + test.get("type", "") == "benchmark" + and test.get("test_script", None) is None + ): + browsertime_script = os.path.join( + browsertime_path, + "browsertime_benchmark.js", + ) + else: + # Custom scripts are treated as pageload tests for now + if test.get("name", "") == "browsertime": + # Check for either a script or a url from the + # --browsertime-arg options + browsertime_script = None + for option in self.browsertime_user_args: + arg, val = option.split("=") + if arg in ("test_script", "url"): + browsertime_script = val + if browsertime_script is None: + raise Exception( + "You must provide a path to the test script or the url like so: " + "`--browsertime-arg test_script=PATH/TO/SCRIPT`, or " + "`--browsertime-arg url=https://www.sitespeed.io`" + ) + + # Make it simple to use our builtin test scripts + if browsertime_script == "pageload": + browsertime_script = os.path.join( + browsertime_path, "browsertime_pageload.js" + ) + elif browsertime_script == "interactive": + browsertime_script = os.path.join( + browsertime_path, "browsertime_interactive.js" + ) + + elif test.get("interactive", False): + browsertime_script = os.path.join( + browsertime_path, + "browsertime_interactive.js", + ) + else: + browsertime_script = os.path.join( + browsertime_path, + test.get("test_script", "browsertime_pageload.js"), + ) + + page_cycle_delay = "1000" + if self.config["live_sites"]: + # Wait a bit longer when we run live site tests + page_cycle_delay = "5000" + + page_cycles = ( + test.get("page_cycles", 1) + if not extra_profiler_run + else test.get("extra_profiler_run_page_cycles", 1) + ) + browser_cycles = ( + test.get("browser_cycles", 1) + if not extra_profiler_run + else test.get("extra_profiler_run_browser_cycles", 1) + ) + + # All the configurations in the browsertime_options variable initialization + # and the secondary_url are priority 3, since none overlap they are grouped together + browsertime_options = [ + "--firefox.noDefaultPrefs", + "--browsertime.page_cycle_delay", + page_cycle_delay, + # Raptor's `pageCycleDelay` delay (ms) between pageload cycles + "--skipHar", + "--pageLoadStrategy", + "none", + "--webdriverPageload", + "true", + "--firefox.disableBrowsertimeExtension", + "true", + "--pageCompleteCheckStartWait", + "5000", + # url load timeout (milliseconds) + "--pageCompleteCheckPollTimeout", + "1000", + # running browser scripts timeout (milliseconds) + "--timeouts.pageLoad", + str(timeout), + "--timeouts.script", + str(timeout * int(page_cycles)), + "--browsertime.page_cycles", + str(page_cycles), + # a delay was added by default to browsertime from 5s -> 8s for iphones, not needed + "--pageCompleteWaitTime", + str(test.get("page_complete_wait_time", "5000")), + "--browsertime.url", + test["test_url"], + # Raptor's `post startup delay` is settle time after the browser has started + "--browsertime.post_startup_delay", + # If we are on the extra profiler run, limit the startup delay to 1 second. + str(min(self.post_startup_delay, 1000)) + if extra_profiler_run + else str(self.post_startup_delay), + "--iterations", + str(browser_cycles), + "--videoParams.androidVideoWaitTime", + "20000", + # running browsertime test in chimera mode + "--browsertime.chimera", + "true" if self.config["chimera"] else "false", + "--browsertime.test_bytecode_cache", + "true" if self.config["test_bytecode_cache"] else "false", + "--firefox.perfStats", + test.get("perfstats", "false"), + ] + + if test.get("perfstats") == "true": + # Take a non-standard approach for perfstats as we + # want to enable them everywhere shortly (bug 1770152) + self.results_handler.perfstats = True + + if self.config["app"] in ("fenix",): + # Fenix can take a lot of time to startup + browsertime_options.extend( + [ + "--browsertime.browserRestartTries", + "10", + "--timeouts.browserStart", + "180000", + ] + ) + + if test.get("secondary_url"): + browsertime_options.extend( + ["--browsertime.secondary_url", test.get("secondary_url")] + ) + + # In this code block we check if any priority 2 argument is in conflict with a priority + # 3 arg if so we overwrite the value with the priority 2 argument, and otherwise we + # simply add the priority 2 arg + if test.get("browsertime_args", None): + split_args = test.get("browsertime_args").strip().split() + for split_arg in split_args: + pairing = split_arg.split("=") + if len(pairing) not in (1, 2): + raise Exception( + "One of the browsertime_args from the test was not split properly. " + f"Expecting a --flag, or a --option=value pairing. Found: {split_arg}" + ) + if pairing[0] in browsertime_options: + # If it's a flag, don't re-add it + if len(pairing) > 1: + ind = browsertime_options.index(pairing[0]) + browsertime_options[ind + 1] = pairing[1] + else: + browsertime_options.extend(pairing) + + priority1_options = self.browsertime_args + if self.config["app"] in ("chrome", "chromium", "chrome-m"): + priority1_options.extend(self.setup_chrome_args(test)) + + if self.debug_mode: + browsertime_options.extend(["-vv", "--debug", "true"]) + + if not extra_profiler_run: + # must happen before --firefox.profileTemplate and --resultDir + self.results_handler.remove_result_dir_for_test(test) + priority1_options.extend( + ["--resultDir", self.results_handler.result_dir_for_test(test)] + ) + else: + priority1_options.extend( + [ + "--resultDir", + self.results_handler.result_dir_for_test_profiling(test), + ] + ) + if self.profile is not None: + priority1_options.extend( + ["--firefox.profileTemplate", str(self.profile.profile)] + ) + + # This argument can have duplicates of the value "--firefox.env" so we do not need + # to check if it conflicts + for var, val in self.config.get("environment", {}).items(): + browsertime_options.extend(["--firefox.env", "{}={}".format(var, val)]) + + # Parse the test commands (if any) from the test manifest + cmds = evaluate_list_from_string(test.get("test_cmds", "[]")) + parsed_cmds = [":::".join([str(i) for i in item]) for item in cmds if item] + browsertime_options.extend(["--browsertime.commands", ";;;".join(parsed_cmds)]) + + if self.verbose: + browsertime_options.append("-vvv") + + if self.browsertime_video: + priority1_options.extend( + [ + "--video", + "true", + "--visualMetrics", + "true" if self.browsertime_visualmetrics else "false", + "--visualMetricsContentful", + "true", + "--visualMetricsPerceptual", + "true", + "--visualMetricsPortable", + "true", + "--videoParams.keepOriginalVideo", + "true", + ] + ) + + if self.browsertime_no_ffwindowrecorder or self.config["app"] in ( + "chromium", + "chrome-m", + "chrome", + ): + priority1_options.extend( + [ + "--firefox.windowRecorder", + "false", + "--xvfbParams.display", + "0", + ] + ) + LOG.info( + "Using adb screenrecord for mobile, or ffmpeg on desktop for videos" + ) + else: + priority1_options.extend( + [ + "--firefox.windowRecorder", + "true", + ] + ) + LOG.info("Using Firefox Window Recorder for videos") + else: + priority1_options.extend(["--video", "false", "--visualMetrics", "false"]) + + if self.config["gecko_profile"] or extra_profiler_run: + self.config[ + "browsertime_result_dir" + ] = self.results_handler.result_dir_for_test(test) + self._init_gecko_profiling(test) + priority1_options.append("--firefox.geckoProfiler") + for option, browser_time_option, default in ( + ( + "gecko_profile_features", + "--firefox.geckoProfilerParams.features", + "js,stackwalk,cpu,screenshots", + ), + ( + "gecko_profile_threads", + "--firefox.geckoProfilerParams.threads", + "GeckoMain,Compositor,Renderer", + ), + ( + "gecko_profile_interval", + "--firefox.geckoProfilerParams.interval", + None, + ), + ( + "gecko_profile_entries", + "--firefox.geckoProfilerParams.bufferSize", + None, + ), + ): + # 0 is a valid value. The setting may be present but set to None. + value = self.config.get(option) + if value is None: + value = test.get(option) + if value is None: + value = default + if option == "gecko_profile_threads": + extra = self.config.get("gecko_profile_extra_threads", []) + value = ",".join(value.split(",") + extra) + if value is not None: + priority1_options.extend([browser_time_option, str(value)]) + + # Add any user-specified flags here, let them override anything + # with no restrictions + for user_arg in self.browsertime_user_args: + arg, val = user_arg.split("=") + priority1_options.extend([f"--{arg}", val]) + + # In this code block we check if any priority 1 arguments are in conflict with a + # priority 2/3/4 argument + MULTI_OPTS = [ + "--firefox.android.intentArgument", + "--firefox.args", + "--firefox.preference", + ] + for index, argument in list(enumerate(priority1_options)): + if argument in MULTI_OPTS: + browsertime_options.extend([argument, priority1_options[index + 1]]) + elif argument.startswith("--"): + if index == len(priority1_options) - 1: + if argument not in browsertime_options: + browsertime_options.append(argument) + else: + arg = priority1_options[index + 1] + try: + ind = browsertime_options.index(argument) + if not arg.startswith("--"): + browsertime_options[ind + 1] = arg + except ValueError: + res = [argument] + if not arg.startswith("--"): + res.append(arg) + browsertime_options.extend(res) + else: + continue + + # Finalize the `browsertime_options` before starting pageload tests + if test.get("type") == "pageload": + self._finalize_pageload_test_setup( + browsertime_options=browsertime_options, test=test + ) + + return ( + [self.browsertime_node, self.browsertime_browsertimejs] + + self.driver_paths + + [browsertime_script] + + browsertime_options + ) + + def _finalize_pageload_test_setup(self, browsertime_options, test): + """This function finalizes remaining configurations for browsertime pageload tests. + + For pageload tests, ensure that the test name is available to the `browsertime_pageload.js` + script. In addition, make the `live_sites` and `login` boolean available as these will be + required in determining the flow of the login-logic. Finally, we disable the verbose mode + as a safety precaution when doing a live login site. + """ + browsertime_options.extend(["--browsertime.testName", str(test.get("name"))]) + browsertime_options.extend( + ["--browsertime.liveSite", str(self.config["live_sites"])] + ) + + login_required = self.is_live_login_site(test.get("name")) + browsertime_options.extend(["--browsertime.loginRequired", str(login_required)]) + + # Turn off verbose if login logic is required and we are running on CI + if ( + login_required + and self.config.get("verbose", False) + and os.environ.get("MOZ_AUTOMATION") + ): + LOG.info("Turning off verbose mode for login-logic.") + LOG.info( + "Please contact the perftest team if you need verbose mode enabled." + ) + self.config["verbose"] = False + for verbose_level in ("-v", "-vv", "-vvv", "-vvvv"): + try: + browsertime_options.remove(verbose_level) + except ValueError: + pass + + @staticmethod + def is_live_login_site(test_name): + """This function checks the login field of a live-site in the pageload_sites.json + + After reading in the json file, perform a brute force search for the matching test name + and return the login field boolean + """ + + # That pathing is different on CI vs locally for the pageload_sites.json file + if os.environ.get("MOZ_AUTOMATION"): + PAGELOAD_SITES = os.path.join( + os.getcwd(), "tests/raptor/browsertime/pageload_sites.json" + ) + else: + base_dir = os.path.dirname(os.path.dirname(os.getcwd())) + pageload_subpath = "raptor/browsertime/pageload_sites.json" + PAGELOAD_SITES = os.path.join(base_dir, pageload_subpath) + + with open(PAGELOAD_SITES, "r") as f: + pageload_data = json.load(f) + + desktop_sites = pageload_data["desktop"] + for site in desktop_sites: + if site["name"] == test_name: + return site["login"] + + return False + + def _compute_process_timeout(self, test, timeout): + if self.debug_mode: + return sys.maxsize + + # bt_timeout will be the overall browsertime cmd/session timeout (seconds) + # browsertime deals with page cycles internally, so we need to give it a timeout + # value that includes all page cycles + # pylint --py3k W1619 + bt_timeout = int(timeout / 1000) * int(test.get("page_cycles", 1)) + + # the post-startup-delay is a delay after the browser has started, to let it settle + # it's handled within browsertime itself by loading a 'preUrl' (about:blank) and having a + # delay after that ('preURLDelay') as the post-startup-delay, so we must add that in sec + # pylint --py3k W1619 + bt_timeout += int(self.post_startup_delay / 1000) + + # add some time for browser startup, time for the browsertime measurement code + # to be injected/invoked, and for exceptions to bubble up; be generous + bt_timeout += 20 + + # When we produce videos, sometimes FFMPEG can take time to process + # large folders of JPEG files into an MP4 file + if self.browsertime_video: + bt_timeout += 30 + + # Visual metrics processing takes another extra 30 seconds + if self.browsertime_visualmetrics: + bt_timeout += 30 + + # browsertime also handles restarting the browser/running all of the browser cycles; + # so we need to multiply our bt_timeout by the number of browser cycles + bt_timeout = bt_timeout * int(test.get("browser_cycles", 1)) + + # if geckoProfile enabled, give browser more time for profiling + if self.config["gecko_profile"] is True: + bt_timeout += 5 * 60 + return bt_timeout + + @staticmethod + def _kill_browsertime_process(msg): + """This method determines if a browsertime process should be killed. + + Examine the error message from the line handler to determine what to do by returning + a boolean. + + In the future, we can extend this method to consider other scenarios. + """ + + # If we encounter an `xpath` & `double click` related error + # message, it is due to a failure in the 2FA checks during the + # login logic since not all websites have 2FA + if "xpath" in msg and "double click" in msg: + LOG.info("Ignoring 2FA error") + return False + + return True + + def run_extra_profiler_run( + self, test, timeout, proc_timeout, output_timeout, line_handler, env + ): + try: + LOG.info( + "Running browsertime with the profiler enabled after the main run." + ) + profiler_test = deepcopy(test) + cmd = self._compose_cmd(profiler_test, timeout, True) + LOG.info( + "browsertime profiling cmd: {}".format(" ".join([str(c) for c in cmd])) + ) + proc = self.process_handler(cmd, processOutputLine=line_handler, env=env) + proc.run(timeout=proc_timeout, outputTimeout=output_timeout) + proc.wait() + + # Do not raise exception for the browsertime failure or timeout for this case. + # Second profiler browsertime run is fallible. + if proc.outputTimedOut: + LOG.info( + "Browsertime process for extra profiler run timed out after " + f"waiting {output_timeout} seconds for output" + ) + if proc.timedOut: + LOG.info( + "Browsertime process for extra profiler run timed out after " + f"{proc_timeout} seconds" + ) + + if self.browsertime_failure: + LOG.info( + f"Browsertime for extra profiler run failure: {self.browsertime_failure}" + ) + + except Exception as e: + LOG.info("Failed during the extra profiler run: " + str(e)) + + def run_test(self, test, timeout): + global BROWSERTIME_PAGELOAD_OUTPUT_TIMEOUT + + self.run_test_setup(test) + # timeout is a single page-load timeout value (ms) from the test INI + # this will be used for btime --timeouts.pageLoad + cmd = self._compose_cmd(test, timeout) + + output_timeout = BROWSERTIME_PAGELOAD_OUTPUT_TIMEOUT + if test.get("type", "") == "scenario": + # Change the timeout for scenarios since they + # don't output much for a long period of time + output_timeout = timeout + elif self.benchmark: + output_timeout = BROWSERTIME_BENCHMARK_OUTPUT_TIMEOUT + + if self.debug_mode: + output_timeout = 2147483647 + + LOG.info("timeout (s): {}".format(timeout)) + LOG.info("browsertime cwd: {}".format(os.getcwd())) + LOG.info("browsertime cmd: {}".format(" ".join([str(c) for c in cmd]))) + if self.browsertime_video: + LOG.info("browsertime_ffmpeg: {}".format(self.browsertime_ffmpeg)) + + # browsertime requires ffmpeg on the PATH for `--video=true`. + # It's easier to configure the PATH here than at the TC level. + env = dict(os.environ) + env["PYTHON"] = sys.executable + if self.browsertime_video and self.browsertime_ffmpeg: + ffmpeg_dir = os.path.dirname(os.path.abspath(self.browsertime_ffmpeg)) + old_path = env.setdefault("PATH", "") + new_path = os.pathsep.join([ffmpeg_dir, old_path]) + if isinstance(new_path, six.text_type): + # Python 2 doesn't like unicode in the environment. + new_path = new_path.encode("utf-8", "strict") + env["PATH"] = new_path + + LOG.info("PATH: {}".format(env["PATH"])) + + try: + line_matcher = re.compile(r".*(\[.*\])\s+([a-zA-Z]+):\s+(.*)") + + def _create_line_handler(extra_profiler_run=False): + def _line_handler(line): + """This function acts as a bridge between browsertime + and raptor. It reforms the lines to get rid of information + that is not needed, and outputs them appropriately based + on the level that is found. (Debug and info all go to info). + + For errors, we set an attribute (self.browsertime_failure) to + it, then raise a generic exception. When we return, we check + if self.browsertime_failure, and raise an Exception if necessary + to stop Raptor execution (preventing the results processing). + """ + + # NOTE: this hack is to workaround encoding issues on windows + # a newer version of browsertime adds a `σ` character to output + line = line.replace(b"\xcf\x83", b"") + + line = line.decode("utf-8") + match = line_matcher.match(line) + if not match: + LOG.info(line) + return + + date, level, msg = match.groups() + level = level.lower() + if "error" in level and not self.debug_mode: + if self._kill_browsertime_process(msg): + self.browsertime_failure = msg + if extra_profiler_run: + # Do not trigger the log parser for extra profiler run. + LOG.info( + "Browsertime failed to run on extra profiler run" + ) + else: + LOG.error("Browsertime failed to run") + proc.kill() + elif "warning" in level: + if extra_profiler_run: + # Do not trigger the log parser for extra profiler run. + LOG.info(msg) + else: + LOG.warning(msg) + elif "metrics" in level: + vals = msg.split(":")[-1].strip() + self.page_count = vals.split(",") + else: + LOG.info(msg) + + return _line_handler + + proc_timeout = self._compute_process_timeout(test, timeout) + output_timeout = BROWSERTIME_PAGELOAD_OUTPUT_TIMEOUT + if self.benchmark: + output_timeout = BROWSERTIME_BENCHMARK_OUTPUT_TIMEOUT + elif test.get("output_timeout", None) is not None: + output_timeout = int(test.get("output_timeout")) + proc_timeout = max(proc_timeout, output_timeout) + + # Double the timeouts on live sites and when running with Fenix + if self.config["live_sites"] or self.config["app"] in ("fenix",): + # Since output_timeout is None for benchmark tests we should + # not perform any operations on it. + if output_timeout is not None: + output_timeout *= 2 + proc_timeout *= 2 + + LOG.info( + f"Calling browsertime with proc_timeout={proc_timeout}, " + f"and output_timeout={output_timeout}" + ) + + proc = self.process_handler( + cmd, processOutputLine=_create_line_handler(), env=env + ) + proc.run(timeout=proc_timeout, outputTimeout=output_timeout) + proc.wait() + + if proc.outputTimedOut: + raise Exception( + f"Browsertime process timed out after waiting {output_timeout} seconds " + "for output" + ) + if proc.timedOut: + raise Exception( + f"Browsertime process timed out after {proc_timeout} seconds" + ) + + if self.browsertime_failure: + raise Exception(self.browsertime_failure) + + # We've run the main browsertime process, now we need to run the + # browsertime one more time if the profiler wasn't enabled already + # in the previous run and user wants this extra run. + if ( + self.config.get("extra_profiler_run") + and not self.config["gecko_profile"] + ): + self.run_extra_profiler_run( + test, + timeout, + proc_timeout, + output_timeout, + _create_line_handler(extra_profiler_run=True), + env, + ) + + except Exception as e: + LOG.critical(str(e)) + raise diff --git a/testing/raptor/raptor/browsertime/desktop.py b/testing/raptor/raptor/browsertime/desktop.py new file mode 100644 index 0000000000..be5e88453c --- /dev/null +++ b/testing/raptor/raptor/browsertime/desktop.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python + +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from logger.logger import RaptorLogger +from perftest import PerftestDesktop + +from .base import Browsertime + +LOG = RaptorLogger(component="raptor-browsertime-desktop") + + +class BrowsertimeDesktop(PerftestDesktop, Browsertime): + def __init__(self, *args, **kwargs): + super(BrowsertimeDesktop, self).__init__(*args, **kwargs) + + @property + def browsertime_args(self): + binary_path = self.config["binary"] + LOG.info("binary_path: {}".format(binary_path)) + + args_list = ["--viewPort", "1280x1024"] + + if self.config["app"] in ( + "chrome", + "chromium", + ): + return args_list + [ + "--browser", + "chrome", + "--chrome.binaryPath", + binary_path, + ] + return args_list + [ + "--browser", + self.config["app"], + "--firefox.binaryPath", + binary_path, + ] + + def setup_chrome_args(self, test): + # Setup required chrome arguments + chrome_args = self.desktop_chrome_args(test) + + # Add this argument here, it's added by mozrunner + # for raptor + chrome_args.extend(["--no-first-run"]) + + btime_chrome_args = [] + for arg in chrome_args: + btime_chrome_args.extend(["--chrome.args=" + str(arg.replace("'", '"'))]) + + return btime_chrome_args diff --git a/testing/raptor/raptor/cmdline.py b/testing/raptor/raptor/cmdline.py new file mode 100644 index 0000000000..0e29134fb7 --- /dev/null +++ b/testing/raptor/raptor/cmdline.py @@ -0,0 +1,718 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. +import argparse +import os +import platform + +import six +from mozlog.commandline import add_logging_group + +(FIREFOX, CHROME, CHROMIUM, SAFARI) = DESKTOP_APPS = [ + "firefox", + "chrome", + "chromium", + "safari", +] +(GECKOVIEW, REFBROW, FENIX, CHROME_ANDROID) = FIREFOX_ANDROID_APPS = [ + "geckoview", + "refbrow", + "fenix", + "chrome-m", +] + +CHROMIUM_DISTROS = [CHROME, CHROMIUM] +APPS = { + FIREFOX: {"long_name": "Firefox Desktop"}, + CHROME: {"long_name": "Google Chrome Desktop"}, + CHROMIUM: {"long_name": "Google Chromium Desktop"}, + SAFARI: {"long_name": "Safari Desktop"}, + GECKOVIEW: { + "long_name": "Firefox GeckoView on Android", + "default_activity": "org.mozilla.geckoview_example.GeckoViewActivity", + "default_intent": "android.intent.action.MAIN", + }, + REFBROW: { + "long_name": "Firefox Android Components Reference Browser", + "default_activity": "org.mozilla.reference.browser.BrowserTestActivity", + "default_intent": "android.intent.action.MAIN", + }, + FENIX: { + "long_name": "Firefox Android Fenix Browser", + "default_activity": "org.mozilla.fenix.IntentReceiverActivity", + "default_intent": "android.intent.action.VIEW", + }, + CHROME_ANDROID: { + "long_name": "Google Chrome on Android", + "default_activity": "com.android.chrome/com.google.android.apps.chrome.Main", + "default_intent": "android.intent.action.VIEW", + }, +} +INTEGRATED_APPS = list(APPS.keys()) + + +def print_all_activities(): + all_activities = [] + for next_app in APPS: + if APPS[next_app].get("default_activity", None) is not None: + _activity = "%s:%s" % (next_app, APPS[next_app]["default_activity"]) + all_activities.append(_activity) + return all_activities + + +def print_all_intents(): + all_intents = [] + for next_app in APPS: + if APPS[next_app].get("default_intent", None) is not None: + _intent = "%s:%s" % (next_app, APPS[next_app]["default_intent"]) + all_intents.append(_intent) + return all_intents + + +def create_parser(mach_interface=False): + parser = argparse.ArgumentParser() + add_arg = parser.add_argument + + add_arg( + "-t", + "--test", + required=True, + dest="test", + help="Name of Raptor test to run (can be a top-level suite name i.e. " + "'--test raptor-speedometer','--test raptor-tp6-1', or for page-load " + "tests a suite sub-test i.e. '--test raptor-tp6-google-firefox')", + ) + add_arg( + "--app", + default="firefox", + dest="app", + help="Name of the application we are testing (default: firefox)", + choices=list(APPS), + ) + add_arg( + "-b", + "--binary", + dest="binary", + help="path to the browser executable that we are testing", + ) + add_arg( + "-a", + "--activity", + dest="activity", + default=None, + help="Name of Android activity used to launch the Android app." + "i.e.: %s" % print_all_activities(), + ) + add_arg( + "-i", + "--intent", + dest="intent", + default=None, + help="Name of Android intent action used to launch the Android app." + "i.e.: %s" % print_all_intents(), + ) + add_arg( + "--host", + dest="host", + help="Hostname from which to serve URLs; defaults to 127.0.0.1. " + "The value HOST_IP will cause the value of host to be " + "loaded from the environment variable HOST_IP.", + default="127.0.0.1", + ) + add_arg( + "--power-test", + dest="power_test", + action="store_true", + help="Use Raptor to measure power usage on Android browsers (Geckoview Example, " + "Fenix, and Refbrow) as well as on Intel-based MacOS machines that have " + "Intel Power Gadget installed.", + ) + add_arg( + "--memory-test", + dest="memory_test", + action="store_true", + help="Use Raptor to measure memory usage.", + ) + add_arg( + "--cpu-test", + dest="cpu_test", + action="store_true", + help="Use Raptor to measure CPU usage. Currently supported for Android only.", + ) + add_arg( + "--live-sites", + dest="live_sites", + action="store_true", + default=False, + help="Run tests using live sites instead of recorded sites.", + ) + add_arg( + "--chimera", + dest="chimera", + action="store_true", + default=False, + help="Run tests in chimera mode. Each browser cycle will run a cold and warm test.", + ) + add_arg( + "--is-release-build", + dest="is_release_build", + default=False, + action="store_true", + help="Whether the build is a release build which requires workarounds " + "using MOZ_DISABLE_NONLOCAL_CONNECTIONS to support installing unsigned " + "webextensions. Defaults to False.", + ) + add_arg( + "--geckoProfile", + action="store_true", + dest="gecko_profile", + help=argparse.SUPPRESS, + ) + add_arg( + "--geckoProfileInterval", + dest="gecko_profile_interval", + type=float, + help=argparse.SUPPRESS, + ) + add_arg( + "--geckoProfileEntries", + dest="gecko_profile_entries", + type=int, + help=argparse.SUPPRESS, + ) + add_arg( + "--gecko-profile", + action="store_true", + dest="gecko_profile", + help="Profile the run and out-put the results in $MOZ_UPLOAD_DIR. " + "After talos is finished, profiler.firefox.com will be launched in Firefox " + "so you can analyze the local profiles. To disable auto-launching of " + "profiler.firefox.com, set the DISABLE_PROFILE_LAUNCH=1 env var.", + ) + add_arg( + "--gecko-profile-entries", + dest="gecko_profile_entries", + type=int, + help="How many samples to take with the profiler", + ) + add_arg( + "--gecko-profile-interval", + dest="gecko_profile_interval", + type=int, + help="How frequently to take samples (milliseconds)", + ) + add_arg( + "--gecko-profile-thread", + dest="gecko_profile_extra_threads", + default=[], + action="append", + help="Name of the extra thread to be profiled", + ) + add_arg( + "--gecko-profile-threads", + dest="gecko_profile_threads", + type=str, + help="Comma-separated list of all threads to sample", + ) + add_arg( + "--gecko-profile-features", + dest="gecko_profile_features", + type=str, + help="What features to enable in the profiler", + ) + add_arg( + "--extra-profiler-run", + dest="extra_profiler_run", + action="store_true", + default=False, + help="Run the tests again with profiler enabled after the main run.", + ) + add_arg( + "--symbolsPath", + dest="symbols_path", + help="Path to the symbols for the build we are testing", + ) + add_arg( + "--page-cycles", + dest="page_cycles", + type=int, + help="How many times to repeat loading the test page (for page load tests); " + "for benchmark tests, this is how many times the benchmark test will be run", + ) + add_arg( + "--page-timeout", + dest="page_timeout", + type=int, + help="How long to wait (ms) for one page_cycle to complete, before timing out", + ) + add_arg( + "--post-startup-delay", + dest="post_startup_delay", + type=int, + default=30000, + help="How long to wait (ms) after browser start-up before starting the tests", + ) + add_arg( + "--browser-cycles", + dest="browser_cycles", + type=int, + help="The number of times a cold load test is repeated (for cold load tests only, " + "where the browser is shutdown and restarted between test iterations)", + ), + add_arg( + "--project", + dest="project", + type=str, + default="mozilla-central", + help="Project name (try, mozilla-central, etc.)", + ), + add_arg( + "--test-url-params", + dest="test_url_params", + help="Parameters to add to the test_url query string", + ) + add_arg( + "--print-tests", action=_PrintTests, help="Print all available Raptor tests" + ) + add_arg( + "--debug-mode", + dest="debug_mode", + action="store_true", + help="Run Raptor in debug mode (open browser console, limited page-cycles, etc.)", + ) + add_arg( + "--disable-e10s", + dest="e10s", + action="store_false", + default=True, + help="Run without multiple processes (e10s).", + ) + add_arg( + "--device-name", + dest="device_name", + default=None, + type=str, + help="Device name of mobile device.", + ) + add_arg( + "--disable-fission", + dest="fission", + action="store_false", + default=True, + help="Disable Fission (site isolation) in Gecko.", + ) + add_arg( + "--enable-fission-mobile", + dest="fission_mobile", + action="store_true", + default=False, + help="Temporary work-around to enable fission on mobile as it is enabled " + "by default for desktop now but not mobile.", + ) + add_arg( + "--setpref", + dest="extra_prefs", + action="append", + default=[], + metavar="PREF=VALUE", + help="Set a browser preference. May be used multiple times.", + ) + add_arg( + "--setenv", + dest="environment", + action="append", + default=[], + metavar="NAME=VALUE", + help="Set a variable in the test environment. May be used multiple times.", + ) + if not mach_interface: + add_arg( + "--run-local", + dest="run_local", + default=False, + action="store_true", + help="Flag which indicates if Raptor is running locally or in production", + ) + add_arg( + "--obj-path", + dest="obj_path", + default=None, + help="Browser-build obj_path (received when running in production)", + ) + add_arg( + "--mozbuild-path", + dest="mozbuild_path", + default=None, + help="This contains the path to mozbuild.", + ) + add_arg( + "--noinstall", + dest="noinstall", + default=False, + action="store_true", + help="Flag which indicates if Raptor should not offer to install Android APK.", + ) + add_arg( + "--installerpath", + dest="installerpath", + default=None, + type=str, + help="Location where Android browser APK was extracted to before installation.", + ) + add_arg( + "--disable-perf-tuning", + dest="disable_perf_tuning", + default=False, + action="store_true", + help="Disable performance tuning on android.", + ) + add_arg( + "--conditioned-profile", + dest="conditioned_profile", + default=None, + type=str, + help="Name of conditioned profile to use. Prefix with `artifact:` " + "if we should obtain the profile from CI.", + ) + add_arg( + "--webext", + dest="webext", + action="store_true", + default=False, + help="Whether to use webextension to execute pageload tests " + "(WebExtension is being deprecated).", + ) + add_arg( + "--test-bytecode-cache", + dest="test_bytecode_cache", + default=False, + action="store_true", + help="If set, the pageload test will set the preference " + "`dom.script_loader.bytecode_cache.strategy=-1` and wait 20 seconds after " + "the first cold pageload to populate the bytecode cache before running " + "a warm pageload test. Only available if `--chimera` is also provided.", + ) + + # for browsertime jobs, cold page load is determined by a '--cold' cmd line argument + add_arg( + "--cold", + dest="cold", + action="store_true", + help="Enable cold page-load for browsertime tp6", + ) + # Arguments for invoking browsertime. + + add_arg( + "--browsertime", + dest="browsertime", + default=True, + action="store_true", + help="Whether to use browsertime to execute pageload tests", + ) + add_arg( + "--browsertime-arg", + dest="browsertime_user_args", + action="append", + default=[], + metavar="OPTION=VALUE", + help="Add extra browsertime arguments to your test run using " + "this option e.g.: --browsertime-arg timeout.scripts=1000", + ) + add_arg( + "--browsertime-node", dest="browsertime_node", help="path to Node.js executable" + ) + add_arg( + "--browsertime-browsertimejs", + dest="browsertime_browsertimejs", + help="path to browsertime.js script", + ) + add_arg( + "--browsertime-vismet-script", + dest="browsertime_vismet_script", + help="path to visualmetrics.py script", + ) + add_arg( + "--browsertime-chromedriver", + dest="browsertime_chromedriver", + help="path to chromedriver executable", + ) + add_arg( + "--browsertime-video", + dest="browsertime_video", + help="records the viewport", + default=False, + action="store_true", + ) + add_arg( + "--browsertime-visualmetrics", + dest="browsertime_visualmetrics", + help="enables visual metrics", + default=False, + action="store_true", + ) + add_arg( + "--browsertime-no-ffwindowrecorder", + dest="browsertime_no_ffwindowrecorder", + help="disable the firefox window recorder", + default=False, + action="store_true", + ) + add_arg( + "--browsertime-ffmpeg", + dest="browsertime_ffmpeg", + help="path to ffmpeg executable (for `--video=true`)", + ) + add_arg( + "--browsertime-geckodriver", + dest="browsertime_geckodriver", + help="path to geckodriver executable", + ) + add_arg( + "--browsertime-existing-results", + dest="browsertime_existing_results", + default=None, + help="load existing results instead of running tests", + ) + add_arg( + "--verbose", + dest="verbose", + action="store_true", + default=False, + help="Verbose output", + ) + add_arg( + "--enable-marionette-trace", + dest="enable_marionette_trace", + action="store_true", + default=False, + help="Enable marionette tracing", + ) + add_arg( + "--clean", + dest="clean", + action="store_true", + default=False, + help="Clean the python virtualenv (remove, and rebuild) for Raptor before running tests.", + ) + add_arg( + "--collect-perfstats", + dest="collect_perfstats", + action="store_true", + default=False, + help="If set, the test will collect perfstats in addition to " + "the regular metrics it gathers.", + ) + add_arg( + "--extra-summary-methods", + dest="extra_summary_methods", + action="append", + default=[], + metavar="OPTION", + help="Alternative methods for summarizing technical and visual pageload metrics. " + "Options: geomean, mean.", + ) + + add_logging_group(parser) + return parser + + +def verify_options(parser, args): + ctx = vars(args) + if args.binary is None and args.app != "chrome-m": + parser.error("--binary is required!") + + # Debug-mode is disabled in CI (check for attribute in case of mach_interface issues) + if hasattr(args, "run_local") and (not args.run_local and args.debug_mode): + parser.error("Cannot run debug mode in CI") + + # If running on webextension, browsertime flag is changed (browsertime is run by default) + if args.webext: + args.browsertime = False + + # make sure that browsertime_video is set if visual metrics are requested + if args.browsertime_visualmetrics and not args.browsertime_video: + args.browsertime_video = True + + # if running chrome android tests, make sure it's on browsertime and + # that the chromedriver path was provided + if args.app == "chrome-m": + if not args.browsertime: + parser.error("--browsertime is required to run android chrome tests") + if not args.browsertime_chromedriver: + parser.error( + "--browsertime-chromedriver path is required for android chrome tests" + ) + + if args.chimera: + if not args.browsertime: + parser.error("--browsertime is required to run tests in chimera mode") + if isinstance(args.page_cycles, int) and args.page_cycles != 2: + parser.error( + "--page-cycles must either not be set, or must be equal to 2 in chimera mode" + ) + # Force cold pageloads with 2 page cycles + args.cold = True + args.page_cycles = 2 + # Create bytecode cache at the first cold load, so that the next warm load uses it. + # This is applicable for chimera mode only + if args.test_bytecode_cache: + args.extra_prefs.append("dom.script_loader.bytecode_cache.strategy=-1") + elif args.test_bytecode_cache: + parser.error("--test-bytecode-cache can only be used in --chimera mode") + + # if running on a desktop browser make sure the binary exists + if args.app in DESKTOP_APPS: + if not os.path.isfile(args.binary): + parser.error("{binary} does not exist!".format(**ctx)) + + # if geckoProfile specified but running on Chrom[e|ium], not supported + if args.gecko_profile and args.app in CHROMIUM_DISTROS: + parser.error("Gecko profiling is not supported on Chrome/Chromium!") + + if args.power_test: + if args.app not in ["geckoview", "refbrow", "fenix"]: + if platform.system().lower() not in ("darwin",): + parser.error( + "Power tests are only available on MacOS desktop machines or " + "Firefox android browers. App requested: %s. Platform " + "detected: %s." % (args.app, platform.system().lower()) + ) + + if args.cpu_test: + if args.app not in ["geckoview", "refbrow", "fenix"]: + parser.error( + "CPU test is only supported when running Raptor on Firefox Android " + "browsers!" + ) + + if args.memory_test: + if args.app not in ["geckoview", "refbrow", "fenix"]: + parser.error( + "Memory test is only supported when running Raptor on Firefox Android " + "browsers!" + ) + + if args.fission: + print("Fission enabled through browser preferences") + args.extra_prefs.append("fission.autostart=true") + else: + print("Fission disabled through browser preferences") + args.extra_prefs.append("fission.autostart=false") + + # if running on geckoview/refbrow/fenix, we need an activity and intent + if args.app in ["geckoview", "refbrow", "fenix"]: + if not args.activity: + # if we have a default activity specified in APPS above, use that + if APPS[args.app].get("default_activity", None) is not None: + args.activity = APPS[args.app]["default_activity"] + else: + # otherwise fail out + parser.error("--activity command-line argument is required!") + if not args.intent: + # if we have a default intent specified in APPS above, use that + if APPS[args.app].get("default_intent", None) is not None: + args.intent = APPS[args.app]["default_intent"] + else: + # otherwise fail out + parser.error("--intent command-line argument is required!") + + +def parse_args(argv=None): + parser = create_parser() + args = parser.parse_args(argv) + if args.host == "HOST_IP": + args.host = os.environ["HOST_IP"] + verify_options(parser, args) + return args + + +class _StopAction(argparse.Action): + def __init__( + self, + option_strings, + dest=argparse.SUPPRESS, + default=argparse.SUPPRESS, + help=None, + ): + super(_StopAction, self).__init__( + option_strings=option_strings, + dest=dest, + default=default, + nargs=0, + help=help, + ) + + +class _PrintTests(_StopAction): + def __init__(self, integrated_apps=INTEGRATED_APPS, *args, **kwargs): + super(_PrintTests, self).__init__(*args, **kwargs) + self.integrated_apps = integrated_apps + + def __call__(self, parser, namespace, values, option_string=None): + from manifestparser import TestManifest + + here = os.path.abspath(os.path.dirname(__file__)) + raptor_ini = os.path.join(here, "raptor.ini") + + for _app in self.integrated_apps: + test_manifest = TestManifest([raptor_ini], strict=False) + info = {"app": _app} + available_tests = test_manifest.active_tests( + exists=False, disabled=False, filters=[self.filter_app], **info + ) + if len(available_tests) == 0: + # none for that app; skip to next + continue + + # print in readable format + if _app == "firefox": + title = "\nRaptor Tests Available for %s" % APPS[_app]["long_name"] + else: + title = "\nRaptor Tests Available for %s (--app=%s)" % ( + APPS[_app]["long_name"], + _app, + ) + + print(title) + print("=" * (len(title) - 1)) + + # build the list of tests for this app + test_list = {} + + for next_test in available_tests: + if next_test.get("name", None) is None: + # no test name; skip it + continue + + suite = os.path.basename(next_test["manifest"])[:-4] + if suite not in test_list: + test_list[suite] = {"type": None, "subtests": []} + + # for page-load tests, we want to list every subtest, so we + # can see which pages are available in which tp6-* sets + if next_test.get("type", None) is not None: + test_list[suite]["type"] = next_test["type"] + if next_test["type"] == "pageload": + subtest = next_test["name"] + measure = next_test.get("measure") + if measure is not None: + subtest = "{0} ({1})".format(subtest, measure) + test_list[suite]["subtests"].append(subtest) + + # print the list in a nice, readable format + for key in sorted(six.iterkeys(test_list)): + print("\n%s" % key) + print(" type: %s" % test_list[key]["type"]) + if len(test_list[key]["subtests"]) != 0: + print(" subtests:") + for _sub in sorted(test_list[key]["subtests"]): + print(" %s" % _sub) + + print("\nDone.") + # exit Raptor + parser.exit() + + def filter_app(self, tests, values): + for test in tests: + if values["app"] in test["apps"]: + yield test diff --git a/testing/raptor/raptor/control_server.py b/testing/raptor/raptor/control_server.py new file mode 100644 index 0000000000..5a1b2f2536 --- /dev/null +++ b/testing/raptor/raptor/control_server.py @@ -0,0 +1,461 @@ +#!/usr/bin/env python + +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# control server for raptor performance framework +# communicates with the raptor browser webextension +import datetime +import json +import os +import shutil +import socket +import threading +import time + +import six + +try: + from http import server # py3 +except ImportError: + import BaseHTTPServer as server # py2 + +from logger.logger import RaptorLogger + +LOG = RaptorLogger(component="raptor-control-server") + +here = os.path.abspath(os.path.dirname(__file__)) + + +def MakeCustomHandlerClass( + results_handler, + error_handler, + startup_handler, + shutdown_browser, + handle_gecko_profile, + background_app, + foreground_app, +): + class MyHandler(server.BaseHTTPRequestHandler, object): + """ + Control server expects messages of the form + {'type': 'messagetype', 'data':...} + + Each message is given a key which is calculated as + + If type is 'webext_status', then + the key is data['type']/data['data'] + otherwise + the key is data['type']. + + The control server can be forced to wait before performing an + action requested via POST by sending a special message + + {'type': 'wait-set', 'data': key} + + where key is the key of the message for which the control server should + perform a wait before processing. The handler will store + this key in the wait_after_messages dict as a True value. + + wait_after_messages[key] = True + + For subsequent requests, the handler will check the key of + the incoming message against wait_for_messages; if found + and its value is True, the handler will assign the key + to waiting_in_state and will loop until the key is removed + or until its value is changed to False. + + The control server will stop waiting for a state to be continued + or cleared after wait_timeout seconds, after which the wait + will be removed and the control server will finish processing + the current POST request. wait_timeout defaults to 60 seconds + but can be set globally for all wait states by sending the + message + + {'type': 'wait-timeout', 'data': timeout} + + The value of waiting_in_state can be retrieved by sending the + message + + {'type': 'wait-get', 'data': ''} + + which will return the value of waiting_in_state in the + content of the response. If the value returned is not + 'None', then the control server has received a message whose + key is recorded in wait_after_messages and is waiting before + completing the request. + + The control server can be told to stop waiting and finish + processing the current request while keeping the wait for + subsequent requests by sending + + {'type': 'wait-continue', 'data': ''} + + The control server can be told to stop waiting and finish + processing the current request while removing the wait for + subsequent requests by sending + + {'type': 'wait-clear', 'data': key} + + if key is the empty string '' + the key in waiting_in_state is removed from wait_after_messages + waiting_in_state is set to None + else if key is 'all' + all keys in wait_after_messages are removed + else key is not in wait_after_messages + the message is ignored + else + the key is removed from wait_after messages + if the key matches the value in waiting_in_state, + then waiting_in_state is set to None + """ + + wait_after_messages = {} + waiting_in_state = None + wait_timeout = 60 + + def __init__(self, *args, **kwargs): + self.results_handler = results_handler + self.error_handler = error_handler + self.startup_handler = startup_handler + self.shutdown_browser = shutdown_browser + self.handle_gecko_profile = handle_gecko_profile + self.background_app = background_app + self.foreground_app = foreground_app + try: + super(MyHandler, self).__init__(*args, **kwargs) + except ValueError: + pass + + def log_request(self, code="-", size="-"): + if code != 200: + super(MyHandler, self).log_request(code, size) + + def do_GET(self): + # get handler, received request for test settings from webext runner + self.send_response(200) + head, tail = os.path.split(self.path) + + if tail.startswith("raptor") and tail.endswith(".json"): + LOG.info("reading test settings from json/" + tail) + try: + with open("json/{}".format(tail)) as json_settings: + self.send_header("Access-Control-Allow-Origin", "*") + self.send_header("Content-type", "application/json") + self.end_headers() + self.wfile.write( + json.dumps(json.load(json_settings)).encode("utf-8") + ) + self.wfile.close() + LOG.info("sent test settings to webext runner") + except Exception as ex: + LOG.info("control server exception") + LOG.info(ex) + else: + LOG.info("received request for unknown file: " + self.path) + + def do_POST(self): + # post handler, received something from webext + self.send_response(200) + self.send_header("Access-Control-Allow-Origin", "*") + self.send_header("Content-type", "text/html") + self.end_headers() + + if six.PY2: + content_len = int(self.headers.getheader("content-length")) + elif six.PY3: + content_len = int(self.headers.get("content-length")) + + post_body = self.rfile.read(content_len) + # could have received a status update or test results + if isinstance(post_body, six.binary_type): + post_body = post_body.decode("utf-8") + data = json.loads(post_body) + + if data["type"] == "webext_status": + wait_key = "%s/%s" % (data["type"], data["data"]) + else: + wait_key = data["type"] + + if MyHandler.wait_after_messages.get(wait_key, None): + LOG.info("Waiting in %s" % wait_key) + MyHandler.waiting_in_state = wait_key + start_time = datetime.datetime.now() + + while MyHandler.wait_after_messages.get(wait_key, None): + time.sleep(1) + elapsed_time = datetime.datetime.now() - start_time + if elapsed_time > datetime.timedelta(seconds=MyHandler.wait_timeout): + del MyHandler.wait_after_messages[wait_key] + MyHandler.waiting_in_state = None + LOG.error( + "TEST-UNEXPECTED-ERROR | " + "control server wait %s exceeded %s seconds" + % (wait_key, MyHandler.wait_timeout) + ) + + if MyHandler.wait_after_messages.get(wait_key, None) is not None: + # If the wait is False, it was continued, so we set it back + # to True for the next time. If removed by clear, we + # leave it alone so it will not cause further waits. + MyHandler.wait_after_messages[wait_key] = True + + if data["type"] == "webext_error": + error, stack = data["data"] + LOG.info("received " + data["type"] + ": " + str(error)) + self.error_handler(error, stack) + + elif data["type"] == "webext_gecko_profile": + # received file name of the saved gecko profile + filename = str(data["data"]) + LOG.info("received gecko profile filename: {}".format(filename)) + self.handle_gecko_profile(filename) + + elif data["type"] == "webext_results": + LOG.info("received " + data["type"] + ": " + str(data["data"])) + self.results_handler.add(data["data"]) + elif data["type"] == "webext_raptor-page-timeout": + LOG.info("received " + data["type"] + ": " + str(data["data"])) + + if len(data["data"]) == 3: + data["data"].append("") + # pageload test has timed out; record it as a failure + self.results_handler.add_page_timeout( + str(data["data"][0]), + str(data["data"][1]), + str(data["data"][2]), + dict(data["data"][3]), + ) + elif data["type"] == "webext_shutdownBrowser": + LOG.info("received request to shutdown the browser") + self.shutdown_browser() + elif data["type"] == "webext_start_background": + LOG.info("received request to background app") + self.background_app() + elif data["type"] == "webext_end_background": + LOG.info("received request to foreground app") + self.foreground_app() + elif data["type"] == "webext_screenshot": + LOG.info("received " + data["type"]) + self.results_handler.add_image( + str(data["data"][0]), str(data["data"][1]), str(data["data"][2]) + ) + elif data["type"] == "webext_status": + LOG.info("received " + data["type"] + ": " + str(data["data"])) + elif data["type"] == "webext_loaded": + LOG.info("received " + data["type"] + ": raptor runner.js is loaded!") + self.startup_handler(True) + elif data["type"] == "wait-set": + LOG.info("received " + data["type"] + ": " + str(data["data"])) + MyHandler.wait_after_messages[str(data["data"])] = True + elif data["type"] == "wait-timeout": + LOG.info("received " + data["type"] + ": " + str(data["data"])) + MyHandler.wait_timeout = data["data"] + elif data["type"] == "wait-get": + state = MyHandler.waiting_in_state + if state is None: + state = "None" + if isinstance(state, six.text_type): + state = state.encode("utf-8") + self.wfile.write(state) + elif data["type"] == "wait-continue": + LOG.info("received " + data["type"] + ": " + str(data["data"])) + if MyHandler.waiting_in_state: + MyHandler.wait_after_messages[MyHandler.waiting_in_state] = False + MyHandler.waiting_in_state = None + elif data["type"] == "wait-clear": + LOG.info("received " + data["type"] + ": " + str(data["data"])) + clear_key = str(data["data"]) + if clear_key == "": + if MyHandler.waiting_in_state: + del MyHandler.wait_after_messages[MyHandler.waiting_in_state] + MyHandler.waiting_in_state = None + else: + pass + elif clear_key == "all": + MyHandler.wait_after_messages = {} + MyHandler.waiting_in_state = None + elif clear_key not in MyHandler.wait_after_messages: + pass + else: + del MyHandler.wait_after_messages[clear_key] + if MyHandler.waiting_in_state == clear_key: + MyHandler.waiting_in_state = None + else: + LOG.info("received " + data["type"] + ": " + str(data["data"])) + + def do_OPTIONS(self): + self.send_response(200, "ok") + self.send_header("Access-Control-Allow-Origin", "*") + self.send_header("Access-Control-Allow-Methods", "GET, POST, OPTIONS") + self.send_header("Access-Control-Allow-Headers", "X-Requested-With") + self.send_header("Access-Control-Allow-Headers", "Content-Type") + self.end_headers() + + return MyHandler + + +class ThreadedHTTPServer(server.HTTPServer): + # See + # https://stackoverflow.com/questions/19537132/threaded-basehttpserver-one-thread-per-request#30312766 + def process_request(self, request, client_address): + thread = threading.Thread( + target=self.__new_request, + args=(self.RequestHandlerClass, request, client_address, self), + ) + thread.start() + + def __new_request(self, handlerClass, request, address, server): + handlerClass(request, address, server) + self.shutdown_request(request) + + +class RaptorControlServer: + """Container class for Raptor Control Server""" + + def __init__(self, results_handler, debug_mode=False): + self.raptor_venv = os.path.join(os.getcwd(), "raptor-venv") + self.server = None + self._server_thread = None + self.port = None + self.results_handler = results_handler + self.browser_proc = None + self._finished = False + self._is_shutting_down = False + self._runtime_error = None + self.device = None + self.app_name = None + self.gecko_profile_dir = None + self.debug_mode = debug_mode + self.user_profile = None + self.is_webextension_loaded = False + + def start(self): + config_dir = os.path.join(here, "tests") + os.chdir(config_dir) + + # pick a free port + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.bind(("", 0)) + self.port = sock.getsockname()[1] + sock.close() + server_address = ("", self.port) + + server_class = ThreadedHTTPServer + handler_class = MakeCustomHandlerClass( + self.results_handler, + self.error_handler, + self.startup_handler, + self.shutdown_browser, + self.handle_gecko_profile, + self.background_app, + self.foreground_app, + ) + + httpd = server_class(server_address, handler_class) + + self._server_thread = threading.Thread(target=httpd.serve_forever) + self._server_thread.setDaemon(True) # don't hang on exit + self._server_thread.start() + LOG.info("raptor control server running on port %d..." % self.port) + self.server = httpd + + def error_handler(self, error, stack): + self._runtime_error = {"error": error, "stack": stack} + + def startup_handler(self, value): + self.is_webextension_loaded = value + + def shutdown_browser(self): + # if debug-mode enabled, leave the browser running - require manual shutdown + # this way the browser console remains open, so we can review the logs etc. + if self.debug_mode: + LOG.info("debug-mode enabled, so NOT shutting down the browser") + self._finished = True + return + + if self.device is not None: + LOG.info("shutting down android app %s" % self.app_name) + else: + LOG.info("shutting down browser (pid: %d)" % self.browser_proc.pid) + self.kill_thread = threading.Thread( + target=self.wait_for_quit, kwargs={"timeout": 0} + ) + self.kill_thread.daemon = True + self.kill_thread.start() + + def handle_gecko_profile(self, filename): + # Move the stored profile to a location outside the Firefox profile + source_path = os.path.join(self.user_profile.profile, "profiler", filename) + target_path = os.path.join(self.gecko_profile_dir, filename) + shutil.move(source_path, target_path) + LOG.info("moved gecko profile to {}".format(target_path)) + + def is_app_in_background(self): + # Get the app view state: foreground->False, background->True + current_focus = self.device.shell_output( + "dumpsys window windows | grep mCurrentFocus" + ).strip() + return self.app_name not in current_focus + + def background_app(self): + # Disable Doze, background the app, then disable App Standby + self.device.shell_output("dumpsys deviceidle whitelist +%s" % self.app_name) + self.device.shell_output("input keyevent 3") + if not self.is_app_in_background(): + LOG.critical( + "%s is still in foreground after background request" % self.app_name + ) + else: + LOG.info("%s was successfully backgrounded" % self.app_name) + + def foreground_app(self): + self.device.shell_output("am start --activity-single-top %s" % self.app_name) + self.device.shell_output("dumpsys deviceidle enable") + if self.is_app_in_background(): + LOG.critical( + "%s is still in background after foreground request" % self.app_name + ) + else: + LOG.info("%s was successfully foregrounded" % self.app_name) + + def wait_for_quit(self, timeout=15): + """Wait timeout seconds for the process to exit. If it hasn't + exited by then, kill it. + + The sleep calls are required to give those new values enough time + to sync-up between threads. It would be better to maybe use signals + for synchronization (bug 1633975) + """ + self._is_shutting_down = True + time.sleep(0.25) + + if self.device is not None: + self.device.stop_application(self.app_name) + else: + try: + self.browser_proc.wait(timeout) + except OSError: + LOG.warning("OSError while shutting down browser", exc_info=True) + finally: + if self.browser_proc.poll() is None: + self.browser_proc.kill() + + self._finished = True + time.sleep(0.25) + self._is_shutting_down = False + + def submit_supporting_data(self, supporting_data): + """ + Allow the submission of supporting data i.e. power data. + This type of data is measured outside of the webext; so + we can submit directly to the control server instead of + doing an http post. + """ + self.results_handler.add_supporting_data(supporting_data) + + def stop(self): + LOG.info("shutting down control server") + self.server.shutdown() + self._server_thread.join() diff --git a/testing/raptor/raptor/cpu.py b/testing/raptor/raptor/cpu.py new file mode 100644 index 0000000000..aa43f06901 --- /dev/null +++ b/testing/raptor/raptor/cpu.py @@ -0,0 +1,169 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +import threading +import time + +from logger.logger import RaptorLogger + +LOG = RaptorLogger(component="raptor-cpu") + + +class AndroidCPUProfiler(object): + """AndroidCPUProfiler is used to measure CPU usage over time for an + app in testing. Polls usage at a defined interval and then submits this + as a supporting perfherder data measurement. + + ``` + # Initialize the profiler and start polling for CPU usage of the app + cpu_profiler = AndroidCPUProfiler(raptor, poll_interval=1) + cpu_profiler.start_polling() + + # Or call the following to perform the commands above + cpu_profiler = start_android_cpu_profiler(raptor) + + # Run test... + + # Stop measuring and generate perfherder data + cpu_profiler.generate_android_cpu_profile("browsertime-tp6m") + ``` + """ + + def __init__(self, raptor, poll_interval=10): + """Initialize the Android CPU Profiler. + + :param RaptorAndroid raptor: raptor object which contains android device. + :param float poll_interval: interval, in seconds, at which we should poll + for CPU usage. Defaults to 10 seconds. + """ + self.raptor = raptor + self.polls = [] + self.poll_interval = poll_interval + self.pause_polling = False + + # Get the android version + android_version = self.raptor.device.shell_output( + "getprop ro.build.version.release" + ).strip() + self.android_version = int(android_version.split(".")[0]) + + # Prepare the polling thread (set as daemon for clean exiting) + self.thread = threading.Thread(target=self.poll_cpu, args=(poll_interval,)) + self.thread.daemon = True + + def start_polling(self): + """Start the thread responsible for polling CPU usage.""" + self.thread.start() + + def stop_polling(self): + """Pauses CPU usage polling.""" + self.pause_polling = True + + def poll_cpu(self, poll_interval): + """Polls CPU usage at an interval of `poll_interval`. + + :param float poll_interval: interval at which we poll for CPU usage. + """ + while True: + time.sleep(self.poll_interval) + if self.pause_polling: + continue + self.polls.append(self.get_app_cpu_usage()) + + def get_app_cpu_usage(self): + """Gather a point on an android app's CPU usage. + + :return float: CPU usage found for the app at this point in time. + """ + cpu_usage = 0 + app_name = self.raptor.config["binary"] + verbose = self.raptor.device._verbose + self.raptor.device._verbose = False + + if self.android_version >= 8: + # On android 8 we can use the -O option to order the entries + # in top by %CPU. + cpuinfo = self.raptor.device.shell_output("top -O %CPU -n 1").split("\n") + + for line in cpuinfo: + # A line looks like: + # 14781 u0_a83 0 92.8 12.4 64:53.04 org.mozilla.geckoview_example + data = line.split() + if data[-1] == app_name: + cpu_usage = float(data[3]) + else: + # On android 7, -O is not required since top already orders them by %CPU. + # top also has different columns on this android version. + cpuinfo = self.raptor.device.shell_output("top -n 1").split("\n") + + # Parse the app-specific entries which look like: + # 21165 u0_a196 10 -10 14% S 67 1442392K 163356K fg org.mozilla.geckoview_example + + # Gather the app-specific entries + appcpuinfo = [] + for line in cpuinfo: + if not line.strip().endswith(app_name): + continue + appcpuinfo.append(line) + + # Get CPU usage + for line in appcpuinfo: + data = line.split() + cpu_usage = float(data[4].strip("%")) + + self.raptor.device._verbose = verbose + return cpu_usage + + def generate_android_cpu_profile(self, test_name): + """Use the CPU usage data which was sampled to produce a supporting + perfherder data measurement and submit it to the raptor control + server. Returns the minimum, maximum, and average CPU usage. + + :param str test_name: name of the test that was run. + """ + self.stop_polling() + if len(self.polls) == 0: + LOG.info("No CPU usage data found, submitting as 0% usage.") + self.polls.append(0) + + avg_cpuinfo_data = { + u"type": u"cpu", + u"test": test_name + "-avg", + u"unit": u"%", + u"values": { + u"avg": sum(self.polls) / len(self.polls) + }, # pylint --py3k W1619 + } + self.raptor.control_server.submit_supporting_data(avg_cpuinfo_data) + + min_cpuinfo_data = { + u"type": u"cpu", + u"test": test_name + "-min", + u"unit": u"%", + u"values": {u"min": min(self.polls)}, + } + self.raptor.control_server.submit_supporting_data(min_cpuinfo_data) + + max_cpuinfo_data = { + u"type": u"cpu", + u"test": test_name + "-max", + u"unit": u"%", + u"values": {u"max": max(self.polls)}, + } + self.raptor.control_server.submit_supporting_data(max_cpuinfo_data) + + +def start_android_cpu_profiler(raptor, **kwargs): + """Start the android CPU profiler and return the profiler + object so measurements can be obtained. + + :return AndroidCPUProfiler: the profiler performing the CPU measurements. + """ + if not raptor.device: + LOG.error("No ADB device found in raptor, not creating AndroidCPUProfiler.") + return + + cpu_profiler = AndroidCPUProfiler(raptor, **kwargs) + cpu_profiler.start_polling() + + return cpu_profiler diff --git a/testing/raptor/raptor/filters.py b/testing/raptor/raptor/filters.py new file mode 100644 index 0000000000..a0c7ffc1a9 --- /dev/null +++ b/testing/raptor/raptor/filters.py @@ -0,0 +1,281 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# originally taken from /testing/talos/talos/filter.py + +import math + +""" +data filters: +takes a series of run data and applies statistical transforms to it + +Each filter is a simple function, but it also have attached a special +`prepare` method that create a tuple with one instance of a +:class:`Filter`; this allow to write stuff like:: + + from raptor import filters + filter_list = filters.ignore_first.prepare(1) + filters.median.prepare() + + for filter in filter_list: + data = filter(data) + # data is filtered +""" + +_FILTERS = {} + + +class Filter(object): + def __init__(self, func, *args, **kwargs): + """ + Takes a filter function, and save args and kwargs that + should be used when the filter is used. + """ + self.func = func + self.args = args + self.kwargs = kwargs + + def apply(self, data): + """ + Apply the filter on the data, and return the new data + """ + return self.func(data, *self.args, **self.kwargs) + + +def define_filter(func): + """ + decorator to attach the prepare method. + """ + + def prepare(*args, **kwargs): + return (Filter(func, *args, **kwargs),) + + func.prepare = prepare + return func + + +def register_filter(func): + """ + all filters defined in this module + should be registered + """ + global _FILTERS + + _FILTERS[func.__name__] = func + return func + + +def filters(*args): + global _FILTERS + + filters_ = [_FILTERS[filter] for filter in args] + return filters_ + + +def apply(data, filters): + for filter in filters: + data = filter(data) + + return data + + +def parse(string_): + def to_number(string_number): + try: + return int(string_number) + except ValueError: + return float(string_number) + + tokens = string_.split(":") + + func = tokens[0] + digits = [] + if len(tokens) > 1: + digits.extend(tokens[1].split(",")) + digits = [to_number(digit) for digit in digits] + + return [func, digits] + + +# filters that return a scalar + + +@register_filter +@define_filter +def mean(series): + """ + mean of data; needs at least one data point + """ + return sum(series) / float(len(series)) + + +@register_filter +@define_filter +def median(series): + """ + median of data; needs at least one data point + """ + series = sorted(series) + if len(series) % 2: + # odd + # pylint --py3k W1619 + # must force to int to use as index. + return series[int(len(series) / 2)] + else: + # even + # pylint --py3k W1619 + middle = int(len(series) / 2) # the higher of the middle 2, actually + return 0.5 * (series[middle - 1] + series[middle]) + + +@register_filter +@define_filter +def variance(series): + """ + variance: http://en.wikipedia.org/wiki/Variance + """ + + _mean = mean(series) + variance = sum([(i - _mean) ** 2 for i in series]) / float(len(series)) + return variance + + +@register_filter +@define_filter +def stddev(series): + """ + standard deviation: http://en.wikipedia.org/wiki/Standard_deviation + """ + return variance(series) ** 0.5 + + +@register_filter +@define_filter +def dromaeo(series): + """ + dromaeo: https://wiki.mozilla.org/Dromaeo, pull the internal calculation + out + * This is for 'runs/s' based tests, not 'ms' tests. + * chunksize: defined in dromaeo: tests/dromaeo/webrunner.js#l8 + """ + means = [] + chunksize = 5 + series = list(dromaeo_chunks(series, chunksize)) + for i in series: + means.append(mean(i)) + return geometric_mean(means) + + +@register_filter +@define_filter +def dromaeo_chunks(series, size): + for i in range(0, len(series), size): + yield series[i : i + size] + + +@register_filter +@define_filter +def geometric_mean(series): + """ + geometric_mean: http://en.wikipedia.org/wiki/Geometric_mean + """ + total = 0 + for i in series: + total += math.log(i + 1) + # pylint --py3k W1619 + return math.exp(total / len(series)) - 1 + + +# filters that return a list + + +@register_filter +@define_filter +def ignore_first(series, number=1): + """ + ignore first datapoint + """ + if len(series) <= number: + # don't modify short series + return series + return series[number:] + + +@register_filter +@define_filter +def ignore(series, function): + """ + ignore the first value of a list given by function + """ + if len(series) <= 1: + # don't modify short series + return series + series = series[:] # do not mutate the original series + value = function(series) + series.remove(value) + return series + + +@register_filter +@define_filter +def ignore_max(series): + """ + ignore maximum data point + """ + return ignore(series, max) + + +@register_filter +@define_filter +def ignore_min(series): + """ + ignore minimum data point + """ + return ignore(series, min) + + +@register_filter +@define_filter +def ignore_negative(series): + """ + ignore data points that have a negative value + caution: if all data values are < 0, this will return an empty list + """ + if len(series) <= 1: + # don't modify short series + return series + series = series[:] # do not mutate the original series + return list(filter(lambda x: x >= 0, series)) + + +@register_filter +@define_filter +def v8_subtest(series, name): + """ + v8 benchmark score - modified for no sub benchmarks. + * removed Crypto and kept Encrypt/Decrypt standalone + * removed EarlyBoyer and kept Earley/Boyer standalone + + this is not 100% in parity but within .3% + """ + reference = { + "Encrypt": 266181.0, + "Decrypt": 266181.0, + "DeltaBlue": 66118.0, + "Earley": 666463.0, + "Boyer": 666463.0, + "NavierStokes": 1484000.0, + "RayTrace": 739989.0, + "RegExp": 910985.0, + "Richards": 35302.0, + "Splay": 81491.0, + } + + # pylint --py3k W1619 + return reference[name] / geometric_mean(series) + + +@register_filter +@define_filter +def responsiveness_Metric(val_list): + return sum([float(x) * float(x) / 1000000.0 for x in val_list]) diff --git a/testing/raptor/raptor/gecko_profile.py b/testing/raptor/raptor/gecko_profile.py new file mode 100644 index 0000000000..94bb703f16 --- /dev/null +++ b/testing/raptor/raptor/gecko_profile.py @@ -0,0 +1,369 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +""" +module to handle Gecko profiling. +""" +import gzip +import json +import os +import tempfile +import zipfile + +import mozfile +from logger.logger import RaptorLogger +from mozgeckoprofiler import ProfileSymbolicator + +here = os.path.dirname(os.path.realpath(__file__)) +LOG = RaptorLogger(component="raptor-gecko-profile") + + +class GeckoProfile(object): + """ + Handle Gecko profiling. + + This allows us to collect Gecko profiling data and to zip results into one file. + """ + + def __init__(self, upload_dir, raptor_config, test_config): + self.upload_dir = upload_dir + self.raptor_config = raptor_config + self.test_config = test_config + self.cleanup = True + + # Create a temporary directory into which the tests can put + # their profiles. These files will be assembled into one big + # zip file later on, which is put into the MOZ_UPLOAD_DIR. + self.gecko_profile_dir = tempfile.mkdtemp() + + # Each test INI can specify gecko_profile_interval and entries but they + # can be overrided by user input. + gecko_profile_interval = raptor_config.get( + "gecko_profile_interval", None + ) or test_config.get("gecko_profile_interval", 1) + gecko_profile_entries = raptor_config.get( + "gecko_profile_entries", None + ) or test_config.get("gecko_profile_entries", 1000000) + + # We need symbols_path; if it wasn't passed in on cmdline, set it + # use objdir/dist/crashreporter-symbols for symbolsPath if none provided + if ( + not self.raptor_config["symbols_path"] + and self.raptor_config["run_local"] + and "MOZ_DEVELOPER_OBJ_DIR" in os.environ + ): + self.raptor_config["symbols_path"] = os.path.join( + os.environ["MOZ_DEVELOPER_OBJ_DIR"], "dist", "crashreporter-symbols" + ) + + # turn on crash reporter if we have symbols + os.environ["MOZ_CRASHREPORTER_NO_REPORT"] = "1" + if self.raptor_config["symbols_path"]: + os.environ["MOZ_CRASHREPORTER"] = "1" + else: + os.environ["MOZ_CRASHREPORTER_DISABLE"] = "1" + + # Make sure no archive already exists in the location where + # we plan to output our profiler archive + self.profile_arcname = os.path.join( + self.upload_dir, "profile_{0}.zip".format(test_config["name"]) + ) + LOG.info("Clearing archive {0}".format(self.profile_arcname)) + mozfile.remove(self.profile_arcname) + + self.symbol_paths = { + "FIREFOX": tempfile.mkdtemp(), + "WINDOWS": tempfile.mkdtemp(), + } + + LOG.info( + "Activating gecko profiling, temp profile dir:" + " {0}, interval: {1}, entries: {2}".format( + self.gecko_profile_dir, gecko_profile_interval, gecko_profile_entries + ) + ) + + def _open_gecko_profile(self, profile_path): + """Open a gecko profile and return the contents.""" + if profile_path.endswith(".gz"): + with gzip.open(profile_path, "r") as profile_file: + profile = json.load(profile_file) + else: + with open(profile_path, "r", encoding="utf-8") as profile_file: + profile = json.load(profile_file) + return profile + + def _symbolicate_profile(self, profile, missing_symbols_zip, symbolicator): + try: + symbolicator.dump_and_integrate_missing_symbols( + profile, missing_symbols_zip + ) + symbolicator.symbolicate_profile(profile) + return profile + except MemoryError: + LOG.critical("Ran out of memory while trying to symbolicate profile") + raise + except Exception: + LOG.critical("Encountered an exception during profile symbolication") + # Do not raise an exception and return the profile so we won't block + # the profile capturing pipeline if symbolication fails. + return profile + + def _is_extra_profiler_run(self): + return self.raptor_config.get("extra_profiler_run", False) + + def collect_profiles(self): + """Returns all profiles files.""" + + def __get_test_type(): + """Returns the type of test that was run. + + For benchmark/scenario tests, we return those specific types, + but for pageloads we return cold or warm depending on the --cold + flag. + """ + if self.test_config.get("type", "pageload") not in ( + "benchmark", + "scenario", + ): + return "cold" if self.raptor_config.get("cold", False) else "warm" + else: + return self.test_config.get("type", "benchmark") + + res = [] + if self.raptor_config.get("browsertime"): + topdir = self.raptor_config.get("browsertime_result_dir") + + # Get the browsertime.json file along with the cold/warm splits + # if they exist from a chimera test + results = {"main": None, "cold": None, "warm": None} + profiling_dir = os.path.join(topdir, "profiling") + is_extra_profiler_run = self._is_extra_profiler_run() + result_dir = profiling_dir if is_extra_profiler_run else topdir + + if not os.path.isdir(result_dir): + # Result directory not found. Return early. Caller will decide + # if this should throw an error or not. + LOG.info("Could not find the result directory.") + return [] + + for filename in os.listdir(result_dir): + if filename == "browsertime.json": + results["main"] = os.path.join(result_dir, filename) + elif filename == "cold-browsertime.json": + results["cold"] = os.path.join(result_dir, filename) + elif filename == "warm-browsertime.json": + results["warm"] = os.path.join(result_dir, filename) + if all(results.values()): + break + + if not any(results.values()): + if is_extra_profiler_run: + LOG.info( + "Could not find any browsertime result JSONs in the artifacts " + " for the extra profiler run" + ) + return [] + else: + raise Exception( + "Could not find any browsertime result JSONs in the artifacts" + ) + + profile_locations = [] + if self.raptor_config.get("chimera", False): + if results["warm"] is None or results["cold"] is None: + if is_extra_profiler_run: + LOG.info( + "The test ran in chimera mode but we found no cold " + "and warm browsertime JSONs. Cannot symbolicate profiles. " + "Failing silently because this is an extra profiler run." + ) + return [] + else: + raise Exception( + "The test ran in chimera mode but we found no cold " + "and warm browsertime JSONs. Cannot symbolicate profiles." + ) + profile_locations.extend( + [("cold", results["cold"]), ("warm", results["warm"])] + ) + else: + # When we don't run in chimera mode, it means that we + # either ran a benchmark, scenario test or separate + # warm/cold pageload tests. + profile_locations.append( + ( + __get_test_type(), + results["main"], + ) + ) + + for testtype, results_json in profile_locations: + with open(results_json, encoding="utf-8") as f: + data = json.load(f) + results_dir = os.path.dirname(results_json) + for entry in data: + try: + for rel_profile_path in entry["files"]["geckoProfiles"]: + res.append( + { + "path": os.path.join(results_dir, rel_profile_path), + "type": testtype, + } + ) + except KeyError: + if is_extra_profiler_run: + LOG.info("Failed to find profiles for extra profiler run.") + else: + LOG.error("Failed to find profiles.") + else: + # Raptor-webext stores its profiles in the self.gecko_profile_dir + # directory + for profile in os.listdir(self.gecko_profile_dir): + res.append( + { + "path": os.path.join(self.gecko_profile_dir, profile), + "type": __get_test_type(), + } + ) + + LOG.info("Found %s profiles: %s" % (len(res), str(res))) + return res + + def symbolicate(self): + """ + Symbolicate Gecko profiling data for one pagecycle. + + """ + profiles = self.collect_profiles() + is_extra_profiler_run = self._is_extra_profiler_run() + if len(profiles) == 0: + if is_extra_profiler_run: + LOG.info("No profiles collected in the extra profiler run") + else: + LOG.error("No profiles collected") + return + + symbolicator = ProfileSymbolicator( + { + # Trace-level logging (verbose) + "enableTracing": 0, + # Fallback server if symbol is not found locally + "remoteSymbolServer": "https://symbols.mozilla.org/symbolicate/v4", + # Maximum number of symbol files to keep in memory + "maxCacheEntries": 2000000, + # Frequency of checking for recent symbols to + # cache (in hours) + "prefetchInterval": 12, + # Oldest file age to prefetch (in hours) + "prefetchThreshold": 48, + # Maximum number of library versions to pre-fetch + # per library + "prefetchMaxSymbolsPerLib": 3, + # Default symbol lookup directories + "defaultApp": "FIREFOX", + "defaultOs": "WINDOWS", + # Paths to .SYM files, expressed internally as a + # mapping of app or platform names to directories + # Note: App & OS names from requests are converted + # to all-uppercase internally + "symbolPaths": self.symbol_paths, + } + ) + + if self.raptor_config.get("symbols_path") is not None: + if mozfile.is_url(self.raptor_config["symbols_path"]): + symbolicator.integrate_symbol_zip_from_url( + self.raptor_config["symbols_path"] + ) + elif os.path.isfile(self.raptor_config["symbols_path"]): + symbolicator.integrate_symbol_zip_from_file( + self.raptor_config["symbols_path"] + ) + elif os.path.isdir(self.raptor_config["symbols_path"]): + sym_path = self.raptor_config["symbols_path"] + symbolicator.options["symbolPaths"]["FIREFOX"] = sym_path + self.cleanup = False + + missing_symbols_zip = os.path.join(self.upload_dir, "missingsymbols.zip") + test_type = self.test_config.get("type", "pageload") + + try: + mode = zipfile.ZIP_DEFLATED + except NameError: + mode = zipfile.ZIP_STORED + + with zipfile.ZipFile(self.profile_arcname, "a", mode) as arc: + for profile_info in profiles: + profile_path = profile_info["path"] + + LOG.info("Opening profile at %s" % profile_path) + try: + profile = self._open_gecko_profile(profile_path) + except FileNotFoundError: + if is_extra_profiler_run: + LOG.info("Profile not found on extra profiler run.") + else: + LOG.error("Profile not found.") + continue + + LOG.info("Symbolicating profile from %s" % profile_path) + symbolicated_profile = self._symbolicate_profile( + profile, missing_symbols_zip, symbolicator + ) + + try: + # Write the profiles into a set of folders formatted as: + # <TEST-NAME>-<TEST-RUN-TYPE>. + # <TEST-RUN-TYPE> can be pageload-{warm,cold} or {test-type} + # only for the tests that are not a pageload test. + # For example, "cnn-pageload-warm". + # The file names are formatted as <ITERATION-TYPE>-<ITERATION> + # to clearly indicate without redundant information. + # For example, "browser-cycle-1". + test_run_type = ( + "{0}-{1}".format(test_type, profile_info["type"]) + if test_type == "pageload" + else test_type + ) + folder_name = "%s-%s" % (self.test_config["name"], test_run_type) + iteration = str(os.path.split(profile_path)[-1].split("-")[-1]) + if test_type == "pageload" and profile_info["type"] == "cold": + iteration_type = "browser-cycle" + elif profile_info["type"] == "warm": + iteration_type = "page-cycle" + else: + iteration_type = "iteration" + profile_name = "-".join([iteration_type, iteration]) + path_in_zip = os.path.join(folder_name, profile_name) + + LOG.info( + "Adding profile %s to archive %s as %s" + % (profile_path, self.profile_arcname, path_in_zip) + ) + arc.writestr( + path_in_zip, + json.dumps(symbolicated_profile, ensure_ascii=False).encode( + "utf-8" + ), + ) + except Exception: + LOG.exception( + "Failed to add symbolicated profile %s to archive %s" + % (profile_path, self.profile_arcname) + ) + raise + + # save the latest gecko profile archive to an env var, so later on + # it can be viewed automatically via the view-gecko-profile tool + os.environ["RAPTOR_LATEST_GECKO_PROFILE_ARCHIVE"] = self.profile_arcname + + def clean(self): + """ + Clean up temp folders created with the instance creation. + """ + mozfile.remove(self.gecko_profile_dir) + if self.cleanup: + for symbol_path in self.symbol_paths.values(): + mozfile.remove(symbol_path) diff --git a/testing/raptor/raptor/gen_test_config.py b/testing/raptor/raptor/gen_test_config.py new file mode 100644 index 0000000000..a655b00ba1 --- /dev/null +++ b/testing/raptor/raptor/gen_test_config.py @@ -0,0 +1,59 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +import os + +from logger.logger import RaptorLogger + +here = os.path.abspath(os.path.dirname(__file__)) +webext_dir = os.path.join(os.path.dirname(here), "webext", "raptor") +LOG = RaptorLogger(component="raptor-gen-test-config") + +FILE_CONTENTS = """// this file is auto-generated by raptor, do not edit directly +function getTestConfig() {{ + return {{ + "cs_port": "{control_server_port}", + "test_name": "{test}", + "test_settings_url": "http://{host}:{control_server_port}/json/{test}.json", + "post_startup_delay": "{post_startup_delay}", + "benchmark_port": "{benchmark_port}", + "host": "{host}", + "debug_mode": "{debug_mode}", + "browser_cycle": "{browser_cycle}" + }}; +}} + +""" + + +def gen_test_config( + test, + cs_port, + post_startup_delay, + host="127.0.0.1", + b_port=0, + debug_mode=0, + browser_cycle=1, +): + LOG.info("writing test settings into background js, so webext can get it") + + if host is None or cs_port is None: + raise ValueError( + "Invalid URL for control server: http://{}:{}".format(host, cs_port) + ) + + config = FILE_CONTENTS.format( + benchmark_port=b_port, + browser_cycle=browser_cycle, + control_server_port=cs_port, + debug_mode=debug_mode, + host=host, + post_startup_delay=post_startup_delay, + test=test, + ) + + webext_background_script = os.path.join(webext_dir, "auto_gen_test_config.js") + with open(webext_background_script, "w") as f: + f.write(config) + + LOG.info("finished writing test config to %s" % webext_background_script) diff --git a/testing/raptor/raptor/manifest.py b/testing/raptor/raptor/manifest.py new file mode 100644 index 0000000000..d18b621722 --- /dev/null +++ b/testing/raptor/raptor/manifest.py @@ -0,0 +1,615 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +import json +import os +import re + +from constants.raptor_tests_constants import YOUTUBE_PLAYBACK_MEASURE +from logger.logger import RaptorLogger +from manifestparser import TestManifest +from six.moves.urllib.parse import parse_qs, unquote, urlencode, urlsplit, urlunsplit +from utils import bool_from_str, transform_platform, transform_subtest + +here = os.path.abspath(os.path.dirname(__file__)) +raptor_ini = os.path.join(here, "raptor.ini") +tests_dir = os.path.join(here, "tests") +LOG = RaptorLogger(component="raptor-manifest") + +LIVE_SITE_TIMEOUT_MULTIPLIER = 1.2 + +required_settings = [ + "alert_threshold", + "apps", + "lower_is_better", + "measure", + "page_cycles", + "test_url", + "scenario_time", + "type", + "unit", +] + +playback_settings = [ + "playback_pageset_manifest", +] + + +def filter_app(tests, values): + for test in tests: + if values["app"] in test["apps"]: + yield test + + +def get_browser_test_list(browser_app, run_local): + LOG.info(raptor_ini) + test_manifest = TestManifest([raptor_ini], strict=False) + info = {"app": browser_app, "run_local": run_local} + return test_manifest.active_tests( + exists=False, disabled=False, filters=[filter_app], **info + ) + + +def validate_test_ini(test_details): + # validate all required test details were found in the test INI + valid_settings = True + + for setting in required_settings: + # measure setting not required for benchmark type tests + if setting == "measure" and test_details["type"] == "benchmark": + continue + if setting == "scenario_time" and test_details["type"] != "scenario": + continue + if test_details.get(setting) is None: + # if page-cycles is not specified, it's ok as long as browser-cycles is there + if ( + setting == "page-cycles" + and test_details.get("browser_cycles") is not None + ): + continue + valid_settings = False + LOG.error( + "setting '%s' is required but not found in %s" + % (setting, test_details["manifest"]) + ) + + test_details.setdefault("page_timeout", 30000) + + # if playback is specified, we need more playback settings + if test_details.get("playback") is not None: + for setting in playback_settings: + if test_details.get(setting) is None: + valid_settings = False + LOG.error( + "setting '%s' is required but not found in %s" + % (setting, test_details["manifest"]) + ) + + # if 'alert-on' is specified, we need to make sure that the value given is valid + # i.e. any 'alert_on' values must be values that exist in the 'measure' ini setting + if test_details.get("alert_on") is not None: + + # support with or without spaces, i.e. 'measure = fcp, loadtime' or '= fcp,loadtime' + # convert to a list; and remove any spaces + # this can also have regexes inside + test_details["alert_on"] = [ + _item.strip() for _item in test_details["alert_on"].split(",") + ] + + # this variable will store all the concrete values for alert_on elements + # that have a match in "measure" list + valid_alerts = [] + + # if test is raptor-youtube-playback and measure is empty, use all the tests + if test_details.get( + "measure" + ) is None and "youtube-playback" in test_details.get("name", ""): + test_details["measure"] = YOUTUBE_PLAYBACK_MEASURE + + # convert "measure" to string, so we can use it inside a regex + measure_as_string = " ".join(test_details["measure"]) + + # now make sure each alert_on value provided is valid + for alert_on_value in test_details["alert_on"]: + # replace the '*' with a valid regex pattern + alert_on_value_pattern = alert_on_value.replace("*", "[a-zA-Z0-9.@_%]*") + # store all elements that have been found in "measure_as_string" + matches = re.findall(alert_on_value_pattern, measure_as_string) + + if len(matches) == 0: + LOG.error( + "The 'alert_on' value of '%s' is not valid because " + "it doesn't exist in the 'measure' test setting!" % alert_on_value + ) + valid_settings = False + else: + # add the matched elements to valid_alerts + valid_alerts.extend(matches) + + # replace old alert_on values with valid elements (no more regexes inside) + # and also remove duplicates if any, by converting valid_alerts to a 'set' first + test_details["alert_on"] = sorted(set(valid_alerts)) + + # if repository is defined, then a revision also needs to be defined + # the path is optional and we'll default to the root of the repo + if test_details.get("repository", None) is not None: + if test_details.get("repository_revision", None) is None: + LOG.error( + "`repository_revision` is required when a `repository` is defined." + ) + valid_settings = False + elif test_details.get("type") not in ("benchmark"): + LOG.error("`repository` is only available for benchmark test types.") + valid_settings = False + + return valid_settings + + +def add_test_url_params(url, extra_params): + # add extra parameters to the test_url query string + # the values that already exist are re-written + + # urlsplit returns a result as a tuple like (scheme, netloc, path, query, fragment) + parsed_url = urlsplit(url) + + parsed_query_params = parse_qs(parsed_url.query) + parsed_extra_params = parse_qs(extra_params) + + for name, value in parsed_extra_params.items(): + # overwrite the old value + parsed_query_params[name] = value + + final_query_string = unquote(urlencode(parsed_query_params, doseq=True)) + + # reconstruct test_url with the changed query string + return urlunsplit( + ( + parsed_url.scheme, + parsed_url.netloc, + parsed_url.path, + final_query_string, + parsed_url.fragment, + ) + ) + + +def write_test_settings_json(args, test_details, oskey): + # write test settings json file with test details that the control + # server will provide for the web ext + test_url = transform_platform(test_details["test_url"], oskey) + # this is needed for raptor browsertime to pick up the replaced + # {platform} argument for motionmark tests + test_details["test_url"] = test_url + + test_settings = { + "raptor-options": { + "type": test_details["type"], + "cold": test_details["cold"], + "test_url": test_url, + "expected_browser_cycles": test_details["expected_browser_cycles"], + "page_cycles": int(test_details["page_cycles"]), + "host": args.host, + } + } + + if test_details["type"] == "pageload": + test_settings["raptor-options"]["measure"] = {} + + # test_details['measure'] was already converted to a list in get_raptor_test_list below + # the 'hero=' line is still a raw string from the test INI + for m in test_details["measure"]: + test_settings["raptor-options"]["measure"][m] = True + if m == "hero": + test_settings["raptor-options"]["measure"][m] = [ + h.strip() for h in test_details["hero"].split(",") + ] + + if test_details.get("alert_on", None) is not None: + # alert_on was already converted to list above + test_settings["raptor-options"]["alert_on"] = test_details["alert_on"] + + if test_details.get("page_timeout", None) is not None: + test_settings["raptor-options"]["page_timeout"] = int( + test_details["page_timeout"] + ) + + test_settings["raptor-options"]["unit"] = test_details.get("unit", "ms") + + test_settings["raptor-options"]["lower_is_better"] = test_details.get( + "lower_is_better", True + ) + + # support optional subtest unit/lower_is_better fields + val = test_details.get("subtest_unit", test_settings["raptor-options"]["unit"]) + test_settings["raptor-options"]["subtest_unit"] = val + subtest_lower_is_better = test_details.get("subtest_lower_is_better") + + if subtest_lower_is_better is None: + # default to main test values if not set + test_settings["raptor-options"]["subtest_lower_is_better"] = test_settings[ + "raptor-options" + ]["lower_is_better"] + else: + test_settings["raptor-options"][ + "subtest_lower_is_better" + ] = subtest_lower_is_better + + if test_details.get("alert_change_type", None) is not None: + test_settings["raptor-options"]["alert_change_type"] = test_details[ + "alert_change_type" + ] + + if test_details.get("alert_threshold", None) is not None: + test_settings["raptor-options"]["alert_threshold"] = float( + test_details["alert_threshold"] + ) + + if test_details.get("screen_capture", None) is not None: + test_settings["raptor-options"]["screen_capture"] = test_details.get( + "screen_capture" + ) + + # if Gecko profiling is enabled, write profiling settings for webext + if test_details.get("gecko_profile", False): + threads = ["GeckoMain", "Compositor"] + threads.extend(["Renderer", "WR"]) + + if test_details.get("gecko_profile_threads"): + # pylint --py3k: W1639 + test_threads = list( + filter(None, test_details["gecko_profile_threads"].split(",")) + ) + threads.extend(test_threads) + + test_settings["raptor-options"].update( + { + "gecko_profile": True, + "gecko_profile_entries": int( + test_details.get("gecko_profile_entries", 1000000) + ), + "gecko_profile_interval": int( + test_details.get("gecko_profile_interval", 1) + ), + "gecko_profile_threads": ",".join(set(threads)), + } + ) + + features = test_details.get("gecko_profile_features") + if features: + test_settings["raptor-options"]["gecko_profile_features"] = features + + if test_details.get("extra_profiler_run", False): + test_settings["raptor-options"]["extra_profiler_run"] = True + + if test_details.get("newtab_per_cycle", None) is not None: + test_settings["raptor-options"]["newtab_per_cycle"] = bool( + test_details["newtab_per_cycle"] + ) + + if test_details["type"] == "scenario": + test_settings["raptor-options"]["scenario_time"] = test_details["scenario_time"] + if "background_test" in test_details: + test_settings["raptor-options"]["background_test"] = bool( + test_details["background_test"] + ) + else: + test_settings["raptor-options"]["background_test"] = False + + jsons_dir = os.path.join(tests_dir, "json") + + if not os.path.exists(jsons_dir): + os.mkdir(os.path.join(tests_dir, "json")) + + settings_file = os.path.join(jsons_dir, test_details["name"] + ".json") + try: + with open(settings_file, "w") as out_file: + json.dump(test_settings, out_file, indent=4, ensure_ascii=False) + out_file.close() + except IOError: + LOG.info("abort: exception writing test settings json!") + + +def get_raptor_test_list(args, oskey): + """ + A test ini (i.e. raptor-firefox-tp6.ini) will have one or more subtests inside, + each with it's own name ([the-ini-file-test-section]). + + We want the ability to eiter: + - run * all * of the subtests listed inside the test ini; - or - + - just run a single one of those subtests that are inside the ini + + A test name is received on the command line. This will either match the name + of a single subtest (within an ini) - or - if there's no matching single + subtest with that name, then the test name provided might be the name of a + test ini itself (i.e. raptor-firefox-tp6) that contains multiple subtests. + + First look for a single matching subtest name in the list of all availble tests, + and if it's found we will just run that single subtest. + + Then look at the list of all available tests - each available test has a manifest + name associated to it - and pull out all subtests whose manifest name matches + the test name provided on the command line i.e. run all subtests in a specified ini. + + If no tests are found at all then the test name is invalid. + """ + tests_to_run = [] + # get list of all available tests for the browser we are testing against + available_tests = get_browser_test_list(args.app, args.run_local) + + # look for single subtest that matches test name provided on cmd line + for next_test in available_tests: + if next_test["name"] == args.test: + tests_to_run.append(next_test) + break + + # no matches, so now look for all subtests that come from a test ini + # manifest that matches the test name provided on the commmand line + if len(tests_to_run) == 0: + _ini = args.test + ".ini" + for next_test in available_tests: + head, tail = os.path.split(next_test["manifest"]) + if tail == _ini: + # subtest comes from matching test ini file name, so add it + tests_to_run.append(next_test) + + if args.collect_perfstats and "chrom" not in args.app.lower(): + for next_test in tests_to_run: + next_test["perfstats"] = "true" + + # enable live sites if requested with --live-sites + if args.live_sites: + for next_test in tests_to_run: + # set use_live_sites to `true` and disable mitmproxy playback + # immediately so we don't follow playback paths below + next_test["use_live_sites"] = "true" + next_test["playback"] = None + + # go through each test and set the page-cycles and page-timeout, and some config flags + # the page-cycles value in the INI can be overriden when debug-mode enabled, when + # gecko-profiling enabled, or when --page-cycles cmd line arg was used (that overrides all) + for next_test in tests_to_run: + LOG.info("configuring settings for test %s" % next_test["name"]) + max_page_cycles = int(next_test.get("page_cycles", 1)) + max_browser_cycles = int(next_test.get("browser_cycles", 1)) + + # If using playback, the playback recording info may need to be transformed. + # This transformation needs to happen before the test name is changed + # below (for cold tests for instance) + if next_test.get("playback") is not None: + next_test["playback_pageset_manifest"] = transform_subtest( + next_test["playback_pageset_manifest"], next_test["name"] + ) + + if args.gecko_profile is True: + next_test["gecko_profile"] = True + LOG.info("gecko-profiling enabled") + max_page_cycles = 3 + max_browser_cycles = 3 + + if ( + "gecko_profile_entries" in args + and args.gecko_profile_entries is not None + ): + next_test["gecko_profile_entries"] = str(args.gecko_profile_entries) + LOG.info( + "gecko-profiling entries set to %s" % args.gecko_profile_entries + ) + + if ( + "gecko_profile_interval" in args + and args.gecko_profile_interval is not None + ): + next_test["gecko_profile_interval"] = str(args.gecko_profile_interval) + LOG.info( + "gecko-profiling interval set to %s" % args.gecko_profile_interval + ) + + if ( + "gecko_profile_threads" in args + and args.gecko_profile_threads is not None + ): + # pylint --py3k: W1639 + threads = list( + filter(None, next_test.get("gecko_profile_threads", "").split(",")) + ) + threads.extend(args.gecko_profile_threads.split(",")) + if ( + "gecko_profile_extra_threads" in args + and args.gecko_profile_extra_threads is not None + ): + threads.extend(getattr(args, "gecko_profile_extra_threads", [])) + next_test["gecko_profile_threads"] = ",".join(threads) + LOG.info("gecko-profiling threads %s" % args.gecko_profile_threads) + if ( + "gecko_profile_features" in args + and args.gecko_profile_features is not None + ): + next_test["gecko_profile_features"] = args.gecko_profile_features + LOG.info("gecko-profiling features %s" % args.gecko_profile_features) + + else: + # if the gecko profiler is not enabled, ignore all of its settings + next_test.pop("gecko_profile_entries", None) + next_test.pop("gecko_profile_interval", None) + next_test.pop("gecko_profile_threads", None) + next_test.pop("gecko_profile_features", None) + + if args.extra_profiler_run is True and args.app == "firefox": + next_test["extra_profiler_run"] = True + LOG.info("extra-profiler-run enabled") + next_test["extra_profiler_run_browser_cycles"] = 1 + if args.chimera: + next_test["extra_profiler_run_page_cycles"] = 2 + else: + next_test["extra_profiler_run_page_cycles"] = 1 + else: + args.extra_profiler_run = False + LOG.info("extra-profiler-run disabled") + + if args.debug_mode is True: + next_test["debug_mode"] = True + LOG.info("debug-mode enabled") + max_page_cycles = 2 + + # if --page-cycles was provided on the command line, use that instead of INI + # if just provided in the INI use that but cap at 3 if gecko-profiling is enabled + if args.page_cycles is not None: + next_test["page_cycles"] = args.page_cycles + LOG.info( + "setting page-cycles to %d as specified on cmd line" % args.page_cycles + ) + else: + if int(next_test.get("page_cycles", 1)) > max_page_cycles: + next_test["page_cycles"] = max_page_cycles + LOG.info( + "setting page-cycles to %d because gecko-profling is enabled" + % next_test["page_cycles"] + ) + + # if --browser-cycles was provided on the command line, use that instead of INI + # if just provided in the INI use that but cap at 3 if gecko-profiling is enabled + if args.browser_cycles is not None: + next_test["browser_cycles"] = args.browser_cycles + LOG.info( + "setting browser-cycles to %d as specified on cmd line" + % args.browser_cycles + ) + else: + if int(next_test.get("browser_cycles", 1)) > max_browser_cycles: + next_test["browser_cycles"] = max_browser_cycles + LOG.info( + "setting browser-cycles to %d because gecko-profilng is enabled" + % next_test["browser_cycles"] + ) + + # if --page-timeout was provided on the command line, use that instead of INI + if args.page_timeout is not None: + LOG.info( + "setting page-timeout to %d as specified on cmd line" + % args.page_timeout + ) + next_test["page_timeout"] = args.page_timeout + + _running_cold = False + + # check command line to see if we set cold page load from command line + if args.cold or next_test.get("cold") == "true": + # for raptor-webext jobs cold page-load is determined by the 'cold' key + # in test manifest INI + _running_cold = True + else: + # if it's a warm load test ignore browser_cycles if set + next_test["browser_cycles"] = 1 + + if _running_cold: + # when running in cold mode, set browser-cycles to the page-cycles value; as we want + # the browser to restart between page-cycles; and set page-cycles to 1 as we only + # want 1 single page-load for every browser-cycle + next_test["cold"] = True + next_test["expected_browser_cycles"] = int(next_test["browser_cycles"]) + if args.chimera: + next_test["page_cycles"] = 2 + else: + next_test["page_cycles"] = 1 + # also ensure '-cold' is in test name so perfherder results indicate warm cold-load + # Bug 1644344 we can remove this condition once we're migrated away from WebExtension + if "-cold" not in next_test["name"] and not args.browsertime: + next_test["name"] += "-cold" + else: + # when running in warm mode, just set test-cycles to 1 and leave page-cycles as/is + next_test["cold"] = False + next_test["expected_browser_cycles"] = 1 + + # either warm or cold-mode, initialize the starting current 'browser-cycle' + next_test["browser_cycle"] = 1 + + # if --test-url-params was provided on the command line, add the params to the test_url + # provided in the INI + if args.test_url_params is not None: + initial_test_url = next_test["test_url"] + next_test["test_url"] = add_test_url_params( + initial_test_url, args.test_url_params + ) + LOG.info( + "adding extra test_url params (%s) as specified on cmd line " + "to the current test_url (%s), resulting: %s" + % (args.test_url_params, initial_test_url, next_test["test_url"]) + ) + + if next_test.get("use_live_sites", "false") == "true": + # when using live sites we want to turn off playback + LOG.info("using live sites so turning playback off!") + next_test["playback"] = None + # Only for raptor-youtube-playback tests until they are removed + # in favor of the browsertime variant + if "raptor-youtube-playback" in next_test["name"]: + next_test["name"] = next_test["name"] + "-live" + # allow a slightly higher page timeout due to remote page loads + next_test["page_timeout"] = ( + int(next_test["page_timeout"]) * LIVE_SITE_TIMEOUT_MULTIPLIER + ) + LOG.info( + "using live sites so using page timeout of %dms" + % next_test["page_timeout"] + ) + + if not args.browsertime and "browsertime" in next_test.get("manifest", ""): + raise Exception( + "%s test can only be run with --browsertime" + % next_test.get("name", "Unknown") + ) + + # browsertime doesn't use the 'measure' test ini setting; however just for the sake + # of supporting both webext and browsertime, just provide a dummy 'measure' setting + # here to prevent having to check in multiple places; it has no effect on what + # browsertime actually measures; remove this when eventually we remove webext support + if ( + args.browsertime + and next_test.get("measure") is None + and next_test.get("type") == "pageload" + ): + next_test["measure"] = ( + "fnbpaint, fcp, dcf, loadtime," + "ContentfulSpeedIndex, PerceptualSpeedIndex," + "SpeedIndex, FirstVisualChange, LastVisualChange" + ) + + # convert 'measure =' test INI line to list + if next_test.get("measure") is not None: + _measures = [] + for measure in [m.strip() for m in next_test["measure"].split(",")]: + # build the 'measures =' list + _measures.append(measure) + next_test["measure"] = _measures + + # if using live sites, don't measure hero element as it only exists in recordings + if ( + "hero" in next_test["measure"] + and next_test.get("use_live_sites", "false") == "true" + ): + # remove 'hero' from the 'measures =' list + next_test["measure"].remove("hero") + # remove the 'hero =' line since no longer measuring hero + del next_test["hero"] + + bool_settings = [ + "lower_is_better", + "subtest_lower_is_better", + "accept_zero_vismet", + "interactive", + ] + for setting in bool_settings: + if next_test.get(setting, None) is not None: + next_test[setting] = bool_from_str(next_test.get(setting)) + + # write out .json test setting files for the control server to read and send to web ext + if len(tests_to_run) != 0: + for test in tests_to_run: + if validate_test_ini(test): + write_test_settings_json(args, test, oskey) + else: + # test doesn't have valid settings, remove it from available list + LOG.info("test %s is not valid due to missing settings" % test["name"]) + tests_to_run.remove(test) + + return tests_to_run diff --git a/testing/raptor/raptor/memory.py b/testing/raptor/raptor/memory.py new file mode 100644 index 0000000000..96dafcf190 --- /dev/null +++ b/testing/raptor/raptor/memory.py @@ -0,0 +1,41 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +import re + + +def get_app_memory_usage(raptor): + app_name = raptor.config["binary"] + total = 0 + re_total_memory = re.compile(r"TOTAL:\s+(\d+)") + verbose = raptor.device._verbose + raptor.device._verbose = False + meminfo = raptor.device.shell_output("dumpsys meminfo %s" % app_name).split("\n") + raptor.device._verbose = verbose + for line in meminfo: + match = re_total_memory.search(line) + if match: + total = int(match.group(1)) + break + return total + + +def generate_android_memory_profile(raptor, test_name): + if not raptor.device or not raptor.config["memory_test"]: + return + foreground = get_app_memory_usage(raptor) + # put app into background + verbose = raptor.device._verbose + raptor.device._verbose = False + raptor.device.shell_output( + "am start -a android.intent.action.MAIN " "-c android.intent.category.HOME" + ) + raptor.device._verbose = verbose + background = get_app_memory_usage(raptor) + meminfo_data = { + "type": "memory", + "test": test_name, + "unit": "KB", + "values": {"foreground": foreground, "background": background}, + } + raptor.control_server.submit_supporting_data(meminfo_data) diff --git a/testing/raptor/raptor/output.py b/testing/raptor/raptor/output.py new file mode 100644 index 0000000000..11c38ff0b8 --- /dev/null +++ b/testing/raptor/raptor/output.py @@ -0,0 +1,1850 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# some parts of this originally taken from /testing/talos/talos/output.py + +"""output raptor test results""" +import copy +import json +import os +from abc import ABCMeta, abstractmethod +from collections.abc import Iterable + +import filters +import six +from logger.logger import RaptorLogger +from utils import flatten + +LOG = RaptorLogger(component="perftest-output") + +VISUAL_METRICS = [ + "SpeedIndex", + "ContentfulSpeedIndex", + "PerceptualSpeedIndex", + "FirstVisualChange", + "LastVisualChange", + "VisualReadiness", + "VisualComplete85", + "VisualComplete95", + "VisualComplete99", +] + +METRIC_BLOCKLIST = [ + "mean", + "median", + "geomean", +] + + +@six.add_metaclass(ABCMeta) +class PerftestOutput(object): + """Abstract base class to handle output of perftest results""" + + def __init__( + self, results, supporting_data, subtest_alert_on, app, extra_summary_methods=[] + ): + """ + - results : list of RaptorTestResult instances + """ + self.app = app + self.results = results + self.summarized_results = {} + self.supporting_data = supporting_data + self.summarized_supporting_data = [] + self.summarized_screenshots = [] + self.subtest_alert_on = subtest_alert_on + self.browser_name = None + self.browser_version = None + self.extra_summary_methods = extra_summary_methods + + @abstractmethod + def summarize(self, test_names): + raise NotImplementedError() + + def set_browser_meta(self, browser_name, browser_version): + # sets the browser metadata for the perfherder data + self.browser_name = browser_name + self.browser_version = browser_version + + def summarize_supporting_data(self): + """ + Supporting data was gathered outside of the main raptor test; it will be kept + separate from the main raptor test results. Summarize it appropriately. + + supporting_data = { + 'type': 'data-type', + 'test': 'raptor-test-ran-when-data-was-gathered', + 'unit': 'unit that the values are in', + 'summarize-values': True/False, + 'suite-suffix-type': True/False, + 'values': { + 'name': value_dict, + 'nameN': value_dictN + } + } + + More specifically, subtest supporting data will look like this: + + supporting_data = { + 'type': 'power', + 'test': 'raptor-speedometer-geckoview', + 'unit': 'mAh', + 'values': { + 'cpu': { + 'values': val, + 'lowerIsBetter': True/False, + 'alertThreshold': 2.0, + 'subtest-prefix-type': True/False, + 'unit': 'mWh' + }, + 'wifi': ... + } + } + + We want to treat each value as a 'subtest'; and for the overall aggregated + test result the summary value is dependent on the unit. An exception is + raised in case we don't know about the specified unit. + """ + if self.supporting_data is None: + return + + self.summarized_supporting_data = [] + support_data_by_type = {} + + for data_set in self.supporting_data: + + data_type = data_set["type"] + LOG.info("summarizing %s data" % data_type) + + if data_type not in support_data_by_type: + support_data_by_type[data_type] = { + "framework": {"name": "raptor"}, + "suites": [], + } + + # suite name will be name of the actual raptor test that ran, plus the type of + # supporting data i.e. 'raptor-speedometer-geckoview-power' + vals = [] + subtests = [] + + suite_name = data_set["test"] + if data_set.get("suite-suffix-type", True): + suite_name = "%s-%s" % (data_set["test"], data_set["type"]) + + suite = { + "name": suite_name, + "type": data_set["type"], + "subtests": subtests, + } + if data_set.get("summarize-values", True): + suite.update( + { + "lowerIsBetter": True, + "unit": data_set["unit"], + "alertThreshold": 2.0, + } + ) + + for result in self.results: + if result["name"] == data_set["test"]: + suite["extraOptions"] = result["extra_options"] + break + + support_data_by_type[data_type]["suites"].append(suite) + for measurement_name, value_info in data_set["values"].items(): + # Subtests are expected to be specified in a dictionary, this + # provides backwards compatibility with the old method + if not isinstance(value_info, dict): + value_info = {"values": value_info} + + new_subtest = {} + if value_info.get("subtest-prefix-type", True): + new_subtest["name"] = data_type + "-" + measurement_name + else: + new_subtest["name"] = measurement_name + + new_subtest["value"] = value_info["values"] + new_subtest["lowerIsBetter"] = value_info.get("lowerIsBetter", True) + new_subtest["alertThreshold"] = value_info.get("alertThreshold", 2.0) + new_subtest["unit"] = value_info.get("unit", data_set["unit"]) + + if "shouldAlert" in value_info: + new_subtest["shouldAlert"] = value_info.get("shouldAlert") + + subtests.append(new_subtest) + vals.append([new_subtest["value"], new_subtest["name"]]) + + if len(subtests) >= 1 and data_set.get("summarize-values", True): + suite["value"] = self.construct_summary( + vals, testname="supporting_data", unit=data_set["unit"] + ) + + # split the supporting data by type, there will be one + # perfherder output per type + for data_type in support_data_by_type: + data = support_data_by_type[data_type] + if self.browser_name: + data["application"] = {"name": self.browser_name} + if self.browser_version: + data["application"]["version"] = self.browser_version + self.summarized_supporting_data.append(data) + + return + + def output(self, test_names): + """output to file and perfherder data json""" + if os.getenv("MOZ_UPLOAD_DIR"): + # i.e. testing/mozharness/build/raptor.json locally; in production it will + # be at /tasks/task_*/build/ (where it will be picked up by mozharness later + # and made into a tc artifact accessible in treeherder as perfherder-data.json) + results_path = os.path.join( + os.path.dirname(os.environ["MOZ_UPLOAD_DIR"]), "raptor.json" + ) + screenshot_path = os.path.join( + os.path.dirname(os.environ["MOZ_UPLOAD_DIR"]), "screenshots.html" + ) + else: + results_path = os.path.join(os.getcwd(), "raptor.json") + screenshot_path = os.path.join(os.getcwd(), "screenshots.html") + + success = True + if self.summarized_results == {}: + success = False + LOG.error( + "no summarized raptor results found for any of %s" + % ", ".join(test_names) + ) + else: + for suite in self.summarized_results["suites"]: + gecko_profiling_enabled = "gecko-profile" in suite.get( + "extraOptions", [] + ) + if gecko_profiling_enabled: + LOG.info("gecko profiling enabled") + suite["shouldAlert"] = False + + # as we do navigation, tname could end in .<alias> + # test_names doesn't have tname, so either add it to test_names, + # or strip it + tname = suite["name"] + parts = tname.split(".") + try: + tname = ".".join(parts[:-1]) + except Exception as e: + LOG.info("no alias found on test, ignoring: %s" % e) + pass + + # Since test names might have been modified, check if + # part of the test name exists in the test_names list entries + found = False + for test in test_names: + if tname in test: + found = True + break + if not found: + success = False + LOG.error("no summarized raptor results found for %s" % (tname)) + + with open(results_path, "w") as f: + for result in self.summarized_results: + f.write("%s\n" % result) + + if len(self.summarized_screenshots) > 0: + with open(screenshot_path, "w") as f: + for result in self.summarized_screenshots: + f.write("%s\n" % result) + LOG.info("screen captures can be found locally at: %s" % screenshot_path) + + # now that we've checked for screen captures too, if there were no actual + # test results we can bail out here + if self.summarized_results == {}: + return success, 0 + + test_type = self.summarized_results["suites"][0].get("type", "") + output_perf_data = True + not_posting = "- not posting regular test results for perfherder" + if test_type == "scenario": + # if a resource-usage flag was supplied the perfherder data + # will still be output from output_supporting_data + LOG.info("scenario test type was run %s" % not_posting) + output_perf_data = False + + if self.browser_name: + self.summarized_results["application"] = {"name": self.browser_name} + if self.browser_version: + self.summarized_results["application"]["version"] = self.browser_version + + total_perfdata = 0 + if output_perf_data: + # if we have supporting data i.e. power, we ONLY want those measurements + # dumped out. TODO: Bug 1515406 - Add option to output both supplementary + # data (i.e. power) and the regular Raptor test result + # Both are already available as separate PERFHERDER_DATA json blobs + if len(self.summarized_supporting_data) == 0: + LOG.info("PERFHERDER_DATA: %s" % json.dumps(self.summarized_results)) + total_perfdata = 1 + else: + LOG.info( + "supporting data measurements exist - only posting those to perfherder" + ) + + json.dump( + self.summarized_results, open(results_path, "w"), indent=2, sort_keys=True + ) + LOG.info("results can also be found locally at: %s" % results_path) + + return success, total_perfdata + + def output_supporting_data(self, test_names): + """ + Supporting data was gathered outside of the main raptor test; it has already + been summarized, now output it appropriately. + + We want to output supporting data in a completely separate perfherder json blob and + in a corresponding file artifact. This way, supporting data can be ingested as its own + test suite in perfherder and alerted upon if desired; kept outside of the test results + from the actual Raptor test which was run when the supporting data was gathered. + """ + if len(self.summarized_supporting_data) == 0: + LOG.error( + "no summarized supporting data found for %s" % ", ".join(test_names) + ) + return False, 0 + + total_perfdata = 0 + for next_data_set in self.summarized_supporting_data: + data_type = next_data_set["suites"][0]["type"] + + if os.environ["MOZ_UPLOAD_DIR"]: + # i.e. testing/mozharness/build/raptor.json locally; in production it will + # be at /tasks/task_*/build/ (where it will be picked up by mozharness later + # and made into a tc artifact accessible in treeherder as perfherder-data.json) + results_path = os.path.join( + os.path.dirname(os.environ["MOZ_UPLOAD_DIR"]), + "raptor-%s.json" % data_type, + ) + else: + results_path = os.path.join(os.getcwd(), "raptor-%s.json" % data_type) + + # dump data to raptor-data.json artifact + json.dump(next_data_set, open(results_path, "w"), indent=2, sort_keys=True) + + # the output that treeherder expects to find + LOG.info("PERFHERDER_DATA: %s" % json.dumps(next_data_set)) + LOG.info( + "%s results can also be found locally at: %s" + % (data_type, results_path) + ) + total_perfdata += 1 + + return True, total_perfdata + + def construct_summary(self, vals, testname, unit=None): + def _filter(vals, value=None): + if value is None: + return [i for i, j in vals] + return [i for i, j in vals if j == value] + + if testname.startswith("raptor-v8_7"): + return 100 * filters.geometric_mean(_filter(vals)) + + if testname == "speedometer3": + score = None + for val, name in vals: + if name == "score": + score = val + if score is None: + raise Exception("Unable to find score for Speedometer 3") + return score + + if "speedometer" in testname: + correctionFactor = 3 + results = _filter(vals) + # speedometer has 16 tests, each of these are made of up 9 subtests + # and a sum of the 9 values. We receive 160 values, and want to use + # the 16 test values, not the sub test values. + if len(results) != 160: + raise Exception( + "Speedometer has 160 subtests, found: %s instead" % len(results) + ) + + results = results[9::10] + # pylint --py3k W1619 + score = 60 * 1000 / filters.geometric_mean(results) / correctionFactor + return score + + if "stylebench" in testname: + # see https://bug-172968-attachments.webkit.org/attachment.cgi?id=319888 + correctionFactor = 3 + results = _filter(vals) + + # stylebench has 5 tests, each of these are made of up 5 subtests + # + # * Adding classes. + # * Removing classes. + # * Mutating attributes. + # * Adding leaf elements. + # * Removing leaf elements. + # + # which are made of two subtests each (sync/async) and repeated 5 times + # each, thus, the list here looks like: + # + # [Test name/Adding classes - 0/ Sync; <x>] + # [Test name/Adding classes - 0/ Async; <y>] + # [Test name/Adding classes - 0; <x> + <y>] + # [Test name/Removing classes - 0/ Sync; <x>] + # [Test name/Removing classes - 0/ Async; <y>] + # [Test name/Removing classes - 0; <x> + <y>] + # ... + # [Test name/Adding classes - 1 / Sync; <x>] + # [Test name/Adding classes - 1 / Async; <y>] + # [Test name/Adding classes - 1 ; <x> + <y>] + # ... + # [Test name/Removing leaf elements - 4; <x> + <y>] + # [Test name; <sum>] <- This is what we want. + # + # So, 5 (subtests) * + # 5 (repetitions) * + # 3 (entries per repetition (sync/async/sum)) = + # 75 entries for test before the sum. + # + # We receive 76 entries per test, which ads up to 380. We want to use + # the 5 test entries, not the rest. + if len(results) != 380: + raise Exception( + "StyleBench requires 380 entries, found: %s instead" % len(results) + ) + results = results[75::76] + # pylint --py3k W1619 + return 60 * 1000 / filters.geometric_mean(results) / correctionFactor + + if testname.startswith("raptor-kraken") or "sunspider" in testname: + return sum(_filter(vals)) + + if "unity-webgl" in testname or "webaudio" in testname: + # webaudio_score and unity_webgl_score: self reported as 'Geometric Mean' + return filters.mean(_filter(vals, "Geometric Mean")) + + if "assorted-dom" in testname: + # pylint: disable=W1633 + return round(filters.geometric_mean(_filter(vals)), 2) + + if "wasm-misc" in testname: + # wasm_misc_score: self reported as '__total__' + return filters.mean(_filter(vals, "__total__")) + + if "wasm-godot" in testname: + # wasm_godot_score: first-interactive mean + return filters.mean(_filter(vals, "first-interactive")) + + if "youtube-playback" in testname: + # pylint: disable=W1633 + return round(filters.mean(_filter(vals)), 2) + + if testname.startswith("supporting_data"): + if not unit: + return sum(_filter(vals)) + + if unit == "%": + return filters.mean(_filter(vals)) + + if unit in ("W", "MHz"): + # For power in Watts and clock frequencies, + # summarize with the sum of the averages + allavgs = [] + for val, subtest in vals: + if "avg" in subtest: + allavgs.append(val) + if allavgs: + return sum(allavgs) + + raise Exception( + "No average measurements found for supporting data with W, or MHz unit ." + ) + + if unit in ["KB", "mAh", "mWh"]: + return sum(_filter(vals)) + + raise NotImplementedError("Unit %s not suported" % unit) + + if len(vals) > 1: + # pylint: disable=W1633 + return round(filters.geometric_mean(_filter(vals)), 2) + + # pylint: disable=W1633 + return round(filters.mean(_filter(vals)), 2) + + def parseUnknown(self, test): + # Attempt to flatten whatever we've been given + # Dictionary keys will be joined by dashes, arrays represent + # represent "iterations" + _subtests = {} + + if not isinstance(test["measurements"], dict): + raise Exception( + "Expected a dictionary with a single entry as the name of the test. " + "The value of this key should be the data." + ) + if test.get("custom_data", False): + # If custom_data is true it means that the data was already flattened + # and the test name is included in the keys (the test might have + # also removed it if it's in the subtest_name_filters option). Handle this + # exception by wrapping it + test["measurements"] = {test["name"]: [test["measurements"]]} + + for iteration in test["measurements"][list(test["measurements"].keys())[0]]: + flattened_metrics = None + if not test.get("custom_data", False): + flattened_metrics = flatten(iteration, ()) + + for metric, value in (flattened_metrics or iteration).items(): + if metric in METRIC_BLOCKLIST: + # TODO: Add an option in the test manifest for this + continue + if metric not in _subtests: + # subtest not added yet, first pagecycle, so add new one + _subtests[metric] = { + "unit": test["subtest_unit"], + "alertThreshold": float(test["alert_threshold"]), + "lowerIsBetter": test["subtest_lower_is_better"], + "name": metric, + "replicates": [], + } + if not isinstance(value, Iterable): + value = [value] + # pylint: disable=W1633 + _subtests[metric]["replicates"].extend([round(x, 3) for x in value]) + + vals = [] + subtests = [] + names = list(_subtests) + names.sort(reverse=True) + summaries = { + "median": filters.median, + "mean": filters.mean, + "geomean": filters.geometric_mean, + } + for name in names: + summary_method = test.get("submetric_summary_method", "median") + _subtests[name]["value"] = round( + summaries[summary_method](_subtests[name]["replicates"]), 3 + ) + subtests.append(_subtests[name]) + vals.append([_subtests[name]["value"], name]) + + return subtests, vals + + def parseSpeedometerOutput(self, test): + # each benchmark 'index' becomes a subtest; each pagecycle / iteration + # of the test has multiple values per index/subtest + + # this is the format we receive the results in from the benchmark + # i.e. this is ONE pagecycle of speedometer: + + # {u'name': u'raptor-speedometer', u'type': u'benchmark', u'measurements': + # {u'speedometer': [[{u'AngularJS-TodoMVC/DeletingAllItems': [147.3000000000011, + # 149.95999999999913, 143.29999999999927, 150.34000000000378, 257.6999999999971], + # u'Inferno-TodoMVC/CompletingAllItems/Sync': [88.03999999999996,# + # 85.60000000000036, 94.18000000000029, 95.19999999999709, 86.47999999999593], + # u'AngularJS-TodoMVC': [518.2400000000016, 525.8199999999997, 610.5199999999968, + # 532.8200000000215, 640.1800000000003], ...(repeated for each index/subtest)}]]}, + # u'browser': u'Firefox 62.0a1 20180528123052', u'lower_is_better': False, u'page': + # u'http://localhost:55019/Speedometer/index.html?raptor', u'unit': u'score', + # u'alert_threshold': 2} + + _subtests = {} + data = test["measurements"]["speedometer"] + for page_cycle in data: + for sub, replicates in page_cycle[0].items(): + # for each pagecycle, build a list of subtests and append all related replicates + if sub not in _subtests: + # subtest not added yet, first pagecycle, so add new one + _subtests[sub] = { + "unit": test["subtest_unit"], + "alertThreshold": float(test["alert_threshold"]), + "lowerIsBetter": test["subtest_lower_is_better"], + "name": sub, + "replicates": [], + } + # pylint: disable=W1633 + _subtests[sub]["replicates"].extend([round(x, 3) for x in replicates]) + + vals = [] + subtests = [] + names = list(_subtests) + names.sort(reverse=True) + for name in names: + _subtests[name]["value"] = filters.median(_subtests[name]["replicates"]) + subtests.append(_subtests[name]) + vals.append([_subtests[name]["value"], name]) + + return subtests, vals + + def parseAresSixOutput(self, test): + """ + https://browserbench.org/ARES-6/ + Every pagecycle will perform the tests from the index page + We have 4 main tests per index page: + - Air, Basic, Babylon, ML + - and from these 4 above, ares6 generates the Overall results + Each test has 3 subtests (firstIteration, steadyState, averageWorstCase): + - _steadyState + - _firstIteration + - _averageWorstCase + Each index page will run 5 cycles, this is set in glue.js + + { + 'expected_browser_cycles': 1, + 'subtest_unit': 'ms', + 'name': 'raptor-ares6-firefox', + 'lower_is_better': False, + 'browser_cycle': '1', + 'subtest_lower_is_better': True, + 'cold': False, + 'browser': 'Firefox 69.0a1 20190531035909', + 'type': 'benchmark', + 'page': 'http://127.0.0.1:35369/ARES-6/index.html?raptor', + 'unit': 'ms', + 'alert_threshold': 2 + 'measurements': { + 'ares6': [[{ + 'Babylon_firstIteration': [ + 123.68, + 168.21999999999997, + 127.34000000000003, + 113.56, + 128.78, + 169.44000000000003 + ], + 'Air_steadyState': [ + 21.184723618090434, + 22.906331658291457, + 19.939396984924624, + 20.572462311557775, + 20.790452261306534, + 18.378693467336696 + ], + etc. + }]] + } + } + + Details on how /ARES6/index.html is showing the mean on subsequent test results: + + I selected just a small part from the metrics just to be easier to explain + what is going on. + + After the raptor GeckoView test finishes, we have these results in the logs: + + Extracted from "INFO - raptor-control-server Info: received webext_results:" + 'Air_firstIteration': [660.8000000000002, 626.4599999999999, 655.6199999999999, + 635.9000000000001, 636.4000000000001] + + Extracted from "INFO - raptor-output Info: PERFHERDER_DATA:" + {"name": "Air_firstIteration", "lowerIsBetter": true, "alertThreshold": 2.0, + "replicates": [660.8, 626.46, 655.62, 635.9, 636.4], "value": 636.4, "unit": "ms"} + + On GeckoView's /ARES6/index.html this is what we see for Air - First Iteration: + + - on 1st test cycle : 660.80 (rounded from 660.8000000000002) + + - on 2nd test cycle : 643.63 , this is coming from + (660.8000000000002 + 626.4599999999999) / 2 , + then rounded up to a precision of 2 decimals + + - on 3rd test cycle : 647.63 this is coming from + (660.8000000000002 + 626.4599999999999 + 655.6199999999999) / 3 , + then rounded up to a precision of 2 decimals + + - and so on + """ + + _subtests = {} + data = test["measurements"]["ares6"] + + for page_cycle in data: + for sub, replicates in page_cycle[0].items(): + # for each pagecycle, build a list of subtests and append all related replicates + if sub not in _subtests: + # subtest not added yet, first pagecycle, so add new one + _subtests[sub] = { + "unit": test["subtest_unit"], + "alertThreshold": float(test["alert_threshold"]), + "lowerIsBetter": test["subtest_lower_is_better"], + "name": sub, + "replicates": [], + } + # pylint: disable=W1633 + _subtests[sub]["replicates"].extend( + [float(round(x, 3)) for x in replicates] + ) + + vals = [] + for name, test in _subtests.items(): + test["value"] = filters.mean(test["replicates"]) + vals.append([test["value"], name]) + + # pylint W1656 + return list(_subtests.values()), sorted(vals, reverse=True) + + def parseMotionmarkOutput(self, test): + # for motionmark we want the frameLength:average value for each test + + # this is the format we receive the results in from the benchmark + # i.e. this is ONE pagecycle of motionmark htmlsuite test:composited Transforms: + + # {u'name': u'raptor-motionmark-firefox', + # u'type': u'benchmark', + # u'measurements': { + # u'motionmark': + # [[{u'HTMLsuite': + # {u'Composited Transforms': + # {u'scoreLowerBound': 272.9947975553528, + # u'frameLength': {u'average': 25.2, u'stdev': 27.0, + # u'percent': 68.2, u'concern': 39.5}, + # u'controller': {u'average': 300, u'stdev': 0, u'percent': 0, u'concern': 3}, + # u'scoreUpperBound': 327.0052024446473, + # u'complexity': {u'segment1': [[300, 16.6], [300, 16.6]], u'complexity': 300, + # u'segment2': [[300, None], [300, None]], u'stdev': 6.8}, + # u'score': 300.00000000000006, + # u'complexityAverage': {u'segment1': [[30, 30], [30, 30]], u'complexity': 30, + # u'segment2': [[300, 300], [300, 300]], u'stdev': None} + # }}}]]}} + + _subtests = {} + data = test["measurements"]["motionmark"] + for page_cycle in data: + page_cycle_results = page_cycle[0] + + # TODO: this assumes a single suite is run + suite = list(page_cycle_results)[0] + for sub in page_cycle_results[suite].keys(): + try: + # pylint: disable=W1633 + replicate = round( + float(page_cycle_results[suite][sub]["frameLength"]["average"]), + 3, + ) + except TypeError as e: + LOG.warning( + "[{}][{}] : {} - {}".format(suite, sub, e.__class__.__name__, e) + ) + + if sub not in _subtests: + # subtest not added yet, first pagecycle, so add new one + _subtests[sub] = { + "unit": test["subtest_unit"], + "alertThreshold": float(test["alert_threshold"]), + "lowerIsBetter": test["subtest_lower_is_better"], + "name": sub, + "replicates": [], + } + _subtests[sub]["replicates"].extend([replicate]) + + vals = [] + subtests = [] + names = list(_subtests) + names.sort(reverse=True) + for name in names: + _subtests[name]["value"] = filters.median(_subtests[name]["replicates"]) + subtests.append(_subtests[name]) + vals.append([_subtests[name]["value"], name]) + + return subtests, vals + + def parseYoutubePlaybackPerformanceOutput(self, test): + """Parse the metrics for the Youtube playback performance test. + + For each video measured values for dropped and decoded frames will be + available from the benchmark site. + + {u'PlaybackPerf.VP9.2160p60@2X': {u'droppedFrames': 1, u'decodedFrames': 796} + + With each page cycle / iteration of the test multiple values can be present. + + Raptor will calculate the percentage of dropped frames to decoded frames. + All those three values will then be emitted as separate sub tests. + """ + _subtests = {} + test_name = [ + measurement + for measurement in test["measurements"].keys() + if "youtube-playback" in measurement + ] + if len(test_name) > 0: + data = test["measurements"].get(test_name[0]) + else: + raise Exception("No measurements found for youtube test!") + + def create_subtest_entry( + name, + value, + unit=test["subtest_unit"], + lower_is_better=test["subtest_lower_is_better"], + ): + # build a list of subtests and append all related replicates + if name not in _subtests: + # subtest not added yet, first pagecycle, so add new one + _subtests[name] = { + "name": name, + "unit": unit, + "lowerIsBetter": lower_is_better, + "replicates": [], + } + + _subtests[name]["replicates"].append(value) + if self.subtest_alert_on is not None: + if name in self.subtest_alert_on: + LOG.info( + "turning on subtest alerting for measurement type: %s" % name + ) + _subtests[name]["shouldAlert"] = True + + failed_tests = [] + for pagecycle in data: + for _sub, _value in six.iteritems(pagecycle[0]): + if _value["decodedFrames"] == 0: + failed_tests.append( + "%s test Failed. decodedFrames %s droppedFrames %s." + % (_sub, _value["decodedFrames"], _value["droppedFrames"]) + ) + + try: + percent_dropped = ( + float(_value["droppedFrames"]) / _value["decodedFrames"] * 100.0 + ) + except ZeroDivisionError: + # if no frames have been decoded the playback failed completely + percent_dropped = 100.0 + + # Remove the not needed "PlaybackPerf." prefix from each test + _sub = _sub.split("PlaybackPerf", 1)[-1] + if _sub.startswith("."): + _sub = _sub[1:] + + # build a list of subtests and append all related replicates + create_subtest_entry( + "{}_decoded_frames".format(_sub), + _value["decodedFrames"], + lower_is_better=False, + ) + create_subtest_entry( + "{}_dropped_frames".format(_sub), _value["droppedFrames"] + ) + create_subtest_entry( + "{}_%_dropped_frames".format(_sub), percent_dropped + ) + + # Check if any youtube test failed and generate exception + if len(failed_tests) > 0: + [LOG.warning("Youtube sub-test FAILED: %s" % test) for test in failed_tests] + # TODO: Change this to raise Exception after we figure out the failing tests + LOG.warning( + "Youtube playback sub-tests failed!!! " + "Not submitting results to perfherder!" + ) + vals = [] + subtests = [] + names = list(_subtests) + names.sort(reverse=True) + for name in names: + # pylint: disable=W1633 + _subtests[name]["value"] = round( + float(filters.median(_subtests[name]["replicates"])), 2 + ) + subtests.append(_subtests[name]) + # only include dropped_frames values, without the %_dropped_frames values + if name.endswith("X_dropped_frames"): + vals.append([_subtests[name]["value"], name]) + + return subtests, vals + + def parseUnityWebGLOutput(self, test): + """ + Example output (this is one page cycle): + + {'name': 'raptor-unity-webgl-firefox', + 'type': 'benchmark', + 'measurements': { + 'unity-webgl': [ + [ + '[{"benchmark":"Mandelbrot GPU","result":1035361},...}]' + ] + ] + }, + 'lower_is_better': False, + 'unit': 'score' + } + """ + _subtests = {} + data = test["measurements"]["unity-webgl"] + for page_cycle in data: + data = json.loads(page_cycle[0]) + for item in data: + # for each pagecycle, build a list of subtests and append all related replicates + sub = item["benchmark"] + if sub not in _subtests: + # subtest not added yet, first pagecycle, so add new one + _subtests[sub] = { + "unit": test["subtest_unit"], + "alertThreshold": float(test["alert_threshold"]), + "lowerIsBetter": test["subtest_lower_is_better"], + "name": sub, + "replicates": [], + } + _subtests[sub]["replicates"].append(item["result"]) + + vals = [] + subtests = [] + names = list(_subtests) + names.sort(reverse=True) + for name in names: + _subtests[name]["value"] = filters.median(_subtests[name]["replicates"]) + subtests.append(_subtests[name]) + vals.append([_subtests[name]["value"], name]) + + return subtests, vals + + def parseWebaudioOutput(self, test): + # each benchmark 'index' becomes a subtest; each pagecycle / iteration + # of the test has multiple values per index/subtest + + # this is the format we receive the results in from the benchmark + # i.e. this is ONE pagecycle of speedometer: + + # {u'name': u'raptor-webaudio-firefox', u'type': u'benchmark', u'measurements': + # {u'webaudio': [[u'[{"name":"Empty testcase","duration":26,"buffer":{}},{"name" + # :"Simple gain test without resampling","duration":66,"buffer":{}},{"name":"Simple + # gain test without resampling (Stereo)","duration":71,"buffer":{}},{"name":"Simple + # gain test without resampling (Stereo and positional)","duration":67,"buffer":{}}, + # {"name":"Simple gain test","duration":41,"buffer":{}},{"name":"Simple gain test + # (Stereo)","duration":59,"buffer":{}},{"name":"Simple gain test (Stereo and positional)", + # "duration":68,"buffer":{}},{"name":"Upmix without resampling (Mono -> Stereo)", + # "duration":53,"buffer":{}},{"name":"Downmix without resampling (Mono -> Stereo)", + # "duration":44,"buffer":{}},{"name":"Simple mixing (same buffer)", + # "duration":288,"buffer":{}} + + _subtests = {} + data = test["measurements"]["webaudio"] + for page_cycle in data: + data = json.loads(page_cycle[0]) + for item in data: + # for each pagecycle, build a list of subtests and append all related replicates + sub = item["name"] + replicates = [item["duration"]] + if sub not in _subtests: + # subtest not added yet, first pagecycle, so add new one + _subtests[sub] = { + "unit": test["subtest_unit"], + "alertThreshold": float(test["alert_threshold"]), + "lowerIsBetter": test["subtest_lower_is_better"], + "name": sub, + "replicates": [], + } + # pylint: disable=W1633 + _subtests[sub]["replicates"].extend( + [float(round(x, 3)) for x in replicates] + ) + + vals = [] + subtests = [] + names = list(_subtests) + names.sort(reverse=True) + for name in names: + _subtests[name]["value"] = filters.median(_subtests[name]["replicates"]) + subtests.append(_subtests[name]) + vals.append([_subtests[name]["value"], name]) + + print(subtests) + return subtests, vals + + def parseWASMGodotOutput(self, test): + """ + {u'wasm-godot': [ + { + "name": "wasm-instantiate", + "time": 349 + },{ + "name": "engine-instantiate", + "time": 1263 + ... + }]} + """ + _subtests = {} + data = test["measurements"]["wasm-godot"] + print(data) + for page_cycle in data: + for item in page_cycle[0]: + # for each pagecycle, build a list of subtests and append all related replicates + sub = item["name"] + if sub not in _subtests: + # subtest not added yet, first pagecycle, so add new one + _subtests[sub] = { + "unit": test["subtest_unit"], + "alertThreshold": float(test["alert_threshold"]), + "lowerIsBetter": test["subtest_lower_is_better"], + "name": sub, + "replicates": [], + } + _subtests[sub]["replicates"].append(item["time"]) + + vals = [] + subtests = [] + names = list(_subtests) + names.sort(reverse=True) + for name in names: + _subtests[name]["value"] = filters.median(_subtests[name]["replicates"]) + subtests.append(_subtests[name]) + vals.append([_subtests[name]["value"], name]) + + return subtests, vals + + def parseSunspiderOutput(self, test): + _subtests = {} + data = test["measurements"]["sunspider"] + for page_cycle in data: + for sub, replicates in page_cycle[0].items(): + # for each pagecycle, build a list of subtests and append all related replicates + if sub not in _subtests: + # subtest not added yet, first pagecycle, so add new one + _subtests[sub] = { + "unit": test["subtest_unit"], + "alertThreshold": float(test["alert_threshold"]), + "lowerIsBetter": test["subtest_lower_is_better"], + "name": sub, + "replicates": [], + } + # pylint: disable=W1633 + _subtests[sub]["replicates"].extend( + [float(round(x, 3)) for x in replicates] + ) + + subtests = [] + vals = [] + + names = list(_subtests) + names.sort(reverse=True) + for name in names: + _subtests[name]["value"] = filters.mean(_subtests[name]["replicates"]) + subtests.append(_subtests[name]) + + vals.append([_subtests[name]["value"], name]) + + return subtests, vals + + def parseAssortedDomOutput(self, test): + # each benchmark 'index' becomes a subtest; each pagecycle / iteration + # of the test has multiple values + + # this is the format we receive the results in from the benchmark + # i.e. this is ONE pagecycle of assorted-dom ('test' is a valid subtest name btw): + + # {u'worker-getname-performance-getter': 5.9, u'window-getname-performance-getter': 6.1, + # u'window-getprop-performance-getter': 6.1, u'worker-getprop-performance-getter': 6.1, + # u'test': 5.8, u'total': 30} + + # the 'total' is provided for us from the benchmark; the overall score will be the mean of + # the totals from all pagecycles; but keep all the subtest values for the logs/json + + _subtests = {} + data = test["measurements"]["assorted-dom"] + for pagecycle in data: + for _sub, _value in pagecycle[0].items(): + # build a list of subtests and append all related replicates + if _sub not in _subtests: + # subtest not added yet, first pagecycle, so add new one + _subtests[_sub] = { + "unit": test["subtest_unit"], + "alertThreshold": float(test["alert_threshold"]), + "lowerIsBetter": test["subtest_lower_is_better"], + "name": _sub, + "replicates": [], + } + _subtests[_sub]["replicates"].extend([_value]) + + vals = [] + subtests = [] + names = list(_subtests) + names.sort(reverse=True) + for name in names: + # pylint: disable=W1633 + _subtests[name]["value"] = float( + round(filters.median(_subtests[name]["replicates"]), 2) + ) + subtests.append(_subtests[name]) + # only use the 'total's to compute the overall result + if name == "total": + vals.append([_subtests[name]["value"], name]) + + return subtests, vals + + def parseJetstreamTwoOutput(self, test): + # https://browserbench.org/JetStream/ + + _subtests = {} + data = test["measurements"]["jetstream2"] + for page_cycle in data: + for sub, replicates in page_cycle[0].items(): + # for each pagecycle, build a list of subtests and append all related replicates + if sub not in _subtests: + # subtest not added yet, first pagecycle, so add new one + _subtests[sub] = { + "unit": test["subtest_unit"], + "alertThreshold": float(test["alert_threshold"]), + "lowerIsBetter": test["subtest_lower_is_better"], + "name": sub, + "replicates": [], + } + # pylint: disable=W1633 + _subtests[sub]["replicates"].extend( + [float(round(x, 3)) for x in replicates] + ) + + vals = [] + subtests = [] + names = list(_subtests) + names.sort(reverse=True) + for name in names: + _subtests[name]["value"] = filters.mean(_subtests[name]["replicates"]) + subtests.append(_subtests[name]) + vals.append([_subtests[name]["value"], name]) + + return subtests, vals + + def parseWASMMiscOutput(self, test): + """ + {u'wasm-misc': [ + [[{u'name': u'validate', u'time': 163.44000000000005}, + ... + {u'name': u'__total__', u'time': 63308.434904788155}]], + ... + [[{u'name': u'validate', u'time': 129.42000000000002}, + {u'name': u'__total__', u'time': 63181.24089257814}]] + ]} + """ + _subtests = {} + data = test["measurements"]["wasm-misc"] + for page_cycle in data: + for item in page_cycle[0]: + # for each pagecycle, build a list of subtests and append all related replicates + sub = item["name"] + if sub not in _subtests: + # subtest not added yet, first pagecycle, so add new one + _subtests[sub] = { + "unit": test["subtest_unit"], + "alertThreshold": float(test["alert_threshold"]), + "lowerIsBetter": test["subtest_lower_is_better"], + "name": sub, + "replicates": [], + } + _subtests[sub]["replicates"].append(item["time"]) + + vals = [] + subtests = [] + names = list(_subtests) + names.sort(reverse=True) + for name in names: + _subtests[name]["value"] = filters.median(_subtests[name]["replicates"]) + subtests.append(_subtests[name]) + vals.append([_subtests[name]["value"], name]) + + return subtests, vals + + def parseMatrixReactBenchOutput(self, test): + # https://github.com/jandem/matrix-react-bench + + _subtests = {} + data = test["measurements"]["matrix-react-bench"] + for page_cycle in data: + # Each cycle is formatted like `[[iterations, val], [iterations, val2], ...]` + for iteration, val in page_cycle: + sub = f"{iteration}-iterations" + _subtests.setdefault( + sub, + { + "unit": test["subtest_unit"], + "alertThreshold": float(test["alert_threshold"]), + "lowerIsBetter": test["subtest_lower_is_better"], + "name": sub, + "replicates": [], + }, + ) + + # The values produced are far too large for perfherder + _subtests[sub]["replicates"].append(val) + + vals = [] + subtests = [] + names = list(_subtests) + names.sort(reverse=True) + for name in names: + _subtests[name]["value"] = filters.mean(_subtests[name]["replicates"]) + subtests.append(_subtests[name]) + vals.append([_subtests[name]["value"], name]) + + return subtests, vals + + def parseTwitchAnimationOutput(self, test): + _subtests = {} + + for metric, data in test["measurements"].items(): + if "perfstat-" not in metric and metric != "twitch-animation": + # Only keep perfstats or the run metric + continue + if metric == "twitch-animation": + metric = "run" + + # data is just an array with a single number + for page_cycle in data: + # Each benchmark cycle is formatted like `[val]`, perfstats + # are not + if not isinstance(page_cycle, list): + page_cycle = [page_cycle] + for val in page_cycle: + _subtests.setdefault( + metric, + { + "unit": test["subtest_unit"], + "alertThreshold": float(test["alert_threshold"]), + "lowerIsBetter": test["subtest_lower_is_better"], + "name": metric, + "replicates": [], + }, + ) + + # The values produced are far too large for perfherder + _subtests[metric]["replicates"].append(val) + + vals = [] + subtests = [] + names = list(_subtests) + names.sort(reverse=True) + for name in names: + _subtests[name]["value"] = filters.mean(_subtests[name]["replicates"]) + subtests.append(_subtests[name]) + vals.append([_subtests[name]["value"], name]) + + return subtests, vals + + +class RaptorOutput(PerftestOutput): + """class for raptor output""" + + def summarize(self, test_names): + suites = [] + test_results = {"framework": {"name": "raptor"}, "suites": suites} + + # check if we actually have any results + if len(self.results) == 0: + LOG.error("no raptor test results found for %s" % ", ".join(test_names)) + return + + for test in self.results: + vals = [] + subtests = [] + suite = { + "name": test["name"], + "type": test["type"], + "tags": test.get("tags", []), + "extraOptions": test["extra_options"], + "subtests": subtests, + "lowerIsBetter": test["lower_is_better"], + "unit": test["unit"], + "alertThreshold": float(test["alert_threshold"]), + } + + # Check if optional properties have been set by the test + if hasattr(test, "alert_change_type"): + suite["alertChangeType"] = test["alert_change_type"] + + # if cold load add that info to the suite result dict; this will be used later + # when combining the results from multiple browser cycles into one overall result + if test["cold"] is True: + suite["cold"] = True + suite["browser_cycle"] = int(test["browser_cycle"]) + suite["expected_browser_cycles"] = int(test["expected_browser_cycles"]) + suite["tags"].append("cold") + else: + suite["tags"].append("warm") + + suites.append(suite) + + # process results for pageloader type of tests + if test["type"] in ("pageload", "scenario"): + # each test can report multiple measurements per pageload + # each measurement becomes a subtest inside the 'suite' + + # this is the format we receive the results in from the pageload test + # i.e. one test (subtest) in raptor-firefox-tp6: + + # {u'name': u'raptor-firefox-tp6-amazon', u'type': u'pageload', u'measurements': + # {u'fnbpaint': [788, 315, 334, 286, 318, 276, 296, 296, 292, 285, 268, 277, 274, + # 328, 295, 290, 286, 270, 279, 280, 346, 303, 308, 398, 281]}, u'browser': + # u'Firefox 62.0a1 20180528123052', u'lower_is_better': True, u'page': + # u'https://www.amazon.com/s/url=search-alias%3Daps&field-keywords=laptop', + # u'unit': u'ms', u'alert_threshold': 2} + + for measurement_name, replicates in test["measurements"].items(): + new_subtest = {} + new_subtest["name"] = measurement_name + new_subtest["replicates"] = replicates + new_subtest["lowerIsBetter"] = test["subtest_lower_is_better"] + new_subtest["alertThreshold"] = float(test["alert_threshold"]) + new_subtest["value"] = 0 + new_subtest["unit"] = test["subtest_unit"] + + if test["cold"] is False: + # for warm page-load, ignore first value due to 1st pageload noise + LOG.info( + "ignoring the first %s value due to initial pageload noise" + % measurement_name + ) + filtered_values = filters.ignore_first( + new_subtest["replicates"], 1 + ) + else: + # for cold-load we want all the values + filtered_values = new_subtest["replicates"] + + # for pageload tests that measure TTFI: TTFI is not guaranteed to be available + # everytime; the raptor measure.js webext will substitute a '-1' value in the + # cases where TTFI is not available, which is acceptable; however we don't want + # to include those '-1' TTFI values in our final results calculations + if measurement_name == "ttfi": + filtered_values = filters.ignore_negative(filtered_values) + # we've already removed the first pageload value; if there aren't any more + # valid TTFI values available for this pageload just remove it from results + if len(filtered_values) < 1: + continue + + # if 'alert_on' is set for this particular measurement, then we want to set the + # flag in the perfherder output to turn on alerting for this subtest + if self.subtest_alert_on is not None: + if measurement_name in self.subtest_alert_on: + LOG.info( + "turning on subtest alerting for measurement type: %s" + % measurement_name + ) + new_subtest["shouldAlert"] = True + else: + # Explicitly set `shouldAlert` to False so that the measurement + # is not alerted on. Otherwise Perfherder defaults to alerting + LOG.info( + "turning off subtest alerting for measurement type: %s" + % measurement_name + ) + new_subtest["shouldAlert"] = False + + new_subtest["value"] = filters.median(filtered_values) + + vals.append([new_subtest["value"], new_subtest["name"]]) + subtests.append(new_subtest) + + elif test["type"] == "benchmark": + + if any( + [ + "youtube-playback" in measurement + for measurement in test["measurements"].keys() + ] + ): + subtests, vals = self.parseYoutubePlaybackPerformanceOutput(test) + elif "assorted-dom" in test["measurements"]: + subtests, vals = self.parseAssortedDomOutput(test) + elif "ares6" in test["measurements"]: + subtests, vals = self.parseAresSixOutput(test) + elif "jetstream2" in test["measurements"]: + subtests, vals = self.parseJetstreamTwoOutput(test) + elif "motionmark" in test["measurements"]: + subtests, vals = self.parseMotionmarkOutput(test) + elif "speedometer" in test["measurements"]: + # this includes stylebench + subtests, vals = self.parseSpeedometerOutput(test) + elif "sunspider" in test["measurements"]: + subtests, vals = self.parseSunspiderOutput(test) + elif "unity-webgl" in test["measurements"]: + subtests, vals = self.parseUnityWebGLOutput(test) + elif "wasm-godot" in test["measurements"]: + subtests, vals = self.parseWASMGodotOutput(test) + elif "wasm-misc" in test["measurements"]: + subtests, vals = self.parseWASMMiscOutput(test) + elif "webaudio" in test["measurements"]: + subtests, vals = self.parseWebaudioOutput(test) + else: + subtests, vals in self.parseUnknown(test) + + suite["subtests"] = subtests + + else: + LOG.error( + "output.summarize received unsupported test results type for %s" + % test["name"] + ) + return + + suite["tags"].append(test["type"]) + + # for benchmarks there is generally more than one subtest in each cycle + # and a benchmark-specific formula is needed to calculate the final score + # we no longer summarise the page load as we alert on individual subtests + # and the geometric mean was found to be of little value + if len(subtests) > 1 and test["type"] != "pageload": + suite["value"] = self.construct_summary(vals, testname=test["name"]) + + subtests.sort(key=lambda subtest: subtest["name"]) + suite["tags"].sort() + + suites.sort(key=lambda suite: suite["name"]) + + self.summarized_results = test_results + + def combine_browser_cycles(self): + """ + At this point the results have been summarized; however there may have been multiple + browser cycles (i.e. cold load). In which case the results have one entry for each + test for each browser cycle. For each test we need to combine the results for all + browser cycles into one results entry. + + For example, this is what the summarized results suites list looks like from a test that + was run with multiple (two) browser cycles: + + [{'expected_browser_cycles': 2, 'extraOptions': [], + 'name': u'raptor-tp6m-amazon-geckoview-cold', 'lowerIsBetter': True, + 'alertThreshold': 2.0, 'value': 1776.94, 'browser_cycle': 1, + 'subtests': [{'name': u'dcf', 'lowerIsBetter': True, 'alertThreshold': 2.0, + 'value': 818, 'replicates': [818], 'unit': u'ms'}, {'name': u'fcp', + 'lowerIsBetter': True, 'alertThreshold': 2.0, 'value': 1131, 'shouldAlert': True, + 'replicates': [1131], 'unit': u'ms'}, {'name': u'fnbpaint', 'lowerIsBetter': True, + 'alertThreshold': 2.0, 'value': 1056, 'replicates': [1056], 'unit': u'ms'}, + {'name': u'ttfi', 'lowerIsBetter': True, 'alertThreshold': 2.0, 'value': 18074, + 'replicates': [18074], 'unit': u'ms'}, {'name': u'loadtime', 'lowerIsBetter': True, + 'alertThreshold': 2.0, 'value': 1002, 'shouldAlert': True, 'replicates': [1002], + 'unit': u'ms'}], + 'cold': True, 'type': u'pageload', 'unit': u'ms'}, + {'expected_browser_cycles': 2, 'extraOptions': [], + 'name': u'raptor-tp6m-amazon-geckoview-cold', 'lowerIsBetter': True, + 'alertThreshold': 2.0, 'value': 840.25, 'browser_cycle': 2, + 'subtests': [{'name': u'dcf', 'lowerIsBetter': True, 'alertThreshold': 2.0, + 'value': 462, 'replicates': [462], 'unit': u'ms'}, {'name': u'fcp', + 'lowerIsBetter': True, 'alertThreshold': 2.0, 'value': 718, 'shouldAlert': True, + 'replicates': [718], 'unit': u'ms'}, {'name': u'fnbpaint', 'lowerIsBetter': True, + 'alertThreshold': 2.0, 'value': 676, 'replicates': [676], 'unit': u'ms'}, + {'name': u'ttfi', 'lowerIsBetter': True, 'alertThreshold': 2.0, 'value': 3084, + 'replicates': [3084], 'unit': u'ms'}, {'name': u'loadtime', 'lowerIsBetter': True, + 'alertThreshold': 2.0, 'value': 605, 'shouldAlert': True, 'replicates': [605], + 'unit': u'ms'}], + 'cold': True, 'type': u'pageload', 'unit': u'ms'}] + + Need to combine those into a single entry. + """ + # check if we actually have any results + if len(self.results) == 0: + LOG.info( + "error: no raptor test results found, so no need to combine browser cycles" + ) + return + + # first build a list of entries that need to be combined; and as we do that, mark the + # original suite entry as up for deletion, so once combined we know which ones to del + # note that summarized results are for all tests that were ran in the session, which + # could include cold and / or warm page-load and / or benchnarks combined + suites_to_be_combined = [] + combined_suites = [] + + for _index, suite in enumerate(self.summarized_results.get("suites", [])): + if suite.get("cold") is None: + continue + + if suite["expected_browser_cycles"] > 1: + _name = suite["name"] + _details = suite.copy() + suites_to_be_combined.append({"name": _name, "details": _details}) + suite["to_be_deleted"] = True + + # now create a new suite entry that will have all the results from + # all of the browser cycles, but in one result entry for each test + combined_suites = {} + + for next_suite in suites_to_be_combined: + suite_name = next_suite["details"]["name"] + browser_cycle = next_suite["details"]["browser_cycle"] + LOG.info( + "combining results from browser cycle %d for %s" + % (browser_cycle, suite_name) + ) + if suite_name not in combined_suites: + # first browser cycle so just take entire entry to start with + combined_suites[suite_name] = next_suite["details"] + LOG.info("created new combined result with intial cycle replicates") + # remove the 'cold', 'browser_cycle', and 'expected_browser_cycles' info + # as we don't want that showing up in perfherder data output + del combined_suites[suite_name]["cold"] + del combined_suites[suite_name]["browser_cycle"] + del combined_suites[suite_name]["expected_browser_cycles"] + else: + # subsequent browser cycles, already have an entry; just add subtest replicates + for next_subtest in next_suite["details"]["subtests"]: + # find the existing entry for that subtest in our new combined test entry + found_subtest = False + for combined_subtest in combined_suites[suite_name]["subtests"]: + if combined_subtest["name"] == next_subtest["name"]: + # add subtest (measurement type) replicates to the combined entry + LOG.info("adding replicates for %s" % next_subtest["name"]) + combined_subtest["replicates"].extend( + next_subtest["replicates"] + ) + found_subtest = True + # the subtest / measurement type wasn't found in our existing combined + # result entry; if it is for the same suite name add it - this could happen + # as ttfi may not be available in every browser cycle + if not found_subtest: + LOG.info("adding replicates for %s" % next_subtest["name"]) + combined_suites[next_suite["details"]["name"]][ + "subtests" + ].append(next_subtest) + + # now we have a single entry for each test; with all replicates from all browser cycles + for i, name in enumerate(combined_suites): + vals = [] + for next_sub in combined_suites[name]["subtests"]: + # calculate sub-test results (i.e. each measurement type) + next_sub["value"] = filters.median(next_sub["replicates"]) + # add to vals; vals is used to calculate overall suite result i.e. the + # geomean of all of the subtests / measurement types + vals.append([next_sub["value"], next_sub["name"]]) + + # calculate overall suite result ('value') which is geomean of all measures + if len(combined_suites[name]["subtests"]) > 1: + combined_suites[name]["value"] = self.construct_summary( + vals, testname=name + ) + + # now add the combined suite entry to our overall summarized results! + self.summarized_results["suites"].append(combined_suites[name]) + + # now it is safe to delete the original entries that were made by each cycle + self.summarized_results["suites"] = [ + item + for item in self.summarized_results["suites"] + if item.get("to_be_deleted") is not True + ] + + def summarize_screenshots(self, screenshots): + if len(screenshots) == 0: + return + + self.summarized_screenshots.append( + """<!DOCTYPE html> + <head> + <style> + table, th, td { + border: 1px solid black; + border-collapse: collapse; + } + </style> + </head> + <html> <body> + <h1>Captured screenshots!</h1> + <table style="width:100%"> + <tr> + <th>Test Name</th> + <th>Pagecycle</th> + <th>Screenshot</th> + </tr>""" + ) + + for screenshot in screenshots: + self.summarized_screenshots.append( + """<tr> + <th>%s</th> + <th> %s</th> + <th> + <img src="%s" alt="%s %s" width="320" height="240"> + </th> + </tr>""" + % ( + screenshot["test_name"], + screenshot["page_cycle"], + screenshot["screenshot"], + screenshot["test_name"], + screenshot["page_cycle"], + ) + ) + + self.summarized_screenshots.append("""</table></body> </html>""") + + +class BrowsertimeOutput(PerftestOutput): + """class for browsertime output""" + + def summarize(self, test_names): + """ + Summarize the parsed browsertime test output, and format accordingly so the output can + be ingested by Perfherder. + + At this point each entry in self.results for browsertime-pageload tests is in this format: + + {'statistics':{'fcp': {u'p99': 932, u'mdev': 10.0941, u'min': 712, u'p90': 810, u'max': + 932, u'median': 758, u'p10': 728, u'stddev': 50, u'mean': 769}, 'dcf': {u'p99': 864, + u'mdev': 11.6768, u'min': 614, u'p90': 738, u'max': 864, u'median': 670, u'p10': 632, + u'stddev': 58, u'mean': 684}, 'fnbpaint': {u'p99': 830, u'mdev': 9.6851, u'min': 616, + u'p90': 719, u'max': 830, u'median': 668, u'p10': 642, u'stddev': 48, u'mean': 680}, + 'loadtime': {u'p99': 5818, u'mdev': 111.7028, u'min': 3220, u'p90': 4450, u'max': 5818, + u'median': 3476, u'p10': 3241, u'stddev': 559, u'mean': 3642}}, 'name': + 'raptor-tp6-guardian-firefox', 'url': 'https://www.theguardian.co.uk', 'lower_is_better': + True, 'measurements': {'fcp': [932, 744, 744, 810, 712, 775, 759, 744, 777, 739, 809, 906, + 734, 742, 760, 758, 728, 792, 757, 759, 742, 759, 775, 726, 730], 'dcf': [864, 679, 637, + 662, 652, 651, 710, 679, 646, 689, 686, 845, 670, 694, 632, 703, 670, 738, 633, 703, 614, + 703, 650, 622, 670], 'fnbpaint': [830, 648, 666, 704, 616, 683, 678, 650, 685, 651, 719, + 820, 634, 664, 681, 664, 642, 703, 668, 670, 669, 668, 681, 652, 642], 'loadtime': [4450, + 3592, 3770, 3345, 3453, 3220, 3434, 3621, 3511, 3416, 3430, 5818, 4729, 3406, 3506, 3588, + 3245, 3381, 3707, 3241, 3595, 3483, 3236, 3390, 3476]}, 'subtest_unit': 'ms', 'bt_ver': + '4.9.2-android', 'alert_threshold': 2, 'cold': True, 'type': 'browsertime-pageload', + 'unit': 'ms', 'browser': "{u'userAgent': u'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.13; + rv:70.0) Gecko/20100101 Firefox/70.0', u'windowSize': u'1366x694'}"} + + Now we must process this further and prepare the result for output suitable for perfherder + ingestion. + + Note: For the overall subtest values/results (i.e. for each measurement type) we will use + the Browsertime-provided statistics, instead of calcuating our own geomeans from the + replicates. + """ + + def _filter(data): + import numpy as np + + # Apply a gaussian filter + data = np.asarray(data) + data = data[np.where(data > np.mean(data) - np.std(data) * 2)[0]] + data = list(data[np.where(data < np.mean(data) + np.std(data) * 2)[0]]) + + return data + + def _process_alt_method(subtest, alternative_method): + # Don't filter with less than 10 data points + data = subtest["replicates"] + if len(subtest["replicates"]) > 10: + data = _filter(data) + if alternative_method == "geomean": + subtest["value"] = round(filters.geometric_mean(data), 1) + elif alternative_method == "mean": + subtest["value"] = round(filters.mean(data), 1) + + # converting suites and subtests into lists, and sorting them + def _process(subtest, alternative_method=""): + if test["type"] == "power": + subtest["value"] = filters.mean(subtest["replicates"]) + elif ( + subtest["name"] in VISUAL_METRICS + or subtest["name"].startswith("perfstat") + or subtest["name"] == "cpuTime" + ): + if alternative_method in ("geomean", "mean"): + _process_alt_method(subtest, alternative_method) + else: + subtest["value"] = filters.median(subtest["replicates"]) + else: + if alternative_method in ("geomean", "mean"): + _process_alt_method(subtest, alternative_method) + else: + subtest["value"] = filters.median( + filters.ignore_first(subtest["replicates"], 1) + ) + return subtest + + def _process_suite(suite): + suite["subtests"] = [ + _process(subtest) + for subtest in suite["subtests"].values() + if subtest["replicates"] + ] + + # Duplicate for different summary values if needed + if self.extra_summary_methods: + new_subtests = [] + for subtest in suite["subtests"]: + try: + for alternative_method in self.extra_summary_methods: + new_subtest = copy.deepcopy(subtest) + new_subtest[ + "name" + ] = f"{new_subtest['name']} ({alternative_method})" + _process(new_subtest, alternative_method) + new_subtests.append(new_subtest) + except Exception as e: + # Ignore failures here + LOG.info(f"Failed to summarize with alternative methods: {e}") + pass + suite["subtests"].extend(new_subtests) + + suite["subtests"].sort(key=lambda subtest: subtest["name"]) + + # for benchmarks there is generally more than one subtest in each cycle + # and a benchmark-specific formula is needed to calculate the final score + # we no longer summarise the page load as we alert on individual subtests + # and the geometric mean was found to be of little value + if len(suite["subtests"]) > 1 and suite["type"] != "pageload": + vals = [ + [subtest["value"], subtest["name"]] for subtest in suite["subtests"] + ] + testname = suite["name"] + if suite["type"] == "power": + testname = "supporting_data" + suite["value"] = self.construct_summary(vals, testname=testname) + return suite + + LOG.info("preparing browsertime results for output") + + # check if we actually have any results + if len(self.results) == 0: + LOG.error( + "no browsertime test results found for %s" % ", ".join(test_names) + ) + return + + test_results = {"framework": {"name": "browsertime"}} + + # using a mapping so we can have a unique set of results given a name + suites = {} + + for test in self.results: + test_name = test["name"] + extra_options = test["extra_options"] + + # If a test with the same name has different extra options, handle it + # by appending the difference in options to the key used in `suites`. We + # need to do a loop here in case we get a conflicting test name again. + prev_name = test_name + while ( + test_name in suites + and suites[test_name]["extraOptions"] != extra_options + ): + missing = set(extra_options) - set(suites[test_name]["extraOptions"]) + if len(missing) == 0: + missing = set(suites[test_name]["extraOptions"]) - set( + extra_options + ) + test_name = test_name + "-".join(list(missing)) + + if prev_name == test_name: + # Kill the loop if we get the same name again + break + else: + prev_name = test_name + + suite = suites.setdefault( + test_name, + { + "name": test["name"], + "type": test["type"], + "extraOptions": extra_options, + # There may be unique options in tags now, but we don't want to remove the + # previous behaviour which includes the extra options in the tags. + "tags": test.get("tags", []) + extra_options, + "lowerIsBetter": test["lower_is_better"], + "unit": test["unit"], + "alertThreshold": float(test["alert_threshold"]), + # like suites, subtests are identified by names + "subtests": {}, + }, + ) + # Setting shouldAlert to False whenever self.app is either chrome, chrome-m, chromium + if self.app in ("chrome", "chrome-m", "chromium"): + suite["shouldAlert"] = False + # Check if the test has set optional properties + if "alert_change_type" in test and "alertChangeType" not in suite: + suite["alertChangeType"] = test["alert_change_type"] + + if test["type"] in ["pageload", "scenario", "power"]: + for measurement_name, replicates in test["measurements"].items(): + if measurement_name not in suite["subtests"]: + subtest = {} + subtest["name"] = measurement_name + subtest["lowerIsBetter"] = test["subtest_lower_is_better"] + subtest["alertThreshold"] = float(test["alert_threshold"]) + subtest["unit"] = test["subtest_unit"] + + # if 'alert_on' is set for this particular measurement, then we want to set + # the flag in the perfherder output to turn on alerting for this subtest + if self.subtest_alert_on is not None: + if measurement_name in self.subtest_alert_on: + LOG.info( + "turning on subtest alerting for measurement type: %s" + % measurement_name + ) + subtest["shouldAlert"] = True + if self.app in ("chrome", "chrome-m", "chromium"): + subtest["shouldAlert"] = False + else: + # Explicitly set `shouldAlert` to False so that the measurement + # is not alerted on. Otherwise Perfherder defaults to alerting. + LOG.info( + "turning off subtest alerting for measurement type: %s" + % measurement_name + ) + subtest["shouldAlert"] = False + subtest["replicates"] = [] + suite["subtests"][measurement_name] = subtest + else: + subtest = suite["subtests"][measurement_name] + + subtest["replicates"].extend(replicates) + + elif "benchmark" in test["type"]: + subtests = None + if "speedometer" in test["measurements"]: + # this includes stylebench + subtests, vals = self.parseSpeedometerOutput(test) + elif "ares6" in test["name"]: + subtests, vals = self.parseAresSixOutput(test) + elif "motionmark" in test["measurements"]: + subtests, vals = self.parseMotionmarkOutput(test) + elif "youtube-playback" in test["name"]: + subtests, vals = self.parseYoutubePlaybackPerformanceOutput(test) + elif "unity-webgl" in test["name"]: + subtests, vals = self.parseUnityWebGLOutput(test) + elif "webaudio" in test["measurements"]: + subtests, vals = self.parseWebaudioOutput(test) + elif "wasm-godot" in test["measurements"]: + subtests, vals = self.parseWASMGodotOutput(test) + elif "wasm-misc" in test["measurements"]: + subtests, vals = self.parseWASMMiscOutput(test) + elif "sunspider" in test["measurements"]: + subtests, vals = self.parseSunspiderOutput(test) + elif "assorted-dom" in test["measurements"]: + subtests, vals = self.parseAssortedDomOutput(test) + elif "jetstream2" in test["measurements"]: + subtests, vals = self.parseJetstreamTwoOutput(test) + elif "matrix-react-bench" in test["name"]: + subtests, vals = self.parseMatrixReactBenchOutput(test) + elif "twitch-animation" in test["name"]: + subtests, vals = self.parseTwitchAnimationOutput(test) + else: + # Attempt to parse the unknown benchmark by flattening the + # given data and merging all the arrays of non-iterable + # data that fall under the same key. + # XXX Note that this is not fully implemented for the summary + # of the metric or test as we don't have a use case for that yet. + subtests, vals = self.parseUnknown(test) + + if subtests is None: + raise Exception("No benchmark metrics found in browsertime results") + + suite["subtests"] = subtests + # summarize results for both benchmark type tests + if len(subtests) > 1: + suite["value"] = self.construct_summary(vals, testname=test["name"]) + subtests.sort(key=lambda subtest: subtest["name"]) + + # convert suites to list + suites = [ + s if "benchmark" in s["type"] else _process_suite(s) + for s in suites.values() + ] + + suites.sort(key=lambda suite: suite["name"]) + + test_results["suites"] = suites + self.summarized_results = test_results diff --git a/testing/raptor/raptor/outputhandler.py b/testing/raptor/raptor/outputhandler.py new file mode 100644 index 0000000000..645b1d18fc --- /dev/null +++ b/testing/raptor/raptor/outputhandler.py @@ -0,0 +1,36 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +# originally from talos_process.py +import json + +from logger.logger import RaptorLogger + +LOG = RaptorLogger(component="raptor-output-handler") + + +class OutputHandler(object): + def __init__(self, verbose=False): + self.proc = None + self.verbose = verbose + + def __call__(self, line): + if not line.strip(): + return + line = line.decode("utf-8", errors="replace") + + try: + data = json.loads(line) + except ValueError: + self.process_output(line) + return + + if isinstance(data, dict) and "action" in data: + LOG.log_raw(data) + else: + self.process_output(json.dumps(data)) + + def process_output(self, line): + if "raptor" in line or self.verbose: + LOG.process_output(self.proc.pid, line) diff --git a/testing/raptor/raptor/perfdocs/browsertime.rst b/testing/raptor/raptor/perfdocs/browsertime.rst new file mode 100644 index 0000000000..e2e29ff2c7 --- /dev/null +++ b/testing/raptor/raptor/perfdocs/browsertime.rst @@ -0,0 +1,236 @@ +################## +Raptor Browsertime +################## + +.. contents:: + :depth: 2 + :local: + +Browsertime is a harness for running performance tests, similar to Mozilla's Raptor testing framework. Browsertime is written in Node.js and uses Selenium WebDriver to drive multiple browsers including Chrome, Chrome for Android, Firefox, and Firefox for Android and GeckoView-based vehicles. + +Source code: + +- Our current Browsertime version uses the canonical repo. +- In-tree: https://searchfox.org/mozilla-central/source/tools/browsertime and https://searchfox.org/mozilla-central/source/taskcluster/scripts/misc/browsertime.sh + +Running Locally +--------------- + +**Prerequisites** + +- A local mozilla repository clone with a `successful Firefox build <https://developer.mozilla.org/en-US/docs/Mozilla/Developer_guide/Build_Instructions>`_ completed + +Running on Firefox Desktop +-------------------------- + +Vanilla Browsertime tests +------------------------- + +If you want to run highly customized tests, you can make use of our customizable ``browsertime`` test. + +With this test, you can customize the page to test, test script to use, and anything else required. It will make use of default settings that Raptor uses in browsertime but these can be overridden with ``--browsertime-arg`` settings. + +For example, here's a test on ``https://www.sitespeed.io`` using this custom test: + +:: + + ./mach raptor --browsertime -t browsertime --browsertime-arg test_script=pageload --browsertime-arg browsertime.url=https://www.sitespeed.io --browsertime-arg iterations=3 + +That test will perform 3 iterations of the given url. Note also that we can use simplified names to make use of test scripts that are built into raptor. You can use ``pageload``, ``interactive``, or provide a path to another test script. + +This custom test is only available locally. + +Page-load tests +--------------- +There are two ways to run performance tests through browsertime listed below. + +**Note that** ``./mach browsertime`` **should not be used when debugging performance issues with profiles as it does not do symbolication.** + +* Raptor-Browsertime (recommended): + +:: + + ./mach raptor --browsertime -t google-search + +* Browsertime-"native": + +:: + + ./mach browsertime https://www.sitespeed.io --firefox.binaryPath '/Users/{userdir}/moz_src/mozilla-unified/obj-x86_64-apple-darwin18.7.0/dist/Nightly.app/Contents/MacOS/firefox' + +Benchmark tests +--------------- +* Raptor-wrapped: + +:: + + ./mach raptor -t raptor-speedometer --browsertime + +Running on Android +------------------ +Running on Raptor-Browsertime (recommended): + +* Running on Fenix + +:: + + ./mach raptor --browsertime -t amazon --app fenix --binary org.mozilla.fenix + +* Running on Geckoview + +:: + + ./mach raptor --browsertime -t amazon --app geckoview --binary org.mozilla.geckoview_example + +Running on vanilla Browsertime: + +* Running on Fenix/Firefox Preview + +:: + + ./mach browsertime --android --browser firefox --firefox.android.package org.mozilla.fenix.debug --firefox.android.activity org.mozilla.fenix.IntentReceiverActivity https://www.sitespeed.io + +* Running on the GeckoView Example app + +:: + + ./mach browsertime --android --browser firefox https://www.sitespeed.io + +Running on Google Chrome +------------------------ +Chrome releases are tied to a specific version of ChromeDriver -- you will need to ensure the two are aligned. + +There are two ways of doing this: + +* Download the ChromeDriver that matches the chrome you wish to run from https://chromedriver.chromium.org/ and specify the path: + +:: + + ./mach browsertime https://www.sitespeed.io -b chrome --chrome.chromedriverPath <PATH/TO/VERSIONED/CHROMEDRIVER> + +* Upgrade the ChromeDriver version in ``tools/browsertime/package-lock.json`` (see https://www.npmjs.com/package/@sitespeed.io/chromedriver for versions). + +Run ``npm install``. + +Launch vanilla Browsertime as follows: + +:: + + ./mach browsertime https://www.sitespeed.io -b chrome + +Or for Raptor-Browsertime (use ``chrome`` for desktop, and ``chrome-m`` for mobile): + +:: + + ./mach raptor --browsertime -t amazon --app chrome --browsertime-chromedriver <PATH/TO/CHROMEDRIVER> + +More Examples +------------- + +`Browsertime docs <https://github.com/mozilla/browsertime/tree/master/docs/examples>`_ + +Passing Additional Arguments to Browsertime +------------------------------------------- + +Browsertime has many command line flags to configure its usage, see `Browsertime configuration <https://www.sitespeed.io/documentation/browsertime/configuration/>`_. + +There are multiple ways of adding additional arguments to Browsertime from Raptor. The primary method is to use ``--browsertime-arg``. For example: ``./mach raptor --browsertime -t amazon --browsertime-arg iterations=10`` + +Other methods for adding additional arguments are: + +* Define additional arguments in `testing/raptor/raptor/browsertime/base.py <https://searchfox.org/mozilla-central/source/testing/raptor/raptor/browsertime/base.py#220-252>`_. + +* Add a ``browsertime_args`` entry to the appropriate manifest with the desired arguments, i.e. `browsertime-tp6.ini <https://searchfox.org/mozilla-central/source/testing/raptor/raptor/tests/tp6/desktop/browsertime-tp6.ini>`_ for desktop page load tests. `Example of browsertime_args format <https://searchfox.org/mozilla-central/source/testing/raptor/raptor/tests/custom/browsertime-process-switch.ini#27>`_. + +Running Browsertime on Try +-------------------------- +You can run all of our browsertime pageload tests through ``./mach try fuzzy --full``. We use chimera mode in these tests which means that both cold and warm pageload variants are running at the same time. + +For example: + +:: + + ./mach try fuzzy -q "'g5 'imdb 'geckoview 'vismet '-wr 'shippable" + +Retriggering Browsertime Visual Metrics Tasks +--------------------------------------------- + +You can retrigger Browsertime tasks just like you retrigger any other tasks from Treeherder (using the retrigger buttons, add-new-jobs, retrigger-multiple, etc.). + +The following metrics are collected each time: ``fcp, loadtime, ContentfulSpeedIndex, PerceptualSpeedIndex, SpeedIndex, FirstVisualChange, LastVisualChange`` + +Further information regarding these metrics can be viewed at `visual-metrics <https://www.sitespeed.io/documentation/sitespeed.io/metrics/#visual-metrics>`_ + +Gecko Profiling with Browsertime +-------------------------------- + +To run gecko profiling using Raptor-Browsertime you can add the ``--gecko-profile`` flag to any command and you will get profiles from the test (with the profiler page opening in the browser automatically). This method also performs symbolication for you. For example: + +:: + + ./mach raptor --browsertime -t amazon --gecko-profile + +Note that vanilla Browsertime does support Gecko Profiling but **it does not symbolicate the profiles** so it is **not recommended** to use for debugging performance regressions/improvements. + +Upgrading Browsertime In-Tree +----------------------------- +To upgrade the browsertime version used in-tree you can run, then commit the changes made to ``package.json`` and ``package-lock.json``: + +:: + + ./mach browsertime --update-upstream-url <TARBALL-URL> + +Here is a sample URL that we can update to: https://github.com/sitespeedio/browsertime/tarball/89771a1d6be54114db190427dbc281582cba3d47 + +To test the upgrade, run a raptor test locally (with and without visual-metrics ``--browsertime-visualmetrics`` if possible) and test it on try with at least one test on desktop and mobile. + +Finding the Geckodriver Being Used +---------------------------------- +If you're looking for the latest geckodriver being used there are two ways: +* Find the latest one from here: https://treeherder.mozilla.org/jobs?repo=mozilla-central&searchStr=geckodriver +* Alternatively, if you're trying to figure out which geckodriver a given CI task is using, you can click on the browsertime task in treeherder, and then click on the ``Task`` id in the bottom left of the pop-up interface. Then in the window that opens up, click on `See more` in the task details tab on the left, this will show you the dependent tasks with the latest toolchain-geckodriver being used. There's an Artifacts drop down on the right hand side for the toolchain-geckodriver task that you can find the latest geckodriver in. + +If you're trying to test Browsertime with a new geckodriver, you can do either of the following: +* Request a new geckodriver build in your try run (i.e. through ``./mach try fuzzy``). +* Trigger a new geckodriver in a try push, then trigger the browsertime tests which will then use the newly built version in the try push. + +Comparing Before/After Browsertime Videos +----------------------------------------- + +We have some scripts that can produce side-by-side comparison videos for you of the worst pairing of videos. You can find the script here: https://github.com/mozilla/mozperftest-tools#browsertime-side-by-side-video-comparisons + +Once the side-by-side comparison is produced, the video on the left is the old/base video, and the video on the right is the new video. + +Mach Browsertime Setup +---------------------- + +**WARNING** + Raptor-Browsertime (i.e. ``./mach raptor --browsertime -t <TEST>``) is currently required to be ran first in order to acquire the Node-16 binary. In general, it is also not recommended to use ``./mach browsertime`` for testing as it will be deprecated soon. + +Note that if you are running Raptor-Browsertime then it will get installed automatically and also update itself. Otherwise, you can run: + +- ``./mach browsertime --clobber --setup --install-vismet-reqs`` + +This will automatically check your setup and install the necessary dependencies if required. If successful, the output should read as something similar to: + +:: + + browsertime installed successfully! + + NOTE: Your local browsertime binary is at <...>/mozilla-unified/tools/browsertime/node_modules/.bin/browsertime + +- To manually check your setup, you can also run ``./mach browsertime --check`` + +Known Issues +^^^^^^^^^^^^ + +With the replacement of ImageMagick, former cross platform installation issues have been resolved. The details of this can be viewed in the meta bug tracker +`Bug 1735410 <https://bugzilla.mozilla.org/show_bug.cgi?id=1735410>`_ + + + +- For other issues, try deleting the ``~/.mozbuild/browsertime`` folder and re-running the browsertime setup command or a Raptor-Browsertime test + +- If you plan on running Browsertime on Android, your Android device must already be set up (see more below in the :ref: `Running on Android` section) + +- **If you encounter any issues not mentioned here, please** `file a bug <https://bugzilla.mozilla.org/enter_bug.cgi?product=Testing&component=Raptor>`_ **in the** ``Testing::Raptor`` **component.** diff --git a/testing/raptor/raptor/perfdocs/config.yml b/testing/raptor/raptor/perfdocs/config.yml new file mode 100644 index 0000000000..b490dc5f73 --- /dev/null +++ b/testing/raptor/raptor/perfdocs/config.yml @@ -0,0 +1,191 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +name: raptor +manifest: testing/raptor/raptor/raptor.ini +static-only: False +metrics: + "First Paint": + aliases: + - fcp + - First Contentful Paint + - First Contentful Composite + description: > + Denotes the first time the browser performs a paint that has content in it. + "Youtube Playback Metrics": + aliases: + - VP9 + - H264 + matcher: VP9.*|H264.* + description: > + Metrics starting with VP9/H264 give the number of frames dropped, and painted. + "In Progress": + aliases: [] + matcher: .* + description: Many metrics have not been documented yet, this is a placeholder. +suites: + desktop: + description: "Tests for page-load performance. The links direct to the actual websites that are being tested." + tests: + amazon: "BT, FF, CH, CU" + bing-search: "BT, FF, CH, CU" + buzzfeed: "BT, FF, CH, CU" + cnn: "BT, FF, CH, CU" + ebay: "BT, FF, CH, CU" + espn: "BT, FF, CH, CU" + expedia: "BT, FF, CH, CU" + facebook: "BT, FF, CH, CU" + fandom: "BT, FF, CH, CU" + google-docs: "BT, FF, CH, CU" + google-mail: "BT, FF, CH, CU" + google-search: "BT, FF, CH, CU" + google-slides: "BT, FF, CH, CU" + imdb: "BT, FF, CH, CU" + imgur: "BT, FF, CH, CU" + instagram: "BT, FF, CH, CU" + linkedin: "BT, FF, CH, CU" + microsoft: "BT, FF, CH, CU" + netflix: "BT, FF, CH, CU" + nytimes: "BT, FF, CH, CU" + office: "BT, FF, CH, CU" + outlook: "BT, FF, CH, CU" + paypal: "BT, FF, CH, CU" + pinterest: "BT, FF, CH, CU" + reddit: "BT, FF, CH, CU" + tumblr: "BT, FF, CH, CU" + twitch: "BT, FF, CH, CU" + twitter: "BT, FF, CH, CU" + wikia: "BT, FF, CH, CU" + wikipedia: "BT, FF, CH, CU" + yahoo-mail: "BT, FF, CH, CU" + youtube: "BT, FF, CH, CU" + mobile: + description: "Page-load performance test suite on Android. The links direct to the actual websites that are being tested." + tests: + amazon: "BT, GV, FE, RB, CH-M" + youtube: "BT, GV, FE, RB, CH-M" + allrecipes: "BT, GV, FE, RB, CH-M" + amazon-search: "BT, GV, FE, RB, CH-M" + bing: "BT, GV, FE, RB, CH-M" + bing-search-restaurants: "BT, GV, FE, RB, CH-M" + booking: "BT, GV, FE, RB, CH-M" + cnn: "BT, GV, FE, RB, CH-M" + cnn-ampstories: "BT, GV, FE, RB, CH-M" + dailymail: "BT, GV, FE, RB, CH-M" + ebay-kleinanzeigen: "BT, GV, FE, RB, CH-M" + ebay-kleinanzeigen-search: "BT, GV, FE, RB, CH-M" + espn: "BT, GV, FE, RB, CH-M" + facebook: "BT, GV, FE, RB, CH-M" + facebook-cristiano: "BT, GV, FE, RB, CH-M" + google: "BT, GV, FE, RB, CH-M" + google-maps: "BT, GV, FE, RB, CH-M" + google-search-restaurants: "BT, GV, FE, RB, CH-M" + imdb: "BT, GV, FE, RB, CH-M" + instagram: "BT, GV, FE, RB, CH-M" + microsoft-support: "BT, GV, FE, RB, CH-M" + reddit: "BT, GV, FE, RB, CH-M" + stackoverflow: "BT, GV, FE, RB, CH-M" + sina: "BT, GV, FE, RB, CH-M" + web-de: "BT, GV, FE, RB, CH-M" + wikipedia: "BT, GV, FE, RB, CH-M" + youtube-watch: "BT, GV, FE, RB, CH-M" + live: + description: "A set of test pages that are run as live sites instead of recorded versions. These tests are available on all browsers, on all platforms." + tests: + booking-sf: "GV, FE, RB, CH-M, FF, CH, CU" + cnn: "GV, FE, RB, CH-M, FF, CH, CU" + cnn-ampstories: "GV, FE, RB, CH-M, FF, CH, CU" + discord: "GV, FE, RB, CH-M, FF, CH, CU" + expedia: "GV, FE, RB, CH-M, FF, CH, CU" + fashionbeans: "GV, FE, RB, CH-M, FF, CH, CU" + google-accounts: "GV, FE, RB, CH-M, FF, CH, CU" + imdb-firefox: "GV, FE, RB, CH-M, FF, CH, CU" + medium-article: "GV, FE, RB, CH-M, FF, CH, CU" + nytimes: "GV, FE, RB, CH-M, FF, CH, CU" + people-article: "GV, FE, RB, CH-M, FF, CH, CU" + reddit-thread: "GV, FE, RB, CH-M, FF, CH, CU" + rumble-fox: "GV, FE, RB, CH-M, FF, CH, CU" + stackoverflow-question: "GV, FE, RB, CH-M, FF, CH, CU" + urbandictionary-define: "GV, FE, RB, CH-M, FF, CH, CU" + wikia-marvel: "GV, FE, RB, CH-M, FF, CH, CU" + benchmarks: + description: "Standard benchmarks are third-party tests (i.e. Speedometer) that we have integrated into Raptor to run per-commit in our production CI." + tests: + ares6: "FF, CH, CU" + assorted-dom: "FF, CH, CU" + jetstream2: "FF, CH, CU" + matrix-react-bench: "FF, CH, CU" + motionmark-animometer: "FF, CH, CU" + motionmark-htmlsuite: "FF, CH, CU" + raptor-speedometer-geckoview: "GV" + raptor-youtube-playback-h264-1080p30-firefox: "FF" + raptor-youtube-playback-h264-1080p60-firefox: "FF" + raptor-youtube-playback-h264-full-1080p30-firefox: "FF" + raptor-youtube-playback-h264-full-1080p60-firefox: "FF" + raptor-youtube-playback-v9-1080p30-firefox: "FF" + raptor-youtube-playback-v9-1080p60-firefox: "FF" + raptor-youtube-playback-v9-full-1080p30-firefox: "FF" + raptor-youtube-playback-v9-full-1080p60-firefox: "FF" + speedometer: "FF, CH, CU, FE, GV, RB, CH-M" + speedometer3: "FF, CH, CU, FE, GV, RB, CH-M" + stylebench: "FF, CH, CU" + sunspider: "FF, CH, CU" + twitch-animation: "FF" + unity-webgl: "FF, CH, CU, FE, RB, FE, CH-M" + wasm-godot-baseline: "FF" + wasm-godot-optimizing: "FF" + wasm-godot: "FF, CH, CU" + wasm-misc-baseline: "FF" + wasm-misc-optimizing: "FF" + wasm-misc: "FF, CH, CU" + webaudio: "FF, CH, CU" + youtube-playback: "FF, GV, FE, RB, CH" + youtube-playback-av1-sfr: "FF , GV, FE, RB, CH" + youtube-playback-h264-1080p30: "FF" + youtube-playback-h264-1080p60: "FF" + youtube-playback-h264-full-1080p30: "FF" + youtube-playback-h264-full-1080p60: "FF" + youtube-playback-h264-sfr: "FF , GV, FE, RB, CH" + youtube-playback-hfr: "FF , GV, FE, RB, CH" + youtube-playback-v9-1080p30: "FF" + youtube-playback-v9-1080p60: "FF" + youtube-playback-v9-full-1080p30: "FF" + youtube-playback-v9-full-1080p60: "FF" + youtube-playback-vp9-sfr: "FF , GV, FE, RB, CH" + youtube-playback-widevine-h264-sfr: "FF , GV, FE, RB, CH" + youtube-playback-widevine-hfr: "FF , GV, FE, RB, CH" + youtube-playback-widevine-vp9-sfr: "FF , GV, FE, RB, CH" + scenario: + description: "Tests that perform a specific action (a scenario), i.e. idle application, idle application in background, etc." + tests: + idle: "FE, GV, RB" + idle-bg: "FE, GV, RB" + raptor-scn-power-idle-bg-fenix: "FE" + raptor-scn-power-idle-bg-geckoview: "GV" + raptor-scn-power-idle-bg-refbrow: "RB" + raptor-scn-power-idle-fenix: "FE" + raptor-scn-power-idle-geckoview: "GV" + raptor-scn-power-idle-refbrow: "RB" + custom: + description: "Browsertime tests that use a custom pageload test script. These use the pageload type, but may have other intentions." + tests: + browsertime: "Used to run vanilla browsertime tests through raptor. For example: `./mach raptor --browsertime -t browsertime --browsertime-arg url=https://www.sitespeed.io --browsertime-arg iterations=3`" + process-switch: "Measures process switch time" + welcome: "Measures pageload metrics for the first-install about:welcome page" + grandprix: "Runs the Grandprix benchmark" + interactive: + description: "Browsertime tests that interact with the webpage. Includes responsiveness tests as they make use of this support for navigation. These form of tests allow the specification of browsertime commands through the test manifest." + tests: + cnn-nav: "Navigates to cnn main page, then to the world sub-page." + facebook-nav: "Navigates to facebook, then the sub-pages friends, marketplace, groups." + reddit-billgates-ama: "Navigates from the Bill Gates AMA to the Reddit members section." + reddit-billgates-post-1: "Navigates the `thisisbillgates` user starting at the main user page, then to the posts, comments, hot, and top sections." + reddit-billgates-post-2: "Navigates the `thisisbillgates` user starting at the main user page, then to the posts, comments, hot, and top sections." + unittests: + description: "These tests aren't used in standard testing, they are only used in the Raptor unit tests (they are similar to raptor-tp6 tests though)." + tests: + raptor-tp6-unittest-amazon-firefox: "FF" + raptor-tp6-unittest-facebook-firefox: "FF" + raptor-tp6-unittest-google-firefox: "FF" + raptor-tp6-unittest-youtube-firefox: "FF" diff --git a/testing/raptor/raptor/perfdocs/contributing.rst b/testing/raptor/raptor/perfdocs/contributing.rst new file mode 100644 index 0000000000..30beac81c3 --- /dev/null +++ b/testing/raptor/raptor/perfdocs/contributing.rst @@ -0,0 +1,42 @@ +############ +Contributing +############ + +Raptor on Mobile projects (Fenix, Reference-Browser) +**************************************************** + +Add new tests +------------- + +For mobile projects, Raptor tests are on the following repositories: + +**Fenix**: + +- Repository: `Github <https://github.com/mozilla-mobile/fenix/>`__ +- Tests results: `Treeherder view <https://treeherder.mozilla.org/#/jobs?repo=fenix>`__ +- Schedule: Every 24 hours `Taskcluster force hook <https://tools.taskcluster.net/hooks/project-releng/cron-task-mozilla-mobile-fenix%2Fraptor>`_ + +**Reference-Browser**: + +- Repository: `Github <https://github.com/mozilla-mobile/reference-browser/>`__ +- Tests results: `Treeherder view <https://treeherder.mozilla.org/#/jobs?repo=reference-browser>`__ +- Schedule: On each push + +Tests are now defined in a similar fashion compared to what exists in mozilla-central. Task definitions are expressed in Yaml: + +* https://github.com/mozilla-mobile/fenix/blob/1c9c5317eb33d92dde3293dfe6a857c279a7ab12/taskcluster/ci/raptor/kind.yml +* https://github.com/mozilla-mobile/reference-browser/blob/4560a83cb559d3d4d06383205a8bb76a44336704/taskcluster/ci/raptor/kind.yml + +If you want to test your changes on a PR, before they land, you need to apply a patch like this one: https://github.com/mozilla-mobile/fenix/pull/5565/files. Don't forget to revert it before merging the patch. Note that the checks will run but the results aren't currently available on treeherder (`bug 1593252 <https://bugzilla.mozilla.org/show_bug.cgi?id=1593252>`_ is expected to address this). + +On Fenix and Reference-Browser, the raptor revision is tied to the latest nightly of mozilla-central + +For more information, please reach out to :mhentges in #cia + +Code formatting on Raptor +************************* +As Raptor is a Mozilla project we follow the general Python coding style: + +* https://firefox-source-docs.mozilla.org/tools/lint/coding-style/coding_style_python.html + +`black <https://github.com/psf/black/>`_ is the tool used to reformat the Python code. diff --git a/testing/raptor/raptor/perfdocs/debugging.rst b/testing/raptor/raptor/perfdocs/debugging.rst new file mode 100644 index 0000000000..755d231d8e --- /dev/null +++ b/testing/raptor/raptor/perfdocs/debugging.rst @@ -0,0 +1,132 @@ +######### +Debugging +######### + +.. contents:: + :depth: 2 + :local: + +Debugging Desktop Product Failures +********************************** + +As of now, there is no easy way to do this. Raptor was not built for debugging functional failures. Hitting these in Raptor is indicative that we lack functional test coverage so regression tests should be added for those failures after they are fixed. + +To debug a functional failure in Raptor you can follow these steps: + +#. If bug 1653617 has not landed yet, apply the patch. +#. Add the --verbose flag to the extra-options list `here <https://searchfox.org/mozilla-central/source/taskcluster/ci/test/raptor.yml#98-101>`__. +#. If the --setenv doesn't exist yet (`bug 1494669 <https://bugzilla.mozilla.org/show_bug.cgi?id=1494669>`_), then add your MOZ_LOG environment variables to give you additional logging `here <https://searchfox.org/mozilla-central/source/testing/raptor/raptor/webextension/desktop.py#42>`_. +#. If the flag does exist, then you can add the MOZ_LOG variables to the `raptor.yml <https://searchfox.org/mozilla-central/source/taskcluster/ci/test/raptor.yml>`_ configuration file. +#. Push to try if you can't reproduce the failure locally. + +You can follow `bug 1655554 <https://bugzilla.mozilla.org/show_bug.cgi?id=1655554>`_ as we work on improving this workflow. + +In some cases, you might not be able to get logging for what you are debugging (browser console logging for instance). In this case, you should make your own debug prints with printf or something along those lines (`see :agi's debugging work for an example <https://matrix.to/#/!LfXZSWEroPFPMQcYmw:mozilla.org/$r_azj7OipkgDzQ75SCns2QIayp4260PIMHLWLApJJNg?via=mozilla.org&via=matrix.org&via=rduce.org>`_). + +Debugging the Raptor Web Extension +********************************** + +When developing on Raptor and debugging, there's often a need to look at the output coming from the `Raptor Web Extension <https://searchfox.org/mozilla-central/source/testing/raptor/webext/raptor>`_. Here are some pointers to help. + +Raptor Debug Mode +----------------- + +The easiest way to debug the Raptor web extension is to run the Raptor test locally and invoke debug mode, i.e. for Firefox: + +:: + + ./mach raptor --test raptor-tp6-amazon-firefox --debug-mode + +Or on Chrome, for example: + +:: + + ./mach raptor --test raptor-tp6-amazon-chrome --app=chrome --binary="/Applications/Google Chrome.app/Contents/MacOS/Google Chrome" --debug-mode + +Running Raptor with debug mode will: + +* Automatically set the number of test page-cycles to 2 maximum +* Reduce the 30 second post-browser startup delay from 30 seconds to 3 seconds +* On Firefox, the devtools browser console will automatically open, where you can view all of the console log messages generated by the Raptor web extension +* On Chrome, the devtools console will automatically open +* The browser will remain open after the Raptor test has finished; you will be prompted in the terminal to manually shutdown the browser when you're finished debugging. + +Manual Debugging on Firefox Desktop +----------------------------------- + +The main Raptor runner is '`runner.js <https://searchfox.org/mozilla-central/source/testing/raptor/webext/raptor/runner.js>`_' which is inside the web extension. The code that actually captures the performance measures is in the web extension content code '`measure.js <https://searchfox.org/mozilla-central/source/testing/raptor/webext/raptor/measure.js>`_'. + +In order to retrieve the console.log() output from the Raptor runner, do the following: + +#. Invoke Raptor locally via ``./mach raptor`` +#. During the 30 second Raptor pause which happens right after Firefox has started up, in the ALREADY OPEN current tab, type "about:debugging" for the URL. +#. On the debugging page that appears, make sure "Add-ons" is selected on the left (default). +#. Turn ON the "Enable add-on debugging" check-box +#. Then scroll down the page until you see the Raptor web extension in the list of currently-loaded add-ons. Under "Raptor" click the blue "Debug" link. +#. A new window will open in a minute, and click the "console" tab + +To retrieve the console.log() output from the Raptor content 'measure.js' code: + +#. As soon as Raptor opens the new test tab (and the test starts running / or the page starts loading), in Firefox just choose "Tools => Web Developer => Web Console", and select the "console' tab. + +Raptor automatically closes the test tab and the entire browser after test completion; which will close any open debug consoles. In order to have more time to review the console logs, Raptor can be temporarily hacked locally in order to prevent the test tab and browser from being closed. Currently this must be done manually, as follows: + +#. In the Raptor web extension runner, comment out the line that closes the test tab in the test clean-up. That line of `code is here <https://searchfox.org/mozilla-central/rev/3c85ea2f8700ab17e38b82d77cd44644b4dae703/testing/raptor/webext/raptor/runner.js#357>`_. +#. Add a return statement at the top of the Raptor control server method that shuts-down the browser, the browser shut-down `method is here <https://searchfox.org/mozilla-central/rev/924e3d96d81a40d2f0eec1db5f74fc6594337128/testing/raptor/raptor/control_server.py#120>`_. + +For **benchmark type tests** (i.e. speedometer, motionmark, etc.) Raptor doesn't inject 'measure.js' into the test page content; instead it injects '`benchmark-relay.js <https://searchfox.org/mozilla-central/source/testing/raptor/webext/raptor/benchmark-relay.js>`_' into the benchmark test content. Benchmark-relay is as it sounds; it basically relays the test results coming from the benchmark test, to the Raptor web extension runner. Viewing the console.log() output from benchmark-relay is done the same was as noted for the 'measure.js' content above. + +Note, `Bug 1470450 <https://bugzilla.mozilla.org/show_bug.cgi?id=1470450>`_ is on file to add a debug mode to Raptor that will automatically grab the web extension console output and dump it to the terminal (if possible) that will make debugging much easier. + +Debugging TP6 and Killing the Mitmproxy Server +---------------------------------------------- + +Regarding debugging Raptor pageload tests that use Mitmproxy (i.e. tp6, gdocs). If Raptor doesn't finish naturally and doesn't stop the Mitmproxy tool, the next time you attempt to run Raptor it might fail out with this error: + +:: + + INFO - Error starting proxy server: OSError(48, 'Address already in use') + NFO - raptor-mitmproxy Aborting: mitmproxy playback process failed to start, poll returned: 1 + +That just means the Mitmproxy server was already running before so it couldn't startup. In this case, you need to kill the Mitmproxy server processes, i.e: + +:: + + mozilla-unified rwood$ ps -ax | grep mitm + 5439 ttys000 0:00.09 /Users/rwood/mozilla-unified/obj-x86_64-apple-darwin17.7.0/testing/raptor/mitmdump -k -q -s /Users/rwood/mozilla-unified/testing/raptor/raptor/playback/alternate-server-replay.py /Users/rwood/mozilla-unified/obj-x86_64-apple-darwin17.7.0/testing/raptor/amazon.mp + 440 ttys000 0:01.64 /Users/rwood/mozilla-unified/obj-x86_64-apple-darwin17.7.0/testing/raptor/mitmdump -k -q -s /Users/rwood/mozilla-unified/testing/raptor/raptor/playback/alternate-server-replay.py /Users/rwood/mozilla-unified/obj-x86_64-apple-darwin17.7.0/testing/raptor/amazon.mp + 5509 ttys000 0:00.01 grep mitm + +Then just kill the first mitm process in the list and that's sufficient: + +:: + + mozilla-unified rwood$ kill 5439 + +Now when you run Raptor again, the Mitmproxy server will be able to start. + +Manual Debugging on Firefox Android +----------------------------------- + +Be sure to read the above section first on how to debug the Raptor web extension when running on Firefox Desktop. + +When running Raptor tests on Firefox on Android (i.e. geckoview), to see the console.log() output from the Raptor web extension, do the following: + +#. With your android device (i.e. Google Pixel 2) all set up and connected to USB, invoke the Raptor test normally via ``./mach raptor`` +#. Start up a local copy of the Firefox Nightly Desktop browser +#. In Firefox Desktop choose "Tools => Web Developer => WebIDE" +#. In the Firefox WebIDE dialog that appears, look under "USB Devices" listed on the top right. If your device is not there, there may be a link to install remote device tools - if that link appears click it and let that install. +#. Under "USB Devices" on the top right your android device should be listed (i.e. "Firefox Custom on Android Pixel 2" - click on your device. +#. The debugger opens. On the left side click on "Main Process", and click the "console" tab below - and the Raptor runner output will be included there. +#. On the left side under "Tabs" you'll also see an option for the active tab/page; select that and the Raptor content console.log() output should be included there. + +Also note: When debugging Raptor on Android, the 'adb logcat' is very useful. More specifically for 'geckoview', the output (including for Raptor) is prefixed with "GeckoConsole" - so this command is very handy: + +:: + + adb logcat | grep GeckoConsole + +Manual Debugging on Google Chrome +--------------------------------- + +Same as on Firefox desktop above, but use the Google Chrome console: View ==> Developer ==> Developer Tools. diff --git a/testing/raptor/raptor/perfdocs/index.rst b/testing/raptor/raptor/perfdocs/index.rst new file mode 100644 index 0000000000..cf5eba4a47 --- /dev/null +++ b/testing/raptor/raptor/perfdocs/index.rst @@ -0,0 +1,40 @@ +###### +Raptor +###### + +Raptor is a performance-testing framework for running browser pageload and browser benchmark tests. Raptor is cross-browser compatible and is currently running in production on Firefox Desktop, Firefox Android GeckoView, Fenix, Reference Browser, Chromium, and Chrome. + +- Contact: Dave Hunt [:davehunt] +- Source code: https://searchfox.org/mozilla-central/source/testing/raptor +- Good first bugs: https://codetribute.mozilla.org/projects/automation?project%3DRaptor + +Raptor currently supports three test types: 1) page-load performance tests, 2) standard benchmark-performance tests, and 3) "scenario"-based tests, such as power, CPU, and memory-usage measurements on Android (and desktop?). + +Locally, Raptor can be invoked with the following command: + +:: + + ./mach raptor + +**We're in the process of migrating away from webextension to browsertime. Currently, raptor supports both of them, but at the end of the migration, the support for webextension will be removed.** + +.. toctree:: + :titlesonly: + + browsertime + webextension + debugging + contributing + test-list + metrics + +.. contents:: + :depth: 2 + :local: + +Raptor tests +************ + +The following documents all testing we have for Raptor. + +{documentation} diff --git a/testing/raptor/raptor/perfdocs/metrics.rst b/testing/raptor/raptor/perfdocs/metrics.rst new file mode 100644 index 0000000000..931456467c --- /dev/null +++ b/testing/raptor/raptor/perfdocs/metrics.rst @@ -0,0 +1,82 @@ +################ +Metrics Gathered +################ + +.. contents:: + :depth: 2 + :local: + +**WARNING: This page is still being actively developed.** + +This document contains information about the metrics gathered in Browsertime tests, as well as detailed information of how they are gathered. + +Pageload Tests +-------------- + +For browsertime pageload tests, there is a limited set of metrics that we collect (which can easily be expanded). Currently we divide these into two sets of metrics: (i) visual metrics, and (ii) technical metrics. These are gathered through two types of tests called warm and cold pageload tests. We have combined these two into a single "chimera" mode which you'll find in the Treeherder tasks. + +Below, you can find the process of how we run Warm, Cold, and Chimera pageload tests. + +Warm Pageload +============== + +In this pageload test type, we open the browser, then repeatedly navigate to the same page without restarting the browser in between cycles. + +* A new, or conditioned, browser profile is created +* The browser is started up +* Post-startup browser settle pause of 30 seconds or 1 second if using a conditioned profile +* A new tab is opened +* The test URL is loaded; measurements taken +* The tab is reloaded ``X`` more times (for ``X`` replicates); measurements taken each time + +NOTES: +- The measurements from the first page-load are not included in overall results metrics b/c of first load noise; however they are listed in the JSON artifacts +- The bytecode cache gets populated on the first test cycle, and subsequent iterations will already have the cache built to reduce noise. + +Cold Pageload +============== + +In this pageload test type, we open the browser, navigate to the page, then restart the browser before performing the next cycle. + +* A new, or conditioned, browser profile is created +* The browser is started up +* Post-startup browser settle pause of 30 seconds or 1 second if using a conditioned profile +* A new tab is opened +* The test URL is loaded; measurements taken +* The browser is shut down +* Entire process is repeated for the remaining browser cycles + +NOTE: The measurements from all browser cycles are used to calculate overall results + +Chimera Pageload +================ + +A new mode for pageload testing is called chimera mode. It combines the warm and cold variants into a single test. This test mode is used in our Taskcluster tasks. + +* A new, or conditioned, browser profile is created +* The browser is started up +* Post-startup browser settle pause of 30 seconds or 1 second if using a conditioned profile +* A new tab is opened +* The test URL is loaded; measurements taken for ``Cold pageload`` +* Navigate to a secondary URL (to preserve the content process) +* The test URL is loaded again; measurements taken for ``Warm pageload`` +* The desktop browser is shut down +* Entire process is repeated for the remaining browser cycles + +NOTE: The bytecode cache mentioned in Warm pageloads still applies here. + +Technical Metrics +================= + +These are metrics that are obtained from the browser. This includes metrics like First Paint, DOM Content Flushed, etc.. + +Visual Metrics +============== + +When you run Raptor Browsertime with ``--browsertime-visualmetrics``, it will record a video of the page being loaded and then process this video to build the metrics. The video is either produced using FFMPEG (with ``--browsertime-no-ffwindowrecorder``) or the Firefox Window Recorder (default). + + +Benchmarks +---------- + +Benchmarks gather their own custom metrics unlike the pageload tests above. Please ping the owners of those benchmarks to determine what they mean and how they are produced, or reach out to the Performance Test and Tooling team in #perftest on Element. diff --git a/testing/raptor/raptor/perfdocs/test-list.rst b/testing/raptor/raptor/perfdocs/test-list.rst new file mode 100644 index 0000000000..a88e4971cf --- /dev/null +++ b/testing/raptor/raptor/perfdocs/test-list.rst @@ -0,0 +1,73 @@ +################ +Raptor Test List +################ + +Currently the following Raptor tests are available. Note: Check the test details below to see which browser (i.e. Firefox, Google Chrome, Android) each test is supported on. + +Page-Load Tests +--------------- + +Raptor page-load test documentation is generated by `PerfDocs <https://firefox-source-docs.mozilla.org/code-quality/lint/linters/perfdocs.html>`_ and available in the `Firefox Source Docs <https://firefox-source-docs.mozilla.org/testing/perfdocs/raptor.html>`_. + +Benchmark Tests +--------------- + +motionmark-animometer, motionmark-htmlsuite +=========================================== + +* summarization: + + * subtest: FPS from the subtest, each subtest is run for 15 seconds, repeat this 5 times and report the median value + * suite: we take a geometric mean of all the subtests (9 for animometer, 11 for html suite) + +speedometer +=========== + +* measuring: responsiveness of web applications +* data: there are 16 subtests in Speedometer; each of these are made up of 9 internal benchmarks. +* summarization: + + * subtest: For all of the 16 subtests, we collect `a summed of all their internal benchmark results <https://searchfox.org/mozilla-central/source/third_party/webkit/PerformanceTests/Speedometer/resources/benchmark-report.js#66-67>`_ for each of them. To obtain a single score per subtest, we take `a median of the replicates <https://searchfox.org/mozilla-central/source/testing/raptor/raptor/output.py#427-470>`_. + * score: `geometric mean of the 16 subtest metrics (along with some special corrections) <https://searchfox.org/mozilla-central/source/testing/raptor/raptor/output.py#317-330>`_. + +This is the `Speedometer v1.0 <http://browserbench.org/Speedometer/>`_ JavaScript benchmark taken verbatim and slightly modified to work with the Raptor harness. + +youtube-playback +================ + +* details: `YouTube playback performance <https://wiki.mozilla.org/TestEngineering/Performance/Raptor/Youtube_playback_performance>`_ +* measuring: media streaming playback performance (dropped video frames) +* reporting: For each video the number of dropped and decoded frames, as well as its percentage value is getting recorded. The overall reported result is the mean value of dropped video frames across all tested video files. +* data: Given the size of the used media files those tests are currently run as live site tests, and are kept up-to-date via the `perf-youtube-playback <https://github.com/mozilla/perf-youtube-playback/>`_ repository on Github. + +This are the `Playback Performance Tests <https://ytlr-cert.appspot.com/2019/main.html?test_type=playbackperf-test>`_ benchmark taken verbatim and slightly modified to work with the Raptor harness. + +webaudio +======== + +* measuring: Rendering speed of various synthetic Web Audio API workloads +* reporting: The time time it took to render the audio of each test case, and a geometric mean of the full test suite. Lower is better +* data: Upstream is https://github.com/padenot/webaudio-benchmark/. Those benchmarks are run by other projects. Upstream is vendored in mozilla-central via an simple update script, at `third_party/webkit/PerformanceTests/webaudio` + +Scenario Tests +-------------- + +This test type runs browser tests that use idle pages for a specified amount of time to gather resource usage information such as power usage. The pages used for testing do not need to be recorded with mitmproxy. + +When creating a new scenario test, ensure that the `page-timeout` is greater than the `scenario-time` to make sure raptor doesn't exit the test before the scenario timer ends. + +This test type can also be used for specialized tests that require communication with the control-server to do things like sending the browser to the background for X minutes. + +Power-Usage Measurement Tests +============================= +These Android power measurement tests output 3 different PERFHERDER_DATA entries. The first contains the power usage of the test itself, the second contains the power usage of the android OS (named os-baseline) over the course of 1 minute, and the third (the name is the test name with '%change-power' appended to it) is a combination of these two measures which shows the percentage increase in power consumption when the test is run, in comparison to when it is not running. In these perfherder data blobs, we provide power consumption attributed to the cpu, wifi, and screen in Milli-ampere-hours (mAh). + +raptor-scn-power-idle +^^^^^^^^^^^^^^^^^^^^^ + +* measuring: Power consumption for idle Android browsers, with about:blank loaded and app foregrounded, over a 20-minute duration + +raptor-scn-power-idle-bg +^^^^^^^^^^^^^^^^^^^^^^^^ + +* measuring: Power consumption for idle Android browsers, with about:blank loaded and app backgrounded, over a 10-minute duration diff --git a/testing/raptor/raptor/perfdocs/webextension.rst b/testing/raptor/raptor/perfdocs/webextension.rst new file mode 100644 index 0000000000..94e770b672 --- /dev/null +++ b/testing/raptor/raptor/perfdocs/webextension.rst @@ -0,0 +1,525 @@ +################### +Raptor WebExtension +################### + + **Note: WebExtension is being deprecated, new tests should be written using Browsertime** + +.. contents:: + :depth: 2 + :local: + +WebExtension Page-Load Tests +---------------------------- + +Page-load tests involve loading a specific web page and measuring the load performance (i.e. `time-to-first-non-blank-paint <https://wiki.mozilla.org/TestEngineering/Performance/Glossary#First_Non-Blank_Paint_.28fnbpaint.29>`_, first-contentful-paint, `dom-content-flushed <https://wiki.mozilla.org/TestEngineering/Performance/Glossary#DOM_Content_Flushed_.28dcf.29>`_). + +For page-load tests by default, instead of using live web pages for performance testing, Raptor uses a tool called `Mitmproxy <https://wiki.mozilla.org/TestEngineering/Performance/Raptor/Mitmproxy>`_. Mitmproxy allows us to record and playback test pages via a local Firefox proxy. The Mitmproxy recordings are stored on `tooltool <https://github.com/mozilla/build-tooltool>`_ and are automatically downloaded by Raptor when they are required for a test. Raptor uses mitmproxy via the `mozproxy <https://searchfox.org/mozilla-central/source/testing/mozbase/mozproxy>`_ package. + +There are two different types of Raptor page-load tests: warm page-load and cold page-load. + +Warm Page-Load +============== +For warm page-load tests, the browser is just started up once; so the browser is warm on each page-load. + +**Raptor warm page-load test process when running on Firefox/Chrome/Chromium desktop:** + +* A new browser profile is created +* The desktop browser is started up +* Post-startup browser settle pause of 30 seconds +* A new tab is opened +* The test URL is loaded; measurements taken +* The tab is reloaded 24 more times; measurements taken each time +* The measurements from the first page-load are not included in overall results metrics b/c of first load noise; however they are listed in the JSON artifacts + +**Raptor warm page-load test process when running on Firefox android browser apps:** + +* The android app data is cleared (via ``adb shell pm clear firefox.app.binary.name``) +* The new browser profile is copied onto the android device SD card +* The Firefox android app is started up +* Post-startup browser settle pause of 30 seconds +* The test URL is loaded; measurements taken +* The tab is reloaded 14 more times; measurements taken each time +* The measurements from the first page-load are not included in overall results metrics b/c of first load noise; however they are listed in the JSON artifacts + +Cold Page-Load +============== +For cold page-load tests, the browser is shut down and restarted between page load cycles, so the browser is cold on each page-load. This is what happens for Raptor cold page-load tests: + +**Raptor cold page-load test process when running on Firefox/Chrome/Chromium desktop:** + +* A new browser profile is created +* The desktop browser is started up +* Post-startup browser settle pause of 30 seconds +* A new tab is opened +* The test URL is loaded; measurements taken +* The tab is closed +* The desktop browser is shut down +* Entire process is repeated for the remaining browser cycles (25 cycles total) +* The measurements from all browser cycles are used to calculate overall results + +**Raptor cold page-load test process when running on Firefox Android browser apps:** + +* The Android app data is cleared (via ``adb shell pm clear firefox.app.binary.name``) +* A new browser profile is created +* The new browser profile is copied onto the Android device SD card +* The Firefox Android app is started up +* Post-startup browser settle pause of 30 seconds +* The test URL is loaded; measurements taken +* The Android app is shut down +* Entire process is repeated for the remaining browser cycles (15 cycles total) +* Note that the SSL cert DB is only created once (browser cycle 1) and copied into the profile for each additional browser cycle, thus not having to use the 'certutil' tool and re-created the db on each cycle +* The measurements from all browser cycles are used to calculate overall results + +Using Live Sites +================ +It is possible to use live web pages for the page-load tests instead of using the mitmproxy recordings. To do this, add ``--live`` to your command line or select one of the 'live' variants when running ``./mach try fuzzy --full``. + +Disabling Alerts +================ +It is possible to disable alerting for all our performance tests. Open the target test manifest such as the raptor-tp6*.ini file (`Raptor tests folder <https://searchfox.org/mozilla-central/source/testing/raptor/raptor/tests>`_), and make sure there are no ``alert_on`` specifications. + +When it's removed there will no longer be a ``shouldAlert`` field in the output Perfherder data (you can find the `schema here <https://searchfox.org/mozilla-central/source/testing/mozharness/external_tools/performance-artifact-schema.json#68,165>`_). As long as ``shouldAlert`` is not in the data, no alerts will be generated. If you need to also disable code sheriffing for the test, then you need to change the tier of the task to 3. + +High value tests +================ + +We have a notion of **high-value** tests in performance testing. These are chosen based on how many alerts they can catch using `this script <https://github.com/gmierz/moz-current-tests/blob/master/high-value-tests/generate_high_value_tests.py>`_ along with `a redash query <https://github.com/gmierz/moz-current-tests/blob/master/high-value-tests/sql_query.txt>`_. The lists below are the minimum set of test pages we should run to catch as many alerts as we can. + +**Desktop** + +Last updated: June 2021 + + - amazon + - bing + - cnn + - fandom + - gslides + - instagram + - twitter + - wikipedia + - yahoo-mail + +**Mobile** + +Last updated: November 2020 + + - allrecipes + - amazon-search + - espn + - facebook + - google-search + - microsoft-support + - youtube-watch + +WebExtension Benchmark Tests +---------------------------- + +Standard benchmarks are third-party tests (i.e. Speedometer) that we have integrated into Raptor to run per-commit in our production CI. + +Scenario Tests +-------------- + +Currently, there are three subtypes of Raptor-run "scenario" tests, all on (and only on) Android: + +#. **power-usage tests** +#. **memory-usage tests** +#. **CPU-usage tests** + +For a combined-measurement run with distinct Perfherder output for each measurement type, you can do: + +:: + + ./mach raptor --test raptor-scn-power-idle-bg-fenix --app fenix --binary org.mozilla.fenix.performancetest --host 10.0.0.16 --power-test --memory-test --cpu-test + +Each measurement subtype (power-, memory-, and cpu-usage) will have a corresponding PERFHERDER_DATA blob: + +:: + + 22:31:05 INFO - raptor-output Info: PERFHERDER_DATA: {"framework": {"name": "raptor"}, "suites": [{"name": "raptor-scn-power-idle-bg-fenix-cpu", "lowerIsBetter": true, "alertThreshold": 2.0, "value": 0, "subtests": [{"lowerIsBetter": true, "unit": "%", "name": "cpu-browser_cpu_usage", "value": 0, "alertThreshold": 2.0}], "type": "cpu", "unit": "%"}]} + 22:31:05 INFO - raptor-output Info: cpu results can also be found locally at: /Users/sdonner/moz_src/mozilla-unified/testing/mozharness/build/raptor-cpu.json + +(repeat for power, memory snippets) + +Power-Use Tests (Android) +========================= +Prerequisites +^^^^^^^^^^^^^ + + +#. rooted (i.e. superuser-capable), bootloader-unlocked Moto G5 or Google Pixel 2: internal (for now) `test-device setup doc. <https://docs.google.com/document/d/1XQLtvVM2U3h1jzzzpcGEDVOp4jMECsgLYJkhCfAwAnc/edit>`_ +#. set up to run Raptor from a Firefox source tree (see `Running Locally <https://wiki.mozilla.org/Performance_sheriffing/Raptor#Running_Locally>`_) +#. `GeckoView-bootstrapped <https://wiki.mozilla.org/Performance_sheriffing/Raptor#Running_on_the_Android_GeckoView_Example_App>`_ environment + +**Raptor power-use measurement test process when running on Firefox Android browser apps:** + +* The Android app data is cleared, via: + +:: + + adb shell pm clear firefox.app.binary.name + + +* The new browser profile is copied onto the Android device's SD card +* We set ``scenario_time`` to **20 minutes** (1200000 milliseconds), and ``page_timeout`` to '22 minutes' (1320000 milliseconds) + +**It's crucial that ``page_timeout`` exceed ``scenario_time``; if not, measurement tests will fail/bail early** + +* We launch the {Fenix, GeckoView, Reference Browser} on-Android app +* Post-startup browser settle pause of 30 seconds +* On Fennec only, a new browser tab is created (other Firefox apps use the single/existing tab) +* Power-use/battery-level measurements (app-specific measurements) are taken, via: + +:: + + adb shell dumpsys batterystats + + +* Raw power-use measurement data is listed in the perfherder-data.json/raptor.json artifacts + +In the Perfherder dashboards for these power usage tests, all data points have milli-Ampere-hour units, with a lower value being better. +Proportional power usage is the total power usage of hidden battery sippers that is proportionally "smeared"/distributed across all open applications. + +Running Scenario Tests Locally +============================== + +To run on a tethered phone via USB from a macOS host, on: + +Fenix +^^^^^ + + +:: + + ./mach raptor --test raptor-scn-power-idle-fenix --app fenix --binary org.mozilla.fenix.performancetest --power-test --host 10.252.27.96 + +GeckoView +^^^^^^^^^ + + +:: + + ./mach raptor --test raptor-scn-power-idle-geckoview --app geckoview --binary org.mozilla.geckoview_example --power-test --host 10.252.27.96 + +Reference Browser +^^^^^^^^^^^^^^^^^ + + +:: + + ./mach raptor --test raptor-scn-power-idle-refbrow --app refbrow --binary org.mozilla.reference.browser.raptor --power-test --host 10.252.27.96 + +**NOTE:** +*it is important that you include* ``--power-test``, *when running power-usage measurement tests, as that will help ensure that local test-measurement data doesn't accidentally get submitted to Perfherder* + +Writing New Tests +================= + +Pushing to Try server +===================== +As an example, a relatively good cross-sampling of builds can be seen in https://hg.mozilla.org/try/rev/6c07631a0c2bf56b51bb82fd5543d1b34d7f6c69. + +* Include both G5 Android 7 (hw-g5-7-0-arm7-api-16/) *and* Pixel 2 Android 8 (p2-8-0-android-aarch64/) target platforms +* pgo builds tend to be -- from my limited empirical evidence -- about 10 - 15 minutes longer to complete than their opt counterparts + +Dashboards +========== + +See `performance results <https://wiki.mozilla.org/TestEngineering/Performance/Results>`_ for our various dashboards. + +Running WebExtension Locally +---------------------------- + +WebExtension Prerequisites +========================== + +In order to run Raptor on a local machine, you need: + +* A local mozilla repository clone with a `successful Firefox build <https://developer.mozilla.org/en-US/docs/Mozilla/Developer_guide/Build_Instructions>`_ completed +* Git needs to be in the path in the terminal/window in which you build Firefox / run Raptor, as Raptor uses Git to check-out a local copy for some of the performance benchmarks' sources. +* If you plan on running Raptor tests on Google Chrome, you need a local install of Google Chrome and know the path to the chrome binary +* If you plan on running Raptor on Android, your Android device must already be set up (see more below in the Android section) + +Getting a List of Raptor Tests +============================== + +To see which Raptor performance tests are currently available on all platforms, use the 'print-tests' option, e.g.: + +:: + + $ ./mach raptor --print-tests + +That will output all available tests on each supported app, as well as each subtest available in each suite (i.e. all the pages in a specific page-load tp6* suite). + +Running on Firefox +================== + +To run Raptor locally, just build Firefox and then run: + +:: + + $ ./mach raptor --test <raptor-test-name> + +For example, to run the raptor-tp6 pageload test locally, just use: + +:: + + $ ./mach raptor --test raptor-tp6-1 + +You can run individual subtests too (i.e. a single page in one of the tp6* suites). For example, to run the amazon page-load test on Firefox: + +:: + + $ ./mach raptor --test raptor-tp6-amazon-firefox + +Raptor test results will be found locally in <your-repo>/testing/mozharness/build/raptor.json. + +Running on the Android GeckoView Example App +============================================ + +When running Raptor tests on a local Android device, Raptor is expecting the device to already be set up and ready to go. + +First, ensure your local host machine has the Android SDK/Tools (i.e. ADB) installed. Check if it is already installed by attaching your Android device to USB and running: + +:: + + $ adb devices + +If your device serial number is listed, then you're all set. If ADB is not found, you can install it by running (in your local mozilla-development repo): + +:: + + $ ./mach bootstrap + +Then, in bootstrap, select the option for "Firefox for Android Artifact Mode," which will install the required tools (no need to do an actual build). + +Next, make sure your Android device is ready to go. Local Android-device prerequisites are: + +* Device is `rooted <https://docs.google.com/document/d/1XQLtvVM2U3h1jzzzpcGEDVOp4jMECsgLYJkhCfAwAnc/edit>`_ + +Note: If you are using Magisk to root your device, use `version 17.3 <https://github.com/topjohnwu/Magisk/releases/tag/v17.3>`_ + +* Device is in 'superuser' mode + +* The geckoview example app is already installed on the device (from ``./mach bootstrap``, above). Download the geckoview_example.apk from the appropriate `android build on treeherder <https://treeherder.mozilla.org/#/jobs?repo=mozilla-central&searchStr=android%2Cbuild>`_, then install it on your device, i.e.: + +:: + + $ adb install -g ../Downloads/geckoview_example.apk + +The '-g' flag will automatically set all application permissions ON, which is required. + +Note, when the Gecko profiler should be run, or a build with build symbols is needed, then use a `Nightly build of geckoview_example.apk <https://treeherder.mozilla.org/#/jobs?repo=mozilla-central&searchStr=nightly%2Candroid>`_. + +When updating the geckoview example app, you MUST uninstall the existing one first, i.e.: + +:: + + $ adb uninstall org.mozilla.geckoview_example + +Once your Android device is ready, and attached to local USB, from within your local mozilla repo use the following command line to run speedometer: + +:: + + $ ./mach raptor --test raptor-speedometer --app=geckoview --binary="org.mozilla.geckoview_example" + +Note: Speedometer on Android GeckoView is currently running on two devices in production - the Google Pixel 2 and the Moto G5 - therefore it is not guaranteed that it will run successfully on all/other untested android devices. There is an intermittent failure on the Moto G5 where speedometer just stalls (`Bug 1492222 <https://bugzilla.mozilla.org/show_bug.cgi?id=1492222>`_). + +To run a Raptor page-load test (i.e. tp6m-1) on the GeckoView Example app, use this command line: + +:: + + $ ./mach raptor --test raptor-tp6m-1 --app=geckoview --binary="org.mozilla.geckoview_example" + +A couple notes about debugging: + +* Raptor browser-extension console messages *do* appear in adb logcat via the GeckoConsole - so this is handy: + +:: + + $ adb logcat | grep GeckoConsole + +* You can also debug Raptor on Android using the Firefox WebIDE; click on the Android device listed under "USB Devices" and then "Main Process" or the 'localhost: Speedometer.." tab process + +Raptor test results will be found locally in <your-repo>/testing/mozharness/build/raptor.json. + +Running on Google Chrome +======================== + +To run Raptor locally on Google Chrome, make sure you already have a local version of Google Chrome installed, and then from within your mozilla-repo run: + +:: + + $ ./mach raptor --test <raptor-test-name> --app=chrome --binary="<path to google chrome binary>" + +For example, to run the raptor-speedometer benchmark on Google Chrome use: + +:: + + $ ./mach raptor --test raptor-speedometer --app=chrome --binary="/Applications/Google Chrome.app/Contents/MacOS/Google Chrome + +Raptor test results will be found locally in <your-repo>/testing/mozharness/build/raptor.json. + +Page-Timeouts +============= + +On different machines the Raptor tests will run at different speeds. The default page-timeout is defined in each Raptor test INI file. On some machines you may see a test failure with a 'raptor page-timeout' which means the page-load timed out, or the benchmark test iteration didn't complete, within the page-timeout limit. + +You can override the default page-timeout by using the --page-timeout command-line arg. In this example, each test page in tp6-1 will be given two minutes to load during each page-cycle: + +:: + + ./mach raptor --test raptor-tp6-1 --page-timeout 120000 + +If an iteration of a benchmark test is not finishing within the allocated time, increase it by: + +:: + + ./mach raptor --test raptor-speedometer --page-timeout 600000 + +Page-Cycles +=========== + +Page-cycles is the number of times a test page is loaded (for page-load tests); for benchmark tests, this is the total number of iterations that the entire benchmark test will be run. The default page-cycles is defined in each Raptor test INI file. + +You can override the default page-cycles by using the --page-cycles command-line arg. In this example, the test page will only be loaded twice: + +:: + + ./mach raptor --test raptor-tp6-google-firefox --page-cycles 2 + +Running Page-Load Tests on Live Sites +===================================== +To use live pages instead of page recordings, just edit the `Raptor tp6* test INI <https://searchfox.org/mozilla-central/source/testing/raptor/raptor/tests>`_ file and add the following attribute either at the top (for all pages in the suite) or under an individual page/subtest heading: + + use_live_pages = true + +With that setting, Raptor will not start the playback tool (i.e. Mitmproxy) and will not turn on the corresponding browser proxy, therefore forcing the test page to load live. + +Running Raptor on Try +--------------------- + +Raptor tests can be run on `try <https://treeherder.mozilla.org/#/jobs?repo=try>`_ on both Firefox and Google Chrome. (Raptor pageload-type tests are not supported on Google Chrome yet, as mentioned above). + +**Note:** Raptor is currently 'tier 2' on `Treeherder <https://treeherder.mozilla.org/#/jobs?repo=try>`_, which means to see the Raptor test jobs you need to ensure 'tier 2' is selected / turned on in the Treeherder 'Tiers' menu. + +The easiest way to run Raptor tests on try is to use mach try fuzzy: + +:: + + $ ./mach try fuzzy --full + +Then type 'raptor' and select which Raptor tests (and on what platforms) you wish to run. + +To see the Raptor test results on your try run: + +#. In treeherder select one of the Raptor test jobs (i.e. 'sp' in 'Rap-e10s', or 'Rap-C-e10s') +#. Below the jobs, click on the "Performance" tab; you'll see the aggregated results listed +#. If you wish to see the raw replicates, click on the "Job Details" tab, and select the "perfherder-data.json" artifact + +Raptor Hardware in Production +============================= + +The Raptor performance tests run on dedicated hardware (the same hardware that the Talos performance tests use). See the `performance platforms <https://wiki.mozilla.org/TestEngineering/Performance/Platforms>`_ for more details. + +Profiling Raptor Jobs +--------------------- + +Raptor tests are able to create Gecko profiles which can be viewed in `profiler.firefox.com. <https://profiler.firefox.com/>`__ This is currently only supported when running Raptor on Firefox desktop. + +Nightly Profiling Jobs in Production +==================================== +We have Firefox desktop Raptor jobs with Gecko-profiling enabled running Nightly in production on Mozilla Central (on Linux64, Win10, and OSX). This provides a steady cache of Gecko profiles for the Raptor tests. Search for the `"Rap-Prof" treeherder group on Mozilla Central <https://treeherder.mozilla.org/#/jobs?repo=mozilla-central&searchStr=Rap-Prof>`_. + +Profiling Locally +================= + +To tell Raptor to create Gecko profiles during a performance test, just add the '--gecko-profile' flag to the command line, i.e.: + +:: + + $ ./mach raptor --test raptor-sunspider --gecko-profile + +When the Raptor test is finished, you will be able to find the resulting gecko profiles (ZIP) located locally in: + + mozilla-central/testing/mozharness/build/blobber_upload_dir/ + +Note: While profiling is turned on, Raptor will automatically reduce the number of pagecycles to 3. If you wish to override this, add the --page-cycles argument to the raptor command line. + +Raptor will automatically launch Firefox and load the latest Gecko profile in `profiler.firefox.com <https://profiler.firefox.com>`__. To turn this feature off, just set the DISABLE_PROFILE_LAUNCH=1 env var. + +If auto-launch doesn't work for some reason, just start Firefox manually and browse to `profiler.firefox.com <https://profiler.firefox.com>`__, click on "Browse" and select the Raptor profile ZIP file noted above. + +If you're on Windows and want to profile a Firefox build that you compiled yourself, make sure it contains profiling information and you have a symbols zip for it, by following the `directions on MDN <https://developer.mozilla.org/en-US/docs/Mozilla/Performance/Profiling_with_the_Built-in_Profiler_and_Local_Symbols_on_Windows#Profiling_local_talos_runs>`_. + +Profiling on Try Server +======================= + +To turn on Gecko profiling for Raptor test jobs on try pushes, just add the '--gecko-profile' flag to your try push i.e.: + +:: + + $ ./mach try fuzzy --gecko-profile + +Then select the Raptor test jobs that you wish to run. The Raptor jobs will be run on try with profiling included. While profiling is turned on, Raptor will automatically reduce the number of pagecycles to 2. + +See below for how to view the gecko profiles from within treeherder. + +Customizing the profiler +======================== +If the default profiling options are not enough, and further information is needed the gecko profiler can be customized. + +Enable profiling of additional threads +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +In some cases it will be helpful to also measure threads which are not part of the default set. Like the **MediaPlayback** thread. This can be accomplished by using: + +#. the **gecko_profile_threads** manifest entry, and specifying the thread names as comma separated list +#. the **--gecko-profile-thread** argument for **mach** for each extra thread to profile + +Add Profiling to Previously Completed Jobs +========================================== + +Note: You might need treeherder 'admin' access for the following. + +Gecko profiles can now be created for Raptor performance test jobs that have already completed in production (i.e. mozilla-central) and on try. To repeat a completed Raptor performance test job on production or try, but add gecko profiling, do the following: + +#. In treeherder, select the symbol for the completed Raptor test job (i.e. 'ss' in 'Rap-e10s') +#. Below, and to the left of the 'Job Details' tab, select the '...' to show the menu +#. On the pop-up menu, select 'Create Gecko Profile' + +The same Raptor test job will be repeated but this time with gecko profiling turned on. A new Raptor test job symbol will be added beside the completed one, with a '-p' added to the symbol name. Wait for that new Raptor profiling job to finish. See below for how to view the gecko profiles from within treeherder. + +Viewing Profiles on Treeherder +============================== +When the Raptor jobs are finished, to view the gecko profiles: + +#. In treeherder, select the symbol for the completed Raptor test job (i.e. 'ss' in 'Rap-e10s') +#. Click on the 'Job Details' tab below +#. The Raptor profile ZIP files will be listed as job artifacts; +#. Select a Raptor profile ZIP artifact, and click the 'view in Firefox Profiler' link to the right + +Recording Pages for Raptor Pageload Tests +----------------------------------------- + +Raptor pageload tests ('tp6' and 'tp6m' suites) use the `Mitmproxy <https://mitmproxy.org/>`__ tool to record and play back page archives. For more information on creating new page playback archives, please see `Raptor and Mitmproxy <https://wiki.mozilla.org/TestEngineering/Performance/Raptor/Mitmproxy>`__. + +Performance Tuning for Android devices +-------------------------------------- + +When the test is run against Android, Raptor executes a series of performance tuning commands over the ADB connection. + +Device agnostic: + +* memory bus +* device remain on when on USB power +* virtual memory (swappiness) +* services (thermal throttling, cpu throttling) +* i/o scheduler + +Device specific: + +* cpu governor +* cpu minimum frequency +* gpu governor +* gpu minimum frequency + +For a detailed list of current tweaks, please refer to `this <https://searchfox.org/mozilla-central/source/testing/raptor/raptor/raptor.py#676>`_ Searchfox page. diff --git a/testing/raptor/raptor/performance_tuning.py b/testing/raptor/raptor/performance_tuning.py new file mode 100644 index 0000000000..1277308d30 --- /dev/null +++ b/testing/raptor/raptor/performance_tuning.py @@ -0,0 +1,233 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +from mozdevice import ADBError + + +def tune_performance(device, log=None, timeout=None): + """Set various performance-oriented parameters, to reduce jitter. + + This includes some device-specific kernel tweaks. + + For more information, see https://bugzilla.mozilla.org/show_bug.cgi?id=1547135. + """ + PerformanceTuner(device, log=log, timeout=timeout).tune_performance() + + +class PerformanceTuner(object): + def __init__(self, device, log=None, timeout=None): + self.device = device + self.log = log or self.device._logger + self.timeout = timeout + + def tune_performance(self): + self.log.info("tuning android device performance") + self.set_svc_power_stayon() + if self.device.is_rooted: + device_name = self.device.shell_output( + "getprop ro.product.model", timeout=self.timeout + ) + # all commands require root shell from here on + self.set_scheduler() + self.set_virtual_memory_parameters() + self.turn_off_services() + self.set_cpu_performance_parameters(device_name) + self.set_gpu_performance_parameters(device_name) + self.set_kernel_performance_parameters() + self.device.clear_logcat(timeout=self.timeout) + self.log.info("android device performance tuning complete") + + def _set_value_and_check_exitcode(self, file_name, value): + self.log.info("setting {} to {}".format(file_name, value)) + try: + self.device.shell_output( + " ".join(["echo", str(value), ">", str(file_name)]), + timeout=self.timeout, + ) + self.log.info("successfully set {} to {}".format(file_name, value)) + except ADBError as e: + self.log.info( + "Ignoring failure to set value {} to {}. {}".format(file_name, value, e) + ) + + def set_svc_power_stayon(self): + self.log.info("set device to stay awake on usb") + self.device.shell_bool("svc power stayon usb", timeout=self.timeout) + + def set_scheduler(self): + self.log.info("setting scheduler to noop") + scheduler_location = "/sys/block/sda/queue/scheduler" + + self._set_value_and_check_exitcode(scheduler_location, "noop") + + def turn_off_services(self): + services = [ + "mpdecision", + "thermal-engine", + "thermald", + ] + for service in services: + try: + self.log.info(" ".join(["turning off service:", service])) + self.device.shell_bool( + " ".join(["stop", service]), timeout=self.timeout + ) + except ADBError as e: + self.log.info( + "Ignoring failure to stop service {}. Error: {}: {}".format( + service, e.__class__.__name__, e + ) + ) + + services_list_output = self.device.shell_output( + "service list", timeout=self.timeout + ) + for service in services: + if service not in services_list_output: + self.log.info(" ".join(["successfully terminated:", service])) + else: + self.log.warning(" ".join(["failed to terminate:", service])) + + def disable_animations(self): + self.log.info("disabling animations") + commands = { + "animator_duration_scale": 0.0, + "transition_animation_scale": 0.0, + "window_animation_scale": 0.0, + } + + for key, value in commands.items(): + command = " ".join(["settings", "put", "global", key, str(value)]) + self.log.info("setting {} to {}".format(key, value)) + self.device.shell_bool(command, timeout=self.timeout) + + def restore_animations(self): + # animation settings are not restored to default by reboot + self.log.info("restoring animations") + commands = { + "animator_duration_scale": 1.0, + "transition_animation_scale": 1.0, + "window_animation_scale": 1.0, + } + + for key, value in commands.items(): + command = " ".join(["settings", "put", "global", key, str(value)]) + self.device.shell_bool(command, timeout=self.timeout) + + def set_virtual_memory_parameters(self): + self.log.info("setting virtual memory parameters") + commands = { + "/proc/sys/vm/swappiness": 0, + "/proc/sys/vm/dirty_ratio": 85, + "/proc/sys/vm/dirty_background_ratio": 70, + } + + for key, value in commands.items(): + self._set_value_and_check_exitcode(key, value) + + def set_cpu_performance_parameters(self, device_name=None): + self.log.info("setting cpu performance parameters") + commands = {} + + if not device_name: + device_name = self.device.shell_output( + "getprop ro.product.model", timeout=self.timeout + ) + + if device_name == "Pixel 2": + # MSM8998 (4x 2.35GHz, 4x 1.9GHz) + # values obtained from: + # /sys/devices/system/cpu/cpufreq/policy0/scaling_available_frequencies + # /sys/devices/system/cpu/cpufreq/policy4/scaling_available_frequencies + commands.update( + { + "/sys/devices/system/cpu/cpufreq/policy0/scaling_governor": "performance", + "/sys/devices/system/cpu/cpufreq/policy4/scaling_governor": "performance", + "/sys/devices/system/cpu/cpufreq/policy0/scaling_min_freq": "1900800", + "/sys/devices/system/cpu/cpufreq/policy4/scaling_min_freq": "2457600", + } + ) + elif device_name == "Moto G (5)": + # MSM8937(8x 1.4GHz) + # values obtained from: + # /sys/devices/system/cpu/cpufreq/policy0/scaling_available_frequencies + for x in range(0, 8): + commands.update( + { + "/sys/devices/system/cpu/cpu{}/" + "cpufreq/scaling_governor".format(x): "performance", + "/sys/devices/system/cpu/cpu{}/" + "cpufreq/scaling_min_freq".format(x): "1401000", + } + ) + else: + self.log.info( + "CPU for device with ro.product.model '{}' unknown, not scaling_governor".format( + device_name + ) + ) + + for key, value in commands.items(): + self._set_value_and_check_exitcode(key, value) + + def set_gpu_performance_parameters(self, device_name=None): + self.log.info("setting gpu performance parameters") + commands = { + "/sys/class/kgsl/kgsl-3d0/bus_split": "0", + "/sys/class/kgsl/kgsl-3d0/force_bus_on": "1", + "/sys/class/kgsl/kgsl-3d0/force_rail_on": "1", + "/sys/class/kgsl/kgsl-3d0/force_clk_on": "1", + "/sys/class/kgsl/kgsl-3d0/force_no_nap": "1", + "/sys/class/kgsl/kgsl-3d0/idle_timer": "1000000", + } + + if not device_name: + device_name = self.device.shell_output( + "getprop ro.product.model", timeout=self.timeout + ) + + if device_name == "Pixel 2": + # Adreno 540 (710MHz) + # values obtained from: + # /sys/devices/soc/5000000.qcom,kgsl-3d0/kgsl/kgsl-3d0/max_clk_mhz + commands.update( + { + "/sys/devices/soc/5000000.qcom,kgsl-3d0/devfreq/" + "5000000.qcom,kgsl-3d0/governor": "performance", + "/sys/devices/soc/soc:qcom,kgsl-busmon/devfreq/" + "soc:qcom,kgsl-busmon/governor": "performance", + "/sys/devices/soc/5000000.qcom,kgsl-3d0/kgsl/kgsl-3d0/min_clock_mhz": "710", + } + ) + elif device_name == "Moto G (5)": + # Adreno 505 (450MHz) + # values obtained from: + # /sys/devices/soc/1c00000.qcom,kgsl-3d0/kgsl/kgsl-3d0/max_clock_mhz + commands.update( + { + "/sys/devices/soc/1c00000.qcom,kgsl-3d0/devfreq/" + "1c00000.qcom,kgsl-3d0/governor": "performance", + "/sys/devices/soc/1c00000.qcom,kgsl-3d0/kgsl/kgsl-3d0/min_clock_mhz": "450", + } + ) + else: + self.log.info( + "GPU for device with ro.product.model '{}' unknown, not setting devfreq".format( + device_name + ) + ) + + for key, value in commands.items(): + self._set_value_and_check_exitcode(key, value) + + def set_kernel_performance_parameters(self): + self.log.info("setting kernel performance parameters") + commands = { + "/sys/kernel/debug/msm-bus-dbg/shell-client/update_request": "1", + "/sys/kernel/debug/msm-bus-dbg/shell-client/mas": "1", + "/sys/kernel/debug/msm-bus-dbg/shell-client/ab": "0", + "/sys/kernel/debug/msm-bus-dbg/shell-client/slv": "512", + } + for key, value in commands.items(): + self._set_value_and_check_exitcode(key, value) diff --git a/testing/raptor/raptor/perftest.py b/testing/raptor/raptor/perftest.py new file mode 100644 index 0000000000..adbc75ef61 --- /dev/null +++ b/testing/raptor/raptor/perftest.py @@ -0,0 +1,869 @@ +#!/usr/bin/env python + +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import json +import os +import pathlib +import re +import shutil +import subprocess +import sys +import tempfile +import traceback +from abc import ABCMeta, abstractmethod + +import mozinfo +import mozprocess +import mozproxy.utils as mpu +import mozversion +import six +from mozprofile import create_profile +from mozproxy import get_playback + +# need this so raptor imports work both from /raptor and via mach +here = os.path.abspath(os.path.dirname(__file__)) +paths = [here] + +webext_dir = os.path.join(here, "..", "webext") +paths.append(webext_dir) + +for path in paths: + if not os.path.exists(path): + raise IOError("%s does not exist. " % path) + sys.path.insert(0, path) + +from cmdline import FIREFOX_ANDROID_APPS +from condprof.client import ProfileNotFoundError, get_profile +from condprof.util import get_current_platform +from gecko_profile import GeckoProfile +from logger.logger import RaptorLogger +from results import RaptorResultsHandler + +LOG = RaptorLogger(component="raptor-perftest") + +# - mozproxy.utils LOG displayed INFO messages even when LOG.error() was used in mitm.py +mpu.LOG = RaptorLogger(component="raptor-mitmproxy") + +try: + from mozbuild.base import MozbuildObject + + build = MozbuildObject.from_environment(cwd=here) +except ImportError: + build = None + +POST_DELAY_CONDPROF = 1000 +POST_DELAY_DEBUG = 3000 +POST_DELAY_DEFAULT = 30000 + + +@six.add_metaclass(ABCMeta) +class Perftest(object): + """Abstract base class for perftests that execute via a subharness, + either Raptor or browsertime.""" + + def __init__( + self, + app, + binary, + run_local=False, + noinstall=False, + obj_path=None, + profile_class=None, + installerpath=None, + gecko_profile=False, + gecko_profile_interval=None, + gecko_profile_entries=None, + gecko_profile_extra_threads=None, + gecko_profile_threads=None, + gecko_profile_features=None, + extra_profiler_run=False, + symbols_path=None, + host=None, + power_test=False, + cpu_test=False, + cold=False, + memory_test=False, + live_sites=False, + is_release_build=False, + debug_mode=False, + post_startup_delay=POST_DELAY_DEFAULT, + interrupt_handler=None, + e10s=True, + results_handler_class=RaptorResultsHandler, + device_name=None, + disable_perf_tuning=False, + conditioned_profile=None, + test_bytecode_cache=False, + chimera=False, + extra_prefs={}, + environment={}, + project="mozilla-central", + verbose=False, + python=None, + fission=True, + extra_summary_methods=[], + **kwargs + ): + self._remote_test_root = None + self._dirs_to_remove = [] + self.verbose = verbose + self.page_count = [] + + # Override the magic --host HOST_IP with the value of the environment variable. + if host == "HOST_IP": + host = os.environ["HOST_IP"] + + self.config = { + "app": app, + "binary": binary, + "platform": mozinfo.os, + "processor": mozinfo.processor, + "run_local": run_local, + "obj_path": obj_path, + "gecko_profile": gecko_profile, + "gecko_profile_interval": gecko_profile_interval, + "gecko_profile_entries": gecko_profile_entries, + "gecko_profile_extra_threads": gecko_profile_extra_threads, + "gecko_profile_threads": gecko_profile_threads, + "gecko_profile_features": gecko_profile_features, + "extra_profiler_run": extra_profiler_run, + "symbols_path": symbols_path, + "host": host, + "power_test": power_test, + "memory_test": memory_test, + "cpu_test": cpu_test, + "cold": cold, + "live_sites": live_sites, + "is_release_build": is_release_build, + "enable_control_server_wait": memory_test or cpu_test, + "e10s": e10s, + "device_name": device_name, + "fission": fission, + "disable_perf_tuning": disable_perf_tuning, + "conditioned_profile": conditioned_profile, + "test_bytecode_cache": test_bytecode_cache, + "chimera": chimera, + "extra_prefs": extra_prefs, + "environment": environment, + "project": project, + "verbose": verbose, + "extra_summary_methods": extra_summary_methods, + } + + self.firefox_android_apps = FIREFOX_ANDROID_APPS + + # We are deactivating the conditioned profiles for: + # - win10-aarch64 : no support for geckodriver see 1582757 + # - reference browser: no conditioned profiles created see 1606767 + if ( + self.config["platform"] == "win" and self.config["processor"] == "aarch64" + ) or self.config["binary"] == "org.mozilla.reference.browser.raptor": + self.config["conditioned_profile"] = None + + if self.config["conditioned_profile"]: + LOG.info("Using a conditioned profile.") + else: + LOG.info("Using an empty profile.") + + # To differentiate between chrome/firefox failures, we + # set an app variable in the logger which prefixes messages + # with the app name + if self.config["app"] in ("chrome", "chrome-m", "chromium"): + LOG.set_app(self.config["app"]) + + self.browser_name = None + self.browser_version = None + + self.raptor_venv = os.path.join(os.getcwd(), "raptor-venv") + self.installerpath = installerpath + self.playback = None + self.benchmark = None + self.gecko_profiler = None + self.device = None + self.runtime_error = None + self.profile_class = profile_class or app + self.conditioned_profile_dir = None + self.interrupt_handler = interrupt_handler + self.results_handler = results_handler_class(**self.config) + + self.browser_name, self.browser_version = self.get_browser_meta() + + browser_name, browser_version = self.get_browser_meta() + self.results_handler.add_browser_meta(self.config["app"], browser_version) + + # debug mode is currently only supported when running locally + self.run_local = self.config["run_local"] + self.debug_mode = debug_mode if self.run_local else False + + # For the post startup delay, we want to max it to 1s when using the + # conditioned profiles. + if self.config.get("conditioned_profile"): + self.post_startup_delay = min(post_startup_delay, POST_DELAY_CONDPROF) + else: + # if running debug-mode reduce the pause after browser startup + if self.debug_mode: + self.post_startup_delay = min(post_startup_delay, POST_DELAY_DEBUG) + else: + self.post_startup_delay = post_startup_delay + + LOG.info("Post startup delay set to %d ms" % self.post_startup_delay) + LOG.info("main raptor init, config is: %s" % str(self.config)) + + # TODO: Move this outside of the perftest initialization, it contains + # platform-specific code + self.build_browser_profile() + + # Crashes counter + self.crashes = 0 + + def _get_temp_dir(self): + tempdir = tempfile.mkdtemp() + self._dirs_to_remove.append(tempdir) + return tempdir + + @property + def is_localhost(self): + return self.config.get("host") in ("localhost", "127.0.0.1") + + @property + def android_external_storage(self): + return "/sdcard/test_root/" + + @property + def conditioned_profile_copy(self): + """Returns a copy of the original conditioned profile that was created.""" + condprof_copy = os.path.join(self._get_temp_dir(), "profile") + shutil.copytree( + self.conditioned_profile_dir, + condprof_copy, + ignore=shutil.ignore_patterns("lock"), + ) + LOG.info("Created a conditioned-profile copy: %s" % condprof_copy) + return condprof_copy + + def build_conditioned_profile(self): + # Late import so python-test doesn't import it + import asyncio + + # The following import patchs an issue with invalid + # content-type, see bug 1655869 + from condprof import patch # noqa + from condprof.runner import Runner + + if not getattr(self, "browsertime"): + raise Exception( + "Building conditioned profiles within a test is only supported " + "when using Browsertime." + ) + + geckodriver = getattr(self, "browsertime_geckodriver", None) + if not geckodriver: + geckodriver = ( + sys.platform.startswith("win") and "geckodriver.exe" or "geckodriver" + ) + + scenario = self.config.get("conditioned_profile") + runner = Runner( + profile=None, + firefox=self.config.get("binary"), + geckodriver=geckodriver, + archive=None, + device_name=self.config.get("device_name"), + strict=True, + visible=True, + force_new=True, + skip_logs=True, + remote_test_root=self.android_external_storage, + ) + + if self.config.get("is_release_build", False): + # Enable non-local connections for building the conditioned profile + self.enable_non_local_connections() + + if scenario == "settled-youtube": + runner.prepare(scenario, "youtube") + + loop = asyncio.get_event_loop() + loop.run_until_complete(runner.one_run(scenario, "youtube")) + else: + runner.prepare(scenario, "default") + + loop = asyncio.get_event_loop() + loop.run_until_complete(runner.one_run(scenario, "default")) + + if self.config.get("is_release_build", False): + self.disable_non_local_connections() + + self.conditioned_profile_dir = runner.env.profile + return self.conditioned_profile_copy + + def get_conditioned_profile(self, binary=None): + """Downloads a platform-specific conditioned profile, using the + condprofile client API; returns a self.conditioned_profile_dir""" + if self.conditioned_profile_dir: + # We already have a directory, so provide a copy that + # will get deleted after it's done with + return self.conditioned_profile_copy + + # Build the conditioned profile before the test + if not self.config.get("conditioned_profile").startswith("artifact:"): + return self.build_conditioned_profile() + + # create a temp file to help ensure uniqueness + temp_download_dir = self._get_temp_dir() + LOG.info( + "Making temp_download_dir from inside get_conditioned_profile {}".format( + temp_download_dir + ) + ) + # call condprof's client API to yield our platform-specific + # conditioned-profile binary + if isinstance(self, PerftestAndroid): + android_app = self.config["binary"].split("org.mozilla.")[-1] + device_name = self.config.get("device_name") + if device_name is None: + device_name = "g5" + platform = "%s-%s" % (device_name, android_app) + else: + platform = get_current_platform() + + LOG.info("Platform used: %s" % platform) + + # when running under mozharness, the --project value + # is set to match the project (try, mozilla-central, etc.) + # By default it's mozilla-central, even on local runs. + # We use it to prioritize conditioned profiles indexed + # into the same project when it runs on the CI + repo = self.config["project"] + + # we fall back to mozilla-central in all cases. If it + # was already mozilla-central, we fall back to try + alternate_repo = "mozilla-central" if repo != "mozilla-central" else "try" + LOG.info("Getting profile from project %s" % repo) + + profile_scenario = self.config.get("conditioned_profile").replace( + "artifact:", "" + ) + try: + cond_prof_target_dir = get_profile( + temp_download_dir, platform, profile_scenario, repo=repo + ) + except ProfileNotFoundError: + cond_prof_target_dir = get_profile( + temp_download_dir, platform, profile_scenario, repo=alternate_repo + ) + except Exception: + # any other error is a showstopper + LOG.critical("Could not get the conditioned profile") + traceback.print_exc() + raise + + # now get the full directory path to our fetched conditioned profile + self.conditioned_profile_dir = os.path.join( + temp_download_dir, cond_prof_target_dir + ) + if not os.path.exists(cond_prof_target_dir): + LOG.critical( + "Can't find target_dir {}, from get_profile()" + "temp_download_dir {}, platform {}, scenario {}".format( + cond_prof_target_dir, temp_download_dir, platform, profile_scenario + ) + ) + raise OSError + + LOG.info( + "Original self.conditioned_profile_dir is now set: {}".format( + self.conditioned_profile_dir + ) + ) + return self.conditioned_profile_copy + + def build_browser_profile(self): + if self.config["app"] in ["safari"]: + self.profile = None + return + elif ( + self.config["app"] in ["chrome", "chromium", "chrome-m"] + or self.config.get("conditioned_profile") is None + ): + self.profile = create_profile(self.profile_class) + else: + # use mozprofile to create a profile for us, from our conditioned profile's path + self.profile = create_profile( + self.profile_class, profile=self.get_conditioned_profile() + ) + # Merge extra profile data from testing/profiles + with open(os.path.join(self.profile_data_dir, "profiles.json"), "r") as fh: + base_profiles = json.load(fh)["raptor"] + + for profile in base_profiles: + path = os.path.join(self.profile_data_dir, profile) + LOG.info("Merging profile: {}".format(path)) + self.profile.merge(path) + + LOG.info("Browser preferences: {}".format(self.config["extra_prefs"])) + self.profile.set_preferences(self.config["extra_prefs"]) + + # share the profile dir with the config and the control server + self.config["local_profile_dir"] = self.profile.profile + LOG.info("Local browser profile: {}".format(self.profile.profile)) + + @property + def profile_data_dir(self): + if "MOZ_DEVELOPER_REPO_DIR" in os.environ: + return os.path.join( + os.environ["MOZ_DEVELOPER_REPO_DIR"], "testing", "profiles" + ) + if build: + return os.path.join(build.topsrcdir, "testing", "profiles") + return os.path.join(here, "profile_data") + + @property + def artifact_dir(self): + artifact_dir = os.getcwd() + if self.config.get("run_local", False): + if "MOZ_DEVELOPER_REPO_DIR" in os.environ: + artifact_dir = os.path.join( + os.environ["MOZ_DEVELOPER_REPO_DIR"], + "testing", + "mozharness", + "build", + ) + else: + artifact_dir = here + elif os.getenv("MOZ_UPLOAD_DIR"): + artifact_dir = os.getenv("MOZ_UPLOAD_DIR") + return artifact_dir + + @abstractmethod + def run_test_setup(self, test): + LOG.info("starting test: %s" % test["name"]) + + # if 'alert_on' was provided in the test INI, add to our config for results/output + self.config["subtest_alert_on"] = test.get("alert_on") + + if test.get("playback") is not None and self.playback is None: + self.start_playback(test) + + if test.get("preferences") is not None and self.config["app"] not in "safari": + self.set_browser_test_prefs(test["preferences"]) + + @abstractmethod + def setup_chrome_args(self): + pass + + @abstractmethod + def get_browser_meta(self): + pass + + def run_tests(self, tests, test_names): + tests_to_run = tests + if self.results_handler.existing_results: + tests_to_run = [] + try: + for test in tests_to_run: + try: + self.run_test(test, timeout=int(test.get("page_timeout"))) + except RuntimeError as e: + # Check for crashes before showing the timeout error. + self.check_for_crashes() + if self.crashes == 0: + LOG.critical(e) + os.sys.exit(1) + finally: + self.run_test_teardown(test) + return self.process_results(tests, test_names) + finally: + self.clean_up() + + @abstractmethod + def run_test(self, test, timeout): + raise NotImplementedError() + + @abstractmethod + def run_test_teardown(self, test): + self.check_for_crashes() + + def process_results(self, tests, test_names): + # when running locally output results in build/raptor.json; when running + # in production output to a local.json to be turned into tc job artifact + raptor_json_path = os.path.join(self.artifact_dir, "raptor.json") + if not self.config.get("run_local", False): + raptor_json_path = os.path.join(os.getcwd(), "local.json") + + self.config["raptor_json_path"] = raptor_json_path + self.config["artifact_dir"] = self.artifact_dir + self.config["page_count"] = self.page_count + res = self.results_handler.summarize_and_output(self.config, tests, test_names) + + # Gecko profiling symbolication + # We enable the gecko profiler either when the profiler is enabled with + # gecko_profile flag form the command line or when an extra profiler-enabled + # run is added with extra_profiler_run flag. + if self.config["gecko_profile"] or self.config.get("extra_profiler_run"): + self.gecko_profiler.symbolicate() + # clean up the temp gecko profiling folders + LOG.info("cleaning up after gecko profiling") + self.gecko_profiler.clean() + + return res + + @abstractmethod + def set_browser_test_prefs(self): + pass + + @abstractmethod + def check_for_crashes(self): + pass + + def clean_up(self): + # Cleanup all of our temporary directories + for dir_to_rm in self._dirs_to_remove: + if not os.path.exists(dir_to_rm): + continue + LOG.info("Removing temporary directory: {}".format(dir_to_rm)) + shutil.rmtree(dir_to_rm, ignore_errors=True) + self._dirs_to_remove = [] + + # Go through the artifact directory and ensure we + # don't have too many JPG/PNG files from a task failure + if ( + not self.run_local + and self.results_handler + and self.results_handler.result_dir() + ): + artifact_dir = pathlib.Path(self.artifact_dir) + for filetype in ("*.png", "*.jpg"): + # Limit the number of images uploaded to the last (newest) 5 + for file in sorted(artifact_dir.rglob(filetype))[:-5]: + try: + file.unlink() + except FileNotFoundError: + pass + + def get_page_timeout_list(self): + return self.results_handler.page_timeout_list + + def delete_proxy_settings_from_profile(self): + # Must delete the proxy settings from the profile if running + # the test with a host different from localhost. + userjspath = os.path.join(self.profile.profile, "user.js") + with open(userjspath) as userjsfile: + prefs = userjsfile.readlines() + prefs = [pref for pref in prefs if "network.proxy" not in pref] + with open(userjspath, "w") as userjsfile: + userjsfile.writelines(prefs) + + def start_playback(self, test): + # creating the playback tool + + playback_dir = os.path.join(here, "tooltool-manifests", "playback") + + self.config.update( + { + "playback_tool": test.get("playback"), + "playback_version": test.get("playback_version", "8.1.1"), + "playback_files": [ + os.path.join(playback_dir, test.get("playback_pageset_manifest")) + ], + } + ) + + LOG.info("test uses playback tool: %s " % self.config["playback_tool"]) + + self.playback = get_playback(self.config) + + # let's start it! + self.playback.start() + + def _init_gecko_profiling(self, test): + LOG.info("initializing gecko profiler") + upload_dir = os.getenv("MOZ_UPLOAD_DIR") + if not upload_dir: + LOG.critical("Profiling ignored because MOZ_UPLOAD_DIR was not set") + else: + self.gecko_profiler = GeckoProfile(upload_dir, self.config, test) + + def disable_non_local_connections(self): + # For Firefox we need to set MOZ_DISABLE_NONLOCAL_CONNECTIONS=1 env var before startup + # when testing release builds from mozilla-beta/release. This is because of restrictions + # on release builds that require webextensions to be signed unless this env var is set + LOG.info("setting MOZ_DISABLE_NONLOCAL_CONNECTIONS=1") + os.environ["MOZ_DISABLE_NONLOCAL_CONNECTIONS"] = "1" + + def enable_non_local_connections(self): + # pageload tests need to be able to access non-local connections via mitmproxy + LOG.info("setting MOZ_DISABLE_NONLOCAL_CONNECTIONS=0") + os.environ["MOZ_DISABLE_NONLOCAL_CONNECTIONS"] = "0" + + +class PerftestAndroid(Perftest): + """Mixin class for Android-specific Perftest subclasses.""" + + def setup_chrome_args(self, test): + """Sets up chrome/chromium cmd-line arguments. + + Needs to be "implemented" here to deal with Python 2 + unittest failures. + """ + raise NotImplementedError + + def get_browser_meta(self): + """Returns the browser name and version in a tuple (name, version). + + Uses mozversion as the primary method to get this meta data and for + android this is the only method which exists to get this data. With android, + we use the installerpath attribute to determine this and this only works + with Firefox browsers. + """ + browser_name = None + browser_version = None + + if self.config["app"] in self.firefox_android_apps: + try: + meta = mozversion.get_version(binary=self.installerpath) + browser_name = meta.get("application_name") + browser_version = meta.get("application_version") + except Exception as e: + LOG.warning( + "Failed to get android browser meta data through mozversion: %s-%s" + % (e.__class__.__name__, e) + ) + + if self.config["app"] == "chrome-m": + # We absolutely need to determine the chrome + # version here so that we can select the correct + # chromedriver for browsertime + from mozdevice import ADBDeviceFactory + + device = ADBDeviceFactory(verbose=True) + binary = "com.android.chrome" + + pkg_info = device.shell_output("dumpsys package %s" % binary) + version_matcher = re.compile(r".*versionName=([\d.]+)") + for line in pkg_info.split("\n"): + match = version_matcher.match(line) + if match: + browser_version = match.group(1) + browser_name = self.config["app"] + # First one found is the non-system + # or latest version. + break + + if not browser_version: + raise Exception( + "Could not determine version for Google Chrome for Android" + ) + + if not browser_name: + LOG.warning("Could not find a browser name") + else: + LOG.info("Browser name: %s" % browser_name) + + if not browser_version: + LOG.warning("Could not find a browser version") + else: + LOG.info("Browser version: %s" % browser_version) + + return (browser_name, browser_version) + + def set_reverse_port(self, port): + tcp_port = "tcp:{}".format(port) + self.device.create_socket_connection("reverse", tcp_port, tcp_port) + + def set_reverse_ports(self): + if self.is_localhost: + + # only raptor-webext uses the control server + if self.config.get("browsertime", False) is False: + LOG.info("making the raptor control server port available to device") + self.set_reverse_port(self.control_server.port) + + if self.playback: + LOG.info("making the raptor playback server port available to device") + self.set_reverse_port(self.playback.port) + + if self.benchmark: + LOG.info("making the raptor benchmarks server port available to device") + self.set_reverse_port(int(self.benchmark.port)) + else: + LOG.info("Reverse port forwarding is used only on local devices") + + def build_browser_profile(self): + super(PerftestAndroid, self).build_browser_profile() + + # Merge in the Android profile. + path = os.path.join(self.profile_data_dir, "raptor-android") + LOG.info("Merging profile: {}".format(path)) + self.profile.merge(path) + self.profile.set_preferences( + {"browser.tabs.remote.autostart": self.config["e10s"]} + ) + + def clear_app_data(self): + LOG.info("clearing %s app data" % self.config["binary"]) + self.device.shell("pm clear %s" % self.config["binary"]) + + def set_debug_app_flag(self): + # required so release apks will read the android config.yml file + LOG.info("setting debug-app flag for %s" % self.config["binary"]) + self.device.shell("am set-debug-app --persistent %s" % self.config["binary"]) + + def copy_profile_to_device(self): + """Copy the profile to the device, and update permissions of all files.""" + if not self.device.is_app_installed(self.config["binary"]): + raise Exception("%s is not installed" % self.config["binary"]) + + try: + LOG.info("copying profile to device: %s" % self.remote_profile) + self.device.rm(self.remote_profile, force=True, recursive=True) + self.device.push(self.profile.profile, self.remote_profile) + self.device.chmod(self.remote_profile, recursive=True) + + except Exception: + LOG.error("Unable to copy profile to device.") + raise + + def turn_on_android_app_proxy(self): + # for geckoview/android pageload playback we can't use a policy to turn on the + # proxy; we need to set prefs instead; note that the 'host' may be different + # than '127.0.0.1' so we must set the prefs accordingly + proxy_prefs = {} + proxy_prefs["network.proxy.type"] = 1 + proxy_prefs["network.proxy.http"] = self.playback.host + proxy_prefs["network.proxy.http_port"] = self.playback.port + proxy_prefs["network.proxy.ssl"] = self.playback.host + proxy_prefs["network.proxy.ssl_port"] = self.playback.port + proxy_prefs["network.proxy.no_proxies_on"] = self.config["host"] + + LOG.info( + "setting profile prefs to turn on the android app proxy: {}".format( + proxy_prefs + ) + ) + self.profile.set_preferences(proxy_prefs) + + +class PerftestDesktop(Perftest): + """Mixin class for Desktop-specific Perftest subclasses""" + + def __init__(self, *args, **kwargs): + super(PerftestDesktop, self).__init__(*args, **kwargs) + + def setup_chrome_args(self, test): + """Sets up chrome/chromium cmd-line arguments. + + Needs to be "implemented" here to deal with Python 2 + unittest failures. + """ + raise NotImplementedError + + def desktop_chrome_args(self, test): + """Returns cmd line options required to run pageload tests on Desktop Chrome + and Chromium. Also add the cmd line options to turn on the proxy and + ignore security certificate errors if using host localhost, 127.0.0.1. + """ + chrome_args = ["--use-mock-keychain", "--no-default-browser-check"] + + if test.get("playback", False): + pb_args = [ + "--proxy-server=%s:%d" % (self.playback.host, self.playback.port), + "--proxy-bypass-list=localhost;127.0.0.1", + "--ignore-certificate-errors", + ] + + if not self.is_localhost: + pb_args[0] = pb_args[0].replace("127.0.0.1", self.config["host"]) + + chrome_args.extend(pb_args) + + if self.debug_mode: + chrome_args.extend(["--auto-open-devtools-for-tabs"]) + + return chrome_args + + def get_browser_meta(self): + """Returns the browser name and version in a tuple (name, version). + + On desktop, we use mozversion but a fallback method also exists + for non-firefox browsers, where mozversion is known to fail. The + methods are OS-specific, with windows being the outlier. + """ + browser_name = None + browser_version = None + + try: + meta = mozversion.get_version(binary=self.config["binary"]) + browser_name = meta.get("application_name") + browser_version = meta.get("application_version") + except Exception as e: + LOG.warning( + "Failed to get browser meta data through mozversion: %s-%s" + % (e.__class__.__name__, e) + ) + LOG.info("Attempting to get version through fallback method...") + + # Fall-back method to get browser version on desktop + try: + if "mac" in self.config["platform"]: + import plistlib + + for plist_file in ("version.plist", "Info.plist"): + try: + binary_path = pathlib.Path(self.config["binary"]) + plist_path = binary_path.parent.parent.joinpath(plist_file) + with plist_path.open("rb") as plist: + plist = plistlib.load(plist) + except FileNotFoundError: + pass + browser_name = self.config["app"] + browser_version = plist.get("CFBundleShortVersionString") + elif "linux" in self.config["platform"]: + command = [self.config["binary"], "--version"] + proc = mozprocess.ProcessHandler(command) + proc.run(timeout=10, outputTimeout=10) + proc.wait() + + bmeta = proc.output + meta_re = re.compile(r"([A-z\s]+)\s+([\w.]*)") + if len(bmeta) != 0: + match = meta_re.match(bmeta[0].decode("utf-8")) + if match: + browser_name = self.config["app"] + browser_version = match.group(2) + else: + LOG.info("Couldn't get browser version and name") + else: + # On windows we need to use wimc to get the version + command = r'wmic datafile where name="{0}"'.format( + self.config["binary"].replace("\\", r"\\") + ) + bmeta = subprocess.check_output(command) + + meta_re = re.compile(r"\s+([\d.a-z]+)\s+") + match = meta_re.findall(bmeta.decode("utf-8")) + if len(match) > 0: + browser_name = self.config["app"] + browser_version = match[-1] + else: + LOG.info("Couldn't get browser version and name") + except Exception as e: + LOG.warning( + "Failed to get browser meta data through fallback method: %s-%s" + % (e.__class__.__name__, e) + ) + + if not browser_name: + LOG.warning("Could not find a browser name") + else: + LOG.info("Browser name: %s" % browser_name) + + if not browser_version: + LOG.warning("Could not find a browser version") + else: + LOG.info("Browser version: %s" % browser_version) + + return (browser_name, browser_version) diff --git a/testing/raptor/raptor/power.py b/testing/raptor/raptor/power.py new file mode 100644 index 0000000000..c34046d761 --- /dev/null +++ b/testing/raptor/raptor/power.py @@ -0,0 +1,389 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import os +import re +import time + +from logger.logger import RaptorLogger +from mozdevice import ADBError, ADBTimeoutError + +LOG = RaptorLogger(component="raptor-power") + +P2_PATH = "/sys/class/power_supply/battery/input_suspend" +G5_PATH = "/sys/class/power_supply/battery/charging_enabled" +S7_PATH = "/sys/class/power_supply/battery/batt_slate_mode" + + +def get_device_type(device, timeout=10): + """Returns the type of device being tested. Currently + it can either be Pixel 2, Moto G5, or Samsung S7.""" + device_type = device.shell_output("getprop ro.product.model", timeout=timeout) + if device_type == "Pixel 2": + pass + elif device_type == "Moto G (5)": + pass + elif device_type == "SM-G930F": + # samsung s7 galaxy (exynos) + pass + else: + raise Exception("TEST-UNEXPECTED-FAIL | Unknown device ('%s')!" % device_type) + return device_type + + +def change_charging_state(device, device_type, enable=True, timeout=10): + """Changes the charging state. If enable is True, charging will be enabled, + otherwise it will be disabled.""" + try: + if device_type == "Pixel 2": + status = 0 if enable else 1 + device.shell_bool("echo %s > %s" % (status, P2_PATH), timeout=timeout) + elif device_type == "Moto G (5)": + status = 1 if enable else 0 + device.shell_bool("echo %s > %s" % (status, G5_PATH), timeout=timeout) + elif device_type == "SM-G930F": + status = 0 if enable else 1 + device.shell_bool("echo %s > %s" % (status, S7_PATH), timeout=timeout) + except (ADBTimeoutError, ADBError) as e: + raise Exception( + "TEST-UNEXPECTED-FAIL | Failed to %s charging. Error: %s" + % ( + "enable" if enable else "disable", + "{}: {}".format(e.__class__.__name__, e), + ) + ) + + +def is_charging_disabled(device, device_type, timeout=10): + """True if charging is already disabled.""" + disabled = False + if device_type == "Pixel 2": + disabled = ( + device.shell_output("cat %s 2>/dev/null" % P2_PATH, timeout=timeout).strip() + == "1" + ) + elif device_type == "Moto G (5)": + disabled = ( + device.shell_output("cat %s 2>/dev/null" % G5_PATH, timeout=timeout).strip() + == "0" + ) + elif device_type == "SM-G930F": + disabled = ( + device.shell_output("cat %s 2>/dev/null" % S7_PATH, timeout=timeout).strip() + == "1" + ) + return disabled + + +def is_charging_enabled(device, device_type): + """True if charging is already enabled.""" + return not is_charging_disabled(device, device_type) + + +def enable_charging(device): + """Enables charging on a supported device.""" + device_type = get_device_type(device) + if is_charging_enabled(device, device_type): + return + + # ProxyLogger returns a RuntimeError when the + # logger isn't initialized. Catching this error + # is the only way to tell if the logger wasn't + # initialized. + enabling_str = "Enabling charging..." + try: + LOG.info(enabling_str) + except RuntimeError: + print(enabling_str) + + change_charging_state(device, device_type, enable=True) + + +def disable_charging(device): + """Disables charging on a supported device.""" + device_type = get_device_type(device) + if is_charging_disabled(device, device_type): + return + + disabling_str = "Disabling charging..." + try: + LOG.info(disabling_str) + except RuntimeError: + print(disabling_str) + + change_charging_state(device, device_type, enable=False) + + +def init_android_power_test(raptor): + upload_dir = os.getenv("MOZ_UPLOAD_DIR") + if not upload_dir: + LOG.critical( + "%s power test ignored; MOZ_UPLOAD_DIR unset" % raptor.config["app"] + ) + return + # Disable adaptive brightness - do not restore the value since this setting + # should always be disabled. + raptor.device.shell_output("settings put system screen_brightness_mode 0") + + # Set the screen-off timeout to two (2) hours, since the device will be running + # disconnected, and would otherwise turn off the screen, thereby halting + # execution of the test. Save the current value so we can restore it later + # since it is a persistent change. + raptor.screen_off_timeout = raptor.device.shell_output( + "settings get system screen_off_timeout" + ).strip() + raptor.device.shell_output("settings put system screen_off_timeout 7200000") + + # Set the screen brightness to ~50% for consistency of measurements across + # devices and save its current value to restore it later. Screen brightness + # values range from 0 to 255. + raptor.screen_brightness = raptor.device.shell_output( + "settings get system screen_brightness" + ).strip() + raptor.device.shell_output("settings put system screen_brightness 127") + + raptor.device.shell_output("dumpsys batterystats --reset") + raptor.device.shell_output("dumpsys batterystats --enable full-wake-history") + + filepath = os.path.join(upload_dir, "battery-before.txt") + with open(filepath, "w") as output: + output.write(raptor.device.shell_output("dumpsys battery")) + + raptor.test_start_time = int(time.time()) + + +# The batterystats output for Estimated power use differs +# for Android 7 and Android 8 and later. +# +# Android 7 +# Estimated power use (mAh): +# Capacity: 2100, Computed drain: 625, actual drain: 1197-1218 +# Unaccounted: 572 ( ) +# Uid u0a78: 329 ( cpu=329 ) +# Screen: 190 +# Cell standby: 87.6 ( radio=87.6 ) +# Idle: 4.10 +# Uid 1000: 1.82 ( cpu=0.537 sensor=1.28 ) +# Wifi: 0.800 ( cpu=0.310 wifi=0.490 ) +# Android 8 +# Estimated power use (mAh): +# Capacity: 2700, Computed drain: 145, actual drain: 135-162 +# Screen: 68.2 Excluded from smearing +# Uid u0a208: 61.7 ( cpu=60.5 wifi=1.28 ) Including smearing: 141 ( screen=67.7 proportional=9. ) +# Cell standby: 2.49 ( radio=2.49 ) Excluded from smearing +# Idle: 1.63 Excluded from smearing +# Bluetooth: 0.527 ( cpu=0.00319 bt=0.524 ) Including smearing: 0.574 ( proportional=... ) +# Wifi: 0.423 ( cpu=0.343 wifi=0.0800 ) Including smearing: 0.461 ( proportional=0.0375 ) +# +# For Android 8, the cpu, wifi, screen, and proportional values are available from +# the Uid line for the app. If the test does not run long enough, it +# appears that the screen value from the Uid will be missing, but the +# standalone Screen value is available. +# +# For Android 7, only the cpu value is available from the Uid line. We +# can use the Screen and Wifi values for Android 7 from the Screen +# and Wifi lines, which might include contributions from the system or +# other apps; however, it should still be useful for spotting changes in power +# usage. +# +# If the energy values from the Uid line for Android 8 are available, they +# will be used. If for any reason either/both screen or wifi power is +# missing, the values from the Screen and Wifi lines will be used. +# +# If only the cpu energy value is available, it will be used +# along with the values from the Screen and Wifi lines. + + +def finish_android_power_test(raptor, test_name, os_baseline=False): + upload_dir = os.getenv("MOZ_UPLOAD_DIR") + if not upload_dir: + LOG.critical( + "%s power test ignored because MOZ_UPLOAD_DIR was not set" % test_name + ) + return + # Restore screen_off_timeout and screen brightness. + raptor.device.shell_output( + "settings put system screen_off_timeout %s" % raptor.screen_off_timeout + ) + raptor.device.shell_output( + "settings put system screen_brightness %s" % raptor.screen_brightness + ) + + test_end_time = int(time.time()) + + filepath = os.path.join(upload_dir, "battery-after.txt") + with open(filepath, "w") as output: + output.write(raptor.device.shell_output("dumpsys battery")) + verbose = raptor.device._verbose + raptor.device._verbose = False + filepath = os.path.join(upload_dir, "batterystats.csv") + with open(filepath, "w") as output: + output.write(raptor.device.shell_output("dumpsys batterystats --checkin")) + filepath = os.path.join(upload_dir, "batterystats.txt") + with open(filepath, "w") as output: + batterystats = raptor.device.shell_output("dumpsys batterystats") + output.write(batterystats) + raptor.device._verbose = verbose + + # Get the android version + android_version = raptor.device.shell_output( + "getprop ro.build.version.release" + ).strip() + major_android_version = int(android_version.split(".")[0]) + + estimated_power = False + uid = None + total = cpu = wifi = smearing = screen = proportional = 0 + full_screen = 0 + full_wifi = 0 + re_uid = re.compile(r'proc=([^:]+):"%s"' % raptor.config["binary"]) + re_wifi = re.compile(r".*wifi=([\d.]+).*") + re_cpu = re.compile(r".*cpu=([\d.]+).*") + re_estimated_power = re.compile(r"\s+Estimated power use [(]mAh[)]") + re_proportional = re.compile(r"proportional=([\d.]+)") + re_screen = re.compile(r"screen=([\d.]+)") + re_full_screen = re.compile(r"\s+Screen:\s+([\d.]+)") + re_full_wifi = re.compile(r"\s+Wifi:\s+([\d.]+)") + + re_smear = re.compile(r".*smearing:\s+([\d.]+)\s+.*") + re_power = re.compile( + r"\s+Uid\s+\w+[:]\s+([\d.]+) [(]([\s\w\d.\=]*)(?:([)] " + r"Including smearing:.*)|(?:[)]))" + ) + + batterystats = batterystats.split("\n") + for line in batterystats: + if uid is None and not os_baseline: + # The proc line containing the Uid and app name appears + # before the Estimated power line. + match = re_uid.search(line) + if match: + uid = match.group(1) + re_power = re.compile( + r"\s+Uid %s[:]\s+([\d.]+) [(]([\s\w\d.\=]*)(?:([)] " + r"Including smearing:.*)|(?:[)]))" % uid + ) + continue + if not estimated_power: + # Do not attempt to parse data until we have seen + # Estimated Power in the output. + match = re_estimated_power.match(line) + if match: + estimated_power = True + continue + if full_screen == 0: + match = re_full_screen.match(line) + if match and match.group(1): + full_screen += float(match.group(1)) + continue + if full_wifi == 0: + match = re_full_wifi.match(line) + if match and match.group(1): + full_wifi += float(match.group(1)) + continue + if re_power: + match = re_power.match(line) + if match: + ttotal, breakdown, smear_info = match.groups() + total += float(ttotal) if ttotal else 0 + + cpu_match = re_cpu.match(breakdown) + if cpu_match and cpu_match.group(1): + cpu += float(cpu_match.group(1)) + + wifi_match = re_wifi.match(breakdown) + if wifi_match and wifi_match.group(1): + wifi += float(wifi_match.group(1)) + + if smear_info: + # Smearing and screen power are only + # available on android 8+ + smear_match = re_smear.match(smear_info) + if smear_match and smear_match.group(1): + smearing += float(smear_match.group(1)) + screen_match = re_screen.search(line) + if screen_match and screen_match.group(1): + screen += float(screen_match.group(1)) + prop_match = re_proportional.search(smear_info) + if prop_match and prop_match.group(1): + proportional += float(prop_match.group(1)) + if full_screen and full_wifi and (cpu and wifi and smearing or total): + # Stop parsing batterystats once we have a full set of data. + # If we are running an OS baseline, stop when we've exhausted + # the list of entries. + if not os_baseline: + break + elif line.replace(" ", "") == "": + break + + cpu = total if cpu == 0 else cpu + screen = full_screen if screen == 0 else screen + wifi = full_wifi if wifi == 0 else wifi + + if os_baseline: + uid = "all" + LOG.info( + "power data for uid: %s, cpu: %s, wifi: %s, screen: %s, proportional: %s" + % (uid, cpu, wifi, screen, proportional) + ) + + # Send power data directly to the control-server results handler + # so it can be formatted and output for perfherder ingestion + + power_data = { + "type": "power", + "test": test_name, + "unit": "mAh", + "values": { + "cpu": float(cpu), + "wifi": float(wifi), + "screen": float(screen), + }, + } + + if major_android_version >= 8: + power_data["values"]["proportional"] = float(proportional) + + if os_baseline: + raptor.os_baseline_data = power_data + else: + LOG.info("submitting power data via control server directly") + + raptor.control_server.submit_supporting_data(power_data) + if raptor.os_baseline_data: + # raptor.power_test_time is only used by test_power.py + # for testing power measurement parsing + test_time = raptor.power_test_time + if not test_time: + test_time = float(test_end_time - raptor.test_start_time) / 60 + LOG.info("Approximate power test time %s" % str(test_time)) + + def calculate_pc(power_measure, baseline_measure): + if not baseline_measure: + LOG.error("Power test baseline_measure is Zero.") + return 0 + # pylint --py3k W1619 + return ( + 100 * ((power_measure + baseline_measure) / baseline_measure) + ) - 100 + + pc_power_data = { + "type": "power", + "test": power_data["test"] + "-%change", + "unit": "%", + "values": {}, + } + for power_measure in power_data["values"]: + # pylint --py3k W1619 + pc_power_data["values"][power_measure] = calculate_pc( + (power_data["values"][power_measure] / test_time), + raptor.os_baseline_data["values"][power_measure], + ) + + raptor.control_server.submit_supporting_data(pc_power_data) + raptor.control_server.submit_supporting_data(raptor.os_baseline_data) + + # Generate power bugreport zip + LOG.info("generating power bugreport zip") + raptor.device.command_output(["bugreport", upload_dir]) diff --git a/testing/raptor/raptor/raptor.ini b/testing/raptor/raptor/raptor.ini new file mode 100644 index 0000000000..a12ec13db5 --- /dev/null +++ b/testing/raptor/raptor/raptor.ini @@ -0,0 +1,60 @@ +# raptor benchmark tests +[include:tests/benchmarks/raptor-speedometer.ini] +[include:tests/benchmarks/raptor-youtube-playback-v9.ini] +[include:tests/benchmarks/raptor-youtube-playback-h264.ini] + +# raptor scenario tests +[include:tests/scenario/idle.ini] +[include:tests/scenario/raptor-scn-power-idle.ini] +[include:tests/scenario/raptor-scn-power-idle-bg.ini] + +# raptor unit tests +# this is required for the manifest unit tests +[include:tests/unittests/raptor-tp6-unittest.ini] + +# raptor-browsertime desktop page-load tests +[include:tests/tp6/desktop/browsertime-tp6.ini] + +# raptor-browsertime android page-load tests +[include:tests/tp6/mobile/browsertime-tp6m.ini] + +# raptor-browsertime live page-load tests +[include:tests/tp6/live/browsertime-live.ini] + +# raptor-browsertime benchmark tests +[include:tests/benchmarks/youtube-playback.ini] +[include:tests/benchmarks/unity-webgl-mobile.ini] +[include:tests/benchmarks/speedometer-desktop.ini] +[include:tests/benchmarks/speedometer-mobile.ini] +[include:tests/benchmarks/ares6.ini] +[include:tests/benchmarks/motionmark-animometer.ini] +[include:tests/benchmarks/motionmark-htmlsuite.ini] +[include:tests/benchmarks/stylebench.ini] +[include:tests/benchmarks/wasm-misc.ini] +[include:tests/benchmarks/wasm-misc-baseline.ini] +[include:tests/benchmarks/wasm-misc-optimizing.ini] +[include:tests/benchmarks/webaudio.ini] +[include:tests/benchmarks/wasm-godot.ini] +[include:tests/benchmarks/wasm-godot-baseline.ini] +[include:tests/benchmarks/wasm-godot-optimizing.ini] +[include:tests/benchmarks/sunspider.ini] +[include:tests/benchmarks/assorted-dom.ini] +[include:tests/benchmarks/jetstream2.ini] +[include:tests/benchmarks/matrix-react-bench.ini] +[include:tests/benchmarks/unity-webgl-desktop.ini] +[include:tests/benchmarks/twitch-animation.ini] + +# Fission process switch time test +[include:tests/custom/browsertime-process-switch.ini] + +# First-install pageload test +[include:tests/custom/browsertime-welcome.ini] + +# Interactive raptor-browsertime tests +[include:tests/interactive/browsertime-responsiveness.ini] + +# Local custom browsertime tests +[include:tests/custom/browsertime-custom.ini] + +# Custom grandprix benchmark test +[include:tests/custom/browsertime-grandprix.ini] diff --git a/testing/raptor/raptor/raptor.py b/testing/raptor/raptor/raptor.py new file mode 100644 index 0000000000..96962eb840 --- /dev/null +++ b/testing/raptor/raptor/raptor.py @@ -0,0 +1,204 @@ +#!/usr/bin/env python + +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import os +import shutil +import sys +import tarfile +import traceback + +import mozinfo + +# need this so raptor imports work both from /raptor and via mach +here = os.path.abspath(os.path.dirname(__file__)) + +try: + from mozbuild.base import MozbuildObject + + build = MozbuildObject.from_environment(cwd=here) +except ImportError: + build = None + +from browsertime import BrowsertimeAndroid, BrowsertimeDesktop +from cmdline import CHROMIUM_DISTROS, DESKTOP_APPS, parse_args +from logger.logger import RaptorLogger +from manifest import get_raptor_test_list +from mozlog import commandline +from mozprofile.cli import parse_key_value, parse_preferences +from signal_handler import SignalHandler +from utils import view_gecko_profile_from_raptor +from webextension import ( + WebExtensionAndroid, + WebExtensionDesktopChrome, + WebExtensionFirefox, +) + +LOG = RaptorLogger(component="raptor-main") + + +def main(args=sys.argv[1:]): + args = parse_args() + + args.extra_prefs = parse_preferences(args.extra_prefs or []) + + if args.enable_marionette_trace: + args.extra_prefs.update( + { + "marionette.log.level": "Trace", + } + ) + + args.environment = dict(parse_key_value(args.environment or [], context="--setenv")) + + commandline.setup_logging("raptor", args, {"tbpl": sys.stdout}) + LOG.info("Python version: %s" % sys.version) + LOG.info("raptor-start") + + if args.debug_mode: + LOG.info("debug-mode enabled") + + LOG.info("received command line arguments: %s" % str(args)) + + # if a test name specified on command line, and it exists, just run that one + # otherwise run all available raptor tests that are found for this browser + raptor_test_list = get_raptor_test_list(args, mozinfo.os) + raptor_test_names = [raptor_test["name"] for raptor_test in raptor_test_list] + + # ensure we have at least one valid test to run + if len(raptor_test_list) == 0: + LOG.critical("test '{}' could not be found for {}".format(args.test, args.app)) + sys.exit(1) + + LOG.info("raptor tests scheduled to run:") + for next_test in raptor_test_list: + LOG.info(next_test["name"]) + + if not args.browsertime: + if args.app == "firefox": + raptor_class = WebExtensionFirefox + elif args.app in CHROMIUM_DISTROS: + raptor_class = WebExtensionDesktopChrome + else: + raptor_class = WebExtensionAndroid + else: + + def raptor_class(*inner_args, **inner_kwargs): + outer_kwargs = vars(args) + # peel off arguments that are specific to browsertime + for key in outer_kwargs.keys(): + if key.startswith("browsertime_"): + inner_kwargs[key] = outer_kwargs.get(key) + + if args.app in DESKTOP_APPS: + klass = BrowsertimeDesktop + else: + klass = BrowsertimeAndroid + + return klass(*inner_args, **inner_kwargs) + + try: + raptor = raptor_class( + args.app, + args.binary, + run_local=args.run_local, + noinstall=args.noinstall, + installerpath=args.installerpath, + obj_path=args.obj_path, + gecko_profile=args.gecko_profile, + gecko_profile_interval=args.gecko_profile_interval, + gecko_profile_entries=args.gecko_profile_entries, + gecko_profile_extra_threads=args.gecko_profile_extra_threads, + gecko_profile_threads=args.gecko_profile_threads, + gecko_profile_features=args.gecko_profile_features, + extra_profiler_run=args.extra_profiler_run, + symbols_path=args.symbols_path, + host=args.host, + power_test=args.power_test, + cpu_test=args.cpu_test, + memory_test=args.memory_test, + live_sites=args.live_sites, + cold=args.cold, + is_release_build=args.is_release_build, + debug_mode=args.debug_mode, + post_startup_delay=args.post_startup_delay, + activity=args.activity, + intent=args.intent, + interrupt_handler=SignalHandler(), + extra_prefs=args.extra_prefs or {}, + environment=args.environment or {}, + device_name=args.device_name, + disable_perf_tuning=args.disable_perf_tuning, + conditioned_profile=args.conditioned_profile, + test_bytecode_cache=args.test_bytecode_cache, + chimera=args.chimera, + project=args.project, + verbose=args.verbose, + fission=args.fission, + extra_summary_methods=args.extra_summary_methods, + ) + except Exception: + traceback.print_exc() + LOG.critical( + "TEST-UNEXPECTED-FAIL: could not initialize the raptor test runner" + ) + os.sys.exit(1) + + raptor.results_handler.use_existing_results(args.browsertime_existing_results) + success = raptor.run_tests(raptor_test_list, raptor_test_names) + + if not success: + # if we have results but one test page timed out (i.e. one tp6 test page didn't load + # but others did) we still dumped PERFHERDER_DATA for the successfull pages but we + # want the overall test job to marked as a failure + pages_that_timed_out = raptor.get_page_timeout_list() + if pages_that_timed_out: + for _page in pages_that_timed_out: + message = [ + ("TEST-UNEXPECTED-FAIL", "test '%s'" % _page["test_name"]), + ("timed out loading test page", "waiting for pending metrics"), + ] + if _page.get("pending_metrics") is not None: + LOG.warning( + "page cycle {} has pending metrics: {}".format( + _page["page_cycle"], _page["pending_metrics"] + ) + ) + + LOG.critical( + " ".join("%s: %s" % (subject, msg) for subject, msg in message) + ) + else: + # we want the job to fail when we didn't get any test results + # (due to test timeout/crash/etc.) + LOG.critical( + "TEST-UNEXPECTED-FAIL: no raptor test results were found for %s" + % ", ".join(raptor_test_names) + ) + os.sys.exit(1) + + # if we're running browsertime in the CI, we want to zip the result dir + if args.browsertime and not args.run_local: + result_dir = raptor.results_handler.result_dir() + if os.path.exists(result_dir): + LOG.info("Creating tarball at %s" % result_dir + ".tgz") + with tarfile.open(result_dir + ".tgz", "w:gz") as tar: + tar.add(result_dir, arcname=os.path.basename(result_dir)) + LOG.info("Removing %s" % result_dir) + shutil.rmtree(result_dir) + + # when running raptor locally with gecko profiling on, use the view-gecko-profile + # tool to automatically load the latest gecko profile in profiler.firefox.com + if args.gecko_profile and args.run_local: + if os.environ.get("DISABLE_PROFILE_LAUNCH", "0") == "1": + LOG.info( + "Not launching profiler.firefox.com because DISABLE_PROFILE_LAUNCH=1" + ) + else: + view_gecko_profile_from_raptor() + + +if __name__ == "__main__": + main() diff --git a/testing/raptor/raptor/results.py b/testing/raptor/raptor/results.py new file mode 100644 index 0000000000..864c60d72f --- /dev/null +++ b/testing/raptor/raptor/results.py @@ -0,0 +1,1082 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# class to process, format, and report raptor test results +# received from the raptor control server +import json +import os +import pathlib +import shutil +from abc import ABCMeta, abstractmethod +from collections.abc import Iterable +from io import open + +import six +from logger.logger import RaptorLogger +from output import BrowsertimeOutput, RaptorOutput +from utils import flatten + +LOG = RaptorLogger(component="perftest-results-handler") +KNOWN_TEST_MODIFIERS = [ + "condprof-settled", + "fission", + "live", + "gecko-profile", + "cold", + "webrender", + "bytecode-cached", +] +NON_FIREFOX_OPTS = ("webrender", "bytecode-cached", "fission") +NON_FIREFOX_BROWSERS = ("chrome", "chromium", "safari") +NON_FIREFOX_BROWSERS_MOBILE = ("chrome-m",) + + +@six.add_metaclass(ABCMeta) +class PerftestResultsHandler(object): + """Abstract base class to handle perftest results""" + + def __init__( + self, + gecko_profile=False, + power_test=False, + cpu_test=False, + memory_test=False, + live_sites=False, + app=None, + conditioned_profile=None, + cold=False, + chimera=False, + fission=True, + perfstats=False, + test_bytecode_cache=False, + extra_summary_methods=[], + **kwargs + ): + self.gecko_profile = gecko_profile + self.power_test = power_test + self.cpu_test = cpu_test + self.memory_test = memory_test + self.live_sites = live_sites + self.app = app + self.conditioned_profile = conditioned_profile + self.results = [] + self.page_timeout_list = [] + self.images = [] + self.supporting_data = None + self.fission_enabled = fission + self.browser_version = None + self.browser_name = None + self.cold = cold + self.chimera = chimera + self.perfstats = perfstats + self.test_bytecode_cache = test_bytecode_cache + self.existing_results = None + self.extra_summary_methods = extra_summary_methods + + @abstractmethod + def add(self, new_result_json): + raise NotImplementedError() + + def result_dir(self): + return None + + def build_extra_options(self, modifiers=None): + extra_options = [] + + # If fields is not None, then we default to + # checking all known fields. Otherwise, we only check + # the fields that were given to us. + if modifiers is None: + if self.conditioned_profile: + # Don't add an option/tag for the base test case + if self.conditioned_profile != "settled": + extra_options.append( + "condprof-%s" + % self.conditioned_profile.replace("artifact:", "") + ) + if self.fission_enabled: + extra_options.append("fission") + if self.live_sites: + extra_options.append("live") + if self.gecko_profile: + extra_options.append("gecko-profile") + if self.cold: + extra_options.append("cold") + if self.test_bytecode_cache: + extra_options.append("bytecode-cached") + extra_options.append("webrender") + else: + for modifier, name in modifiers: + if not modifier: + continue + if name in KNOWN_TEST_MODIFIERS: + extra_options.append(name) + else: + raise Exception( + "Unknown test modifier %s was provided as an extra option" + % name + ) + + # Bug 1770225: Make this more dynamic, this will fail us again in the future + self._clean_up_browser_options(extra_options=extra_options) + + return extra_options + + def _clean_up_browser_options(self, extra_options): + """Remove certain firefox specific options from different browsers""" + if self.app.lower() in NON_FIREFOX_BROWSERS + NON_FIREFOX_BROWSERS_MOBILE: + for opts in NON_FIREFOX_OPTS: + if opts in extra_options: + extra_options.remove(opts) + + def add_browser_meta(self, browser_name, browser_version): + # sets the browser metadata for the perfherder data + self.browser_name = browser_name + self.browser_version = browser_version + + def add_image(self, screenshot, test_name, page_cycle): + # add to results + LOG.info("received screenshot") + self.images.append( + {"screenshot": screenshot, "test_name": test_name, "page_cycle": page_cycle} + ) + + def add_page_timeout(self, test_name, page_url, page_cycle, pending_metrics): + timeout_details = { + "test_name": test_name, + "url": page_url, + "page_cycle": page_cycle, + } + if pending_metrics: + pending_metrics = [key for key, value in pending_metrics.items() if value] + timeout_details["pending_metrics"] = ", ".join(pending_metrics) + + self.page_timeout_list.append(timeout_details) + + def add_supporting_data(self, supporting_data): + """Supporting data is additional data gathered outside of the regular + Raptor test run (i.e. power data). Will arrive in a dict in the format of: + + supporting_data = {'type': 'data-type', + 'test': 'raptor-test-ran-when-data-was-gathered', + 'unit': 'unit that the values are in', + 'values': { + 'name': value, + 'nameN': valueN}} + + More specifically, power data will look like this: + + supporting_data = {'type': 'power', + 'test': 'raptor-speedometer-geckoview', + 'unit': 'mAh', + 'values': { + 'cpu': cpu, + 'wifi': wifi, + 'screen': screen, + 'proportional': proportional}} + """ + LOG.info( + "RaptorResultsHandler.add_supporting_data received %s data" + % supporting_data["type"] + ) + if self.supporting_data is None: + self.supporting_data = [] + self.supporting_data.append(supporting_data) + + def use_existing_results(self, directory): + self.existing_results = directory + + def _get_expected_perfherder(self, output): + def is_resource_test(): + if self.power_test or self.cpu_test or self.memory_test: + return True + return False + + # if results exists, determine if any test is of type 'scenario' + is_scenario = False + if output.summarized_results or output.summarized_supporting_data: + data = output.summarized_supporting_data + if not data: + data = [output.summarized_results] + for next_data_set in data: + data_type = next_data_set["suites"][0]["type"] + if data_type == "scenario": + is_scenario = True + break + + if is_scenario and not is_resource_test(): + # skip perfherder check when a scenario test-type is run without + # a resource flag + return None + + expected_perfherder = 1 + + if is_resource_test(): + # when resource tests are run, no perfherder data is output + # for the regular raptor tests (i.e. speedometer) so we + # expect one per resource-type, starting with 0 + expected_perfherder = 0 + if self.power_test: + expected_perfherder += 1 + if self.memory_test: + expected_perfherder += 1 + if self.cpu_test: + expected_perfherder += 1 + + return expected_perfherder + + def _validate_treeherder_data(self, output, output_perfdata): + # late import is required, because install is done in create_virtualenv + import jsonschema + + expected_perfherder = self._get_expected_perfherder(output) + if expected_perfherder is None: + LOG.info( + "Skipping PERFHERDER_DATA check " + "because no perfherder data output is expected" + ) + return True + elif output_perfdata != expected_perfherder: + LOG.critical( + "PERFHERDER_DATA was seen %d times, expected %d." + % (output_perfdata, expected_perfherder) + ) + return False + + external_tools_path = os.environ["EXTERNALTOOLSPATH"] + schema_path = os.path.join( + external_tools_path, "performance-artifact-schema.json" + ) + LOG.info("Validating PERFHERDER_DATA against %s" % schema_path) + try: + with open(schema_path, encoding="utf-8") as f: + schema = json.load(f) + if output.summarized_results: + data = output.summarized_results + else: + data = output.summarized_supporting_data[0] + jsonschema.validate(data, schema) + except Exception as e: + LOG.exception("Error while validating PERFHERDER_DATA") + LOG.info(str(e)) + return False + return True + + @abstractmethod + def summarize_and_output(self, test_config, tests, test_names): + raise NotImplementedError() + + +class RaptorResultsHandler(PerftestResultsHandler): + """Process Raptor results""" + + def add(self, new_result_json): + LOG.info("received results in RaptorResultsHandler.add") + new_result_json.setdefault("extra_options", []).extend( + self.build_extra_options( + [ + ( + self.conditioned_profile, + "condprof-%s" % self.conditioned_profile, + ), + (self.fission_enabled, "fission"), + ] + ) + ) + if self.live_sites: + new_result_json.setdefault("tags", []).append("live") + new_result_json["extra_options"].append("live") + self.results.append(new_result_json) + + def summarize_and_output(self, test_config, tests, test_names): + # summarize the result data, write to file and output PERFHERDER_DATA + LOG.info("summarizing raptor test results") + output = RaptorOutput( + self.results, + self.supporting_data, + test_config.get("subtest_alert_on", []), + self.app, + ) + output.set_browser_meta(self.browser_name, self.browser_version) + output.summarize(test_names) + # that has each browser cycle separate; need to check if there were multiple browser + # cycles, and if so need to combine results from all cycles into one overall result + output.combine_browser_cycles() + output.summarize_screenshots(self.images) + + # only dump out supporting data (i.e. power) if actual Raptor test completed + out_sup_perfdata = 0 + sup_success = True + if self.supporting_data is not None and len(self.results) != 0: + output.summarize_supporting_data() + sup_success, out_sup_perfdata = output.output_supporting_data(test_names) + + success, out_perfdata = output.output(test_names) + + validate_success = True + if not self.gecko_profile: + validate_success = self._validate_treeherder_data( + output, out_sup_perfdata + out_perfdata + ) + + return ( + sup_success and success and validate_success and not self.page_timeout_list + ) + + +class BrowsertimeResultsHandler(PerftestResultsHandler): + """Process Browsertime results""" + + def __init__(self, config, root_results_dir=None): + super(BrowsertimeResultsHandler, self).__init__(**config) + self._root_results_dir = root_results_dir + self.browsertime_visualmetrics = False + self.failed_vismets = [] + if not os.path.exists(self._root_results_dir): + os.mkdir(self._root_results_dir) + + def result_dir(self): + return self._root_results_dir + + def result_dir_for_test(self, test): + if self.existing_results is None: + results_root = self._root_results_dir + else: + results_root = self.existing_results + return os.path.join(results_root, test["name"]) + + def remove_result_dir_for_test(self, test): + test_result_dir = self.result_dir_for_test(test) + if os.path.exists(test_result_dir): + shutil.rmtree(test_result_dir) + return test_result_dir + + def result_dir_for_test_profiling(self, test): + profiling_dir = os.path.join(self.result_dir_for_test(test), "profiling") + if not os.path.exists(profiling_dir): + os.mkdir(profiling_dir) + return profiling_dir + + def add(self, new_result_json): + # not using control server with bt + pass + + def build_tags(self, test={}): + """Use to build the tags option for our perfherder data. + + This should only contain items that will only be shown within + the tags section and excluded from the extra options. + """ + tags = [] + LOG.info(test) + if test.get("interactive", False): + tags.append("interactive") + return tags + + def parse_browsertime_json( + self, + raw_btresults, + page_cycles, + cold, + browser_cycles, + measure, + page_count, + test_name, + accept_zero_vismet, + load_existing, + test_summary, + subtest_name_filters, + handle_custom_data, + ): + """ + Receive a json blob that contains the results direct from the browsertime tool. Parse + out the values that we wish to use and add those to our result object. That object will + then be further processed in the BrowsertimeOutput class. + + The values that we care about in the browsertime.json are structured as follows. + The 'browserScripts' section has one entry for each page-load / browsertime cycle! + + [ + { + "info": { + "browsertime": { + "version": "4.9.2-android" + }, + "url": "https://www.theguardian.co.uk", + }, + "browserScripts": [ + { + "browser": { + "userAgent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.13; rv:70.0) + Gecko/20100101 Firefox/70.0", + "windowSize": "1366x694" + }, + "timings": { + "firstPaint": 830, + "loadEventEnd": 4450, + "timeToContentfulPaint": 932, + "timeToDomContentFlushed": 864, + } + } + }, + { + <repeated for every page-load cycle> + }, + ], + "statistics": { + "timings": { + "firstPaint": { + "median": 668, + "mean": 680, + "mdev": 9.6851, + "stddev": 48, + "min": 616, + "p10": 642, + "p90": 719, + "p99": 830, + "max": 830 + }, + "loadEventEnd": { + "median": 3476, + "mean": 3642, + "mdev": 111.7028, + "stddev": 559, + "min": 3220, + "p10": 3241, + "p90": 4450, + "p99": 5818, + "max": 5818 + }, + "timeToContentfulPaint": { + "median": 758, + "mean": 769, + "mdev": 10.0941, + "stddev": 50, + "min": 712, + "p10": 728, + "p90": 810, + "p99": 932, + "max": 932 + }, + "timeToDomContentFlushed": { + "median": 670, + "mean": 684, + "mdev": 11.6768, + "stddev": 58, + "min": 614, + "p10": 632, + "p90": 738, + "p99": 864, + "max": 864 + }, + } + }, + "geckoPerfStats": [ + { + "Compositing": 71, + "MajorGC": 144 + }, + { + "Compositing": 13, + "MajorGC": 126 + } + ] + } + ] + + For benchmark tests the browserScripts tag will look like: + "browserScripts":[ + { + "browser":{ }, + "pageinfo":{ }, + "timings":{ }, + "custom":{ + "benchmarks":{ + "speedometer":[ + { + "Angular2-TypeScript-TodoMVC":[ + 326.41999999999825, + 238.7799999999952, + 211.88000000000463, + 186.77999999999884, + 191.47999999999593 + ], + etc... + } + ] + } + } + """ + LOG.info("parsing results from browsertime json") + + # bt to raptor names + conversion = ( + ("fnbpaint", "firstPaint"), + ("fcp", ["paintTiming", "first-contentful-paint"]), + ("dcf", "timeToDomContentFlushed"), + ("loadtime", "loadEventEnd"), + ) + + def _get_raptor_val(mdict, mname, retval=False): + # gets the measurement requested, returns the value + # if one was found, or retval if it couldn't be found + # + # mname: either a path to follow (as a list) to get to + # a requested field value, or a string to check + # if mdict contains it. i.e. + # 'first-contentful-paint'/'fcp' is found by searching + # in mdict['paintTiming']. + # mdict: a dictionary to look through to find the mname + # value. + + if type(mname) != list: + if mname in mdict: + return mdict[mname] + return retval + target = mname[-1] + tmpdict = mdict + for name in mname[:-1]: + tmpdict = tmpdict.get(name, {}) + if target in tmpdict: + return tmpdict[target] + + return retval + + results = [] + + # Do some preliminary results validation. When running cold page-load, the results will + # be all in one entry already, as browsertime groups all cold page-load iterations in + # one results entry with all replicates within. When running warm page-load, there will + # be one results entry for every warm page-load iteration; with one single replicate + # inside each. + if cold: + if len(raw_btresults) == 0: + raise MissingResultsError( + "Missing results for all cold browser cycles." + ) + elif load_existing: + pass # Use whatever is there. + else: + if len(raw_btresults) != int(page_cycles): + raise MissingResultsError( + "Missing results for at least 1 warm page-cycle." + ) + + # now parse out the values + page_counter = 0 + for raw_result in raw_btresults: + if not raw_result["browserScripts"]: + raise MissingResultsError("Browsertime cycle produced no measurements.") + + if raw_result["browserScripts"][0].get("timings") is None: + raise MissingResultsError("Browsertime cycle is missing all timings") + + # Desktop chrome doesn't have `browser` scripts data available for now + bt_browser = raw_result["browserScripts"][0].get("browser", None) + bt_ver = raw_result["info"]["browsertime"]["version"] + + # when doing actions, we append a .X for each additional pageload in a scenario + extra = "" + if len(page_count) > 0: + extra = ".%s" % page_count[page_counter % len(page_count)] + url_parts = raw_result["info"]["url"].split("/") + page_counter += 1 + + bt_url = "%s%s/%s," % ("/".join(url_parts[:-1]), extra, url_parts[-1]) + bt_result = { + "bt_ver": bt_ver, + "browser": bt_browser, + "url": (bt_url,), + "name": "%s%s" % (test_name, extra), + "measurements": {}, + "statistics": {}, + } + + if self.power_test: + power_result = { + "bt_ver": bt_ver, + "browser": bt_browser, + "url": bt_url, + "measurements": {}, + "statistics": {}, + "power_data": True, + } + for cycle in raw_result["android"]["power"]: + for metric in cycle: + if "total" in metric: + continue + power_result["measurements"].setdefault(metric, []).append( + cycle[metric] + ) + power_result["statistics"] = raw_result["statistics"]["android"][ + "power" + ] + results.append(power_result) + + custom_types = raw_result["extras"][0] + if custom_types: + for custom_type in custom_types: + data = custom_types[custom_type] + if handle_custom_data: + if test_summary in ("flatten",): + data = flatten(data, ()) + for k, v in data.items(): + + def _ignore_metric(*args): + if any(type(arg) not in (int, float) for arg in args): + return True + return False + + # Ignore any non-numerical results + if _ignore_metric(v) and _ignore_metric(*v): + continue + + # Clean up the name if requested + for name_filter in subtest_name_filters.split(","): + k = k.replace(name_filter, "") + + if isinstance(v, Iterable): + bt_result["measurements"].setdefault(k, []).extend(v) + else: + bt_result["measurements"].setdefault(k, []).append(v) + bt_result["custom_data"] = True + else: + for k, v in data.items(): + bt_result["measurements"].setdefault(k, []).append(v) + if self.perfstats: + for cycle in raw_result["geckoPerfStats"]: + for metric in cycle: + bt_result["measurements"].setdefault( + "perfstat-" + metric, [] + ).append(cycle[metric]) + else: + # extracting values from browserScripts and statistics + for bt, raptor in conversion: + if measure is not None and bt not in measure: + continue + # chrome and safari we just measure fcp and loadtime; skip fnbpaint and dcf + if ( + self.app + and self.app.lower() in NON_FIREFOX_BROWSERS + and bt + in ( + "fnbpaint", + "dcf", + ) + ): + continue + + # FCP uses a different path to get the timing, so we need to do + # some checks here + if bt == "fcp" and not _get_raptor_val( + raw_result["browserScripts"][0]["timings"], + raptor, + ): + continue + + # XXX looping several times in the list, could do better + for cycle in raw_result["browserScripts"]: + if bt not in bt_result["measurements"]: + bt_result["measurements"][bt] = [] + val = _get_raptor_val(cycle["timings"], raptor) + if not val: + raise MissingResultsError( + "Browsertime cycle missing {} measurement".format( + raptor + ) + ) + bt_result["measurements"][bt].append(val) + + # let's add the browsertime statistics; we'll use those for overall values + # instead of calculating our own based on the replicates + bt_result["statistics"][bt] = _get_raptor_val( + raw_result["statistics"]["timings"], raptor, retval={} + ) + + # Bug 1806402 - Handle chrome cpu data properly + cpu_vals = raw_result.get("cpu", None) + if ( + cpu_vals + and self.app + not in NON_FIREFOX_BROWSERS + NON_FIREFOX_BROWSERS_MOBILE + ): + bt_result["measurements"].setdefault("cpuTime", []).extend(cpu_vals) + + if self.perfstats: + for cycle in raw_result["geckoPerfStats"]: + for metric in cycle: + bt_result["measurements"].setdefault( + "perfstat-" + metric, [] + ).append(cycle[metric]) + + if self.browsertime_visualmetrics: + for cycle in raw_result["visualMetrics"]: + for metric in cycle: + if "progress" in metric.lower(): + # Bug 1665750 - Determine if we should display progress + continue + + if metric not in measure: + continue + + val = cycle[metric] + if not accept_zero_vismet: + if val == 0: + self.failed_vismets.append(metric) + continue + + bt_result["measurements"].setdefault(metric, []).append(val) + bt_result["statistics"][metric] = raw_result["statistics"][ + "visualMetrics" + ][metric] + + results.append(bt_result) + + return results + + def _extract_vmetrics( + self, + test_name, + browsertime_json, + json_name="browsertime.json", + tags=[], + extra_options=[], + accept_zero_vismet=False, + ): + # The visual metrics task expects posix paths. + def _normalized_join(*args): + path = os.path.join(*args) + return path.replace(os.path.sep, "/") + + name = browsertime_json.split(os.path.sep)[-2] + reldir = _normalized_join("browsertime-results", name) + + return { + "browsertime_json_path": _normalized_join(reldir, json_name), + "test_name": test_name, + "tags": tags, + "extra_options": extra_options, + "accept_zero_vismet": accept_zero_vismet, + } + + def _label_video_folder(self, result_data, base_dir, kind="warm"): + for filetype in result_data["files"]: + for idx, data in enumerate(result_data["files"][filetype]): + parts = list(pathlib.Path(data).parts) + lable_idx = parts.index("data") + if "query-" in parts[lable_idx - 1]: + lable_idx -= 1 + + src_dir = pathlib.Path(base_dir).joinpath(*parts[: lable_idx + 1]) + parts.insert(lable_idx, kind) + dst_dir = pathlib.Path(base_dir).joinpath(*parts[: lable_idx + 1]) + + if src_dir.exists() and not dst_dir.exists(): + pathlib.Path(dst_dir).mkdir(parents=True, exist_ok=True) + shutil.move(str(src_dir), str(dst_dir)) + + result_data["files"][filetype][idx] = str(pathlib.Path(*parts[:])) + + def summarize_and_output(self, test_config, tests, test_names): + """ + Retrieve, process, and output the browsertime test results. Currently supports page-load + type tests only. + + The Raptor framework either ran a single page-load test (one URL) - or - an entire suite + of page-load tests (multiple test URLs). Regardless, every test URL measured will + have its own 'browsertime.json' results file, located in a sub-folder names after the + Raptor test name, i.e.: + + browsertime-results/ + raptor-tp6-amazon-firefox + browsertime.json + raptor-tp6-facebook-firefox + browsertime.json + raptor-tp6-google-firefox + browsertime.json + raptor-tp6-youtube-firefox + browsertime.json + + For each test URL that was measured, find the resulting 'browsertime.json' file, and pull + out the values that we care about. + """ + # summarize the browsertime result data, write to file and output PERFHERDER_DATA + LOG.info("retrieving browsertime test results") + + # video_jobs is populated with video files produced by browsertime, we + # will send to the visual metrics task + video_jobs = [] + run_local = test_config.get("run_local", False) + + for test in tests: + test_name = test["name"] + accept_zero_vismet = test.get("accept_zero_vismet", False) + + bt_res_json = os.path.join( + self.result_dir_for_test(test), "browsertime.json" + ) + if os.path.exists(bt_res_json): + LOG.info("found browsertime results at %s" % bt_res_json) + else: + LOG.critical("unable to find browsertime results at %s" % bt_res_json) + return False + + try: + with open(bt_res_json, "r", encoding="utf8") as f: + raw_btresults = json.load(f) + except Exception as e: + LOG.error("Exception reading %s" % bt_res_json) + # XXX this should be replaced by a traceback call + LOG.error("Exception: %s %s" % (type(e).__name__, str(e))) + raise + + # Split the chimera videos here for local testing + def split_browsertime_results(result_json_path, raw_btresults): + # First result is cold, second is warm + cold_data = raw_btresults[0] + warm_data = raw_btresults[1] + + dirpath = os.path.dirname(os.path.abspath(result_json_path)) + _cold_path = os.path.join(dirpath, "cold-browsertime.json") + _warm_path = os.path.join(dirpath, "warm-browsertime.json") + + self._label_video_folder(cold_data, dirpath, "cold") + self._label_video_folder(warm_data, dirpath, "warm") + + with open(_cold_path, "w") as f: + json.dump([cold_data], f) + with open(_warm_path, "w") as f: + json.dump([warm_data], f) + + raw_btresults[0] = cold_data + raw_btresults[1] = warm_data + + return _cold_path, _warm_path + + cold_path = None + warm_path = None + if self.chimera: + cold_path, warm_path = split_browsertime_results( + bt_res_json, raw_btresults + ) + + # Overwrite the contents of the browsertime.json file + # to update it with the new file paths + try: + with open(bt_res_json, "w", encoding="utf8") as f: + json.dump(raw_btresults, f) + except Exception as e: + LOG.error("Exception reading %s" % bt_res_json) + # XXX this should be replaced by a traceback call + LOG.error("Exception: %s %s" % (type(e).__name__, str(e))) + raise + + # If extra profiler run is enabled, split its browsertime.json + # file to cold and warm json files as well. + bt_profiling_res_json = os.path.join( + self.result_dir_for_test_profiling(test), "browsertime.json" + ) + has_extra_profiler_run = test_config.get( + "extra_profiler_run", False + ) and os.path.exists(bt_profiling_res_json) + if has_extra_profiler_run: + try: + with open(bt_profiling_res_json, "r", encoding="utf8") as f: + raw_profiling_btresults = json.load(f) + split_browsertime_results( + bt_profiling_res_json, raw_profiling_btresults + ) + except Exception as e: + LOG.info( + "Exception reading and writing %s" % bt_profiling_res_json + ) + LOG.info("Exception: %s %s" % (type(e).__name__, str(e))) + + if not run_local: + extra_options = self.build_extra_options() + tags = self.build_tags(test=test) + + if self.chimera: + if cold_path is None or warm_path is None: + raise Exception("Cold and warm paths were not created") + + video_jobs.append( + self._extract_vmetrics( + test_name, + cold_path, + json_name="cold-browsertime.json", + tags=list(tags), + extra_options=list(extra_options), + accept_zero_vismet=accept_zero_vismet, + ) + ) + + extra_options.remove("cold") + extra_options.append("warm") + video_jobs.append( + self._extract_vmetrics( + test_name, + warm_path, + json_name="warm-browsertime.json", + tags=list(tags), + extra_options=list(extra_options), + accept_zero_vismet=accept_zero_vismet, + ) + ) + else: + video_jobs.append( + self._extract_vmetrics( + test_name, + bt_res_json, + tags=list(tags), + extra_options=list(extra_options), + accept_zero_vismet=accept_zero_vismet, + ) + ) + + for new_result in self.parse_browsertime_json( + raw_btresults, + test["page_cycles"], + test["cold"], + test["browser_cycles"], + test.get("measure"), + test_config.get("page_count", []), + test["name"], + accept_zero_vismet, + self.existing_results is not None, + test.get("test_summary", "pageload"), + test.get("subtest_name_filters", ""), + test.get("custom_data", False) == "true", + ): + + def _new_standard_result(new_result, subtest_unit="ms"): + # add additional info not from the browsertime json + for field in ( + "name", + "unit", + "lower_is_better", + "alert_threshold", + "cold", + ): + if field in new_result: + continue + new_result[field] = test[field] + + # All Browsertime measurements are elapsed times in milliseconds. + new_result["subtest_lower_is_better"] = test.get( + "subtest_lower_is_better", True + ) + new_result["subtest_unit"] = subtest_unit + + new_result["extra_options"] = self.build_extra_options() + new_result.setdefault("tags", []).extend(self.build_tags(test=test)) + + # Split the chimera + if self.chimera and "run=2" in new_result["url"][0]: + new_result["extra_options"].remove("cold") + new_result["extra_options"].append("warm") + + return new_result + + def _new_powertest_result(new_result): + new_result["type"] = "power" + new_result["unit"] = "mAh" + new_result["lower_is_better"] = True + + new_result = _new_standard_result(new_result, subtest_unit="mAh") + new_result["extra_options"].append("power") + + LOG.info("parsed new power result: %s" % str(new_result)) + return new_result + + def _new_pageload_result(new_result): + new_result["type"] = "pageload" + new_result = _new_standard_result(new_result) + + LOG.info("parsed new pageload result: %s" % str(new_result)) + return new_result + + def _new_benchmark_result(new_result): + new_result["type"] = "benchmark" + + new_result = _new_standard_result( + new_result, subtest_unit=test.get("subtest_unit", "ms") + ) + + LOG.info("parsed new benchmark result: %s" % str(new_result)) + return new_result + + def _is_supporting_data(res): + if res.get("power_data", False): + return True + return False + + if new_result.get("power_data", False): + self.results.append(_new_powertest_result(new_result)) + elif test["type"] == "pageload": + self.results.append(_new_pageload_result(new_result)) + elif test["type"] == "benchmark": + for i, item in enumerate(self.results): + if item["name"] == test["name"] and not _is_supporting_data( + item + ): + # add page cycle custom measurements to the existing results + for measurement in six.iteritems( + new_result["measurements"] + ): + self.results[i]["measurements"][measurement[0]].extend( + measurement[1] + ) + break + else: + self.results.append(_new_benchmark_result(new_result)) + + # now have all results gathered from all browsertime test URLs; format them for output + output = BrowsertimeOutput( + self.results, + self.supporting_data, + test_config.get("subtest_alert_on", []), + self.app, + self.extra_summary_methods, + ) + output.set_browser_meta(self.browser_name, self.browser_version) + output.summarize(test_names) + success, out_perfdata = output.output(test_names) + + if len(self.failed_vismets) > 0: + LOG.critical( + "TEST-UNEXPECTED-FAIL | Some visual metrics have an erroneous value of 0." + ) + LOG.info("Visual metric tests failed: %s" % str(self.failed_vismets)) + + validate_success = True + if not self.gecko_profile: + validate_success = self._validate_treeherder_data(output, out_perfdata) + + if len(video_jobs) > 0: + # The video list and application metadata (browser name and + # optionally version) that will be used in the visual metrics task. + jobs_json = { + "jobs": video_jobs, + "application": {"name": self.browser_name}, + "extra_options": output.summarized_results["suites"][0]["extraOptions"], + } + + if self.browser_version is not None: + jobs_json["application"]["version"] = self.browser_version + + jobs_file = os.path.join(self.result_dir(), "jobs.json") + LOG.info( + "Writing video jobs and application data {} into {}".format( + jobs_json, jobs_file + ) + ) + with open(jobs_file, "w") as f: + f.write(json.dumps(jobs_json)) + + return (success and validate_success) and len(self.failed_vismets) == 0 + + +class MissingResultsError(Exception): + pass diff --git a/testing/raptor/raptor/signal_handler.py b/testing/raptor/raptor/signal_handler.py new file mode 100644 index 0000000000..2a7538da62 --- /dev/null +++ b/testing/raptor/raptor/signal_handler.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python + +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import signal + + +class SignalHandler: + def __init__(self): + signal.signal(signal.SIGINT, self.handle_signal) + signal.signal(signal.SIGTERM, self.handle_signal) + + def handle_signal(self, signum, frame): + raise SignalHandlerException("Program aborted due to signal %s" % signum) + + +class SignalHandlerException(Exception): + pass diff --git a/testing/raptor/raptor/tests/benchmarks/ares6.ini b/testing/raptor/raptor/tests/benchmarks/ares6.ini new file mode 100644 index 0000000000..a2cc22d791 --- /dev/null +++ b/testing/raptor/raptor/tests/benchmarks/ares6.ini @@ -0,0 +1,22 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# ARES-6 benchmark for firefox and chromium distributions + +[DEFAULT] +alert_threshold = 2.0 +apps = firefox, chrome, chromium, safari +gecko_profile_entries = 14000000 +gecko_profile_interval = 1 +lower_is_better = true +owner = :jandem and SpiderMonkey Team +page_cycles = 4 +page_timeout = 270000 +subtest_lower_is_better = true +subtest_unit = ms +test_url = http://<host>:<port>/ARES-6/index.html?raptor +type = benchmark +unit = ms + +[ares6] diff --git a/testing/raptor/raptor/tests/benchmarks/assorted-dom.ini b/testing/raptor/raptor/tests/benchmarks/assorted-dom.ini new file mode 100644 index 0000000000..a5e7953bcf --- /dev/null +++ b/testing/raptor/raptor/tests/benchmarks/assorted-dom.ini @@ -0,0 +1,24 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# assorted-dom benchmark for firefox and chromium distributions + +[DEFAULT] +alert_threshold = 2.0 +apps = firefox, chrome, chromium, safari +gecko_profile_entries = 2000000 +gecko_profile_interval = 1 +lower_is_better = true +owner = PerfTest Team +page_cycles = 1 +page_timeout = 60000 +screen_capture = true +test_url = http://<host>:<port>/assorted/driver.html?raptor +type = benchmark +unit = ms + +[assorted-dom] +repository = https://github.com/mozilla/perf-automation +repository_revision = 61332db584026b73e37066d717a162825408c36b +repository_path = benchmarks/assorted-dom diff --git a/testing/raptor/raptor/tests/benchmarks/jetstream2.ini b/testing/raptor/raptor/tests/benchmarks/jetstream2.ini new file mode 100644 index 0000000000..82849d231d --- /dev/null +++ b/testing/raptor/raptor/tests/benchmarks/jetstream2.ini @@ -0,0 +1,25 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# JetStream-2 benchmark for firefox and chromium distributions + +[DEFAULT] +alert_threshold = 2.0 +apps = firefox, chrome, chromium, safari +gecko_profile_entries = 14000000 +gecko_profile_interval = 1 +lower_is_better = false +owner = :jandem and SpiderMonkey Team +page_cycles = 4 +page_timeout = 2000000 +subtest_lower_is_better = false +subtest_unit = score +test_url = http://<host>:<port>/index.html?raptor +type = benchmark +unit = score + +[jetstream2] +repository = https://github.com/mozilla/perf-automation +repository_revision = 61332db584026b73e37066d717a162825408c36b +repository_path = benchmarks/JetStream2 diff --git a/testing/raptor/raptor/tests/benchmarks/matrix-react-bench.ini b/testing/raptor/raptor/tests/benchmarks/matrix-react-bench.ini new file mode 100644 index 0000000000..93851d9492 --- /dev/null +++ b/testing/raptor/raptor/tests/benchmarks/matrix-react-bench.ini @@ -0,0 +1,25 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# Matrix-react-bench benchmark for firefox and chromium distributions + +[DEFAULT] +alert_threshold = 2.0 +apps = firefox, chrome, chromium +gecko_profile_entries = 14000000 +gecko_profile_interval = 1 +lower_is_better = true +owner = :jandem and SpiderMonkey Team +page_cycles = 30 +page_timeout = 2000000 +subtest_lower_is_better = true +subtest_unit = ms +test_url = http://<host>:<port>/matrix_demo.html +type = benchmark +unit = ms + +[matrix-react-bench] +repository = https://github.com/mozilla/perf-automation +repository_revision = 61332db584026b73e37066d717a162825408c36b +repository_path = benchmarks/matrix-react-bench diff --git a/testing/raptor/raptor/tests/benchmarks/motionmark-animometer.ini b/testing/raptor/raptor/tests/benchmarks/motionmark-animometer.ini new file mode 100644 index 0000000000..3fb8935833 --- /dev/null +++ b/testing/raptor/raptor/tests/benchmarks/motionmark-animometer.ini @@ -0,0 +1,20 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# motionmark benchmark for firefox and chromium distributions + +[DEFAULT] +alert_threshold = 2.0 +apps = firefox, chrome, chromium, safari +gecko_profile_entries = 8000000 +gecko_profile_interval = 1 +lower_is_better = false +page_cycles = 1 +page_timeout = 600000 +owner = :jgilbert and Graphics(gfx) Team +test_url = http://<host>:<port>/MotionMark/developer.html?test-interval=15&display=minimal&tiles=big&controller=fixed&frame-rate=30&kalman-process-error=1&kalman-measurement-error=4&time-measurement=performance&suite-name=Animometer&raptor=true&oskey={platform} +type = benchmark +unit = score + +[motionmark-animometer] diff --git a/testing/raptor/raptor/tests/benchmarks/motionmark-htmlsuite.ini b/testing/raptor/raptor/tests/benchmarks/motionmark-htmlsuite.ini new file mode 100644 index 0000000000..83a8110d58 --- /dev/null +++ b/testing/raptor/raptor/tests/benchmarks/motionmark-htmlsuite.ini @@ -0,0 +1,20 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# motionmark benchmark for firefox and chromium distributions + +[DEFAULT] +alert_threshold = 2.0 +apps = firefox, chrome, chromium, safari +gecko_profile_entries = 8000000 +gecko_profile_interval = 1 +lower_is_better = false +owner = :jgilbert and Graphics(gfx) Team +page_cycles = 5 +page_timeout = 600000 +test_url = http://<host>:<port>/MotionMark/developer.html?test-interval=15&display=minimal&tiles=big&controller=fixed&frame-rate=30&kalman-process-error=1&kalman-measurement-error=4&time-measurement=performance&suite-name=HTMLsuite&raptor=true&oskey={platform} +type = benchmark +unit = score + +[motionmark-htmlsuite] diff --git a/testing/raptor/raptor/tests/benchmarks/raptor-speedometer.ini b/testing/raptor/raptor/tests/benchmarks/raptor-speedometer.ini new file mode 100644 index 0000000000..31d1bb5322 --- /dev/null +++ b/testing/raptor/raptor/tests/benchmarks/raptor-speedometer.ini @@ -0,0 +1,24 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# speedometer benchmark for firefox and chromium distributions + +[DEFAULT] +alert_threshold = 2.0 +gecko_profile_entries = 14000000 +gecko_profile_interval = 1 +lower_is_better = false +owner = SpiderMonkey Team +page_cycles = 5 +page_timeout = 180000 +subtest_lower_is_better = true +subtest_unit = ms +test_url = http://<host>:<port>/Speedometer/index.html?raptor +type = benchmark +unit = score + +[raptor-speedometer-geckoview] +apps = geckoview +page_cycles = 1 +page_timeout = 420000 # 7 mins diff --git a/testing/raptor/raptor/tests/benchmarks/raptor-youtube-playback-h264.ini b/testing/raptor/raptor/tests/benchmarks/raptor-youtube-playback-h264.ini new file mode 100644 index 0000000000..4243d374d7 --- /dev/null +++ b/testing/raptor/raptor/tests/benchmarks/raptor-youtube-playback-h264.ini @@ -0,0 +1,56 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# Fullscreen YouTube-playback performance benchmark +# +# Original location of source and media files: +# https://ytlr-cert.appspot.com/2019/main.html?test_type=playbackperf-test + +[DEFAULT] +alert_threshold = 2.0 +apps = firefox +lower_is_better = true +owner = PerfTest Team +page_cycles = 20 +# account for a page-cycle-duration of at maximum 45 minutes +page_timeout = 2700000 +subtest_lower_is_better = true +subtest_unit = score +type = benchmark +unit = score +use_live_sites = true +preferences = {"media.autoplay.default": 0, + "media.autoplay.ask-permission": false, + "media.autoplay.blocking_policy": 0, + "media.autoplay.block-webaudio": false, + "media.allowed-to-play.enabled": true, + "media.block-autoplay-until-in-foreground": false} + +[raptor-youtube-playback-h264-1080p30-firefox] +test_url = https://yttest.prod.mozaws.net/2020/main.html?test_type=playbackperf-sfr-h264-test&tests=18&raptor=true&muted=true&command=run&exclude=1,2 + +[raptor-youtube-playback-h264-1080p60-firefox] +test_url = https://yttest.prod.mozaws.net/2020/main.html?test_type=playbackperf-hfr-test&raptor=true&tests=46&muted=true&command=run&exclude=1,2 + +[raptor-youtube-playback-h264-full-1080p30-firefox] +preferences = {"media.autoplay.default": 0, + "media.autoplay.ask-permission": false, + "media.autoplay.blocking_policy": 0, + "media.autoplay.block-webaudio": false, + "media.allowed-to-play.enabled": true, + "media.block-autoplay-until-in-foreground": false, + "full-screen-api.allow-trusted-requests-only": false, + "full-screen-api.warning.timeout": 0} +test_url = https://yttest.prod.mozaws.net/2020/main.html?test_type=playbackperf-sfr-h264-test&tests=18&raptor=true&muted=true&command=run&fullscreen=true&exclude=1,2 + +[raptor-youtube-playback-h264-full-1080p60-firefox] +preferences = {"media.autoplay.default": 0, + "media.autoplay.ask-permission": false, + "media.autoplay.blocking_policy": 0, + "media.autoplay.block-webaudio": false, + "media.allowed-to-play.enabled": true, + "media.block-autoplay-until-in-foreground": false, + "full-screen-api.allow-trusted-requests-only": false, + "full-screen-api.warning.timeout": 0} +test_url = https://yttest.prod.mozaws.net/2020/main.html?test_type=playbackperf-hfr-test&raptor=true&tests=46&muted=true&command=run&fullscreen=true&exclude=1,2 diff --git a/testing/raptor/raptor/tests/benchmarks/raptor-youtube-playback-v9.ini b/testing/raptor/raptor/tests/benchmarks/raptor-youtube-playback-v9.ini new file mode 100644 index 0000000000..af79803326 --- /dev/null +++ b/testing/raptor/raptor/tests/benchmarks/raptor-youtube-playback-v9.ini @@ -0,0 +1,56 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# Fullscreen YouTube-playback performance benchmark +# +# Original location of source and media files: +# https://ytlr-cert.appspot.com/2019/main.html?test_type=playbackperf-test + +[DEFAULT] +alert_threshold = 2.0 +apps = firefox +lower_is_better = true +owner = PerfTest Team +page_cycles = 20 +# account for a page-cycle-duration of at maximum 45 minutes +page_timeout = 2700000 +subtest_lower_is_better = true +subtest_unit = score +type = benchmark +unit = score +use_live_sites = true +preferences = {"media.autoplay.default": 0, + "media.autoplay.ask-permission": false, + "media.autoplay.blocking_policy": 0, + "media.autoplay.block-webaudio": false, + "media.allowed-to-play.enabled": true, + "media.block-autoplay-until-in-foreground": false} + +[raptor-youtube-playback-v9-1080p30-firefox] +test_url = https://yttest.prod.mozaws.net/2020/main.html?test_type=playbackperf-sfr-vp9-test&raptor=true&tests=18&muted=true&command=run&exclude=1,2 + +[raptor-youtube-playback-v9-1080p60-firefox] +test_url = https://yttest.prod.mozaws.net/2020/main.html?test_type=playbackperf-hfr-test&raptor=true&tests=14&muted=true&command=run&exclude=1,2 + +[raptor-youtube-playback-v9-full-1080p30-firefox] +preferences = {"media.autoplay.default": 0, + "media.autoplay.ask-permission": false, + "media.autoplay.blocking_policy": 0, + "media.autoplay.block-webaudio": false, + "media.allowed-to-play.enabled": true, + "media.block-autoplay-until-in-foreground": false, + "full-screen-api.allow-trusted-requests-only": false, + "full-screen-api.warning.timeout": 0} +test_url = https://yttest.prod.mozaws.net/2020/main.html?test_type=playbackperf-sfr-vp9-test&raptor=true&tests=18&muted=true&command=run&fullscreen=true&exclude=1,2 + +[raptor-youtube-playback-v9-full-1080p60-firefox] +preferences = {"media.autoplay.default": 0, + "media.autoplay.ask-permission": false, + "media.autoplay.blocking_policy": 0, + "media.autoplay.block-webaudio": false, + "media.allowed-to-play.enabled": true, + "media.block-autoplay-until-in-foreground": false, + "full-screen-api.allow-trusted-requests-only": false, + "full-screen-api.warning.timeout": 0} +test_url = https://yttest.prod.mozaws.net/2020/main.html?test_type=playbackperf-hfr-test&raptor=true&tests=14&muted=true&command=run&fullscreen=true&exclude=1,2 diff --git a/testing/raptor/raptor/tests/benchmarks/speedometer-desktop.ini b/testing/raptor/raptor/tests/benchmarks/speedometer-desktop.ini new file mode 100644 index 0000000000..4d3c2951a4 --- /dev/null +++ b/testing/raptor/raptor/tests/benchmarks/speedometer-desktop.ini @@ -0,0 +1,35 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# speedometer benchmark for desktop browsers + +[DEFAULT] +alert_threshold = 2.0 +apps = firefox, chrome, chromium, safari +gecko_profile_entries = 14000000 +gecko_profile_interval = 1 +lower_is_better = false +owner = SpiderMonkey Team +page_cycles = 5 +page_timeout = 180000 +subtest_lower_is_better = true +subtest_unit = ms +test_url = http://<host>:<port>/Speedometer/index.html?raptor +type = benchmark +unit = score + +[speedometer] + +[speedometer3] +browsertime_args = --browsertime.speedometer_iterations=5 +custom_data = true +owner = Performance Team +repository = https://github.com/camillobruni/Speedometer +repository_branch = 2022-12-08_interactive_runner +repository_revision = 9e2d82c75a66cae636db1888e916276075a137b8 +submetric_summary_method = median +subtest_name_filters = tests/,s3/ +test_script = speedometer3.js +test_summary = flatten +test_url = http://<host>:<port>/InteractiveRunner.html diff --git a/testing/raptor/raptor/tests/benchmarks/speedometer-mobile.ini b/testing/raptor/raptor/tests/benchmarks/speedometer-mobile.ini new file mode 100644 index 0000000000..f6f9215dbd --- /dev/null +++ b/testing/raptor/raptor/tests/benchmarks/speedometer-mobile.ini @@ -0,0 +1,35 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# speedometer benchmark for mobile browsers + +[DEFAULT] +alert_threshold = 2.0 +apps = fenix, geckoview, refbrow, chrome-m +gecko_profile_entries = 14000000 +gecko_profile_interval = 1 +lower_is_better = false +owner = SpiderMonkey Team +page_cycles = 5 +page_timeout = 420000 +subtest_lower_is_better = true +subtest_unit = ms +test_url = http://<host>:<port>/Speedometer/index.html?raptor +type = benchmark +unit = score + +[speedometer] + +[speedometer3] +browsertime_args = --browsertime.speedometer_iterations=5 +custom_data = true +owner = Performance Team +repository = https://github.com/camillobruni/Speedometer +repository_branch = 2022-12-08_interactive_runner +repository_revision = 9e2d82c75a66cae636db1888e916276075a137b8 +submetric_summary_method = median +subtest_name_filters = tests/,s3/ +test_script = speedometer3.js +test_summary = flatten +test_url = http://<host>:<port>/InteractiveRunner.html diff --git a/testing/raptor/raptor/tests/benchmarks/stylebench.ini b/testing/raptor/raptor/tests/benchmarks/stylebench.ini new file mode 100644 index 0000000000..3b24d35c2e --- /dev/null +++ b/testing/raptor/raptor/tests/benchmarks/stylebench.ini @@ -0,0 +1,22 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# speedometer benchmark for firefox and chromium distributions + +[DEFAULT] +alert_threshold = 2.0 +apps = firefox, chrome, chromium, safari +gecko_profile_entries = 8000000 +gecko_profile_interval = 1 +lower_is_better = false +owner = :emelio and Layout Team +page_cycles = 5 +page_timeout = 120000 +subtest_lower_is_better = true +subtest_unit = ms +test_url = http://<host>:<port>/StyleBench/index.html?raptor +type = benchmark +unit = score + +[stylebench] diff --git a/testing/raptor/raptor/tests/benchmarks/sunspider.ini b/testing/raptor/raptor/tests/benchmarks/sunspider.ini new file mode 100644 index 0000000000..15ef1eef06 --- /dev/null +++ b/testing/raptor/raptor/tests/benchmarks/sunspider.ini @@ -0,0 +1,20 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# sunspider benchmark for firefox and chromium distributions + +[DEFAULT] +alert_threshold = 2.0 +apps = firefox, chrome, chromium, safari +gecko_profile_entries = 8000000 +gecko_profile_interval = 1 +lower_is_better = true +owner = :jandem and SpiderMonkey Team +page_cycles = 5 +page_timeout = 55000 +test_url = http://<host>:<port>/SunSpider/sunspider-1.0.1/sunspider-1.0.1/driver.html?raptor +type = benchmark +unit = ms + +[sunspider] diff --git a/testing/raptor/raptor/tests/benchmarks/twitch-animation.ini b/testing/raptor/raptor/tests/benchmarks/twitch-animation.ini new file mode 100644 index 0000000000..2dcede8a17 --- /dev/null +++ b/testing/raptor/raptor/tests/benchmarks/twitch-animation.ini @@ -0,0 +1,26 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# Matrix-react-bench benchmark for firefox and chromium distributions + +[DEFAULT] +alert_threshold = 2.0 +apps = firefox +gecko_profile_entries = 14000000 +gecko_profile_interval = 1 +lower_is_better = true +owner = :jrmuizel +page_cycles = 1 +page_timeout = 2000000 +subtest_lower_is_better = true +subtest_unit = ms +test_url = http://<host>:<port>/index.html +type = benchmark +unit = ms +perfstats = true + +[twitch-animation] +repository = https://github.com/mozilla/perf-automation +repository_revision = 61332db584026b73e37066d717a162825408c36b +repository_path = benchmarks/twitch-animation diff --git a/testing/raptor/raptor/tests/benchmarks/unity-webgl-desktop.ini b/testing/raptor/raptor/tests/benchmarks/unity-webgl-desktop.ini new file mode 100644 index 0000000000..86b4a91676 --- /dev/null +++ b/testing/raptor/raptor/tests/benchmarks/unity-webgl-desktop.ini @@ -0,0 +1,23 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# unity-webgl benchmark for firefox and chromium distributions + +[DEFAULT] +alert_threshold = 2.0 +apps = firefox, chrome, chromium, safari +gecko_profile_entries = 8000000 +gecko_profile_interval = 1 +lower_is_better = false +owner = :jgilbert and Graphics(gfx) Team +page_cycles = 5 +page_timeout = 420000 # 7 mins +test_url = http://<host>:<port>/index.html?raptor +type = benchmark +unit = score + +[unity-webgl] +repository = https://github.com/mozilla/perf-automation +repository_revision = 61332db584026b73e37066d717a162825408c36b +repository_path = benchmarks/unity-webgl diff --git a/testing/raptor/raptor/tests/benchmarks/unity-webgl-mobile.ini b/testing/raptor/raptor/tests/benchmarks/unity-webgl-mobile.ini new file mode 100644 index 0000000000..90b07c2155 --- /dev/null +++ b/testing/raptor/raptor/tests/benchmarks/unity-webgl-mobile.ini @@ -0,0 +1,23 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# unity-webgl benchmark for mobile browsers + +[DEFAULT] +alert_threshold = 2.0 +apps = geckoview, refbrow, fenix, chrome-m +gecko_profile_entries = 8000000 +gecko_profile_interval = 1 +lower_is_better = false +owner = :jgilbert and Graphics(gfx) Team +page_cycles = 1 +page_timeout = 420000 # 7 mins +test_url = http://<host>:<port>/index.html?raptor +type = benchmark +unit = score + +[unity-webgl] +repository = https://github.com/mozilla/perf-automation +repository_revision = 61332db584026b73e37066d717a162825408c36b +repository_path = benchmarks/unity-webgl diff --git a/testing/raptor/raptor/tests/benchmarks/wasm-godot-baseline.ini b/testing/raptor/raptor/tests/benchmarks/wasm-godot-baseline.ini new file mode 100644 index 0000000000..845dda3729 --- /dev/null +++ b/testing/raptor/raptor/tests/benchmarks/wasm-godot-baseline.ini @@ -0,0 +1,23 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# Wasm-godot benchmark for firefox and chromium distributions + +[DEFAULT] +alert_threshold = 2.0 +gecko_profile_entries = 8000000 +gecko_profile_interval = 1 +lower_is_better = true +newtab_per_cycle = true +owner = :lth and SpiderMonkey Team +page_cycles = 5 +page_timeout = 120000 +test_url = http://localhost:<port>/wasm-godot/index.html +type = benchmark +unit = ms + +[wasm-godot-baseline] +apps = firefox +preferences = {"javascript.options.wasm_baselinejit": true, + "javascript.options.wasm_optimizingjit": false} diff --git a/testing/raptor/raptor/tests/benchmarks/wasm-godot-optimizing.ini b/testing/raptor/raptor/tests/benchmarks/wasm-godot-optimizing.ini new file mode 100644 index 0000000000..3b598c58d8 --- /dev/null +++ b/testing/raptor/raptor/tests/benchmarks/wasm-godot-optimizing.ini @@ -0,0 +1,23 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# Wasm-godot benchmark for firefox and chromium distributions + +[DEFAULT] +alert_threshold = 2.0 +gecko_profile_entries = 8000000 +gecko_profile_interval = 1 +lower_is_better = true +newtab_per_cycle = true +owner = :lth and SpiderMonkey Team +page_cycles = 5 +page_timeout = 120000 +test_url = http://localhost:<port>/wasm-godot/index.html +type = benchmark +unit = ms + +[wasm-godot-optimizing] +apps = firefox +preferences = {"javascript.options.wasm_baselinejit": false, + "javascript.options.wasm_optimizingjit": true} diff --git a/testing/raptor/raptor/tests/benchmarks/wasm-godot.ini b/testing/raptor/raptor/tests/benchmarks/wasm-godot.ini new file mode 100644 index 0000000000..42a9c9357d --- /dev/null +++ b/testing/raptor/raptor/tests/benchmarks/wasm-godot.ini @@ -0,0 +1,21 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# Wasm-godot benchmark for firefox and chromium distributions + +[DEFAULT] +alert_threshold = 2.0 +apps = firefox, chrome, chromium +gecko_profile_entries = 8000000 +gecko_profile_interval = 1 +lower_is_better = true +newtab_per_cycle = true +owner = :lth and SpiderMonkey Team +page_cycles = 5 +page_timeout = 120000 +test_url = http://localhost:<port>/wasm-godot/index.html +type = benchmark +unit = ms + +[wasm-godot] diff --git a/testing/raptor/raptor/tests/benchmarks/wasm-misc-baseline.ini b/testing/raptor/raptor/tests/benchmarks/wasm-misc-baseline.ini new file mode 100644 index 0000000000..3e09c58e81 --- /dev/null +++ b/testing/raptor/raptor/tests/benchmarks/wasm-misc-baseline.ini @@ -0,0 +1,25 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# Wasm-Misc benchmark for firefox and chromium distributions + +[DEFAULT] +alert_threshold = 2.0 +gecko_profile_entries = 4000000 +gecko_profile_interval = 1 +lower_is_better = true +owner = :lth and SpiderMonkey Team +page_cycles = 5 +page_timeout = 1200000 +test_url = http://<host>:<port>/index.html?raptor +type = benchmark +unit = ms + +[wasm-misc-baseline] +apps = firefox +preferences = {"javascript.options.wasm_baselinejit": true, + "javascript.options.wasm_optimizingjit": false} +repository = https://github.com/mozilla/perf-automation +repository_revision = 61332db584026b73e37066d717a162825408c36b +repository_path = benchmarks/wasm-misc diff --git a/testing/raptor/raptor/tests/benchmarks/wasm-misc-optimizing.ini b/testing/raptor/raptor/tests/benchmarks/wasm-misc-optimizing.ini new file mode 100644 index 0000000000..396d9a4621 --- /dev/null +++ b/testing/raptor/raptor/tests/benchmarks/wasm-misc-optimizing.ini @@ -0,0 +1,25 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# Wasm-Misc benchmark for firefox and chromium distributions + +[DEFAULT] +alert_threshold = 2.0 +gecko_profile_entries = 4000000 +gecko_profile_interval = 1 +lower_is_better = true +owner = :lth and SpiderMonkey Team +page_cycles = 5 +page_timeout = 1200000 +test_url = http://<host>:<port>/index.html?raptor +type = benchmark +unit = ms + +[wasm-misc-optimizing] +apps = firefox +preferences = {"javascript.options.wasm_baselinejit": false, + "javascript.options.wasm_optimizingjit": true} +repository = https://github.com/mozilla/perf-automation +repository_revision = 61332db584026b73e37066d717a162825408c36b +repository_path = benchmarks/wasm-misc diff --git a/testing/raptor/raptor/tests/benchmarks/wasm-misc.ini b/testing/raptor/raptor/tests/benchmarks/wasm-misc.ini new file mode 100644 index 0000000000..30e12e823e --- /dev/null +++ b/testing/raptor/raptor/tests/benchmarks/wasm-misc.ini @@ -0,0 +1,23 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# Wasm-Misc benchmark for firefox and chromium distributions + +[DEFAULT] +alert_threshold = 2.0 +apps = firefox, chrome, chromium +gecko_profile_entries = 4000000 +gecko_profile_interval = 1 +lower_is_better = true +owner = :lth and SpiderMonkey Team +page_cycles = 5 +page_timeout = 1200000 +test_url = http://<host>:<port>/index.html?raptor +type = benchmark +unit = ms + +[wasm-misc] +repository = https://github.com/mozilla/perf-automation +repository_revision = 61332db584026b73e37066d717a162825408c36b +repository_path = benchmarks/wasm-misc diff --git a/testing/raptor/raptor/tests/benchmarks/webaudio.ini b/testing/raptor/raptor/tests/benchmarks/webaudio.ini new file mode 100644 index 0000000000..099a748a9f --- /dev/null +++ b/testing/raptor/raptor/tests/benchmarks/webaudio.ini @@ -0,0 +1,26 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# speedometer benchmark for firefox and chromium distributions + +[DEFAULT] +alert_threshold = 2.0 +apps = firefox, chrome, chromium +gecko_profile_entries = 4000000 +gecko_profile_interval = 1 +lower_is_better = true +owner = :padenot and Media Team +page_cycles = 5 +page_timeout = 360000 +test_url = http://<host>:<port>/webaudio/index.html?raptor +type = benchmark +unit = score +preferences = {"media.autoplay.default": 0, + "media.autoplay.ask-permission": false, + "media.autoplay.blocking_policy": 0, + "media.autoplay.block-webaudio": false, + "media.allowed-to-play.enabled": true, + "media.block-autoplay-until-in-foreground": false} + +[webaudio] diff --git a/testing/raptor/raptor/tests/benchmarks/youtube-playback.ini b/testing/raptor/raptor/tests/benchmarks/youtube-playback.ini new file mode 100644 index 0000000000..83b301df50 --- /dev/null +++ b/testing/raptor/raptor/tests/benchmarks/youtube-playback.ini @@ -0,0 +1,183 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# Youtube playback performance benchmark Browsertime tests +# +# Original location of source and media files: +# https://yttest.prod.mozaws.net/2020/main.html?test_type=playbackperf-hfr-test&raptor=true&exclude=1,2&muted=true&command=run +# +# Bug 1547717 - Cannot override autoplay preference due to GeckoRuntime Settings +# Bug 1554966 - With GeckoView there is no way yet to get around the problem that autoplay is blocked +# As such using muted playback everywhere the numbers across platforms should be closer +# Bug 1618566 - change test_url to production server, once all is done +# test_url = https://yttest.prod.mozaws.net/2019/main.html?test_type=playbackperf-hfr-test&raptor=true&exclude=1,2&muted=true&command=run + +[DEFAULT] +alert_threshold = 2.0 +gecko_profile_entries = 50000000 +gecko_profile_interval = 1000 +gecko_profile_threads = MediaPlayback +lower_is_better = true +owner = PerfTest Team +page_cycles = 1 +# account for a page cycle duration of at maximum 45 minutes +page_timeout = 2700000 +subtest_lower_is_better = true +subtest_unit = score +type = benchmark +unit = score +use_live_sites = true +preferences = {"media.autoplay.default": 0, + "media.autoplay.ask-permission": false, + "media.autoplay.blocking_policy": 0, + "media.autoplay.block-webaudio": false, + "media.allowed-to-play.enabled": true, + "media.block-autoplay-until-in-foreground": false} + +[youtube-playback] +alert_on = H264.1080p30@1X_dropped_frames, + H264.1080p60@1X_dropped_frames, + H264.1440p30@1X_dropped_frames, + H264.144p15@1X_dropped_frames, + H264.2160p30@1X_dropped_frames, + H264.240p30@1X_dropped_frames, + H264.360p30@1X_dropped_frames, + H264.480p30@1X_dropped_frames, + H264.720p30@1X_dropped_frames, + H264.720p60@1X_dropped_frames, + VP9.1080p30@1X_dropped_frames, + VP9.1080p60@1X_dropped_frames, + VP9.1440p30@1X_dropped_frames, + VP9.1440p60@1X_dropped_frames, + VP9.144p30@1X_dropped_frames, + VP9.2160p30@1X_dropped_frames, + VP9.2160p60@1X_dropped_frames, + VP9.240p30@1X_dropped_frames, + VP9.360p30@1X_dropped_frames, + VP9.480p30@1X_dropped_frames, + VP9.720p30@1X_dropped_frames, + VP9.720p60@1X_dropped_frames +apps = firefox, geckoview, fenix,refbrow, chrome +test_url = http://yttest.prod.mozaws.net/2019/main.html?test_type=playbackperf-test&raptor=true&command=run&exclude=1,2&muted=true + +[youtube-playback-av1-sfr] +apps = firefox, geckoview, fenix, refbrow, chrome +test_url = https://yttest.prod.mozaws.net/2020/main.html?test_type=playbackperf-sfr-av1-test&raptor=true&exclude=1,2&muted=true&command=run + +[youtube-playback-h264-1080p30] +apps = firefox +test_url = https://yttest.prod.mozaws.net/2020/main.html?test_type=playbackperf-sfr-h264-test&tests=18&raptor=true&muted=true&command=run&exclude=1,2 + +[youtube-playback-h264-1080p60] +apps = firefox +test_url = https://yttest.prod.mozaws.net/2020/main.html?test_type=playbackperf-hfr-test&raptor=true&tests=46&muted=true&command=run&exclude=1,2 + +[youtube-playback-h264-full-1080p30] +apps = firefox +preferences = {"media.autoplay.default": 0, + "media.autoplay.ask-permission": false, + "media.autoplay.blocking_policy": 0, + "media.autoplay.block-webaudio": false, + "media.allowed-to-play.enabled": true, + "media.block-autoplay-until-in-foreground": false, + "full-screen-api.allow-trusted-requests-only": false, + "full-screen-api.warning.timeout": 0} +test_url = https://yttest.prod.mozaws.net/2020/main.html?test_type=playbackperf-sfr-h264-test&tests=18&raptor=true&muted=true&command=run&fullscreen=true&exclude=1,2 + +[youtube-playback-h264-full-1080p60] +apps = firefox +preferences = {"media.autoplay.default": 0, + "media.autoplay.ask-permission": false, + "media.autoplay.blocking_policy": 0, + "media.autoplay.block-webaudio": false, + "media.allowed-to-play.enabled": true, + "media.block-autoplay-until-in-foreground": false, + "full-screen-api.allow-trusted-requests-only": false, + "full-screen-api.warning.timeout": 0} +test_url = https://yttest.prod.mozaws.net/2020/main.html?test_type=playbackperf-hfr-test&raptor=true&tests=46&muted=true&command=run&fullscreen=true&exclude=1,2 + +[youtube-playback-h264-sfr] +apps = firefox, geckoview, fenix, refbrow, chrome +test_url = https://yttest.prod.mozaws.net/2020/main.html?test_type=playbackperf-sfr-h264-test&raptor=true&exclude=1,2&muted=true&command=run + +[youtube-playback-hfr] +alert_on = H2641080p60fps@1X_dropped_frames, + H264720p60fps@1X_dropped_frames +apps = firefox, geckoview, fenix, refbrow, chrome +test_url = https://yttest.prod.mozaws.net/2020/main.html?test_type=playbackperf-hfr-test&raptor=true&exclude=1,2&muted=true&command=run + +[youtube-playback-v9-1080p30] +apps = firefox +test_url = https://yttest.prod.mozaws.net/2020/main.html?test_type=playbackperf-sfr-vp9-test&raptor=true&tests=18&muted=true&command=run&exclude=1,2 + +[youtube-playback-v9-1080p60] +apps = firefox +test_url = https://yttest.prod.mozaws.net/2020/main.html?test_type=playbackperf-hfr-test&raptor=true&tests=14&muted=true&command=run&exclude=1,2 + +[youtube-playback-v9-full-1080p30] +apps = firefox +preferences = {"media.autoplay.default": 0, + "media.autoplay.ask-permission": false, + "media.autoplay.blocking_policy": 0, + "media.autoplay.block-webaudio": false, + "media.allowed-to-play.enabled": true, + "media.block-autoplay-until-in-foreground": false, + "full-screen-api.allow-trusted-requests-only": false, + "full-screen-api.warning.timeout": 0} +test_url = https://yttest.prod.mozaws.net/2020/main.html?test_type=playbackperf-sfr-vp9-test&raptor=true&tests=18&muted=true&command=run&fullscreen=true&exclude=1,2 + +[youtube-playback-v9-full-1080p60] +apps = firefox +preferences = {"media.autoplay.default": 0, + "media.autoplay.ask-permission": false, + "media.autoplay.blocking_policy": 0, + "media.autoplay.block-webaudio": false, + "media.allowed-to-play.enabled": true, + "media.block-autoplay-until-in-foreground": false, + "full-screen-api.allow-trusted-requests-only": false, + "full-screen-api.warning.timeout": 0} +test_url = https://yttest.prod.mozaws.net/2020/main.html?test_type=playbackperf-hfr-test&raptor=true&tests=14&muted=true&command=run&fullscreen=true&exclude=1,2 + +[youtube-playback-vp9-sfr] +apps = firefox, geckoview, fenix, refbrow, chrome +test_url = https://yttest.prod.mozaws.net/2020/main.html?test_type=playbackperf-sfr-vp9-test&raptor=true&exclude=1,2&muted=true&command=run + +[youtube-playback-widevine-h264-sfr] +apps = firefox, geckoview, fenix, refbrow, chrome +preferences = {"media.autoplay.default": 0, + "media.autoplay.ask-permission": false, + "media.autoplay.blocking_policy": 0, + "media.autoplay.block-webaudio": false, + "media.allowed-to-play.enabled": true, + "media.block-autoplay-until-in-foreground": false, + "media.eme.enabled": true, + "media.gmp-manager.updateEnabled": true, + "media.eme.require-app-approval": false} +test_url = https://yttest.prod.mozaws.net/2020/main.html?test_type=playbackperf-widevine-sfr-h264-test&raptor=true&exclude=1,2&muted=true&command=run + +[youtube-playback-widevine-hfr] +apps = firefox, geckoview, fenix, refbrow, chrome +preferences = {"media.autoplay.default": 0, + "media.autoplay.ask-permission": false, + "media.autoplay.blocking_policy": 0, + "media.autoplay.block-webaudio": false, + "media.allowed-to-play.enabled": true, + "media.block-autoplay-until-in-foreground": false, + "media.eme.enabled": true, + "media.gmp-manager.updateEnabled": true, + "media.eme.require-app-approval": false} +test_url = https://yttest.prod.mozaws.net/2020/main.html?test_type=playbackperf-widevine-hfr-test&raptor=true&exclude=1,2&muted=true&command=run + +[youtube-playback-widevine-vp9-sfr] +apps = firefox, geckoview, fenix, refbrow, chrome +preferences = {"media.autoplay.default": 0, + "media.autoplay.ask-permission": false, + "media.autoplay.blocking_policy": 0, + "media.autoplay.block-webaudio": false, + "media.allowed-to-play.enabled": true, + "media.block-autoplay-until-in-foreground": false, + "media.eme.enabled": true, + "media.gmp-manager.updateEnabled": true, + "media.eme.require-app-approval": false} +test_url = https://yttest.prod.mozaws.net/2020/main.html?test_type=playbackperf-widevine-sfr-vp9-test&raptor=true&exclude=1,2&muted=true&command=run diff --git a/testing/raptor/raptor/tests/custom/browsertime-custom.ini b/testing/raptor/raptor/tests/custom/browsertime-custom.ini new file mode 100644 index 0000000000..3b02de3bce --- /dev/null +++ b/testing/raptor/raptor/tests/custom/browsertime-custom.ini @@ -0,0 +1,29 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +[DEFAULT] +alert_on = fcp, loadtime +alert_threshold = 2.0 +apps = firefox +browser_cycles = 1 +gecko_profile_entries = 14000000 +gecko_profile_interval = 1 +lower_is_better = true +measure = fnbpaint, fcp, dcf, loadtime +owner = PerfTest Team +page_cycles = 1 +page_timeout = 60000 +playback = mitmproxy +playback_version = 8.1.1 +type = pageload +unit = ms +use_live_sites = true + +# Use this to run a custom browsertime test locally (with a custom url). +# Essentially, this is "vanilla" browsertime in a raptor wrapper. + +[browsertime] +playback_pageset_manifest = null.manifest +test_script = None +test_url = None diff --git a/testing/raptor/raptor/tests/custom/browsertime-grandprix.ini b/testing/raptor/raptor/tests/custom/browsertime-grandprix.ini new file mode 100644 index 0000000000..408620f130 --- /dev/null +++ b/testing/raptor/raptor/tests/custom/browsertime-grandprix.ini @@ -0,0 +1,29 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +[DEFAULT] +apps = firefox, chrome, chromium, safari +alert_threshold = 2.0 +browser_cycles = 1 +gecko_profile_entries = 14000000 +gecko_profile_interval = 1 +lower_is_better = true +owner = PerfTest Team +page_cycles = 1 +page_timeout = 1800000 +output_timeout = 2000000 +subtest_unit = ms +type = pageload +unit = score +use_live_sites = true + +# grandprix custom benchmark test + +[grandprix] +browsertime_args = --browsertime.grandprix_iterations=25 +custom_data = true +subtest_name_filters = tests/,iterations/ +test_script = grandprix.js +test_summary = flatten +test_url = None diff --git a/testing/raptor/raptor/tests/custom/browsertime-process-switch.ini b/testing/raptor/raptor/tests/custom/browsertime-process-switch.ini new file mode 100644 index 0000000000..ff008932eb --- /dev/null +++ b/testing/raptor/raptor/tests/custom/browsertime-process-switch.ini @@ -0,0 +1,29 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +[DEFAULT] +alert_on = fcp, loadtime, ContentfulSpeedIndex, PerceptualSpeedIndex, SpeedIndex, FirstVisualChange, LastVisualChange +alert_threshold = 2.0 +apps = firefox, chrome, chromium +browser_cycles = 25 +gecko_profile_entries = 14000000 +gecko_profile_interval = 1 +lower_is_better = true +owner = PerfTest Team +page_cycles = 25 +page_timeout = 60000 +playback = mitmproxy +playback_version = 5.1.1 +type = pageload +unit = ms +use_live_sites = false + +# raptor-browsertime fission process switch time test + +[process-switch] +accept_zero_vismet = true +browsertime_args = --pageCompleteWaitTime=1000 --pageCompleteCheckInactivity=true +playback_pageset_manifest = mitm5-linux-firefox-proc-switch.manifest +test_script = process_switch.js +test_url = https://mozilla.seanfeng.dev/files/red.html,https://mozilla.pettay.fi/moztests/blue.html diff --git a/testing/raptor/raptor/tests/custom/browsertime-welcome.ini b/testing/raptor/raptor/tests/custom/browsertime-welcome.ini new file mode 100644 index 0000000000..97086fecc1 --- /dev/null +++ b/testing/raptor/raptor/tests/custom/browsertime-welcome.ini @@ -0,0 +1,27 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +[DEFAULT] +alert_on = fcp, loadtime, ContentfulSpeedIndex, PerceptualSpeedIndex, SpeedIndex, FirstVisualChange, LastVisualChange +alert_threshold = 2.0 +apps = firefox +browser_cycles = 25 +gecko_profile_entries = 14000000 +gecko_profile_interval = 1 +lower_is_better = true +owner = PerfTest Team +page_cycles = 25 +page_timeout = 60000 +playback = mitmproxy +playback_version = 8.1.1 +type = pageload +unit = ms +use_live_sites = false + +# raptor-browsertime first-install about:welcome pageload test + +[welcome] +playback_pageset_manifest = mitm5-linux-firefox-welcome.manifest +test_script = welcome.js +test_url = about:welcome diff --git a/testing/raptor/raptor/tests/interactive/browsertime-responsiveness.ini b/testing/raptor/raptor/tests/interactive/browsertime-responsiveness.ini new file mode 100644 index 0000000000..9df1ac9366 --- /dev/null +++ b/testing/raptor/raptor/tests/interactive/browsertime-responsiveness.ini @@ -0,0 +1,123 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# Raptor-browsertime interactive responsiveness tests + +[DEFAULT] +accept_zero_vismet = true +alert_on = fcp, loadtime, ContentfulSpeedIndex, PerceptualSpeedIndex, SpeedIndex, FirstVisualChange, LastVisualChange +alert_threshold = 2.0 +apps = firefox, chrome, chromium, safari +browser_cycles = 25 +gecko_profile_entries = 14000000 +gecko_profile_interval = 1 +interactive = true +lower_is_better = true +owner = PerfTest Team +page_cycles = 25 +page_timeout = 60000 +playback = mitmproxy +playback_version = 8.1.1 +type = pageload +unit = ms +use_live_sites = false + +# Keep this list in alphabetical order +# Do not use `measure.start(URL)` in interactive tests if they need to be recorded, +# see bug 1737822 for more information. Instead, use `navigate(URL)` after starting +# calling `measure.start(ALIAS)`. + +[cnn-nav] +browser_cycles = 10 # used with --cold +playback_pageset_manifest = mitm7-linux-firefox-cnn-nav.manifest +test_cmds = + ["measure.start", "landing"], + ["navigate", "https://www.cnn.com"], + ["wait.byTime", 4000], + ["measure.stop", ""], + ["measure.start", "world"], + ["click.byXpathAndWait", "/html/body/div[5]/div/div/header/div/div[1]/div/div[2]/nav/ul/li[2]/a"], + ["wait.byTime", 1000], + ["measure.stop", ""], +test_url = https://www.cnn.com/ + +[facebook-nav] +browser_cycles = 10 # used with --cold +page_timeout = 90000 +playback_pageset_manifest = mitm6-windows-firefox-facebook-nav.manifest +test_cmds = + ["measure.start", "landing"], + ["navigate", "https://www.facebook.com/"], + ["wait.byTime", 5000], + ["measure.stop", ""], + ["measure.start", "marketplace"], + ["navigate", "https://www.facebook.com/marketplace"], + ["wait.byTime", 5000], + ["measure.stop", ""], + ["measure.start", "groups"], + ["navigate", "https://www.facebook.com/groups/discover/"], + ["wait.byTime", 5000], + ["measure.stop", ""], + ["measure.start", "friends"], + ["navigate", "https://www.facebook.com/friends/"], + ["wait.byTime", 5000], + ["measure.stop", ""], +test_url = https://www.facebook.com + +[reddit-billgates-ama] +page_timeout = 240000 # at most 4 minutes required per browser cycle +browser_cycles = 10 # used with --cold +playback_pageset_manifest = mitm6-windows-firefox-reddit-billgates-ama.manifest +test_cmds = + ["measure.start", "billg-ama"], + ["navigate", "https://www.reddit.com/r/IAmA/comments/m8n4vt/im_bill_gates_cochair_of_the_bill_and_melinda/"], + ["wait.byTime", 5000], + ["measure.stop", ""], + ["measure.start", "members"], + ["click.byXpathAndWait", "/html/body/div[1]/div/div[2]/div[2]/div/div[3]/div[2]/div/div[1]/div/div[4]/div[1]/div"], + ["wait.byTime", 1000], + ["measure.stop", ""], +test_url = https://www.reddit.com/ + +[reddit-billgates-post-1] +browser_cycles = 10 # used with --cold +page_timeout = 90000 +playback_pageset_manifest = mitm6-windows-firefox-reddit-billgates-post.manifest +test_cmds = + ["measure.start", "billg"], + ["navigate", "https://www.reddit.com/user/thisisbillgates/"], + ["wait.byTime", 500], + ["measure.stop", ""], + ["measure.start", "posts"], + ["click.byXpathAndWait", "/html/body/div[1]/div/div[2]/div[2]/div/div/div/div[2]/div[2]/div/div/div/a[2]"], + ["wait.byTime", 500], + ["measure.stop", ""], + ["measure.start", "comments"], + ["click.byXpathAndWait", "/html/body/div[1]/div/div[2]/div[2]/div/div/div/div[2]/div[2]/div/div/div/a[3]"], + ["wait.byTime", 500], + ["measure.stop", ""], + ["wait.byTime", 500], +test_url = https://www.reddit.com/user/thisisbillgates/ + +[reddit-billgates-post-2] +browser_cycles = 10 # used with --cold +page_timeout = 90000 +playback_pageset_manifest = mitm6-windows-firefox-reddit-billgates-post.manifest +test_cmds = + ["measure.start", "billg"], + ["navigate", "https://www.reddit.com/user/thisisbillgates/"], + ["wait.byTime", 500], + ["measure.stop", ""], + ["click.byXpathAndWait", "/html/body/div[1]/div/div[2]/div[2]/div/div/div/div[2]/div[2]/div/div/div/a[3]"], + ["wait.byTime", 500], + ["measure.start", "hot"], + ["click.byXpathAndWait", "/html/body/div[1]/div/div[2]/div[2]/div/div/div/div[2]/div[4]/div[1]/div[1]/div[2]/a[2]"], + ["wait.byTime", 500], + ["measure.stop", ""], + ["measure.start", "top"], + ["click.byXpathAndWait", "/html/body/div[1]/div/div[2]/div[2]/div/div/div/div[2]/div[4]/div[1]/div[1]/div[2]/a[3]"], + ["wait.byTime", 500], + ["measure.stop", ""], + ["wait.byTime", 500], +test_url = https://www.reddit.com/user/thisisbillgates/ diff --git a/testing/raptor/raptor/tests/scenario/idle.ini b/testing/raptor/raptor/tests/scenario/idle.ini new file mode 100644 index 0000000000..cfa7fa6400 --- /dev/null +++ b/testing/raptor/raptor/tests/scenario/idle.ini @@ -0,0 +1,16 @@ +[DEFAULT] +alert_threshold = 2.0 +apps = fenix, geckoview, refbrow +lower_is_better = true +measure = fakeMeasure +owner = PerfTest Team +page_cycles = 1 +page_timeout = 1320000 +scenario_time = 1200000 +test_url = about:blank +type = scenario +unit = scenarioComplete + +[idle] +[idle-bg] +browsertime_args = --browsertime.scenario_time=60000 --browsertime.background_app=false diff --git a/testing/raptor/raptor/tests/scenario/raptor-scn-power-idle-bg.ini b/testing/raptor/raptor/tests/scenario/raptor-scn-power-idle-bg.ini new file mode 100644 index 0000000000..1d9301c8eb --- /dev/null +++ b/testing/raptor/raptor/tests/scenario/raptor-scn-power-idle-bg.ini @@ -0,0 +1,21 @@ +[DEFAULT] +alert_threshold = 2.0 +background_test = true +lower_is_better = true +measure = fakeMeasure +owner = PerfTest Team +page_cycles = 1 +page_timeout = 660000 +scenario_time = 600000 +test_url = about:blank +type = scenario +unit = scenarioComplete + +[raptor-scn-power-idle-bg-fenix] +apps = fenix + +[raptor-scn-power-idle-bg-geckoview] +apps = geckoview + +[raptor-scn-power-idle-bg-refbrow] +apps = refbrow diff --git a/testing/raptor/raptor/tests/scenario/raptor-scn-power-idle.ini b/testing/raptor/raptor/tests/scenario/raptor-scn-power-idle.ini new file mode 100644 index 0000000000..3e27709506 --- /dev/null +++ b/testing/raptor/raptor/tests/scenario/raptor-scn-power-idle.ini @@ -0,0 +1,20 @@ +[DEFAULT] +alert_threshold = 2.0 +lower_is_better = true +measure = fakeMeasure +owner = PerfTest Team +page_cycles = 1 +page_timeout = 1320000 +scenario_time = 1200000 +test_url = about:blank +type = scenario +unit = scenarioComplete + +[raptor-scn-power-idle-fenix] +apps = fenix + +[raptor-scn-power-idle-geckoview] +apps = geckoview + +[raptor-scn-power-idle-refbrow] +apps = refbrow diff --git a/testing/raptor/raptor/tests/tp6/desktop/browsertime-tp6.ini b/testing/raptor/raptor/tests/tp6/desktop/browsertime-tp6.ini new file mode 100644 index 0000000000..c010d228ea --- /dev/null +++ b/testing/raptor/raptor/tests/tp6/desktop/browsertime-tp6.ini @@ -0,0 +1,176 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# raptor-browsertime tp6 desktop page-load test specification + +[DEFAULT] +alert_on = fcp, loadtime, ContentfulSpeedIndex, PerceptualSpeedIndex, SpeedIndex, FirstVisualChange, LastVisualChange +alert_threshold = 2.0 +apps = firefox, chrome, chromium, safari +browser_cycles = 25 +gecko_profile_entries = 14000000 +gecko_profile_interval = 1 +lower_is_better = true +owner = PerfTest Team +page_cycles = 25 +page_timeout = 60000 +playback = mitmproxy +playback_pageset_manifest = mitm7-linux-firefox-{subtest}.manifest +playback_version = 8.1.1 +type = pageload +unit = ms +use_live_sites = false + +# Keep this list in alphabetical order + +[amazon] +test_url = https://www.amazon.com/s?k=laptop&ref=nb_sb_noss_1 +secondary_url = https://www.amazon.com/Acer-A515-46-R14K-Quad-Core-Processor-Backlit/dp/B08VKNVDDR/ref=sr_1_3?dchild=1&keywords=laptop&qid=1627047187&sr=8-3 + +[bing-search] +playback_pageset_manifest = mitm5-linux-firefox-bing-search.manifest +test_url = https://www.bing.com/search?q=barack+obama + +[buzzfeed] +secondary_url = https://www.buzzfeed.com/quizzes +test_url = https://www.buzzfeed.com/ + +[cnn] +secondary_url = https://www.cnn.com/weather +test_url = https://www.cnn.com/2021/03/22/weather/climate-change-warm-waters-lake-michigan/index.html + +[ebay] +secondary_url = https://www.ebay.com/deals +test_url = https://www.ebay.com/ + +[espn] +test_url = http://www.espn.com/nba/story/_/page/allstarweekend25788027/the-comparison-lebron-james-michael-jordan-their-own-words +secondary_url = https://www.espn.com/nba/draft/news + +[expedia] +secondary_url = https://groups.expedia.com/Group-Rate/?locale=en_US&ol=1 +test_url = https://expedia.com/Hotel-Search?destination=New+York%2C+New+York&latLong=40.756680%2C-73.986470®ionId=178293&startDate=&endDate=&rooms=1&_xpid=11905%7C1&adults=2 + +[facebook] +playback_pageset_manifest = mitm6-linux-firefox-facebook.manifest +secondary_url = https://www.facebook.com/marketplace/?ref=bookmark +test_url = https://www.facebook.com + +[fandom] +playback_pageset_manifest = mitm5-linux-firefox-fandom.manifest +test_url = https://www.fandom.com/articles/fallout-76-will-live-and-die-on-the-creativity-of-its-playerbase + +[google-docs] +page_complete_wait_time = 8000 +secondary_url = https://docs.google.com/document/d/1vUnn0ePU-ynArE1OdxyEHXR2G0sl74ja_st_4OOzlgE/preview +test_url = https://docs.google.com/document/d/1US-07msg12slQtI_xchzYxcKlTs6Fp7WqIc6W5GK5M8/edit?usp=sharing + +[google-mail] +playback_pageset_manifest = mitm5-linux-firefox-google-mail.manifest +test_url = https://mail.google.com/ + +[google-search] +playback_pageset_manifest = mitm5-linux-firefox-google-search.manifest +test_url = https://www.google.com/search?hl=en&q=barack+obama&cad=h + +[google-slides] +playback_pageset_manifest = mitm6-linux-firefox-google-slides.manifest +playback_version = 6.0.2 +secondary_url = https://docs.google.com/document/d/1vUnn0ePU-ynArE1OdxyEHXR2G0sl74ja_st_4OOzlgE/preview +test_url = https://docs.google.com/presentation/d/1Ici0ceWwpFvmIb3EmKeWSq_vAQdmmdFcWqaiLqUkJng/edit?usp=sharing + +[imdb] +test_url = https://www.imdb.com/title/tt0084967/?ref_=nv_sr_2 +secondary_url = https://www.imdb.com/title/tt0084967/episodes/?ref_=tt_ov_epl + +[imgur] +playback_pageset_manifest = mitm7-linux-firefox-imgur.manifest +preferences = {"media.autoplay.default": 5, + "media.autoplay.ask-permission": true, + "media.autoplay.blocking_policy": 1, + "media.autoplay.block-webaudio": true, + "media.allowed-to-play.enabled": false, + "media.block-autoplay-until-in-foreground": true} +secondary_url = https://imgur.com/gallery/L13Ci +test_url = https://imgur.com/gallery/m5tYJL6 + +[instagram] +playback_pageset_manifest = mitm6-linux-firefox-instagram.manifest +secondary_url = https://www.instagram.com/nobelprize_org/ +test_url = https://www.instagram.com/ + +[linkedin] +playback_pageset_manifest = mitm6-linux-firefox-linkedin.manifest +secondary_url = https://www.linkedin.com/in/thommy-harris-hk-385723106/ +test_url = https://www.linkedin.com/feed/ + +[microsoft] +secondary_url = https://support.microsoft.com/en-us +test_url = https://www.microsoft.com/en-us/ + +[netflix] +playback_pageset_manifest = mitm6-linux-firefox-netflix.manifest +secondary_url = https://www.netflix.com/title/699257 +test_url = https://www.netflix.com/title/80117263 + +[nytimes] +secondary_url = https://www.nytimes.com/section/opinion/columnists +test_url = https://www.nytimes.com/2020/02/19/opinion/surprise-medical-bill.html + +[office] +secondary_url = https://www.office.com/ +test_url = https://www.office.com/launch/powerpoint?auth=1 + +[outlook] +playback_pageset_manifest = mitm5-linux-firefox-live.manifest +test_url = https://outlook.live.com/mail/inbox + +[paypal] +playback_pageset_manifest = mitm5-linux-firefox-paypal.manifest +test_url = https://www.paypal.com/myaccount/summary/ + +[pinterest] +playback_pageset_manifest = mitm6-linux-firefox-pinterest.manifest +secondary_url = https://www.pinterest.com/today/best/halloween-costumes-for-your-furry-friends/75787/ +test_url = https://pinterest.com/ + +[reddit] +playback_pageset_manifest = mitm6-linux-firefox-reddit.manifest +secondary_url = https://www.reddit.com/r/technology/ +test_url = https://www.reddit.com/r/technology/comments/9sqwyh/we_posed_as_100_senators_to_run_ads_on_facebook/ + +[tumblr] +playback_pageset_manifest = mitm6-linux-firefox-tumblr.manifest +secondary_url = https://www.tumblr.com/tagged/funny+cats?sort=top +test_url = https://www.tumblr.com/dashboard + +[twitch] +secondary_url = https://www.twitch.tv/gmashley +test_url = https://www.twitch.tv/videos/894226211 +preferences = {"media.autoplay.default": 5, + "media.autoplay.ask-permission": true, + "media.autoplay.blocking_policy": 1, + "media.autoplay.block-webaudio": true, + "media.allowed-to-play.enabled": false, + "media.block-autoplay-until-in-foreground": true} + +[twitter] +playback_pageset_manifest = mitm5-linux-firefox-twitter.manifest +test_url = https://twitter.com/BarackObama + +[wikia] +secondary_url = https://marvel.fandom.com/wiki/Celestials +test_url = https://marvel.fandom.com/wiki/Black_Panther + +[wikipedia] +secondary_url = https://en.wikipedia.org/wiki/Joe_Biden +test_url = https://en.wikipedia.org/wiki/Barack_Obama + +[yahoo-mail] +playback_pageset_manifest = mitm5-linux-firefox-yahoo-mail.manifest +test_url = https://mail.yahoo.com/ + +[youtube] +secondary_url = https://www.youtube.com/watch?v=JrdEMERq8MA +test_url = https://www.youtube.com diff --git a/testing/raptor/raptor/tests/tp6/live/browsertime-live.ini b/testing/raptor/raptor/tests/tp6/live/browsertime-live.ini new file mode 100644 index 0000000000..2ef2e69cf1 --- /dev/null +++ b/testing/raptor/raptor/tests/tp6/live/browsertime-live.ini @@ -0,0 +1,70 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# Raptor-Browsertime tp6 tests running on live sites only. There should +# not be any playback settings in this file. If there are, please move the +# tests to `mobile/browsertime-tp6m.ini` or `desktop/browsertime-tp6.ini`. + +[DEFAULT] +alert_on = fcp, loadtime, ContentfulSpeedIndex, PerceptualSpeedIndex, SpeedIndex, FirstVisualChange, LastVisualChange +alert_threshold = 2.0 +apps = geckoview, fenix, refbrow, chrome-m, firefox, chrome, chromium +browser_cycles = 15 +lower_is_better = true +owner = PerfTest Team +page_cycles = 25 +page_timeout = 60000 +type = pageload +unit = ms +use_live_sites = true + +## Keep this test list in alphabetical order + +[booking-sf] +test_url = https://www.booking.com/hotel/us/edwardian-san-francisco.html + +[cnn] +test_url = https://www.cnn.com/2021/03/22/weather/climate-change-warm-waters-lake-michigan/index.html + +[cnn-ampstories] +test_url = https://cnn.com/ampstories/us/why-hurricane-michael-is-a-monster-unlike-any-other + +[discord] +test_url = https://discordapp.com/ + +[expedia] +test_url = https://expedia.com/Hotel-Search?destination=New+York%2C+New+York&latLong=40.756680%2C-73.986470®ionId=178293&startDate=&endDate=&rooms=1&_xpid=11905%7C1&adults=2 #noqa + +[fashionbeans] +test_url = https://fashionbeans.com/article/coolest-menswear-stores-in-the-world + +[google-accounts] +test_url = https://accounts.google.com + +[imdb-firefox] +test_url = https://m.imdb.com/title/tt0083943/ + +[medium-article] +test_url = https://medium.com/s/coincidences-are-a-lie/could-america-have-also-been-the-birthplace-of-impressionism-cb3d31a2e22d #noqa + +[nytimes] +test_url = https://www.nytimes.com/2020/02/19/opinion/surprise-medical-bill.html + +[people-article] +test_url = https://people.com/amp-stories/royal-a-to-z/ + +[reddit-thread] +test_url = https://www.reddit.com/r/firefox/comments/7dkq03/its_been_a_while/ + +[rumble-fox] +test_url = https://rumble.com/v3c44t-foxes-jumping-on-my-trampoline.html + +[stackoverflow-question] +test_url = https://stackoverflow.com/questions/927358/how-do-i-undo-the-most-recent-commits-in-git + +[urbandictionary-define] +test_url = https://urbandictionary.com/define.php?term=awesome%20sauce + +[wikia-marvel] +test_url = https://marvel.wikia.com/wiki/Black_Panther diff --git a/testing/raptor/raptor/tests/tp6/mobile/browsertime-tp6m.ini b/testing/raptor/raptor/tests/tp6/mobile/browsertime-tp6m.ini new file mode 100644 index 0000000000..c3a09fe00e --- /dev/null +++ b/testing/raptor/raptor/tests/tp6/mobile/browsertime-tp6m.ini @@ -0,0 +1,113 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# raptor-browsertime tp6 tests on android + +[DEFAULT] +alert_on = fcp, loadtime, ContentfulSpeedIndex, PerceptualSpeedIndex, SpeedIndex, FirstVisualChange, LastVisualChange +alert_threshold = 2.0 +apps = geckoview, fenix, refbrow, chrome-m +browser_cycles = 15 +lower_is_better = true +owner = PerfTest Team +page_cycles = 25 +page_timeout = 60000 +playback = mitmproxy-android +playback_pageset_manifest = mitm6-android-fenix-{subtest}.manifest +playback_version = 8.1.1 +type = pageload +unit = ms +use_live_sites = false + +## Keep this test list in alphabetical order + +[allrecipes] +test_url = https://www.allrecipes.com/ + +[amazon] +test_url = https://www.amazon.com + +[amazon-search] +test_url = https://www.amazon.com/s/ref=nb_sb_noss_2/139-6317191-5622045?url=search-alias%3Daps&field-keywords=mobile+phone + +[bing] +test_url = https://www.bing.com/ + +[bing-search-restaurants] +test_url = https://www.bing.com/search?q=restaurants+in+exton+pa+19341 + +[booking] +test_url = https://www.booking.com/ + +[cnn] +test_url = https://cnn.com + +[cnn-ampstories] +test_url = https://cnn.com/ampstories/us/why-hurricane-michael-is-a-monster-unlike-any-other + +[dailymail] +test_url = https://www.dailymail.co.uk/sciencetech/article-9749081/Experts-say-Hubble-repair-despite-NASA-insisting-multiple-options-fix.html + +[ebay-kleinanzeigen] +test_url = https://m.ebay-kleinanzeigen.de + +[ebay-kleinanzeigen-search] +test_url = https://m.ebay-kleinanzeigen.de/s-anzeigen/auf-zeit-wg-berlin/zimmer/c199-l3331 + +[espn] +test_url = http://www.espn.com/nba/story/_/page/allstarweekend25788027/the-comparison-lebron-james-michael-jordan-their-own-words + +[facebook] +login = true +playback_pageset_manifest = mitm6-g5-fenix-{subtest}.manifest +test_url = https://m.facebook.com + +[facebook-cristiano] +test_url = https://m.facebook.com/Cristiano + +[google] +login = true +playback_pageset_manifest = mitm6-g5-fenix-{subtest}.manifest +test_url = https://www.google.com + +[google-maps] +test_url = https://www.google.com/maps?force=pwa + +[google-search-restaurants] +login = true +playback_pageset_manifest = mitm6-g5-fenix-{subtest}.manifest +test_url = https://www.google.com/search?q=restaurants+near+me + +[imdb] +test_url = https://m.imdb.com/ + +[instagram] +login = true +playback_pageset_manifest = mitm8-android-gve-{subtest}.manifest +test_url = https://www.instagram.com + +[microsoft-support] +test_url = https://support.microsoft.com/en-us + +[reddit] +test_url = https://www.reddit.com + +[sina] +test_url = https://www.sina.com.cn/ + +[stackoverflow] +test_url = https://stackoverflow.com/ + +[web-de] +playback_pageset_manifest = mitm7-android-gve-p2-web-de.manifest +test_url = https://web.de/magazine/politik/politologe-glaubt-grossen-koalition-herbst-knallen-33563566 + +[wikipedia] +test_url = https://en.m.wikipedia.org/wiki/Main_Page + +[youtube] +test_url = https://m.youtube.com + +[youtube-watch] +test_url = https://www.youtube.com/watch?v=COU5T-Wafa4 diff --git a/testing/raptor/raptor/tests/unittests/raptor-tp6-unittest.ini b/testing/raptor/raptor/tests/unittests/raptor-tp6-unittest.ini new file mode 100644 index 0000000000..29fd76edd8 --- /dev/null +++ b/testing/raptor/raptor/tests/unittests/raptor-tp6-unittest.ini @@ -0,0 +1,42 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# raptor tp6-unittest + +[DEFAULT] +alert_on = fcp, loadtime +alert_threshold = 2.0 +gecko_profile_entries = 14000000 +gecko_profile_interval = 1 +lower_is_better = true +owner = PerfTest Team +page_cycles = 25 +page_timeout = 60000 +playback = mitmproxy +type = pageload +unit = ms + +[raptor-tp6-unittest-amazon-firefox] +apps = firefox +measure = fnbpaint, fcp, dcf, loadtime +playback_pageset_manifest = mitm5-linux-firefox-amazon.manifest +test_url = https://www.amazon.com/s?k=laptop&ref=nb_sb_noss_1 + +[raptor-tp6-unittest-facebook-firefox] +apps = firefox +measure = fnbpaint, fcp, dcf, loadtime +playback_pageset_manifest = mitm5-linux-firefox-facebook.manifest +test_url = https://www.facebook.com + +[raptor-tp6-unittest-google-firefox] +apps = firefox +measure = fnbpaint, fcp, dcf, loadtime +playback_pageset_manifest = mitm5-linux-firefox-google-search.manifest +test_url = https://www.google.com/search?hl=en&q=barack+obama&cad=h + +[raptor-tp6-unittest-youtube-firefox] +apps = firefox +measure = fnbpaint, fcp, dcf, loadtime +playback_pageset_manifest = mitm5-linux-firefox-youtube.manifest +test_url = https://www.youtube.com diff --git a/testing/raptor/raptor/tooltool-manifests/chrome-android/chrome80.manifest b/testing/raptor/raptor/tooltool-manifests/chrome-android/chrome80.manifest new file mode 100644 index 0000000000..9c1e6185c4 --- /dev/null +++ b/testing/raptor/raptor/tooltool-manifests/chrome-android/chrome80.manifest @@ -0,0 +1,10 @@ +[ + { + "algorithm": "sha512", + "visibility": "internal", + "filename": "com.android.chrome-v.80.0.3987.162.apk", + "unpack": false, + "digest": "2e41de938372ba2658479e2f1e7f8392659efe9d39f0200308f2b2adbf18654421e2878eb4a74ae9c75ff5987b03b61553f6f8fcc3e0c39cf24de320e35c2168", + "size": 131273700 + } +] diff --git a/testing/raptor/raptor/tooltool-manifests/chrome-android/chrome85.manifest b/testing/raptor/raptor/tooltool-manifests/chrome-android/chrome85.manifest new file mode 100644 index 0000000000..a8bfc5f703 --- /dev/null +++ b/testing/raptor/raptor/tooltool-manifests/chrome-android/chrome85.manifest @@ -0,0 +1,10 @@ +[ + { + "algorithm": "sha512", + "visibility": "internal", + "filename": "com.android.chrome-v.85.0.4183.101.apk", + "unpack": false, + "digest": "fa8688e5c61c74774abc6a79994983b4b8fed0fb9ee092981d4e8f22ffc600bcb9d03b4f30a0e63564e66db0eadc5999713a178724b5ff54fd709b080f87d587", + "size": 139131604 + } +] diff --git a/testing/raptor/raptor/tooltool-manifests/chrome-android/chrome86.manifest b/testing/raptor/raptor/tooltool-manifests/chrome-android/chrome86.manifest new file mode 100644 index 0000000000..9b04259512 --- /dev/null +++ b/testing/raptor/raptor/tooltool-manifests/chrome-android/chrome86.manifest @@ -0,0 +1,10 @@ +[ + { + "algorithm": "sha512", + "visibility": "internal", + "filename": "com.android.chrome-v.86.0.4240.185.apk", + "unpack": false, + "digest": "4b8e30d571a4a1bf28d7ab0dc599b5ee73518276d82770190ce80cd2009560af1e86bb39c179e585d327bf209ff72e6e09354bbdf702d5c6bf4d35bc3f6fb3ef", + "size": 141256746 + } +] diff --git a/testing/raptor/raptor/tooltool-manifests/chrome-android/chrome87.manifest b/testing/raptor/raptor/tooltool-manifests/chrome-android/chrome87.manifest new file mode 100644 index 0000000000..664c7e90ba --- /dev/null +++ b/testing/raptor/raptor/tooltool-manifests/chrome-android/chrome87.manifest @@ -0,0 +1,10 @@ +[ + { + "algorithm": "sha512", + "visibility": "internal", + "filename": "com.android.chrome-v.87.0.4280.86.apk", + "unpack": false, + "digest": "1d84f52ccce55f7ca73ec205bdda2a560a15b532a992c6748f51d6703083db96bf35fab3a686edcc64708e2d4013031b2f8f89eb441b5ab6b150e3e8ed0a61e8", + "size": 144747152 + } +] diff --git a/testing/raptor/raptor/tooltool-manifests/playback/mitm5-linux-firefox-bing-search.manifest b/testing/raptor/raptor/tooltool-manifests/playback/mitm5-linux-firefox-bing-search.manifest new file mode 100644 index 0000000000..060782a039 --- /dev/null +++ b/testing/raptor/raptor/tooltool-manifests/playback/mitm5-linux-firefox-bing-search.manifest @@ -0,0 +1,9 @@ +[ + { + "size": 2413805, + "visibility": "public", + "digest": "5ab1a9e1c7c4d5925bdac9e87123ec0aaeee477d0b2e910d37ba0df70f6c8278688910584b2874d19d8f360af14b6b37ac707012d6b8969a1877c64fed233489", + "algorithm": "sha512", + "filename": "mitm5-linux-firefox-bing-search.zip" + } +] diff --git a/testing/raptor/raptor/tooltool-manifests/playback/mitm5-linux-firefox-espn.manifest b/testing/raptor/raptor/tooltool-manifests/playback/mitm5-linux-firefox-espn.manifest new file mode 100644 index 0000000000..7022f1c969 --- /dev/null +++ b/testing/raptor/raptor/tooltool-manifests/playback/mitm5-linux-firefox-espn.manifest @@ -0,0 +1,9 @@ +[ + { + "filename": "mitm5-linux-firefox-espn.zip", + "size": 7665306, + "algorithm": "sha512", + "digest": "ca0a6d9561bbac193162f797ac3e6226446408ca0d8268313956414db9170be6d7c0d7dc316df4de15a3ed4e0e8c948b60472511339313e58cfc331b556db522", + "visibility": "public" + } +] diff --git a/testing/raptor/raptor/tooltool-manifests/playback/mitm5-linux-firefox-fandom.manifest b/testing/raptor/raptor/tooltool-manifests/playback/mitm5-linux-firefox-fandom.manifest new file mode 100644 index 0000000000..d5b0d974cc --- /dev/null +++ b/testing/raptor/raptor/tooltool-manifests/playback/mitm5-linux-firefox-fandom.manifest @@ -0,0 +1,9 @@ +[ + { + "size": 2875881, + "visibility": "public", + "digest": "3faf88b190c58b5988d1b7ecbfd7cac7d9cabbed7b07badb732e88d320ac8b9262a1aec67022cc0c5dd8ccfbb6a3f8d0e208a1e5d4426ec143b2dafc0b616346", + "algorithm": "sha512", + "filename": "mitm5-linux-firefox-fandom.zip" + } +] diff --git a/testing/raptor/raptor/tooltool-manifests/playback/mitm5-linux-firefox-google-mail.manifest b/testing/raptor/raptor/tooltool-manifests/playback/mitm5-linux-firefox-google-mail.manifest new file mode 100644 index 0000000000..5568d04b73 --- /dev/null +++ b/testing/raptor/raptor/tooltool-manifests/playback/mitm5-linux-firefox-google-mail.manifest @@ -0,0 +1,9 @@ +[ + { + "size": 15505653, + "visibility": "public", + "digest": "22b289c5a054413baa8fe5df047abd430b82a4275882c6247822fa437d3c59516e66dea1deaa01f2568adccf83b3975d75f34a3081dfc92a3d9a4ab54530f1c9", + "algorithm": "sha512", + "filename": "mitm5-linux-firefox-google-mail.zip" + } +] diff --git a/testing/raptor/raptor/tooltool-manifests/playback/mitm5-linux-firefox-google-search.manifest b/testing/raptor/raptor/tooltool-manifests/playback/mitm5-linux-firefox-google-search.manifest new file mode 100644 index 0000000000..fd695cf1ac --- /dev/null +++ b/testing/raptor/raptor/tooltool-manifests/playback/mitm5-linux-firefox-google-search.manifest @@ -0,0 +1,9 @@ +[ + { + "size": 8938916, + "visibility": "public", + "digest": "ad5e9b0a201eded3fa06ce2d646e9cbb774fbe29c236766aaad6d60fe588cade6a02a393493fd9c953910da4a511dd992d18f432ccba55a89f1a5f2d3b10cb3a", + "algorithm": "sha512", + "filename": "mitm5-linux-firefox-google-search.zip" + } +] diff --git a/testing/raptor/raptor/tooltool-manifests/playback/mitm5-linux-firefox-imdb.manifest b/testing/raptor/raptor/tooltool-manifests/playback/mitm5-linux-firefox-imdb.manifest new file mode 100644 index 0000000000..1b58016acf --- /dev/null +++ b/testing/raptor/raptor/tooltool-manifests/playback/mitm5-linux-firefox-imdb.manifest @@ -0,0 +1,9 @@ +[ + { + "size": 2904905, + "visibility": "public", + "digest": "2ea8757271b162713fdd400f68858dc4e9c03a6572edda591af6dbc2927a8efa47c0b9a86de51dd4d19b0bc0d3b7c14ddbe55e37a9e6cdc45b12b3f6479f43bb", + "algorithm": "sha512", + "filename": "mitm5-linux-firefox-imdb.zip" + } +] diff --git a/testing/raptor/raptor/tooltool-manifests/playback/mitm5-linux-firefox-live.manifest b/testing/raptor/raptor/tooltool-manifests/playback/mitm5-linux-firefox-live.manifest new file mode 100644 index 0000000000..8c3f8a4cee --- /dev/null +++ b/testing/raptor/raptor/tooltool-manifests/playback/mitm5-linux-firefox-live.manifest @@ -0,0 +1,9 @@ +[ + { + "size": 12907525, + "visibility": "public", + "digest": "5c892f7b1630a9d34a682ebee3d7564bbf9ce2295a290af8e423187b0f0e2c52dabf9b640114f7920367fbab9ed0bb92fc30f3c2b43df63ce08e63dae9b23a63", + "algorithm": "sha512", + "filename": "mitm5-linux-firefox-live.zip" + } +] diff --git a/testing/raptor/raptor/tooltool-manifests/playback/mitm5-linux-firefox-paypal.manifest b/testing/raptor/raptor/tooltool-manifests/playback/mitm5-linux-firefox-paypal.manifest new file mode 100644 index 0000000000..22c6f17982 --- /dev/null +++ b/testing/raptor/raptor/tooltool-manifests/playback/mitm5-linux-firefox-paypal.manifest @@ -0,0 +1,9 @@ +[ + { + "size": 3606449, + "visibility": "public", + "digest": "531f4ec6c3c475c4d14d198abc3bd8a02181a4759477714b3f30d4b97cf25065ccb6e373d78ec7b9549d1cdd410a94af7112701c2a6cdcb53bf8a24be2996144", + "algorithm": "sha512", + "filename": "mitm5-linux-firefox-paypal.zip" + } +] diff --git a/testing/raptor/raptor/tooltool-manifests/playback/mitm5-linux-firefox-proc-switch.manifest b/testing/raptor/raptor/tooltool-manifests/playback/mitm5-linux-firefox-proc-switch.manifest new file mode 100644 index 0000000000..2cda18abfc --- /dev/null +++ b/testing/raptor/raptor/tooltool-manifests/playback/mitm5-linux-firefox-proc-switch.manifest @@ -0,0 +1,16 @@ +[ + { + "filename": "mitm5-linux-firefox-pettay.zip", + "size": 162774, + "algorithm": "sha512", + "digest": "3b4bf399eca881bd586e1d8ce49758f1b8033d0fdbcb360663c0855ed9b2926020cecb9cb53ad2f141022dbbd73be4a229e2a3048461aefaca9d9be117b1705a", + "visibility": "public" + }, + { + "filename": "mitm5-linux-firefox-seanfeng.zip", + "size": 191769, + "algorithm": "sha512", + "digest": "05cfb31ecbc4d911b367fef1e1112cb2dbdcca827876a2e82a00b9dd723ec205631e81b2db8aa0c827888d484f9d576baec7c18a14483fe188566a7febeadebd", + "visibility": "public" + } +] diff --git a/testing/raptor/raptor/tooltool-manifests/playback/mitm5-linux-firefox-twitter.manifest b/testing/raptor/raptor/tooltool-manifests/playback/mitm5-linux-firefox-twitter.manifest new file mode 100644 index 0000000000..582373ab91 --- /dev/null +++ b/testing/raptor/raptor/tooltool-manifests/playback/mitm5-linux-firefox-twitter.manifest @@ -0,0 +1,9 @@ +[ + { + "size": 7176713, + "visibility": "public", + "digest": "af6ad361d2c027b48d72048d43b8fe7bed1efc307c9fdbb596b4e8e4f4c3247c1161bb21e888d2a4ca75f1a3ee47219c8289070f96ea7ab6e626d1d31f39bc75", + "algorithm": "sha512", + "filename": "mitm5-linux-firefox-twitter.zip" + } +] diff --git a/testing/raptor/raptor/tooltool-manifests/playback/mitm5-linux-firefox-welcome.manifest b/testing/raptor/raptor/tooltool-manifests/playback/mitm5-linux-firefox-welcome.manifest new file mode 100644 index 0000000000..1d17f28a7b --- /dev/null +++ b/testing/raptor/raptor/tooltool-manifests/playback/mitm5-linux-firefox-welcome.manifest @@ -0,0 +1,9 @@ +[ + { + "size": 1413171, + "visibility": "public", + "digest": "078d0022e2a13e415c64d316b62894015d2ce7b0e562dfa5aaf5502f73cad14d8753b7c73133f4655a4417514672d9d6cfb316f49f782161277ee50c21595b72", + "algorithm": "sha512", + "filename": "mitm5-linux-firefox-about.zip" + } +] diff --git a/testing/raptor/raptor/tooltool-manifests/playback/mitm5-linux-firefox-wikia.manifest b/testing/raptor/raptor/tooltool-manifests/playback/mitm5-linux-firefox-wikia.manifest new file mode 100644 index 0000000000..fd640e26e1 --- /dev/null +++ b/testing/raptor/raptor/tooltool-manifests/playback/mitm5-linux-firefox-wikia.manifest @@ -0,0 +1,9 @@ +[ + { + "filename": "mitm5-linux-firefox-wikia.zip", + "size": 4847101, + "algorithm": "sha512", + "digest": "c53c45c374cd924d03972f56b6a7416451861147f0a14b8c51c2c722a82582ae7ce8927a500981152aa6f4d3617b5ce455b85910981a9aabbd672e1ea47c0ff7", + "visibility": "public" + } +] diff --git a/testing/raptor/raptor/tooltool-manifests/playback/mitm5-linux-firefox-yahoo-mail.manifest b/testing/raptor/raptor/tooltool-manifests/playback/mitm5-linux-firefox-yahoo-mail.manifest new file mode 100644 index 0000000000..c133d8c261 --- /dev/null +++ b/testing/raptor/raptor/tooltool-manifests/playback/mitm5-linux-firefox-yahoo-mail.manifest @@ -0,0 +1,9 @@ +[ + { + "size": 11922543, + "visibility": "public", + "digest": "e690faac0d78b330b8c2f22fe23920fcf101c2775beae9575c191477023b2cb4f8c12b4948ece4e04ccbc65b20f68aa6c8acedfb2e5144089e192a7047de76b9", + "algorithm": "sha512", + "filename": "mitm5-linux-firefox-yahoo-mail.zip" + } +] diff --git a/testing/raptor/raptor/tooltool-manifests/playback/mitm6-android-fenix-allrecipes.manifest b/testing/raptor/raptor/tooltool-manifests/playback/mitm6-android-fenix-allrecipes.manifest new file mode 100644 index 0000000000..c0b1875d4f --- /dev/null +++ b/testing/raptor/raptor/tooltool-manifests/playback/mitm6-android-fenix-allrecipes.manifest @@ -0,0 +1,9 @@ +[ + { + "algorithm": "sha512", + "digest": "48e950574a2df1c3f9bd429f4eca90289c722b8ec9a0e01aa6dac7028bbfdba080c091188edf0b70bee7244a27c79edc5231c022569297267757e50b728ea1f1", + "filename": "mitm6-android-fenix-allrecipes.zip", + "size": 4825383, + "visibility": "public" + } +] diff --git a/testing/raptor/raptor/tooltool-manifests/playback/mitm6-android-fenix-amazon-search.manifest b/testing/raptor/raptor/tooltool-manifests/playback/mitm6-android-fenix-amazon-search.manifest new file mode 100644 index 0000000000..7a22ecbfd3 --- /dev/null +++ b/testing/raptor/raptor/tooltool-manifests/playback/mitm6-android-fenix-amazon-search.manifest @@ -0,0 +1,9 @@ +[ + { + "algorithm": "sha512", + "digest": "a00fefe48b37bc84306dc4ae634e6b1fb6f2ff688f9ac579126be1b8025e1d4201e72ea3e9ad581e77e98157c27d4e6cdbb8f2cf7f6044eff5079b91e5de8575", + "filename": "mitm6-android-fenix-amazon-search.zip", + "size": 3334287, + "visibility": "public" + } +] diff --git a/testing/raptor/raptor/tooltool-manifests/playback/mitm6-android-fenix-amazon.manifest b/testing/raptor/raptor/tooltool-manifests/playback/mitm6-android-fenix-amazon.manifest new file mode 100644 index 0000000000..64ce6f84ab --- /dev/null +++ b/testing/raptor/raptor/tooltool-manifests/playback/mitm6-android-fenix-amazon.manifest @@ -0,0 +1,9 @@ +[ + { + "algorithm": "sha512", + "digest": "4b8f1d77389390f6ad582ada8b322350ea7652e4dc15b3e09d91fed33b2cc3c58e87dd1a6939d6785b0ff5be52b05572c46845a61e80c6ab73a3bcd70a338c22", + "filename": "mitm6-android-fenix-amazon.zip", + "size": 3199995, + "visibility": "public" + } +] diff --git a/testing/raptor/raptor/tooltool-manifests/playback/mitm6-android-fenix-bing-search-restaurants.manifest b/testing/raptor/raptor/tooltool-manifests/playback/mitm6-android-fenix-bing-search-restaurants.manifest new file mode 100644 index 0000000000..90bead2849 --- /dev/null +++ b/testing/raptor/raptor/tooltool-manifests/playback/mitm6-android-fenix-bing-search-restaurants.manifest @@ -0,0 +1,9 @@ +[ + { + "algorithm": "sha512", + "digest": "205b696884448ae5a0b8a667b102bf6b1cb3bcdc56df25ff9d546770cd16149c7edb3ea7082b308c4160e71d89bad5343108d9f0f33dba4f30a15f700e6d0ed6", + "filename": "mitm6-android-fenix-bing-search-restaurants.zip", + "size": 1187014, + "visibility": "public" + } +] diff --git a/testing/raptor/raptor/tooltool-manifests/playback/mitm6-android-fenix-bing.manifest b/testing/raptor/raptor/tooltool-manifests/playback/mitm6-android-fenix-bing.manifest new file mode 100644 index 0000000000..f65a483327 --- /dev/null +++ b/testing/raptor/raptor/tooltool-manifests/playback/mitm6-android-fenix-bing.manifest @@ -0,0 +1,9 @@ +[ + { + "algorithm": "sha512", + "digest": "ecc1b12a7c169e9a4269770b83a1a69dc8bcfba300db3e00e4b57bb6b4a252f7e88f3e0adb3faaa68bc3540410db225c8e2f8d644ff5e171f4b34302d0fc1446", + "filename": "mitm6-android-fenix-bing.zip", + "size": 1283380, + "visibility": "public" + } +] diff --git a/testing/raptor/raptor/tooltool-manifests/playback/mitm6-android-fenix-booking.manifest b/testing/raptor/raptor/tooltool-manifests/playback/mitm6-android-fenix-booking.manifest new file mode 100644 index 0000000000..d2ce11da3c --- /dev/null +++ b/testing/raptor/raptor/tooltool-manifests/playback/mitm6-android-fenix-booking.manifest @@ -0,0 +1,9 @@ +[ + { + "algorithm": "sha512", + "digest": "77d2f2b4f7dbf10261a0a23225b05f66015121c0605fda3436487576e2fe43fbc83b6c97bcd5f551ba99be36707a994af0d481608319c15d16c6906fff9989c5", + "filename": "mitm6-android-fenix-booking.zip", + "size": 1584815, + "visibility": "public" + } +] diff --git a/testing/raptor/raptor/tooltool-manifests/playback/mitm6-android-fenix-cnn-ampstories.manifest b/testing/raptor/raptor/tooltool-manifests/playback/mitm6-android-fenix-cnn-ampstories.manifest new file mode 100644 index 0000000000..e26390b233 --- /dev/null +++ b/testing/raptor/raptor/tooltool-manifests/playback/mitm6-android-fenix-cnn-ampstories.manifest @@ -0,0 +1,9 @@ +[ + { + "algorithm": "sha512", + "digest": "d9044a8a7c114f438b5eebd66ffbfc2fdb0baa191a2ceb758798150d0f42bbf5d4058b8a69217a5304355e1776a47531126f7507d89796c5e2a9ec60324ff12f", + "filename": "mitm6-android-fenix-cnn-ampstories.zip", + "size": 1244428, + "visibility": "public" + } +] diff --git a/testing/raptor/raptor/tooltool-manifests/playback/mitm6-android-fenix-cnn.manifest b/testing/raptor/raptor/tooltool-manifests/playback/mitm6-android-fenix-cnn.manifest new file mode 100644 index 0000000000..c5e54323ce --- /dev/null +++ b/testing/raptor/raptor/tooltool-manifests/playback/mitm6-android-fenix-cnn.manifest @@ -0,0 +1,9 @@ +[ + { + "algorithm": "sha512", + "digest": "0ab8ee644b8de452d8cfb43932e7222f2cdf57010b4002921f4d37c63b5c694b1f9955a2586d5cccfc54fae051c7f771f808f801c2592503bbd278c53a65faf4", + "filename": "mitm6-android-fenix-cnn.zip", + "size": 5185843, + "visibility": "public" + } +] diff --git a/testing/raptor/raptor/tooltool-manifests/playback/mitm6-android-fenix-dailymail.manifest b/testing/raptor/raptor/tooltool-manifests/playback/mitm6-android-fenix-dailymail.manifest new file mode 100644 index 0000000000..4aae3ba75e --- /dev/null +++ b/testing/raptor/raptor/tooltool-manifests/playback/mitm6-android-fenix-dailymail.manifest @@ -0,0 +1,9 @@ +[ + { + "algorithm": "sha512", + "digest": "58186e45a1a43234e6b6e30cae230e98ba528cf63fba822e43306010832e57462aefc631a493e6e75f1011c41cc54c7ce9ebbeed0292a22f3e867292d044250a", + "filename": "mitm6-android-fenix-dailymail.zip", + "size": 8427999, + "visibility": "public" + } +] diff --git a/testing/raptor/raptor/tooltool-manifests/playback/mitm6-android-fenix-ebay-kleinanzeigen-search.manifest b/testing/raptor/raptor/tooltool-manifests/playback/mitm6-android-fenix-ebay-kleinanzeigen-search.manifest new file mode 100644 index 0000000000..eb63b59b24 --- /dev/null +++ b/testing/raptor/raptor/tooltool-manifests/playback/mitm6-android-fenix-ebay-kleinanzeigen-search.manifest @@ -0,0 +1,9 @@ +[ + { + "algorithm": "sha512", + "digest": "65e2addf5ab59a0616151b70bd5a0b3facc58188c67915b80b2ea6a15060573b36f32279f7f22680f1773946cefa2d9307da0dec52362afd000c53f6c8b4561f", + "filename": "mitm6-android-fenix-ebay-kleinanzeigen-search.zip", + "size": 2027392, + "visibility": "public" + } +] diff --git a/testing/raptor/raptor/tooltool-manifests/playback/mitm6-android-fenix-ebay-kleinanzeigen.manifest b/testing/raptor/raptor/tooltool-manifests/playback/mitm6-android-fenix-ebay-kleinanzeigen.manifest new file mode 100644 index 0000000000..13d80e9a1d --- /dev/null +++ b/testing/raptor/raptor/tooltool-manifests/playback/mitm6-android-fenix-ebay-kleinanzeigen.manifest @@ -0,0 +1,9 @@ +[ + { + "algorithm": "sha512", + "digest": "9c64cab2938dd256f6106b7bdc0af8f2a374dac03125c1ab6327093c6b3a5af4779691d394a5363f0d7a04ae63e0cdc315087bd373514a0e68bba117bb8d7fd7", + "filename": "mitm6-android-fenix-ebay-kleinanzeigen.zip", + "size": 2452686, + "visibility": "public" + } +] diff --git a/testing/raptor/raptor/tooltool-manifests/playback/mitm6-android-fenix-espn.manifest b/testing/raptor/raptor/tooltool-manifests/playback/mitm6-android-fenix-espn.manifest new file mode 100644 index 0000000000..3585b24520 --- /dev/null +++ b/testing/raptor/raptor/tooltool-manifests/playback/mitm6-android-fenix-espn.manifest @@ -0,0 +1,9 @@ +[ + { + "algorithm": "sha512", + "digest": "51c15a37525595783e06df93c94d91c7a130d7c7d88dbfce26bb066f2ee06d8642311dc9e0cae35a2786c75c0da69078a7f8829c9b87c3bd99b1cf858b013161", + "filename": "mitm6-android-fenix-espn.zip", + "size": 4359513, + "visibility": "public" + } +] diff --git a/testing/raptor/raptor/tooltool-manifests/playback/mitm6-android-fenix-facebook-cristiano.manifest b/testing/raptor/raptor/tooltool-manifests/playback/mitm6-android-fenix-facebook-cristiano.manifest new file mode 100644 index 0000000000..2242a942c9 --- /dev/null +++ b/testing/raptor/raptor/tooltool-manifests/playback/mitm6-android-fenix-facebook-cristiano.manifest @@ -0,0 +1,9 @@ +[ + { + "algorithm": "sha512", + "digest": "d2a01952861a72c74587bed062fa7c547d9acfbe4b9a7ebe512d0dee8508b723553ab5db65a4c5301b933f91467ac9de41ffe1c46f7136379b4ea2a2e7ddc7c8", + "filename": "mitm6-android-fenix-facebook-cristiano.zip", + "size": 1026780, + "visibility": "public" + } +] diff --git a/testing/raptor/raptor/tooltool-manifests/playback/mitm6-android-fenix-google-maps.manifest b/testing/raptor/raptor/tooltool-manifests/playback/mitm6-android-fenix-google-maps.manifest new file mode 100644 index 0000000000..32e403dac3 --- /dev/null +++ b/testing/raptor/raptor/tooltool-manifests/playback/mitm6-android-fenix-google-maps.manifest @@ -0,0 +1,9 @@ +[ + { + "algorithm": "sha512", + "digest": "e35c4fb06a1e79fa42fd75d2be15f557c125fb1e24cdc28d2f0b25e36b12ed2e50d6df304365583b2f1109ea8a6360f5ef76b23e4e133d07813ba61df5482456", + "filename": "mitm6-android-fenix-google-maps.zip", + "size": 82227, + "visibility": "public" + } +] diff --git a/testing/raptor/raptor/tooltool-manifests/playback/mitm6-android-fenix-imdb.manifest b/testing/raptor/raptor/tooltool-manifests/playback/mitm6-android-fenix-imdb.manifest new file mode 100644 index 0000000000..99e6b1dfd7 --- /dev/null +++ b/testing/raptor/raptor/tooltool-manifests/playback/mitm6-android-fenix-imdb.manifest @@ -0,0 +1,9 @@ +[ + { + "algorithm": "sha512", + "digest": "ec96303a6c0c019282f1f808fe99de78303b45063f9106fe42e61f1a146e6f2485431a39946b57595e33d645c358c699a8bf7983755bccc123fb88c6eb301dea", + "filename": "mitm6-android-fenix-imdb.zip", + "size": 3934494, + "visibility": "public" + } +] diff --git a/testing/raptor/raptor/tooltool-manifests/playback/mitm6-android-fenix-microsoft-support.manifest b/testing/raptor/raptor/tooltool-manifests/playback/mitm6-android-fenix-microsoft-support.manifest new file mode 100644 index 0000000000..cd5767abaf --- /dev/null +++ b/testing/raptor/raptor/tooltool-manifests/playback/mitm6-android-fenix-microsoft-support.manifest @@ -0,0 +1,9 @@ +[ + { + "algorithm": "sha512", + "digest": "854cc6ef33fa03214fe5341812ab54e2f833834e3e5169961bebea7797e138c0de6d3d2bb7be847806e0570d10394bd9928ea143c3c2f8f7c9c6a4b43de45ec8", + "filename": "mitm6-android-fenix-microsoft-support.zip", + "size": 2746707, + "visibility": "public" + } +] diff --git a/testing/raptor/raptor/tooltool-manifests/playback/mitm6-android-fenix-reddit.manifest b/testing/raptor/raptor/tooltool-manifests/playback/mitm6-android-fenix-reddit.manifest new file mode 100644 index 0000000000..217621f85d --- /dev/null +++ b/testing/raptor/raptor/tooltool-manifests/playback/mitm6-android-fenix-reddit.manifest @@ -0,0 +1,9 @@ +[ + { + "algorithm": "sha512", + "digest": "27e286a3370ba0c17acf2a2433156772be79eb0e72e9018d9dcae64a3c30cb3a52d9bc750f2c183d99fe532ae2a216058e05c6ec5db35d033942b1aac143b280", + "filename": "mitm6-android-fenix-reddit.zip", + "size": 2258353, + "visibility": "public" + } +] diff --git a/testing/raptor/raptor/tooltool-manifests/playback/mitm6-android-fenix-sina.manifest b/testing/raptor/raptor/tooltool-manifests/playback/mitm6-android-fenix-sina.manifest new file mode 100644 index 0000000000..b52d47df45 --- /dev/null +++ b/testing/raptor/raptor/tooltool-manifests/playback/mitm6-android-fenix-sina.manifest @@ -0,0 +1,9 @@ +[ + { + "algorithm": "sha512", + "digest": "f3b55b61e6c38a342fee68fef04196de39b209053348cf58f5d9cc289100d82f271008f065adfd4ec02df564f6de12726f198e151af24aace9f654bf008a7b86", + "filename": "mitm6-android-fenix-sina.zip", + "size": 14757830, + "visibility": "public" + } +] diff --git a/testing/raptor/raptor/tooltool-manifests/playback/mitm6-android-fenix-stackoverflow.manifest b/testing/raptor/raptor/tooltool-manifests/playback/mitm6-android-fenix-stackoverflow.manifest new file mode 100644 index 0000000000..ab1abfffb2 --- /dev/null +++ b/testing/raptor/raptor/tooltool-manifests/playback/mitm6-android-fenix-stackoverflow.manifest @@ -0,0 +1,9 @@ +[ + { + "algorithm": "sha512", + "digest": "9f80131d6654656b82a560885b5b88b1767dfe47b65a32b2c4dba091a86ba38dfc4d7060a930c78eb51dcb589440ed779d91b7cd69b7b01d5ae54ae17bd9bc85", + "filename": "mitm6-android-fenix-stackoverflow.zip", + "size": 1026600, + "visibility": "public" + } +] diff --git a/testing/raptor/raptor/tooltool-manifests/playback/mitm6-android-fenix-web-de.manifest b/testing/raptor/raptor/tooltool-manifests/playback/mitm6-android-fenix-web-de.manifest new file mode 100644 index 0000000000..9bde5db272 --- /dev/null +++ b/testing/raptor/raptor/tooltool-manifests/playback/mitm6-android-fenix-web-de.manifest @@ -0,0 +1,9 @@ +[ + { + "algorithm": "sha512", + "digest": "9f10db4616f7415dba946f056658126297928d1229e5d2e525f9f1dd539c954b9a90b7d81fa574002fd1127321d8f918a3102cae12054b218118b059702cde7f", + "filename": "mitm6-android-fenix-web-de.zip", + "size": 1891502, + "visibility": "public" + } +] diff --git a/testing/raptor/raptor/tooltool-manifests/playback/mitm6-android-fenix-wikipedia.manifest b/testing/raptor/raptor/tooltool-manifests/playback/mitm6-android-fenix-wikipedia.manifest new file mode 100644 index 0000000000..633c984c1b --- /dev/null +++ b/testing/raptor/raptor/tooltool-manifests/playback/mitm6-android-fenix-wikipedia.manifest @@ -0,0 +1,9 @@ +[ + { + "algorithm": "sha512", + "digest": "23b6923c11004c22a7e021b8eeaa62a330a5f7f864bd9cbeb6373113a74edcb3fffc0633b491b24c2046f27a77a2ae15a4d68db34b3367479eef13165e1b59fb", + "filename": "mitm6-android-fenix-wikipedia.zip", + "size": 763266, + "visibility": "public" + } +] diff --git a/testing/raptor/raptor/tooltool-manifests/playback/mitm6-android-fenix-youtube-watch.manifest b/testing/raptor/raptor/tooltool-manifests/playback/mitm6-android-fenix-youtube-watch.manifest new file mode 100644 index 0000000000..302734fae4 --- /dev/null +++ b/testing/raptor/raptor/tooltool-manifests/playback/mitm6-android-fenix-youtube-watch.manifest @@ -0,0 +1,9 @@ +[ + { + "algorithm": "sha512", + "digest": "f8fa91a60295603ae26b6be0579887dd2a8f21c0b91c34c134d589bba5f49b5820dfc40f30a2fead51bbe989d420e7f928b6b713c59bd112489c04f95ed1fc46", + "filename": "mitm6-android-fenix-youtube-watch.zip", + "size": 782446, + "visibility": "public" + } +] diff --git a/testing/raptor/raptor/tooltool-manifests/playback/mitm6-android-fenix-youtube.manifest b/testing/raptor/raptor/tooltool-manifests/playback/mitm6-android-fenix-youtube.manifest new file mode 100644 index 0000000000..b8b6f531b2 --- /dev/null +++ b/testing/raptor/raptor/tooltool-manifests/playback/mitm6-android-fenix-youtube.manifest @@ -0,0 +1,9 @@ +[ + { + "algorithm": "sha512", + "digest": "1a885e088a6111822d8a772168b51ec19a7ebbe03e36c2867cffbd2704199b723cee2af85e04c4923ef8bc368716f2f6d63ec304589dc769807ee60c20dc2ae8", + "filename": "mitm6-android-fenix-youtube.zip", + "size": 10518039, + "visibility": "public" + } +] diff --git a/testing/raptor/raptor/tooltool-manifests/playback/mitm6-darwin-firefox-ebay.manifest b/testing/raptor/raptor/tooltool-manifests/playback/mitm6-darwin-firefox-ebay.manifest new file mode 100644 index 0000000000..d851988e61 --- /dev/null +++ b/testing/raptor/raptor/tooltool-manifests/playback/mitm6-darwin-firefox-ebay.manifest @@ -0,0 +1,9 @@ +[ + { + "filename": "mitm6-darwin-firefox-ebay.zip", + "size": 6451851, + "algorithm": "sha512", + "digest": "c433ca59de6938298970f70124f1da25b76870d743b6df20e5601ede85be4bb6acdd663bf5ef6bb4bdf172d48e9962895cc55068a2d1daffee22e94082167639", + "visibility": "public" + } +] diff --git a/testing/raptor/raptor/tooltool-manifests/playback/mitm6-g5-fenix-facebook.manifest b/testing/raptor/raptor/tooltool-manifests/playback/mitm6-g5-fenix-facebook.manifest new file mode 100644 index 0000000000..c858ebd680 --- /dev/null +++ b/testing/raptor/raptor/tooltool-manifests/playback/mitm6-g5-fenix-facebook.manifest @@ -0,0 +1,9 @@ +[ + { + "size": 8922738, + "visibility": "public", + "digest": "f48caf821b8e2b2f1f6038a6936f397ccae212e20310545ed10302921a02b8d0f90ced2c2e6a2609e76e886d36a1e01c4049b2cbdd7d272fb5550f866e0d9b7b", + "algorithm": "sha512", + "filename": "mitm6-g5-fenix-facebook.zip" + } +] diff --git a/testing/raptor/raptor/tooltool-manifests/playback/mitm6-g5-fenix-google-search-restaurants.manifest b/testing/raptor/raptor/tooltool-manifests/playback/mitm6-g5-fenix-google-search-restaurants.manifest new file mode 100644 index 0000000000..c945ad8fd2 --- /dev/null +++ b/testing/raptor/raptor/tooltool-manifests/playback/mitm6-g5-fenix-google-search-restaurants.manifest @@ -0,0 +1,9 @@ +[ + { + "size": 7941483, + "visibility": "public", + "digest": "9c9ff8f2a4c78536f6c1f4405fe01422c8b110889d185b89161ff96a518702c2320186fb2961df7f187447b9a23e7cbf6f01b8079427784fb3af0dfbd63321c4", + "algorithm": "sha512", + "filename": "mitm6-g5-fenix-google-search-restaurants.zip" + } +] diff --git a/testing/raptor/raptor/tooltool-manifests/playback/mitm6-g5-fenix-google.manifest b/testing/raptor/raptor/tooltool-manifests/playback/mitm6-g5-fenix-google.manifest new file mode 100644 index 0000000000..e8b12d77f8 --- /dev/null +++ b/testing/raptor/raptor/tooltool-manifests/playback/mitm6-g5-fenix-google.manifest @@ -0,0 +1,9 @@ +[ + { + "size": 8866092, + "visibility": "public", + "digest": "6ab38726ceda5212d34f83c6db97402eaed77bc3179355c51115d9b6350cbd5af398b955eb0562a489bb935deed51fb221c5097d5c48f0b99c23fc1f28df0f40", + "algorithm": "sha512", + "filename": "mitm6-g5-fenix-google.zip" + } +] diff --git a/testing/raptor/raptor/tooltool-manifests/playback/mitm6-linux-firefox-buzzfeed.manifest b/testing/raptor/raptor/tooltool-manifests/playback/mitm6-linux-firefox-buzzfeed.manifest new file mode 100644 index 0000000000..c71344fdc7 --- /dev/null +++ b/testing/raptor/raptor/tooltool-manifests/playback/mitm6-linux-firefox-buzzfeed.manifest @@ -0,0 +1,9 @@ +[ + { + "filename": "mitm6-linux-firefox-buzzfeed.zip", + "size": 10499368, + "algorithm": "sha512", + "digest": "968f3b4d6b056e99ab37e51f12c4f9a1f9914f5d02fb41f169f5304bb0835f7eb7aa1390418a9143665d61bf93719244790bd6f34edcbf683ba6341f43037481", + "visibility": "public" + } +] diff --git a/testing/raptor/raptor/tooltool-manifests/playback/mitm6-linux-firefox-cnn.manifest b/testing/raptor/raptor/tooltool-manifests/playback/mitm6-linux-firefox-cnn.manifest new file mode 100644 index 0000000000..9ce5bf7288 --- /dev/null +++ b/testing/raptor/raptor/tooltool-manifests/playback/mitm6-linux-firefox-cnn.manifest @@ -0,0 +1,9 @@ +[ + { + "size": 21536340, + "visibility": "public", + "digest": "28feb0edff87a243edac6c651f15474f828b2cf005c725aa5f47df0819494fc6caf3438cd940318010a2b2847be92199ff5355bc59e8ae115a18c77eb4af17f7", + "algorithm": "sha512", + "filename": "mitm6-linux-firefox-cnn.zip" + } +] diff --git a/testing/raptor/raptor/tooltool-manifests/playback/mitm6-linux-firefox-expedia.manifest b/testing/raptor/raptor/tooltool-manifests/playback/mitm6-linux-firefox-expedia.manifest new file mode 100644 index 0000000000..e1ef76a43d --- /dev/null +++ b/testing/raptor/raptor/tooltool-manifests/playback/mitm6-linux-firefox-expedia.manifest @@ -0,0 +1,9 @@ +[ + { + "filename": "mitm6-linux-firefox-expedia.zip", + "size": 9447080, + "algorithm": "sha512", + "digest": "c9665a107e90b9b57de7784230f71184356cde2beaf022f96c4629ebc43712d2214fb83131424faec25b95eee5c7b88ca26b28817230040bf98156e96776720f", + "visibility": "public" + } +] diff --git a/testing/raptor/raptor/tooltool-manifests/playback/mitm6-linux-firefox-facebook.manifest b/testing/raptor/raptor/tooltool-manifests/playback/mitm6-linux-firefox-facebook.manifest new file mode 100644 index 0000000000..e5e2a96868 --- /dev/null +++ b/testing/raptor/raptor/tooltool-manifests/playback/mitm6-linux-firefox-facebook.manifest @@ -0,0 +1,9 @@ +[ + { + "size": 9660242, + "visibility": "public", + "digest": "c9586853fb8e755f388c08bbb69fc2574cc9b57354c86af9369458d2fe34c05e29077f851eca20008a365699c32da5cfac0de363b48f4491056dc32e10c4421c", + "algorithm": "sha512", + "filename": "mitm6-linux-firefox-facebook.zip" + } +] diff --git a/testing/raptor/raptor/tooltool-manifests/playback/mitm6-linux-firefox-google-docs.manifest b/testing/raptor/raptor/tooltool-manifests/playback/mitm6-linux-firefox-google-docs.manifest new file mode 100644 index 0000000000..4cd87049e3 --- /dev/null +++ b/testing/raptor/raptor/tooltool-manifests/playback/mitm6-linux-firefox-google-docs.manifest @@ -0,0 +1,9 @@ +[ + { + "size": 53846719, + "visibility": "public", + "digest": "fb29688612efb51485781b4a90c69fff51825f9fc6e3c393425ad22be55b3c4819083c837331869049574b818c60ee062ef8ac5d3afd84ddef34ce9f3cdff929", + "algorithm": "sha512", + "filename": "mitm6-linux-firefox-google-docs.zip" + } +] diff --git a/testing/raptor/raptor/tooltool-manifests/playback/mitm6-linux-firefox-google-slides.manifest b/testing/raptor/raptor/tooltool-manifests/playback/mitm6-linux-firefox-google-slides.manifest new file mode 100644 index 0000000000..499a7d614e --- /dev/null +++ b/testing/raptor/raptor/tooltool-manifests/playback/mitm6-linux-firefox-google-slides.manifest @@ -0,0 +1,9 @@ +[ + { + "size": 21729995, + "visibility": "public", + "digest": "e91b82a3813f768de551a5481f29c30c9108070e7a47fc1372ef716d1d1496f9436804d45a61f5f6c841dabb8e6b2c0ed2e01e704d367c215cdce3232fdef058", + "algorithm": "sha512", + "filename": "mitm6-linux-firefox-google-slides.zip" + } +] diff --git a/testing/raptor/raptor/tooltool-manifests/playback/mitm6-linux-firefox-instagram.manifest b/testing/raptor/raptor/tooltool-manifests/playback/mitm6-linux-firefox-instagram.manifest new file mode 100644 index 0000000000..f992a624c0 --- /dev/null +++ b/testing/raptor/raptor/tooltool-manifests/playback/mitm6-linux-firefox-instagram.manifest @@ -0,0 +1,9 @@ +[ + { + "size": 6548847, + "visibility": "public", + "digest": "30d26a7bd8343255bf5e9e58f9315a56719be25dcb32214de1604563a9fbd49e17e477286a9d9c4b941ba20591385ac523c153e6f290ef57f679bd9261fa9078", + "algorithm": "sha512", + "filename": "mitm6-linux-firefox-instagram.zip" + } +] diff --git a/testing/raptor/raptor/tooltool-manifests/playback/mitm6-linux-firefox-linkedin.manifest b/testing/raptor/raptor/tooltool-manifests/playback/mitm6-linux-firefox-linkedin.manifest new file mode 100644 index 0000000000..de8e4d4996 --- /dev/null +++ b/testing/raptor/raptor/tooltool-manifests/playback/mitm6-linux-firefox-linkedin.manifest @@ -0,0 +1,9 @@ +[ + { + "size": 8958594, + "visibility": "public", + "digest": "5a69c1966d8a6787a3d26f7e2ac1a6a5d68ea693a707e2a954865d489276db1752c8a562d332cce40ffcd30604c7345e0beb375c0c76c6fe82d79541094c7b97", + "algorithm": "sha512", + "filename": "mitm6-linux-firefox-linkedin.zip" + } +] diff --git a/testing/raptor/raptor/tooltool-manifests/playback/mitm6-linux-firefox-microsoft.manifest b/testing/raptor/raptor/tooltool-manifests/playback/mitm6-linux-firefox-microsoft.manifest new file mode 100644 index 0000000000..a54b860243 --- /dev/null +++ b/testing/raptor/raptor/tooltool-manifests/playback/mitm6-linux-firefox-microsoft.manifest @@ -0,0 +1,9 @@ +[ + { + "size": 4595339, + "visibility": "public", + "digest": "0cfd03b0cd8b4a33656da1a6e7651fd2ea584ae4182f46646f61d7e018f4b04223ed65574d884598328dd1d34c629294836375c16798e4cf3c78533accdf2a53", + "algorithm": "sha512", + "filename": "mitm6-linux-firefox-microsoft.zip" + } +] diff --git a/testing/raptor/raptor/tooltool-manifests/playback/mitm6-linux-firefox-netflix.manifest b/testing/raptor/raptor/tooltool-manifests/playback/mitm6-linux-firefox-netflix.manifest new file mode 100644 index 0000000000..f9249afc77 --- /dev/null +++ b/testing/raptor/raptor/tooltool-manifests/playback/mitm6-linux-firefox-netflix.manifest @@ -0,0 +1,9 @@ +[ + { + "size": 22840067, + "visibility": "public", + "digest": "5703af9b5fb6130c9e1008db37f927c7bb287e862160a50bb09459c7539b9b847f49465f9dfab1b0091e21f2c8572d429084fea3762de5d1cd06e0b02640c6d2", + "algorithm": "sha512", + "filename": "mitm6-linux-firefox-netflix.zip" + } +] diff --git a/testing/raptor/raptor/tooltool-manifests/playback/mitm6-linux-firefox-nytimes.manifest b/testing/raptor/raptor/tooltool-manifests/playback/mitm6-linux-firefox-nytimes.manifest new file mode 100644 index 0000000000..cc3472fe40 --- /dev/null +++ b/testing/raptor/raptor/tooltool-manifests/playback/mitm6-linux-firefox-nytimes.manifest @@ -0,0 +1,9 @@ +[ + { + "size": 18492628, + "visibility": "public", + "digest": "a89aed408cb19274c151cd70c17eadd5634485985f70fb71cf90e07038a4eaa89013d555430c41bb3c98b4aaf3492798a6012c789d610f0b8549a611b94a2ee6", + "algorithm": "sha512", + "filename": "mitm6-linux-firefox-nytimes.zip" + } +] diff --git a/testing/raptor/raptor/tooltool-manifests/playback/mitm6-linux-firefox-pinterest.manifest b/testing/raptor/raptor/tooltool-manifests/playback/mitm6-linux-firefox-pinterest.manifest new file mode 100644 index 0000000000..b91f98f2bf --- /dev/null +++ b/testing/raptor/raptor/tooltool-manifests/playback/mitm6-linux-firefox-pinterest.manifest @@ -0,0 +1,9 @@ +[ + { + "size": 15058950, + "visibility": "public", + "digest": "081f9edd82e728c35408337ee2fd5af25ec6a964c74bc3853f12a3a48103baddbfd452571401af96134309040edda1b7208d155ecbeadb3532bca8bde02767df", + "algorithm": "sha512", + "filename": "mitm6-linux-firefox-pinterest.zip" + } +] diff --git a/testing/raptor/raptor/tooltool-manifests/playback/mitm6-linux-firefox-reddit.manifest b/testing/raptor/raptor/tooltool-manifests/playback/mitm6-linux-firefox-reddit.manifest new file mode 100644 index 0000000000..cf5558b025 --- /dev/null +++ b/testing/raptor/raptor/tooltool-manifests/playback/mitm6-linux-firefox-reddit.manifest @@ -0,0 +1,9 @@ +[ + { + "size": 7412555, + "visibility": "public", + "digest": "f8b66ff69aa5ef09c345c2cfd28b88bbfe106f545cc4990779442f9a403369bb409200df1f17ae23d906b30e55de49cfdd125fa54aa8c22a52e8b05d8454e792", + "algorithm": "sha512", + "filename": "mitm6-linux-firefox-reddit.zip" + } +] diff --git a/testing/raptor/raptor/tooltool-manifests/playback/mitm6-linux-firefox-tumblr.manifest b/testing/raptor/raptor/tooltool-manifests/playback/mitm6-linux-firefox-tumblr.manifest new file mode 100644 index 0000000000..a3b9a92e99 --- /dev/null +++ b/testing/raptor/raptor/tooltool-manifests/playback/mitm6-linux-firefox-tumblr.manifest @@ -0,0 +1,9 @@ +[ + { + "size": 23452530, + "visibility": "public", + "digest": "9381e9b6c17c5e05cb1249c43cabd0fae65d15f74c5a7a44bf14e3ea8a0b67559790932ecc38137c806f5e0eaaecf1cfe82d9adddf9994ae2b44b32b81435db0", + "algorithm": "sha512", + "filename": "mitm6-linux-firefox-tumblr.zip" + } +] diff --git a/testing/raptor/raptor/tooltool-manifests/playback/mitm6-linux-firefox-wikipedia.manifest b/testing/raptor/raptor/tooltool-manifests/playback/mitm6-linux-firefox-wikipedia.manifest new file mode 100644 index 0000000000..617f6427d5 --- /dev/null +++ b/testing/raptor/raptor/tooltool-manifests/playback/mitm6-linux-firefox-wikipedia.manifest @@ -0,0 +1,9 @@ +[ + { + "size": 6563660, + "visibility": "public", + "digest": "4e685e571ea5a4a1b6533d6a53360b3e1701191a352de421e75edd7959d8f356e9c5c514ebd37ac507e0a61544d900ba10e4f4fddb64cca66e6515e0e8d1cf03", + "algorithm": "sha512", + "filename": "mitm6-linux-firefox-wikipedia.zip" + } +] diff --git a/testing/raptor/raptor/tooltool-manifests/playback/mitm6-linux-firefox-youtube.manifest b/testing/raptor/raptor/tooltool-manifests/playback/mitm6-linux-firefox-youtube.manifest new file mode 100644 index 0000000000..60f08aa151 --- /dev/null +++ b/testing/raptor/raptor/tooltool-manifests/playback/mitm6-linux-firefox-youtube.manifest @@ -0,0 +1,9 @@ +[ + { + "size": 8559123, + "visibility": "public", + "digest": "91aa7e05a362443646ba17e277202f10bd8d50cfc7dae99c9cb32c27c9dbc35a6e88de7c0e665681424a4241f2925f3c643fdfd09952bab8e49489a6b69ec87a", + "algorithm": "sha512", + "filename": "mitm6-linux-firefox-youtube.zip" + } +] diff --git a/testing/raptor/raptor/tooltool-manifests/playback/mitm6-windows-firefox-cnn-nav.manifest b/testing/raptor/raptor/tooltool-manifests/playback/mitm6-windows-firefox-cnn-nav.manifest new file mode 100644 index 0000000000..b68277f269 --- /dev/null +++ b/testing/raptor/raptor/tooltool-manifests/playback/mitm6-windows-firefox-cnn-nav.manifest @@ -0,0 +1,9 @@ +[ + { + "filename": "mitm6-windows-firefox-cnn-nav.zip", + "size": 11461220, + "algorithm": "sha512", + "digest": "201615126b59bd08df0a85226e6ee1aa3409e9b99b028b59ae61cce66fba05cbb7d33a7b2d2997370b4215325931b30bd6190023855c6d0d2415a9fdcba77c77", + "visibility": "public" + } +] diff --git a/testing/raptor/raptor/tooltool-manifests/playback/mitm6-windows-firefox-facebook-nav.manifest b/testing/raptor/raptor/tooltool-manifests/playback/mitm6-windows-firefox-facebook-nav.manifest new file mode 100644 index 0000000000..e237eaeafc --- /dev/null +++ b/testing/raptor/raptor/tooltool-manifests/playback/mitm6-windows-firefox-facebook-nav.manifest @@ -0,0 +1,9 @@ +[ + { + "size": 27164775, + "visibility": "public", + "digest": "ededd69afd6740712c778b969db37bfd2711664bf07e55596dce615de824d4dc19a008a9c5d9a9a62eb12252d99a00004d77f95c5aeaeb83cd4b39a753330008", + "algorithm": "sha512", + "filename": "mitm6-linux-firefox-facebook-nav.zip" + } +] diff --git a/testing/raptor/raptor/tooltool-manifests/playback/mitm6-windows-firefox-reddit-billgates-ama.manifest b/testing/raptor/raptor/tooltool-manifests/playback/mitm6-windows-firefox-reddit-billgates-ama.manifest new file mode 100644 index 0000000000..635fbff494 --- /dev/null +++ b/testing/raptor/raptor/tooltool-manifests/playback/mitm6-windows-firefox-reddit-billgates-ama.manifest @@ -0,0 +1,9 @@ +[ + { + "filename": "mitm6-windows-firefox-reddit-billgates-ama.zip", + "size": 10560692, + "algorithm": "sha512", + "digest": "77648b7d0499c97e7b4f48a8861566ab3c66c7b74ae5b842d62c59fbdb782332cd8b830a8bbf3645904772c7e43b98aaa4d807ae9150a4be9da137b768fa67aa", + "visibility": "public" + } +] diff --git a/testing/raptor/raptor/tooltool-manifests/playback/mitm6-windows-firefox-reddit-billgates-post.manifest b/testing/raptor/raptor/tooltool-manifests/playback/mitm6-windows-firefox-reddit-billgates-post.manifest new file mode 100644 index 0000000000..9782f22866 --- /dev/null +++ b/testing/raptor/raptor/tooltool-manifests/playback/mitm6-windows-firefox-reddit-billgates-post.manifest @@ -0,0 +1,9 @@ +[ + { + "size": 6426903, + "visibility": "public", + "digest": "4c79c415f5b0f2101bff014d20a176b4ec6e0d967b2658730070ccf287e25958fedde7fa8d8051156a126adb6a9249e6c3cadb95c8450bf8468ab16297307849", + "algorithm": "sha512", + "filename": "mitm6-linux-firefox-reddit-billgates-post.zip" + } +] diff --git a/testing/raptor/raptor/tooltool-manifests/playback/mitm7-android-gve-p2-web-de.manifest b/testing/raptor/raptor/tooltool-manifests/playback/mitm7-android-gve-p2-web-de.manifest new file mode 100644 index 0000000000..3ebf0eea60 --- /dev/null +++ b/testing/raptor/raptor/tooltool-manifests/playback/mitm7-android-gve-p2-web-de.manifest @@ -0,0 +1,9 @@ +[ + { + "filename": "mitm7-android-gve-web-de.zip", + "size": 10329318, + "algorithm": "sha512", + "digest": "6eb6c1563409325142adf8fcacf66d7e9d748df452edff2f918a662e008507c48feb884fe7fbe184d0ebffcae04f8e66b4042c41b541f56c6c4cc2b9b94c411d", + "visibility": "public" + } +] diff --git a/testing/raptor/raptor/tooltool-manifests/playback/mitm7-linux-firefox-amazon.manifest b/testing/raptor/raptor/tooltool-manifests/playback/mitm7-linux-firefox-amazon.manifest new file mode 100644 index 0000000000..5af2865fc4 --- /dev/null +++ b/testing/raptor/raptor/tooltool-manifests/playback/mitm7-linux-firefox-amazon.manifest @@ -0,0 +1,9 @@ +[ + { + "size": 20477615, + "visibility": "public", + "digest": "4f588d80825cd570a702d060a8e75443f5ba564f33a37fa75179829308b7258c7989cd13966d09e3258e87ca7da18b138041d0723d28f9f0397c72bdac559bf4", + "algorithm": "sha512", + "filename": "mitm7-linux-firefox-amazon.zip" + } +] diff --git a/testing/raptor/raptor/tooltool-manifests/playback/mitm7-linux-firefox-buzzfeed.manifest b/testing/raptor/raptor/tooltool-manifests/playback/mitm7-linux-firefox-buzzfeed.manifest new file mode 100644 index 0000000000..ec153b7d62 --- /dev/null +++ b/testing/raptor/raptor/tooltool-manifests/playback/mitm7-linux-firefox-buzzfeed.manifest @@ -0,0 +1,9 @@ +[ + { + "size": 26221048, + "visibility": "public", + "digest": "e3ec4b43e356a410a4af069414d68d248be0736e6cab21cfcfeca658673d27553dfa94a1718c9f0e6b18c71674173bc7ae46d1f700250166199b797759083fd1", + "algorithm": "sha512", + "filename": "mitm7-linux-firefox-buzzfeed.zip" + } +] diff --git a/testing/raptor/raptor/tooltool-manifests/playback/mitm7-linux-firefox-cnn-nav.manifest b/testing/raptor/raptor/tooltool-manifests/playback/mitm7-linux-firefox-cnn-nav.manifest new file mode 100644 index 0000000000..4620ade84a --- /dev/null +++ b/testing/raptor/raptor/tooltool-manifests/playback/mitm7-linux-firefox-cnn-nav.manifest @@ -0,0 +1,9 @@ +[ + { + "size": 14290513, + "visibility": "public", + "digest": "a012950f671b87db78bb9a19232770368990c0affb005c638a95c4f28fef03b249c2929fcd5fb926d52cc37eaeb22101800c223bf82d9251fa5416960ddef629", + "algorithm": "sha512", + "filename": "mitm7-linux-firefox-cnn-nav.zip" + } +] diff --git a/testing/raptor/raptor/tooltool-manifests/playback/mitm7-linux-firefox-cnn.manifest b/testing/raptor/raptor/tooltool-manifests/playback/mitm7-linux-firefox-cnn.manifest new file mode 100644 index 0000000000..ea226526f5 --- /dev/null +++ b/testing/raptor/raptor/tooltool-manifests/playback/mitm7-linux-firefox-cnn.manifest @@ -0,0 +1,9 @@ +[ + { + "size": 12160103, + "visibility": "public", + "digest": "21233727203b8b7a253d9b1836688ac4b4de1ce0b9e7d494403ae5de569bac6baa221581cb761d1f9d72e324fa2a94cfd97b7ffe79f4fc73768b7392c1dd2524", + "algorithm": "sha512", + "filename": "mitm7-linux-firefox-cnn.zip" + } +] diff --git a/testing/raptor/raptor/tooltool-manifests/playback/mitm7-linux-firefox-ebay.manifest b/testing/raptor/raptor/tooltool-manifests/playback/mitm7-linux-firefox-ebay.manifest new file mode 100644 index 0000000000..c003f63800 --- /dev/null +++ b/testing/raptor/raptor/tooltool-manifests/playback/mitm7-linux-firefox-ebay.manifest @@ -0,0 +1,9 @@ +[ + { + "size": 8107058, + "visibility": "public", + "digest": "6fef796da1ad5b5e582893e305577291133a224216dd1781fada2fdef87b0b5258dc43d9340dafcbf5a8edbf238c987692554fc6bc54b75f74e4b29be426487b", + "algorithm": "sha512", + "filename": "mitm7-linux-firefox-ebay.zip" + } +] diff --git a/testing/raptor/raptor/tooltool-manifests/playback/mitm7-linux-firefox-espn.manifest b/testing/raptor/raptor/tooltool-manifests/playback/mitm7-linux-firefox-espn.manifest new file mode 100644 index 0000000000..f85c6104a0 --- /dev/null +++ b/testing/raptor/raptor/tooltool-manifests/playback/mitm7-linux-firefox-espn.manifest @@ -0,0 +1,9 @@ +[ + { + "size": 15427319, + "visibility": "public", + "digest": "aaabccff6617653e3333aa4df9171c7d658208cde73cba31aebe6c86445bc6c573ca301a51779ef5dc6e93bb360d12126930a22bea71aa93caa7266a994e1a28", + "algorithm": "sha512", + "filename": "mitm7-linux-firefox-espn.zip" + } +] diff --git a/testing/raptor/raptor/tooltool-manifests/playback/mitm7-linux-firefox-expedia.manifest b/testing/raptor/raptor/tooltool-manifests/playback/mitm7-linux-firefox-expedia.manifest new file mode 100644 index 0000000000..e1654025d4 --- /dev/null +++ b/testing/raptor/raptor/tooltool-manifests/playback/mitm7-linux-firefox-expedia.manifest @@ -0,0 +1,9 @@ +[ + { + "size": 12966035, + "visibility": "public", + "digest": "cd867226b3e27cebc4176fec0261b2d1750118933f5c66735ba8627ba90d42580e11d0f2012ef4be66a853604bf1e039e7f84d005871cd0ec98585c1133054c8", + "algorithm": "sha512", + "filename": "mitm7-linux-firefox-expedia.zip" + } +] diff --git a/testing/raptor/raptor/tooltool-manifests/playback/mitm7-linux-firefox-google-docs.manifest b/testing/raptor/raptor/tooltool-manifests/playback/mitm7-linux-firefox-google-docs.manifest new file mode 100644 index 0000000000..0da20ea782 --- /dev/null +++ b/testing/raptor/raptor/tooltool-manifests/playback/mitm7-linux-firefox-google-docs.manifest @@ -0,0 +1,9 @@ +[ + { + "size": 53014906, + "visibility": "public", + "digest": "5b56e1ce9fb02fcc815a4200c8fa8b6ac50ecce675efc858d114df2a027f36500dea8b583b01f7021adc86e94d6e39f47c4a699010732fa3e48995e63832c346", + "algorithm": "sha512", + "filename": "mitm7-linux-firefox-google-docs.zip" + } +] diff --git a/testing/raptor/raptor/tooltool-manifests/playback/mitm7-linux-firefox-imdb.manifest b/testing/raptor/raptor/tooltool-manifests/playback/mitm7-linux-firefox-imdb.manifest new file mode 100644 index 0000000000..369cf2045c --- /dev/null +++ b/testing/raptor/raptor/tooltool-manifests/playback/mitm7-linux-firefox-imdb.manifest @@ -0,0 +1,9 @@ +[ + { + "size": 8168718, + "visibility": "public", + "digest": "a3d622feca8085a2eef969e0866e4dbb6eb31571446db61108dcb66a7e0c0c4ae2b3b1ab200c36fcbdbd19ac0e44ceba97630b251282dbf6406009b446168793", + "algorithm": "sha512", + "filename": "mitm7-linux-firefox-imdb.zip" + } +] diff --git a/testing/raptor/raptor/tooltool-manifests/playback/mitm7-linux-firefox-imgur.manifest b/testing/raptor/raptor/tooltool-manifests/playback/mitm7-linux-firefox-imgur.manifest new file mode 100644 index 0000000000..1889baa927 --- /dev/null +++ b/testing/raptor/raptor/tooltool-manifests/playback/mitm7-linux-firefox-imgur.manifest @@ -0,0 +1,9 @@ +[ + { + "size": 33166483, + "visibility": "public", + "digest": "3d1dc0e1b5d1ba2f7004cdd975c62a8cb010a53aeba74da48225f9cb22d0091f2109e9b29053198c0c6df268665ed919f69cca04d0efe161c35a234b59272fb8", + "algorithm": "sha512", + "filename": "mitm7-linux-firefox-imgur.zip" + } +] diff --git a/testing/raptor/raptor/tooltool-manifests/playback/mitm7-linux-firefox-microsoft.manifest b/testing/raptor/raptor/tooltool-manifests/playback/mitm7-linux-firefox-microsoft.manifest new file mode 100644 index 0000000000..092958e287 --- /dev/null +++ b/testing/raptor/raptor/tooltool-manifests/playback/mitm7-linux-firefox-microsoft.manifest @@ -0,0 +1,9 @@ +[ + { + "size": 5078634, + "visibility": "public", + "digest": "315399f0773837ef13b9ba768a8a72a001ba103e8c09992ebef0421b6639beb5d48bb71bf47c29bfe8484bed19bfc4f9b97b9513f6593c9afb01f078a3a9b7f4", + "algorithm": "sha512", + "filename": "mitm7-linux-firefox-microsoft.zip" + } +] diff --git a/testing/raptor/raptor/tooltool-manifests/playback/mitm7-linux-firefox-nytimes.manifest b/testing/raptor/raptor/tooltool-manifests/playback/mitm7-linux-firefox-nytimes.manifest new file mode 100644 index 0000000000..720a0b04eb --- /dev/null +++ b/testing/raptor/raptor/tooltool-manifests/playback/mitm7-linux-firefox-nytimes.manifest @@ -0,0 +1,9 @@ +[ + { + "size": 20218123, + "visibility": "public", + "digest": "92fa5f3e55ce1ab9fcdb4fb0447492c3a61339a8f00043d6eb6294ce8bd83a9af4656ed858af84b08c86cc147a3f2d9d43d66489cc2b68cef4c7048b7172202f", + "algorithm": "sha512", + "filename": "mitm7-linux-firefox-nytimes.zip" + } +] diff --git a/testing/raptor/raptor/tooltool-manifests/playback/mitm7-linux-firefox-office.manifest b/testing/raptor/raptor/tooltool-manifests/playback/mitm7-linux-firefox-office.manifest new file mode 100644 index 0000000000..88b5c4e126 --- /dev/null +++ b/testing/raptor/raptor/tooltool-manifests/playback/mitm7-linux-firefox-office.manifest @@ -0,0 +1,9 @@ +[ + { + "size": 8703303, + "visibility": "public", + "digest": "e802cfddd43299dc4bea3d1db304ac6713e2c4bc96c91039df3efc95db9aeefaac4aad1d88d282b49ed6039e58ff6deb451766c9f2c810d0d006851be7c3d00f", + "algorithm": "sha512", + "filename": "mitm7-linux-firefox-office.zip" + } +] diff --git a/testing/raptor/raptor/tooltool-manifests/playback/mitm7-linux-firefox-twitch.manifest b/testing/raptor/raptor/tooltool-manifests/playback/mitm7-linux-firefox-twitch.manifest new file mode 100644 index 0000000000..a332a531fa --- /dev/null +++ b/testing/raptor/raptor/tooltool-manifests/playback/mitm7-linux-firefox-twitch.manifest @@ -0,0 +1,9 @@ +[ + { + "size": 17618621, + "visibility": "public", + "digest": "e88b8ac72e981e745c5eba9dddc801688678571188ba15d4da7ae80558e167a95ba77bb4540be94be85644329f0fdfb56076ec14ada41fe97f90a56eed41c42c", + "algorithm": "sha512", + "filename": "mitm7-linux-firefox-twitch.zip" + } +] diff --git a/testing/raptor/raptor/tooltool-manifests/playback/mitm7-linux-firefox-wikia.manifest b/testing/raptor/raptor/tooltool-manifests/playback/mitm7-linux-firefox-wikia.manifest new file mode 100644 index 0000000000..d91deff3d4 --- /dev/null +++ b/testing/raptor/raptor/tooltool-manifests/playback/mitm7-linux-firefox-wikia.manifest @@ -0,0 +1,9 @@ +[ + { + "size": 20316960, + "visibility": "public", + "digest": "f00c10e0c0d6f3fa29b770cff775504f365bcf8034e2876ee9b7adb9974db17b62cc352d787ca90d548682f68a9258a8704573de6c05e1951eceafb34e9a9a03", + "algorithm": "sha512", + "filename": "mitm7-linux-firefox-wikia.zip" + } +] diff --git a/testing/raptor/raptor/tooltool-manifests/playback/mitm7-linux-firefox-wikipedia.manifest b/testing/raptor/raptor/tooltool-manifests/playback/mitm7-linux-firefox-wikipedia.manifest new file mode 100644 index 0000000000..131ce07c22 --- /dev/null +++ b/testing/raptor/raptor/tooltool-manifests/playback/mitm7-linux-firefox-wikipedia.manifest @@ -0,0 +1,9 @@ +[ + { + "size": 8154902, + "visibility": "public", + "digest": "28965ce327f342e2ae62a1cab6262d1023bc294c764322adf96e13b3d633aaabbbc2dbe4ea5647b32485f63db7cd10c142c2293415b0af81646ee5b853ee744c", + "algorithm": "sha512", + "filename": "mitm7-linux-firefox-wikipedia.zip" + } +] diff --git a/testing/raptor/raptor/tooltool-manifests/playback/mitm7-linux-firefox-youtube.manifest b/testing/raptor/raptor/tooltool-manifests/playback/mitm7-linux-firefox-youtube.manifest new file mode 100644 index 0000000000..c227f28b02 --- /dev/null +++ b/testing/raptor/raptor/tooltool-manifests/playback/mitm7-linux-firefox-youtube.manifest @@ -0,0 +1,9 @@ +[ + { + "size": 10521203, + "visibility": "public", + "digest": "83bc724fb2d1c2377e717184de0a2cb85ea197703c63596aece38602fe16994a461b74ff48e5cdf3475712a7c6ff5fc0b7268a751d5cbf87b1f71c96b4d86e94", + "algorithm": "sha512", + "filename": "mitm7-linux-firefox-youtube.zip" + } +] diff --git a/testing/raptor/raptor/tooltool-manifests/playback/mitm8-android-gve-instagram.manifest b/testing/raptor/raptor/tooltool-manifests/playback/mitm8-android-gve-instagram.manifest new file mode 100644 index 0000000000..5a9e8d83f6 --- /dev/null +++ b/testing/raptor/raptor/tooltool-manifests/playback/mitm8-android-gve-instagram.manifest @@ -0,0 +1,9 @@ +[ + { + "filename": "mitm8-android-gve-instagram.zip", + "size": 20641917, + "algorithm": "sha512", + "digest": "88abb3f89a4e54acb34998395c6768f1a8f36d24f78da71da2505cbfef6b5ab532a0614f7d4f076714a3189cd660ea2a07ec283468910a678f2cf2fefe7d7644", + "visibility": "public" + } +] diff --git a/testing/raptor/raptor/utils.py b/testing/raptor/raptor/utils.py new file mode 100644 index 0000000000..dd78eac1a0 --- /dev/null +++ b/testing/raptor/raptor/utils.py @@ -0,0 +1,146 @@ +"""Utility functions for Raptor""" +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import os +from collections.abc import Iterable +from distutils.util import strtobool + +import yaml +from logger.logger import RaptorLogger +from mozgeckoprofiler import view_gecko_profile + +LOG = RaptorLogger(component="raptor-utils") +here = os.path.dirname(os.path.realpath(__file__)) + +external_tools_path = os.environ.get("EXTERNALTOOLSPATH", None) + +if external_tools_path is not None: + # running in production via mozharness + TOOLTOOL_PATH = os.path.join(external_tools_path, "tooltool.py") + + +def flatten(data, parent_dir, sep="/"): + """ + Converts a dictionary with nested entries like this + { + "dict1": { + "dict2": { + "key1": value1, + "key2": value2, + ... + }, + ... + }, + ... + "dict3": { + "key3": value3, + "key4": value4, + ... + } + ... + } + + to a "flattened" dictionary like this that has no nested entries: + { + "dict1-dict2-key1": value1, + "dict1-dict2-key2": value2, + ... + "dict3-key3": value3, + "dict3-key4": value4, + ... + } + + :param Iterable data : json data. + :param tuple parent_dir: json fields. + + :return dict: {subtest: value} + """ + result = {} + + if not data: + return result + + if isinstance(data, list): + for item in data: + for k, v in flatten(item, parent_dir, sep=sep).items(): + result.setdefault(k, []).extend(v) + + if isinstance(data, dict): + for k, v in data.items(): + current_dir = parent_dir + (k,) + subtest = sep.join(current_dir) + if isinstance(v, Iterable) and not isinstance(v, str): + for x, y in flatten(v, current_dir, sep=sep).items(): + result.setdefault(x, []).extend(y) + elif v or v == 0: + result.setdefault(subtest, []).append(v) + + return result + + +def transform_platform(str_to_transform, config_platform, config_processor=None): + """Transform platform name i.e. 'mitmproxy-rel-bin-{platform}.manifest' + transforms to 'mitmproxy-rel-bin-osx.manifest'. + Also transform '{x64}' if needed for 64 bit / win 10""" + if "{platform}" not in str_to_transform and "{x64}" not in str_to_transform: + return str_to_transform + + if "win" in config_platform: + platform_id = "win" + elif config_platform == "mac": + platform_id = "osx" + else: + platform_id = "linux64" + + if "{platform}" in str_to_transform: + str_to_transform = str_to_transform.replace("{platform}", platform_id) + + if "{x64}" in str_to_transform and config_processor is not None: + if "x86_64" in config_processor: + str_to_transform = str_to_transform.replace("{x64}", "_x64") + else: + str_to_transform = str_to_transform.replace("{x64}", "") + + return str_to_transform + + +def transform_subtest(str_to_transform, subtest_name): + """Transform subtest name i.e. 'mitm5-linux-firefox-{subtest}.manifest' + transforms to 'mitm5-linux-firefox-amazon.manifest'.""" + if "{subtest}" not in str_to_transform: + return str_to_transform + + return str_to_transform.replace("{subtest}", subtest_name) + + +def view_gecko_profile_from_raptor(): + # automatically load the latest raptor gecko-profile archive in profiler.firefox.com + LOG_GECKO = RaptorLogger(component="raptor-view-gecko-profile") + + profile_zip_path = os.environ.get("RAPTOR_LATEST_GECKO_PROFILE_ARCHIVE", None) + if profile_zip_path is None or not os.path.exists(profile_zip_path): + LOG_GECKO.info( + "No local raptor gecko profiles were found so not " + "launching profiler.firefox.com" + ) + return + + LOG_GECKO.info("Profile saved locally to: %s" % profile_zip_path) + view_gecko_profile(profile_zip_path) + + +def write_yml_file(yml_file, yml_data): + # write provided data to specified local yaml file + LOG.info("writing %s to %s" % (yml_data, yml_file)) + + try: + with open(yml_file, "w") as outfile: + yaml.dump(yml_data, outfile, default_flow_style=False) + except Exception as e: + LOG.critical("failed to write yaml file, exeption: %s" % e) + + +def bool_from_str(boolean_string): + return bool(strtobool(boolean_string)) diff --git a/testing/raptor/raptor/webextension/__init__.py b/testing/raptor/raptor/webextension/__init__.py new file mode 100644 index 0000000000..b17fc38e56 --- /dev/null +++ b/testing/raptor/raptor/webextension/__init__.py @@ -0,0 +1,7 @@ +# flake8: noqa +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +from .android import WebExtensionAndroid +from .desktop import WebExtensionDesktopChrome, WebExtensionFirefox diff --git a/testing/raptor/raptor/webextension/android.py b/testing/raptor/raptor/webextension/android.py new file mode 100644 index 0000000000..5aadc5e4a5 --- /dev/null +++ b/testing/raptor/raptor/webextension/android.py @@ -0,0 +1,396 @@ +#!/usr/bin/env python + +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import os +import posixpath +import shutil +import tempfile +import time + +import mozcrash +from cpu import start_android_cpu_profiler +from logger.logger import RaptorLogger +from mozdevice import ADBDeviceFactory, ADBProcessError +from performance_tuning import tune_performance +from perftest import PerftestAndroid +from power import ( + disable_charging, + enable_charging, + finish_android_power_test, + init_android_power_test, +) +from signal_handler import SignalHandlerException +from utils import write_yml_file +from webextension.base import WebExtension + +LOG = RaptorLogger(component="raptor-webext-android") + + +class WebExtensionAndroid(PerftestAndroid, WebExtension): + def __init__(self, app, binary, activity=None, intent=None, **kwargs): + super(WebExtensionAndroid, self).__init__( + app, binary, profile_class="firefox", **kwargs + ) + + self.config.update({"activity": activity, "intent": intent}) + + self.os_baseline_data = None + self.power_test_time = None + self.screen_off_timeout = 0 + self.screen_brightness = 127 + self.app_launched = False + + def setup_adb_device(self): + if self.device is None: + self.device = ADBDeviceFactory(verbose=True) + if not self.config.get("disable_perf_tuning", False): + tune_performance(self.device, log=LOG) + + self.device.run_as_package = self.config["binary"] + self.remote_test_root = os.path.join(self.device.test_root, "raptor") + self.remote_profile = os.path.join(self.remote_test_root, "profile") + if self.config["power_test"]: + disable_charging(self.device) + + LOG.info("creating remote root folder for raptor: %s" % self.remote_test_root) + self.device.rm(self.remote_test_root, force=True, recursive=True) + self.device.mkdir(self.remote_test_root, parents=True) + + self.clear_app_data() + self.set_debug_app_flag() + + def process_exists(self): + return self.device is not None and self.device.process_exist( + self.config["binary"] + ) + + def write_android_app_config(self): + # geckoview supports having a local on-device config file; use this file + # to tell the app to use the specified browser profile, as well as other opts + # on-device: /data/local/tmp/com.yourcompany.yourapp-geckoview-config.yaml + # https://mozilla.github.io/geckoview/tutorials/automation.html#configuration-file-format + + LOG.info("creating android app config.yml") + + yml_config_data = dict( + args=[ + "--profile", + self.remote_profile, + "--allow-downgrade", + ], + env=dict( + LOG_VERBOSE=1, + R_LOG_LEVEL=6, + ), + ) + + yml_name = "%s-geckoview-config.yaml" % self.config["binary"] + yml_on_host = os.path.join(tempfile.mkdtemp(), yml_name) + write_yml_file(yml_on_host, yml_config_data) + yml_on_device = os.path.join("/data", "local", "tmp", yml_name) + + try: + LOG.info("copying %s to device: %s" % (yml_on_host, yml_on_device)) + self.device.rm(yml_on_device, force=True, recursive=True) + self.device.push(yml_on_host, yml_on_device) + + except Exception: + LOG.critical("failed to push %s to device!" % yml_on_device) + raise + + def log_android_device_temperature(self): + # retrieve and log the android device temperature + try: + # use sort since cat gives I/O Error on Pixel 2 - 10. + thermal_zone0 = self.device.shell_output( + "sort /sys/class/thermal/thermal_zone0/temp" + ) + try: + thermal_zone0 = "%.3f" % (float(thermal_zone0) / 1000) + except ValueError: + thermal_zone0 = "Unknown" + except ADBProcessError: + thermal_zone0 = "Unknown" + try: + zone_type = self.device.shell_output( + "cat /sys/class/thermal/thermal_zone0/type" + ) + except ADBProcessError: + zone_type = "Unknown" + LOG.info( + "(thermal_zone0) device temperature: %s zone type: %s" + % (thermal_zone0, zone_type) + ) + + def launch_firefox_android_app(self, test_name): + LOG.info("starting %s" % self.config["app"]) + + try: + # make sure the android app is not already running + self.device.stop_application(self.config["binary"]) + + # command line 'extra' args not used with geckoview apps; instead we use + # an on-device config.yml file (see write_android_app_config) + + self.device.launch_application( + self.config["binary"], + self.config["activity"], + self.config["intent"], + extras=None, + url="about:blank", + fail_if_running=False, + ) + + # Check if app has started and it's running + if not self.process_exists: + raise Exception( + "Error launching %s. App did not start properly!" + % self.config["binary"] + ) + self.app_launched = True + except Exception as e: + LOG.error("Exception launching %s" % self.config["binary"]) + LOG.error("Exception: %s %s" % (type(e).__name__, str(e))) + if self.config["power_test"]: + finish_android_power_test(self, test_name) + raise + + # give our control server the device and app info + self.control_server.device = self.device + self.control_server.app_name = self.config["binary"] + + def copy_cert_db(self, source_dir, target_dir): + # copy browser cert db (that was previously created via certutil) from source to target + cert_db_files = ["pkcs11.txt", "key4.db", "cert9.db"] + for next_file in cert_db_files: + _source = os.path.join(source_dir, next_file) + _dest = os.path.join(target_dir, next_file) + if os.path.exists(_source): + LOG.info("copying %s to %s" % (_source, _dest)) + shutil.copyfile(_source, _dest) + else: + LOG.critical("unable to find ssl cert db file: %s" % _source) + + def run_tests(self, tests, test_names): + self.setup_adb_device() + + return super(WebExtensionAndroid, self).run_tests(tests, test_names) + + def run_test_setup(self, test): + super(WebExtensionAndroid, self).run_test_setup(test) + self.set_reverse_ports() + + def run_test_teardown(self, test): + LOG.info("removing reverse socket connections") + self.device.remove_socket_connections("reverse") + + super(WebExtensionAndroid, self).run_test_teardown(test) + + def run_test(self, test, timeout): + # tests will be run warm (i.e. NO browser restart between page-cycles) + # unless otheriwse specified in the test INI by using 'cold = true' + try: + + if self.config["power_test"]: + # gather OS baseline data + init_android_power_test(self) + LOG.info("Running OS baseline, pausing for 1 minute...") + time.sleep(60) + LOG.info("Finishing baseline...") + finish_android_power_test(self, "os-baseline", os_baseline=True) + + # initialize for the test + init_android_power_test(self) + + if self.config.get("cold") or test.get("cold"): + self.__run_test_cold(test, timeout) + else: + self.__run_test_warm(test, timeout) + + except SignalHandlerException: + self.device.stop_application(self.config["binary"]) + if self.config["power_test"]: + enable_charging(self.device) + + finally: + if self.config["power_test"]: + finish_android_power_test(self, test["name"]) + + def __run_test_cold(self, test, timeout): + """ + Run the Raptor test but restart the entire browser app between page-cycles. + + Note: For page-load tests, playback will only be started once - at the beginning of all + browser cycles, and then stopped after all cycles are finished. The proxy is set via prefs + in the browser profile so those will need to be set again in each new profile/cycle. + Note that instead of using the certutil tool each time to create a db and import the + mitmproxy SSL cert (it's done in mozbase/mozproxy) we will simply copy the existing + cert db from the first cycle's browser profile into the new clean profile; this way + we don't have to re-create the cert db on each browser cycle. + + Since we're running in cold-mode, before this point (in manifest.py) the + 'expected-browser-cycles' value was already set to the initial 'page-cycles' value; + and the 'page-cycles' value was set to 1 as we want to perform one page-cycle per + browser restart. + + The 'browser-cycle' value is the current overall browser start iteration. The control + server will receive the current 'browser-cycle' and the 'expected-browser-cycles' in + each results set received; and will pass that on as part of the results so that the + results processing will know results for multiple browser cycles are being received. + + The default will be to run in warm mode; unless 'cold = true' is set in the test INI. + """ + LOG.info( + "test %s is running in cold mode; browser WILL be restarted between " + "page cycles" % test["name"] + ) + + for test["browser_cycle"] in range(1, test["expected_browser_cycles"] + 1): + + LOG.info( + "begin browser cycle %d of %d for test %s" + % (test["browser_cycle"], test["expected_browser_cycles"], test["name"]) + ) + + self.run_test_setup(test) + + self.clear_app_data() + self.set_debug_app_flag() + + if test["browser_cycle"] == 1: + if test.get("playback") is not None: + # an ssl cert db has now been created in the profile; copy it out so we + # can use the same cert db in future test cycles / browser restarts + local_cert_db_dir = tempfile.mkdtemp() + LOG.info( + "backing up browser ssl cert db that was created via certutil" + ) + self.copy_cert_db( + self.config["local_profile_dir"], local_cert_db_dir + ) + + if not self.is_localhost: + self.delete_proxy_settings_from_profile() + + else: + # double-check to ensure app has been shutdown + self.device.stop_application(self.config["binary"]) + + # initial browser profile was already created before run_test was called; + # now additional browser cycles we want to create a new one each time + self.build_browser_profile() + + if test.get("playback") is not None: + # get cert db from previous cycle profile and copy into new clean profile + # this saves us from having to start playback again / recreate cert db etc. + LOG.info("copying existing ssl cert db into new browser profile") + self.copy_cert_db( + local_cert_db_dir, self.config["local_profile_dir"] + ) + + self.run_test_setup(test) + + if test.get("playback") is not None: + self.turn_on_android_app_proxy() + + self.copy_profile_to_device() + self.log_android_device_temperature() + + # write android app config.yml + self.write_android_app_config() + + # now start the browser/app under test + self.launch_firefox_android_app(test["name"]) + + # set our control server flag to indicate we are running the browser/app + self.control_server._finished = False + + if self.config["cpu_test"]: + # start measuring CPU usage + self.cpu_profiler = start_android_cpu_profiler(self) + + self.wait_for_test_finish(test, timeout, self.process_exists) + + # in debug mode, and running locally, leave the browser running + if self.debug_mode and self.config["run_local"]: + LOG.info( + "* debug-mode enabled - please shutdown the browser manually..." + ) + self.runner.wait(timeout=None) + + # break test execution if a exception is present + if len(self.results_handler.page_timeout_list) > 0: + break + + def __run_test_warm(self, test, timeout): + LOG.info( + "test %s is running in warm mode; browser will NOT be restarted between " + "page cycles" % test["name"] + ) + + self.run_test_setup(test) + + if not self.is_localhost: + self.delete_proxy_settings_from_profile() + + if test.get("playback") is not None: + self.turn_on_android_app_proxy() + + self.clear_app_data() + self.set_debug_app_flag() + self.copy_profile_to_device() + self.log_android_device_temperature() + + # write android app config.yml + self.write_android_app_config() + + # now start the browser/app under test + self.launch_firefox_android_app(test["name"]) + + # set our control server flag to indicate we are running the browser/app + self.control_server._finished = False + + if self.config["cpu_test"]: + # start measuring CPU usage + self.cpu_profiler = start_android_cpu_profiler(self) + + self.wait_for_test_finish(test, timeout, self.process_exists) + + # in debug mode, and running locally, leave the browser running + if self.debug_mode and self.config["run_local"]: + LOG.info("* debug-mode enabled - please shutdown the browser manually...") + + def check_for_crashes(self): + super(WebExtensionAndroid, self).check_for_crashes() + + if not self.app_launched: + LOG.info("skipping check_for_crashes: application has not been launched") + return + self.app_launched = False + + try: + dump_dir = tempfile.mkdtemp() + remote_dir = posixpath.join(self.remote_profile, "minidumps") + if not self.device.is_dir(remote_dir): + return + self.device.pull(remote_dir, dump_dir) + self.crashes += mozcrash.log_crashes( + LOG, dump_dir, self.config["symbols_path"] + ) + finally: + try: + shutil.rmtree(dump_dir) + except Exception: + LOG.warning("unable to remove directory: %s" % dump_dir) + + def clean_up(self): + LOG.info("removing test folder for raptor: %s" % self.remote_test_root) + self.device.rm(self.remote_test_root, force=True, recursive=True) + + if self.config["power_test"]: + enable_charging(self.device) + + super(WebExtensionAndroid, self).clean_up() diff --git a/testing/raptor/raptor/webextension/base.py b/testing/raptor/raptor/webextension/base.py new file mode 100644 index 0000000000..ab21c5cb19 --- /dev/null +++ b/testing/raptor/raptor/webextension/base.py @@ -0,0 +1,249 @@ +#!/usr/bin/env python + +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import json +import os +import time + +import requests +from benchmark import Benchmark +from cmdline import CHROMIUM_DISTROS +from control_server import RaptorControlServer +from gen_test_config import gen_test_config +from logger.logger import RaptorLogger +from perftest import Perftest +from results import RaptorResultsHandler + +from memory import generate_android_memory_profile + +LOG = RaptorLogger(component="raptor-webext") + +here = os.path.abspath(os.path.dirname(__file__)) +webext_dir = os.path.join(here, "..", "..", "webext") + + +class WebExtension(Perftest): + """Container class for WebExtension.""" + + def __init__(self, *args, **kwargs): + self.raptor_webext = None + self.control_server = None + self.cpu_profiler = None + + super(WebExtension, self).__init__(*args, **kwargs) + + # set up the results handler + self.results_handler = RaptorResultsHandler(**self.config) + browser_name, browser_version = self.get_browser_meta() + self.results_handler.add_browser_meta(self.config["app"], browser_version) + + self.start_control_server() + + def run_test_setup(self, test): + super(WebExtension, self).run_test_setup(test) + + LOG.info("starting web extension test: %s" % test["name"]) + LOG.info("test settings: %s" % str(test)) + LOG.info("web extension config: %s" % str(self.config)) + + if test.get("type") == "benchmark": + self.serve_benchmark_source(test) + + gen_test_config( + test["name"], + self.control_server.port, + self.post_startup_delay, + host=self.config["host"], + b_port=int(self.benchmark.port) if self.benchmark else 0, + debug_mode=1 if self.debug_mode else 0, + browser_cycle=test.get("browser_cycle", 1), + ) + + self.install_raptor_webext() + + def wait_for_test_finish(self, test, timeout, process_exists_callback=None): + # this is a 'back-stop' i.e. if for some reason Raptor doesn't finish for some + # serious problem; i.e. the test was unable to send a 'page-timeout' to the control + # server, etc. Therefore since this is a 'back-stop' we want to be generous here; + # we don't want this timeout occurring unless abosultely necessary + + # convert timeout to seconds and account for page cycles + # pylint --py3k W1619 + timeout = int(timeout / 1000) * int(test.get("page_cycles", 1)) + # account for the pause the raptor webext runner takes after browser startup + # and the time an exception is propagated through the framework + # pylint --py3k W1619 + timeout += int(self.post_startup_delay / 1000) + 10 + + # for page-load tests we don't start the page-timeout timer until the pageload.js content + # is successfully injected and invoked; which differs per site being tested; therefore we + # need to be generous here - let's add 10 seconds extra per page-cycle + if test.get("type") == "pageload": + timeout += 10 * int(test.get("page_cycles", 1)) + + # if geckoProfile enabled, give browser more time for profiling + if self.config["gecko_profile"] is True: + timeout += 5 * 60 + + # we also need to give time for results processing, not just page/browser cycles! + timeout += 60 + + # stop 5 seconds early + end_time = time.time() + timeout - 5 + + while not self.control_server._finished: + # Ignore check if the control server shutdown the app + if not self.control_server._is_shutting_down: + # If the application is no longer running immediately bail out + if callable(process_exists_callback) and not process_exists_callback(): + raise RuntimeError("Process has been unexpectedly closed") + + if self.config["enable_control_server_wait"]: + response = self.control_server_wait_get() + if response == "webext_shutdownBrowser": + if self.config["memory_test"]: + generate_android_memory_profile(self, test["name"]) + if self.cpu_profiler: + self.cpu_profiler.generate_android_cpu_profile(test["name"]) + + self.control_server_wait_continue() + + # Sleep for a moment to not check the process too often + time.sleep(1) + + # we only want to force browser-shutdown on timeout if not in debug mode; + # in debug-mode we leave the browser running (require manual shutdown) + if not self.debug_mode and end_time < time.time(): + self.control_server.wait_for_quit() + + if not self.control_server.is_webextension_loaded: + raise RuntimeError("Connection to Raptor webextension failed!") + + raise RuntimeError( + "Test failed to finish. " + "Application timed out after {} seconds".format(timeout) + ) + + if self.control_server._runtime_error: + raise RuntimeError( + "Failed to run {}: {}\nStack:\n{}".format( + test["name"], + self.control_server._runtime_error["error"], + self.control_server._runtime_error["stack"], + ) + ) + + def run_test_teardown(self, test): + super(WebExtension, self).run_test_teardown(test) + + if self.playback is not None: + self.playback.stop() + self.playback = None + + self.remove_raptor_webext() + + def set_browser_test_prefs(self, raw_prefs): + # add test specific preferences + LOG.info("setting test-specific Firefox preferences") + self.profile.set_preferences(json.loads(raw_prefs)) + + def build_browser_profile(self): + super(WebExtension, self).build_browser_profile() + + if self.control_server: + # The control server and the browser profile are not well factored + # at this time, so the start-up process overlaps. Accommodate. + self.control_server.user_profile = self.profile + + def start_control_server(self): + self.control_server = RaptorControlServer(self.results_handler, self.debug_mode) + self.control_server.user_profile = self.profile + self.control_server.start() + + if self.config["enable_control_server_wait"]: + self.control_server_wait_set("webext_shutdownBrowser") + + def serve_benchmark_source(self, test): + # benchmark-type tests require the benchmark test to be served out + self.benchmark = Benchmark(self.config, test) + + def install_raptor_webext(self): + # must intall raptor addon each time because we dynamically update some content + # the webext is installed into the browser profile + # note: for chrome the addon is just a list of paths that ultimately are added + # to the chromium command line '--load-extension' argument + self.raptor_webext = os.path.join(webext_dir, "raptor") + LOG.info("installing webext %s" % self.raptor_webext) + self.profile.addons.install(self.raptor_webext) + + # on firefox we can get an addon id; chrome addon actually is just cmd line arg + try: + self.webext_id = self.profile.addons.addon_details(self.raptor_webext)["id"] + except AttributeError: + self.webext_id = None + + self.control_server.startup_handler(False) + + def remove_raptor_webext(self): + # remove the raptor webext; as it must be reloaded with each subtest anyway + if not self.raptor_webext: + LOG.info("raptor webext not installed - not attempting removal") + return + + LOG.info("removing webext %s" % self.raptor_webext) + if self.config["app"] in ["firefox", "geckoview", "refbrow", "fenix"]: + self.profile.addons.remove_addon(self.webext_id) + + # for chrome the addon is just a list (appended to cmd line) + chrome_apps = CHROMIUM_DISTROS + ["chrome-android", "chromium-android"] + if self.config["app"] in chrome_apps: + self.profile.addons.remove(self.raptor_webext) + + self.raptor_webext = None + + def clean_up(self): + super(WebExtension, self).clean_up() + + if self.config["enable_control_server_wait"]: + self.control_server_wait_clear("all") + + self.control_server.stop() + LOG.info("finished") + + def control_server_wait_set(self, state): + response = requests.post( + "http://127.0.0.1:%s/" % self.control_server.port, + json={"type": "wait-set", "data": state}, + ) + return response.text + + def control_server_wait_timeout(self, timeout): + response = requests.post( + "http://127.0.0.1:%s/" % self.control_server.port, + json={"type": "wait-timeout", "data": timeout}, + ) + return response.text + + def control_server_wait_get(self): + response = requests.post( + "http://127.0.0.1:%s/" % self.control_server.port, + json={"type": "wait-get", "data": ""}, + ) + return response.text + + def control_server_wait_continue(self): + response = requests.post( + "http://127.0.0.1:%s/" % self.control_server.port, + json={"type": "wait-continue", "data": ""}, + ) + return response.text + + def control_server_wait_clear(self, state): + response = requests.post( + "http://127.0.0.1:%s/" % self.control_server.port, + json={"type": "wait-clear", "data": state}, + ) + return response.text diff --git a/testing/raptor/raptor/webextension/desktop.py b/testing/raptor/raptor/webextension/desktop.py new file mode 100644 index 0000000000..5903e09546 --- /dev/null +++ b/testing/raptor/raptor/webextension/desktop.py @@ -0,0 +1,263 @@ +#!/usr/bin/env python + +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import os +import shutil + +from logger.logger import RaptorLogger +from mozpower import MozPower +from mozrunner import runners +from outputhandler import OutputHandler +from perftest import PerftestDesktop + +from .base import WebExtension + +LOG = RaptorLogger(component="raptor-webext-desktop") + + +class WebExtensionDesktop(PerftestDesktop, WebExtension): + def __init__(self, *args, **kwargs): + super(WebExtensionDesktop, self).__init__(*args, **kwargs) + + # create the desktop browser runner + LOG.info("creating browser runner using mozrunner") + self.output_handler = OutputHandler(verbose=self.config["verbose"]) + process_args = {"processOutputLine": [self.output_handler]} + firefox_args = ["--allow-downgrade"] + runner_cls = runners[self.config["app"]] + self.runner = runner_cls( + self.config["binary"], + profile=self.profile, + cmdargs=firefox_args, + process_args=process_args, + symbols_path=self.config["symbols_path"], + ) + + # Force Firefox to immediately exit for content crashes + self.runner.env["MOZ_CRASHREPORTER_SHUTDOWN"] = "1" + + self.runner.env.update(self.config.get("environment", {})) + + def launch_desktop_browser(self, test): + raise NotImplementedError + + def start_runner_proc(self): + # launch the browser via our previously-created runner + self.runner.start() + + proc = self.runner.process_handler + self.output_handler.proc = proc + + # give our control server the browser process so it can shut it down later + self.control_server.browser_proc = proc + + def process_exists(self): + return self.runner.is_running() + + def run_test(self, test, timeout): + # tests will be run warm (i.e. NO browser restart between page-cycles) + # unless otheriwse specified in the test INI by using 'cold = true' + mozpower_measurer = None + if self.config.get("power_test", False): + powertest_name = test["name"].replace("/", "-").replace("\\", "-") + output_dir = os.path.join( + self.artifact_dir, "power-measurements-%s" % powertest_name + ) + test_dir = os.path.join(output_dir, powertest_name) + + try: + if not os.path.exists(output_dir): + os.mkdir(output_dir) + if not os.path.exists(test_dir): + os.mkdir(test_dir) + except Exception: + LOG.critical( + "Could not create directories to store power testing data." + ) + raise + + # Start power measurements with IPG creating a power usage log + # every 30 seconds with 1 data point per second (or a 1000 milli- + # second sampling rate). + mozpower_measurer = MozPower( + ipg_measure_duration=30, + sampling_rate=1000, + output_file_path=os.path.join(test_dir, "power-usage"), + ) + mozpower_measurer.initialize_power_measurements() + + if self.config.get("cold") or test.get("cold"): + self.__run_test_cold(test, timeout) + else: + self.__run_test_warm(test, timeout) + + if mozpower_measurer: + mozpower_measurer.finalize_power_measurements(test_name=test["name"]) + perfherder_data = mozpower_measurer.get_perfherder_data() + + if not self.config.get("run_local", False): + # when not running locally, zip the data and delete the folder which + # was placed in the zip + powertest_name = test["name"].replace("/", "-").replace("\\", "-") + power_data_path = os.path.join( + self.artifact_dir, "power-measurements-%s" % powertest_name + ) + shutil.make_archive(power_data_path, "zip", power_data_path) + shutil.rmtree(power_data_path) + + for data_type in perfherder_data: + self.control_server.submit_supporting_data(perfherder_data[data_type]) + + def __run_test_cold(self, test, timeout): + """ + Run the Raptor test but restart the entire browser app between page-cycles. + + Note: For page-load tests, playback will only be started once - at the beginning of all + browser cycles, and then stopped after all cycles are finished. That includes the import + of the mozproxy ssl cert and turning on the browser proxy. + + Since we're running in cold-mode, before this point (in manifest.py) the + 'expected-browser-cycles' value was already set to the initial 'page-cycles' value; + and the 'page-cycles' value was set to 1 as we want to perform one page-cycle per + browser restart. + + The 'browser-cycle' value is the current overall browser start iteration. The control + server will receive the current 'browser-cycle' and the 'expected-browser-cycles' in + each results set received; and will pass that on as part of the results so that the + results processing will know results for multiple browser cycles are being received. + + The default will be to run in warm mode; unless 'cold = true' is set in the test INI. + """ + LOG.info( + "test %s is running in cold mode; browser WILL be restarted between " + "page cycles" % test["name"] + ) + + for test["browser_cycle"] in range(1, test["expected_browser_cycles"] + 1): + + LOG.info( + "begin browser cycle %d of %d for test %s" + % (test["browser_cycle"], test["expected_browser_cycles"], test["name"]) + ) + + self.run_test_setup(test) + + if test["browser_cycle"] == 1: + + if not self.is_localhost: + self.delete_proxy_settings_from_profile() + + else: + # initial browser profile was already created before run_test was called; + # now additional browser cycles we want to create a new one each time + self.build_browser_profile() + + # Update runner profile + self.runner.profile = self.profile + + self.run_test_setup(test) + + # now start the browser/app under test + self.launch_desktop_browser(test) + + # set our control server flag to indicate we are running the browser/app + self.control_server._finished = False + + self.wait_for_test_finish(test, timeout, self.process_exists) + + def __run_test_warm(self, test, timeout): + self.run_test_setup(test) + + if not self.is_localhost: + self.delete_proxy_settings_from_profile() + + # start the browser/app under test + self.launch_desktop_browser(test) + + # set our control server flag to indicate we are running the browser/app + self.control_server._finished = False + + self.wait_for_test_finish(test, timeout, self.process_exists) + + def run_test_teardown(self, test): + # browser should be closed by now but this is a backup-shutdown (if not in debug-mode) + if not self.debug_mode: + # If the runner was not started in the first place, stop() will silently + # catch RunnerNotStartedError + self.runner.stop() + else: + # in debug mode, and running locally, leave the browser running + if self.config["run_local"]: + LOG.info( + "* debug-mode enabled - please shutdown the browser manually..." + ) + self.runner.wait(timeout=None) + + super(WebExtensionDesktop, self).run_test_teardown(test) + + def check_for_crashes(self): + super(WebExtensionDesktop, self).check_for_crashes() + + try: + self.runner.check_for_crashes() + except NotImplementedError: # not implemented for Chrome + pass + + self.crashes += self.runner.crashed + + def clean_up(self): + self.runner.stop() + + super(WebExtensionDesktop, self).clean_up() + + +class WebExtensionFirefox(WebExtensionDesktop): + def launch_desktop_browser(self, test): + LOG.info("starting %s" % self.config["app"]) + if self.config["is_release_build"]: + self.disable_non_local_connections() + + # if running debug-mode, tell Firefox to open the browser console on startup + if self.debug_mode: + self.runner.cmdargs.extend(["-jsconsole"]) + + self.start_runner_proc() + + if self.config["is_release_build"] and test.get("playback") is not None: + self.enable_non_local_connections() + + # if geckoProfile is enabled, initialize it + if self.config["gecko_profile"] is True: + self._init_gecko_profiling(test) + # tell the control server the gecko_profile dir; the control server + # will receive the filename of the stored gecko profile from the web + # extension, and will move it out of the browser user profile to + # this directory; where it is picked-up by gecko_profile.symbolicate + self.control_server.gecko_profile_dir = ( + self.gecko_profiler.gecko_profile_dir + ) + + +class WebExtensionDesktopChrome(WebExtensionDesktop): + def setup_chrome_args(self, test): + # Setup chrome args and add them to the runner's args + chrome_args = self.desktop_chrome_args(test) + if " ".join(chrome_args) not in " ".join(self.runner.cmdargs): + self.runner.cmdargs.extend(chrome_args) + + def launch_desktop_browser(self, test): + LOG.info("starting %s" % self.config["app"]) + + # Setup chrome/chromium specific arguments then start the runner + self.setup_chrome_args(test) + self.start_runner_proc() + + def set_browser_test_prefs(self, raw_prefs): + # add test-specific preferences + LOG.info( + "preferences were configured for the test, however \ + we currently do not install them on non-Firefox browsers." + ) diff --git a/testing/raptor/requirements.txt b/testing/raptor/requirements.txt new file mode 100644 index 0000000000..316338c1ea --- /dev/null +++ b/testing/raptor/requirements.txt @@ -0,0 +1,15 @@ +mozcrash ~= 2.0 +mozrunner ~= 7.0 +mozprofile ~= 2.1 +manifestparser >= 1.1 +../tools/wptserve +../tools/wpt_third_party/pywebsocket3 +mozdevice >= 4.0.0 +mozpower >= 1.0.0 +mozversion >= 1.0 +../condprofile +aiohttp==3.7.4.post0 +https://pypi.pub.build.mozilla.org/pub/arsenic-19.1-py3-none-any.whl +requests==2.22.0 +pyyaml==5.1.2 +structlog==15.2.0 diff --git a/testing/raptor/source_requirements.txt b/testing/raptor/source_requirements.txt new file mode 100644 index 0000000000..a7099816e6 --- /dev/null +++ b/testing/raptor/source_requirements.txt @@ -0,0 +1,17 @@ +# These dependencies are required for running raptor tests from a source checkout + +mozcrash ~= 2.0 +mozrunner ~= 7.0 +mozprofile ~= 2.1 +manifestparser >= 1.1 +../web-platform/tests/tools/wptserve +../web-platform/tests/tools/third_party/pywebsocket3 +mozdevice >= 4.0.0 +mozpower >= 1.0.0 +mozversion >= 1.0 +../condprofile +aiohttp==3.7.4.post0 +https://pypi.pub.build.mozilla.org/pub/arsenic-19.1-py3-none-any.whl +requests==2.22.0 +pyyaml==5.1.2 +structlog==15.2.0 diff --git a/testing/raptor/test/__init__.py b/testing/raptor/test/__init__.py new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/testing/raptor/test/__init__.py diff --git a/testing/raptor/test/conftest.py b/testing/raptor/test/conftest.py new file mode 100644 index 0000000000..969270ac65 --- /dev/null +++ b/testing/raptor/test/conftest.py @@ -0,0 +1,165 @@ +import json +import os +import sys +from argparse import Namespace + +import pytest + +# need this so the raptor unit tests can find raptor/raptor classes +here = os.path.abspath(os.path.dirname(__file__)) +raptor_dir = os.path.join(os.path.dirname(here), "raptor") +sys.path.insert(0, raptor_dir) + +from browsertime import Browsertime +from perftest import Perftest +from webextension import WebExtensionFirefox + + +@pytest.fixture +def options(request): + opts = { + "app": "firefox", + "binary": "path/to/dummy/browser", + "browsertime_visualmetrics": False, + "extra_prefs": {}, + } + + if hasattr(request.module, "OPTIONS"): + opts.update(request.module.OPTIONS) + return opts + + +@pytest.fixture +def browsertime_options(options): + options["browsertime_node"] = "browsertime_node" + options["browsertime_browsertimejs"] = "browsertime_browsertimejs" + options["browsertime_ffmpeg"] = "browsertime_ffmpeg" + options["browsertime_geckodriver"] = "browsertime_geckodriver" + options["browsertime_chromedriver"] = "browsertime_chromedriver" + options["browsertime_video"] = "browsertime_video" + options["browsertime_visualmetrics"] = "browsertime_visualmetrics" + options["browsertime_no_ffwindowrecorder"] = "browsertime_no_ffwindowrecorder" + return options + + +@pytest.fixture +def raptor(options): + return WebExtensionFirefox(**options) + + +@pytest.fixture +def mock_test(): + return { + "name": "raptor-firefox-tp6", + "test_url": "/dummy/url", + "secondary_url": "/dummy/url-2", + } + + +@pytest.fixture(scope="session") +def get_prefs(): + def _inner(browser): + import raptor + + prefs_dir = os.path.join(raptor.__file__, "preferences") + with open(os.path.join(prefs_dir, "{}.json".format(browser)), "r") as fh: + return json.load(fh) + + +@pytest.fixture(scope="session") +def filedir(): + return os.path.join(here, "files") + + +@pytest.fixture +def get_binary(): + from moztest.selftest import fixtures + + def inner(app): + if app != "firefox": + pytest.xfail(reason="{} support not implemented".format(app)) + + binary = fixtures.binary() + if not binary: + pytest.skip("could not find a {} binary".format(app)) + return binary + + return inner + + +@pytest.fixture +def create_args(): + args = Namespace( + app="firefox", + test="raptor-tp6-unittest", + binary="path/to/binary", + gecko_profile=False, + extra_profiler_run=False, + debug_mode=False, + page_cycles=None, + page_timeout=None, + test_url_params=None, + host=None, + run_local=True, + browsertime=True, + cold=False, + live_sites=False, + enable_marionette_trace=False, + collect_perfstats=False, + chimera=False, + ) + + def inner(**kwargs): + for next_arg in kwargs: + print(next_arg) + print(kwargs[next_arg]) + setattr(args, next_arg, kwargs[next_arg]) + return args + + return inner + + +@pytest.fixture(scope="module") +def ConcretePerftest(): + class PerftestImplementation(Perftest): + def check_for_crashes(self): + super(PerftestImplementation, self).check_for_crashes() + + def clean_up(self): + super(PerftestImplementation, self).clean_up() + + def run_test(self, test, timeout): + super(PerftestImplementation, self).run_test(test, timeout) + + def run_test_setup(self, test): + super(PerftestImplementation, self).run_test_setup(test) + + def run_test_teardown(self, test): + super(PerftestImplementation, self).run_test_teardown(test) + + def set_browser_test_prefs(self): + super(PerftestImplementation, self).set_browser_test_prefs() + + def get_browser_meta(self): + return (), () + + def setup_chrome_args(self, test): + super(PerftestImplementation, self).setup_chrome_args(test) + + return PerftestImplementation + + +@pytest.fixture(scope="module") +def ConcreteBrowsertime(): + class BrowsertimeImplementation(Browsertime): + @property + def browsertime_args(self): + return [] + + def get_browser_meta(self): + return (), () + + def setup_chrome_args(self, test): + super(BrowsertimeImplementation, self).setup_chrome_args(test) + + return BrowsertimeImplementation diff --git a/testing/raptor/test/files/batterystats-android-7.txt b/testing/raptor/test/files/batterystats-android-7.txt new file mode 100644 index 0000000000..c23ab1b691 --- /dev/null +++ b/testing/raptor/test/files/batterystats-android-7.txt @@ -0,0 +1,1589 @@ +Battery History (2% used, 5316 used of 256KB, 81 strings using 5834): + 0 (10) RESET:TIME: 2019-07-03-23-07-19 + 0 (1) 099 status=discharging health=good plug=none temp=360 volt=4274 charge=2795 +running +wake_lock +sensor +wifi_radio +screen phone_signal_strength=poor brightness=medium +wifi_running +wifi wifi_signal_strength=4 wifi_suppl=completed + 0 (2) 099 proc=u0a13:"com.google.android.ims" + 0 (2) 099 proc=u0a65:"com.google.android.googlequicksearchbox:interactor" + 0 (2) 099 proc=1000:"com.motorola.process.system" + 0 (2) 099 proc=u0a119:"com.bitbar.testdroid.monitor:data" + 0 (2) 099 proc=1000:"WebViewLoader-armeabi-v7a" + 0 (2) 099 proc=u0a28:"com.android.externalstorage" + 0 (2) 099 proc=u0a62:"com.motorola.slpc" + 0 (2) 099 proc=u0a25:"com.android.mtp" + 0 (2) 099 proc=u0a69:"com.motorola.config.wifi" + 0 (2) 099 proc=u0a109:"com.motorola.timeweatherwidget" + 0 (2) 099 proc=u0a24:"com.android.documentsui" + 0 (2) 099 proc=u0a30:"com.google.process.gapps" + 0 (2) 099 proc=u0a119:"com.bitbar.testdroid.monitor" + 0 (2) 099 proc=u0a77:"com.android.chrome" + 0 (2) 099 proc=u0a46:"com.motorola.actions" + 0 (2) 099 proc=u0a74:"com.google.android.calendar" + 0 (2) 099 proc=u0a98:"com.google.android.inputmethod.latin" + 0 (2) 099 proc=1000:"com.fingerprints.serviceext" + 0 (2) 099 proc=u0a31:"com.google.android.ext.services" + 0 (2) 099 proc=1001:"com.qualcomm.qcrilmsgtunnel" + 0 (2) 099 proc=u0a107:"com.google.android.apps.photos" + 0 (2) 099 proc=u0a119:"com.bitbar.testdroid.monitor:monitor" + 0 (2) 099 proc=u0a30:"com.google.android.gms" + 0 (2) 099 proc=u0a30:"com.google.android.gms.persistent" + 0 (2) 099 proc=u0a102:"com.google.android.apps.messaging:rcs" + 0 (2) 099 proc=u0a47:"com.motorola.motocare" + 0 (2) 099 proc=u0a40:"com.android.launcher3" + 0 (2) 099 proc=1000:"com.motorola.modemservice" + 0 (2) 099 proc=u0a6:"com.motorola.ccc" + 0 (2) 099 proc=u0a83:"com.google.android.apps.docs" + 0 (2) 099 proc=u0a50:"com.motorola.motokey" + 0 (2) 099 proc=u0a21:"com.android.defcontainer" + 0 (2) 099 proc=u0a61:"com.motorola.frameworks.singlehand" + 0 (2) 099 proc=u0a25:"android.process.media" + 0 (2) 099 proc=1000:"com.motorola.process.slpc" + 0 (2) 099 proc=1000:".dataservices" + 0 (2) 099 proc=u0a64:"com.android.systemui" + 0 (2) 099 proc=1001:"com.android.phone" + 0 (2) 099 proc=1000:"com.qualcomm.telephony" + 0 (2) 099 proc=u0a11:"com.android.providers.calendar" + 0 (2) 099 proc=u0a54:"com.android.vending" + 0 (2) 099 proc=u0a65:"com.google.android.googlequicksearchbox:search" + 0 (2) 099 top=u0a6:"com.motorola.ccc.ota" + 0 (2) 099 user=0:"0" + 0 (2) 099 userfg=0:"0" + +7ms (2) 099 +wake_lock_in=1001:"RILJ" + +12ms (2) 099 -wake_lock_in=1001:"RILJ" + +12ms (2) 099 stats=0:"dump" + +94ms (2) 099 +proc=u0a13:"com.google.android.ims" + +94ms (2) 099 +proc=u0a65:"com.google.android.googlequicksearchbox:interactor" + +94ms (2) 099 +proc=1000:"com.motorola.process.system" + +94ms (2) 099 +proc=u0a119:"com.bitbar.testdroid.monitor:data" + +94ms (2) 099 +proc=1000:"WebViewLoader-armeabi-v7a" + +94ms (2) 099 +proc=u0a28:"com.android.externalstorage" + +94ms (2) 099 +proc=u0a62:"com.motorola.slpc" + +94ms (2) 099 +proc=u0a25:"com.android.mtp" + +94ms (2) 099 +proc=u0a69:"com.motorola.config.wifi" + +94ms (2) 099 +proc=u0a109:"com.motorola.timeweatherwidget" + +94ms (2) 099 +proc=u0a24:"com.android.documentsui" + +94ms (2) 099 +proc=u0a30:"com.google.process.gapps" + +94ms (2) 099 +proc=u0a119:"com.bitbar.testdroid.monitor" + +94ms (2) 099 +proc=u0a77:"com.android.chrome" + +94ms (2) 099 +proc=u0a46:"com.motorola.actions" + +94ms (2) 099 +proc=u0a74:"com.google.android.calendar" + +94ms (2) 099 +proc=u0a98:"com.google.android.inputmethod.latin" + +94ms (2) 099 +proc=1000:"com.fingerprints.serviceext" + +94ms (2) 099 +proc=u0a31:"com.google.android.ext.services" + +94ms (2) 099 +proc=1001:"com.qualcomm.qcrilmsgtunnel" + +94ms (2) 099 +proc=u0a107:"com.google.android.apps.photos" + +94ms (2) 099 +proc=u0a119:"com.bitbar.testdroid.monitor:monitor" + +94ms (2) 099 +proc=u0a30:"com.google.android.gms" + +94ms (2) 099 +proc=u0a30:"com.google.android.gms.persistent" + +94ms (2) 099 +proc=u0a102:"com.google.android.apps.messaging:rcs" + +94ms (2) 099 +proc=u0a47:"com.motorola.motocare" + +94ms (2) 099 +proc=u0a40:"com.android.launcher3" + +94ms (2) 099 +proc=1000:"com.motorola.modemservice" + +94ms (2) 099 +proc=u0a6:"com.motorola.ccc" + +94ms (2) 099 +proc=u0a83:"com.google.android.apps.docs" + +94ms (2) 099 +proc=u0a50:"com.motorola.motokey" + +94ms (2) 099 +proc=u0a21:"com.android.defcontainer" + +94ms (2) 099 +proc=u0a61:"com.motorola.frameworks.singlehand" + +94ms (2) 099 +proc=u0a25:"android.process.media" + +94ms (2) 099 +proc=1000:"com.motorola.process.slpc" + +94ms (2) 099 +proc=1000:".dataservices" + +94ms (2) 099 +proc=u0a64:"com.android.systemui" + +94ms (2) 099 +proc=1001:"com.android.phone" + +94ms (2) 099 +proc=1000:"com.qualcomm.telephony" + +94ms (2) 099 +proc=u0a11:"com.android.providers.calendar" + +94ms (2) 099 +proc=u0a54:"com.android.vending" + +94ms (2) 099 +proc=u0a65:"com.google.android.googlequicksearchbox:search" + +1s189ms (2) 099 +wake_lock_in=u0a30:"Wakeful StateMachine: GeofencerStateMachine" + +1s192ms (2) 099 -wake_lock_in=u0a30:"Wakeful StateMachine: GeofencerStateMachine" + +1s232ms (2) 099 +wake_lock_in=u0a30:"Icing" + +1s238ms (2) 099 +wake_lock_in=u0a118:"Icing" + +1s238ms (2) 099 -wake_lock_in=u0a30:"Icing" + +1s241ms (2) 099 -wake_lock_in=u0a118:"Icing" + +1s274ms (2) 099 +wake_lock_in=u0a30:"Icing" + +1s274ms (2) 099 +wake_lock_in=u0a118:"Icing" + +1s274ms (2) 099 -wake_lock_in=u0a118:"Icing" + +1s289ms (2) 099 -wake_lock_in=u0a30:"Icing" + +4s582ms (2) 099 +wake_lock_in=1000:"GnssLocationProvider" + +4s592ms (2) 099 -wake_lock_in=1000:"GnssLocationProvider" + +5s590ms (2) 099 +wake_lock_in=1000:"GnssLocationProvider" + +5s592ms (2) 099 -wake_lock_in=1000:"GnssLocationProvider" + +6s585ms (2) 099 +wake_lock_in=1000:"GnssLocationProvider" + +6s586ms (2) 099 -wake_lock_in=1000:"GnssLocationProvider" + +7s582ms (2) 099 +wake_lock_in=1000:"GnssLocationProvider" + +7s583ms (2) 099 -wake_lock_in=1000:"GnssLocationProvider" + +8s603ms (2) 099 +wake_lock_in=1000:"GnssLocationProvider" + +8s604ms (2) 099 -wake_lock_in=1000:"GnssLocationProvider" + +9s587ms (2) 099 +wake_lock_in=1000:"GnssLocationProvider" + +9s588ms (2) 099 -wake_lock_in=1000:"GnssLocationProvider" + +10s603ms (2) 099 +wake_lock_in=u0a118:"*launch*" + +10s624ms (2) 099 +proc=u0a118:"org.mozilla.geckoview_example" + +10s669ms (2) 099 -top=u0a6:"com.motorola.ccc.ota" + +10s669ms (2) 099 +top=u0a118:"org.mozilla.geckoview_example" + +11s026ms (2) 099 +proc=u0a118:"org.mozilla.geckoview_example:tab" + +11s355ms (2) 099 -wake_lock_in=u0a118:"*launch*" + +14s948ms (2) 099 +wake_lock_in=1001:"RILJ" + +14s951ms (2) 099 -wake_lock_in=1001:"RILJ" + +34s975ms (2) 099 +wake_lock_in=1001:"RILJ" + +34s977ms (2) 099 -wake_lock_in=1001:"RILJ" + +36s953ms (2) 099 +alarm=1000:"*walarm*:DhcpClient.wlan0.RENEW" + +36s954ms (2) 099 +wake_lock_in=1000:"*walarm*:DhcpClient.wlan0.RENEW" + +36s954ms (2) 099 -alarm=1000:"*walarm*:DhcpClient.wlan0.RENEW" + +36s954ms (2) 099 -wake_lock_in=1000:"*walarm*:DhcpClient.wlan0.RENEW" + +40s192ms (2) 099 +alarm=1000:"*alarm*:android.intent.action.TIME_TICK" + +40s193ms (2) 099 +wake_lock_in=1000:"*alarm*:android.intent.action.TIME_TICK" + +40s282ms (2) 099 -alarm=1000:"*alarm*:android.intent.action.TIME_TICK" + +40s282ms (2) 099 -wake_lock_in=1000:"*alarm*:android.intent.action.TIME_TICK" + +49s516ms (2) 099 volt=4251 charge=2790 + +54s982ms (2) 099 +wake_lock_in=1001:"RILJ" + +54s985ms (2) 099 -wake_lock_in=1001:"RILJ" + +1m11s259ms (2) 099 -proc=u0a54:"com.android.vending" + +1m15s006ms (2) 099 +wake_lock_in=1001:"RILJ" + +1m15s009ms (2) 099 -wake_lock_in=1001:"RILJ" + +1m35s030ms (2) 099 +wake_lock_in=1001:"RILJ" + +1m35s033ms (2) 099 -wake_lock_in=1001:"RILJ" + +1m40s190ms (2) 099 +alarm=1000:"*alarm*:android.intent.action.TIME_TICK" + +1m40s191ms (2) 099 +wake_lock_in=1000:"*alarm*:android.intent.action.TIME_TICK" + +1m40s267ms (2) 099 -alarm=1000:"*alarm*:android.intent.action.TIME_TICK" + +1m40s267ms (2) 099 -wake_lock_in=1000:"*alarm*:android.intent.action.TIME_TICK" + +1m49s577ms (2) 098 charge=2785 + Details: cpu=2070u+2570s + /proc/stat=7370 usr, 2540 sys, 370 io, 0 irq, 50 sirq, 524790 idle (1.9% of 1h 29m 11s 200ms), PlatformIdleStat null + +1m49s585ms (2) 098 +wake_lock_in=1001:"RILJ" + +1m49s590ms (2) 098 -wake_lock_in=1001:"RILJ" + +1m49s591ms (2) 098 charge=2783 stats=0:"battery-level" + +1m55s036ms (2) 098 +wake_lock_in=1001:"RILJ" + +1m55s038ms (2) 098 -wake_lock_in=1001:"RILJ" + +2m12s231ms (2) 098 +alarm=u0a30:"*walarm*:com.google.android.location.ALARM_WAKEUP_LOCATOR" + +2m12s232ms (2) 098 +wake_lock_in=1000:"*walarm*:com.google.android.location.ALARM_WAKEUP_LOCATOR" + +2m12s232ms (2) 098 +alarm=1000:"*walarm*:WifiConnectivityManager Schedule Periodic Scan Timer" + +2m12s232ms (2) 098 +alarm=u0a30:"*walarm*:com.google.android.location.ALARM_WAKEUP_ACTIVITY_DETECTION" + +2m12s234ms (2) 098 -alarm=1000:"*walarm*:WifiConnectivityManager Schedule Periodic Scan Timer" + +2m12s234ms (2) 098 +wifi_scan +wake_lock_in=u0a30:"NlpWakeLock" + +2m12s242ms (2) 098 +wifi_full_lock +wake_lock_in=u0a30:"NlpCollectorWakeLock" + +2m12s254ms (2) 098 -alarm=u0a30:"*walarm*:com.google.android.location.ALARM_WAKEUP_LOCATOR" + +2m12s258ms (2) 098 -wake_lock_in=u0a30:"NlpWakeLock" + +2m12s262ms (2) 098 +wake_lock_in=1001:"RILJ" + +2m12s269ms (2) 098 -wake_lock_in=1001:"RILJ" + +2m12s269ms (2) 098 +wake_lock_in=1001:"RILJ" + +2m12s273ms (2) 098 -wake_lock_in=1001:"RILJ" + +2m12s275ms (2) 098 +wake_lock_in=u0a30:"NlpWakeLock" + +2m12s277ms (2) 098 -wake_lock_in=u0a30:"NlpWakeLock" + +2m12s278ms (2) 098 -alarm=u0a30:"*walarm*:com.google.android.location.ALARM_WAKEUP_ACTIVITY_DETECTION" + +2m12s278ms (2) 098 -wifi_scan -wake_lock_in=1000:"*walarm*:com.google.android.location.ALARM_WAKEUP_LOCATOR" + +2m12s345ms (2) 098 +wifi_scan +wake_lock_in=u0a30:"NlpWakeLock" + +2m12s787ms (2) 098 -wake_lock_in=u0a30:"NlpWakeLock" + +2m12s788ms (2) 098 +wake_lock_in=u0a30:"NlpWakeLock" + +2m12s790ms (2) 098 -wake_lock_in=u0a30:"NlpCollectorWakeLock" + +2m12s799ms (2) 098 -wake_lock_in=u0a30:"NlpWakeLock" + +2m12s803ms (2) 098 +wake_lock_in=u0a30:"NlpWakeLock" + +2m12s806ms (2) 098 -wake_lock_in=u0a30:"NlpWakeLock" + +2m12s807ms (2) 098 +wake_lock_in=u0a30:"GCoreFlp" + +2m12s809ms (2) 098 -wake_lock_in=u0a30:"GCoreFlp" + +2m12s809ms (2) 098 +wake_lock_in=u0a30:"GCoreFlp" + +2m12s812ms (2) 098 -wake_lock_in=u0a30:"GCoreFlp" + +2m14s035ms (2) 098 -wifi_scan +wake_lock_in=u0a30:"NlpWakeLock" + +2m14s116ms (2) 098 +wake_lock_in=1000:"LocationManagerService" + +2m14s116ms (2) 098 -wake_lock_in=1000:"LocationManagerService" + +2m14s118ms (2) 098 -wake_lock_in=u0a30:"NlpWakeLock" + +2m14s120ms (2) 098 +wake_lock_in=u0a30:"NlpWakeLock" + +2m14s121ms (2) 098 +wake_lock_in=1000:"LocationManagerService" + +2m14s122ms (2) 098 +wake_lock_in=u0a30:"LocationManagerService" + +2m14s122ms (2) 098 -wake_lock_in=1000:"LocationManagerService" + +2m14s122ms (2) 098 -wake_lock_in=u0a30:"LocationManagerService" + +2m14s123ms (2) 098 -wake_lock_in=u0a30:"NlpWakeLock" + +2m14s124ms (2) 098 +wake_lock_in=u0a30:"GCoreFlp" + +2m14s125ms (2) 098 -wifi_full_lock -wake_lock_in=u0a30:"GCoreFlp" + +2m14s126ms (2) 098 +wake_lock_in=u0a30:"GCoreFlp" + +2m14s146ms (2) 098 -wake_lock_in=u0a30:"GCoreFlp" + +2m15s043ms (2) 098 +wake_lock_in=1001:"RILJ" + +2m15s045ms (2) 098 -wake_lock_in=1001:"RILJ" + +2m35s052ms (2) 098 +wake_lock_in=1001:"RILJ" + +2m35s054ms (2) 098 -wake_lock_in=1001:"RILJ" + +2m40s190ms (2) 098 +alarm=1000:"*alarm*:android.intent.action.TIME_TICK" + +2m40s191ms (2) 098 +wake_lock_in=1000:"*alarm*:android.intent.action.TIME_TICK" + +2m40s267ms (2) 098 -alarm=1000:"*alarm*:android.intent.action.TIME_TICK" + +2m40s268ms (2) 098 -wake_lock_in=1000:"*alarm*:android.intent.action.TIME_TICK" + +2m49s723ms (1) 098 charge=2777 + +2m55s062ms (2) 098 +wake_lock_in=1001:"RILJ" + +2m55s064ms (2) 098 -wake_lock_in=1001:"RILJ" + +3m12s636ms (2) 098 +alarm=u0a30:"*alarm*:null" + +3m12s636ms (2) 098 +alarm=u0a65:"*alarm*:null" + +3m12s636ms (2) 098 +wake_lock_in=u0a30:"*alarm*:null" + +3m12s636ms (2) 098 +wake_lock_in=u0a65:"*alarm*:null" + +3m12s639ms (2) 098 -alarm=u0a30:"*alarm*:null" + +3m12s639ms (2) 098 -alarm=u0a65:"*alarm*:null" + +3m12s640ms (2) 098 -wake_lock_in=u0a30:"*alarm*:null" + +3m12s640ms (2) 098 -wake_lock_in=u0a65:"*alarm*:null" + +3m15s086ms (2) 098 +wake_lock_in=1001:"RILJ" + +3m15s088ms (2) 098 -wake_lock_in=1001:"RILJ" + +3m35s109ms (2) 098 +wake_lock_in=1001:"RILJ" + +3m35s112ms (2) 098 -wake_lock_in=1001:"RILJ" + +3m40s191ms (2) 098 +alarm=1000:"*alarm*:android.intent.action.TIME_TICK" + +3m40s192ms (2) 098 +wake_lock_in=1000:"*alarm*:android.intent.action.TIME_TICK" + +3m40s268ms (2) 098 -alarm=1000:"*alarm*:android.intent.action.TIME_TICK" + +3m40s268ms (2) 098 -wake_lock_in=1000:"*alarm*:android.intent.action.TIME_TICK" + +3m49s897ms (2) 098 temp=370 charge=2770 + +3m55s118ms (2) 098 +wake_lock_in=1001:"RILJ" + +3m55s120ms (2) 098 -wake_lock_in=1001:"RILJ" + +4m15s142ms (2) 098 +wake_lock_in=1001:"RILJ" + +4m15s144ms (2) 098 -wake_lock_in=1001:"RILJ" + +4m35s166ms (2) 098 +wake_lock_in=1001:"RILJ" + +4m35s168ms (2) 098 -wake_lock_in=1001:"RILJ" + +4m40s191ms (2) 098 +alarm=1000:"*alarm*:android.intent.action.TIME_TICK" + +4m40s192ms (2) 098 +wake_lock_in=1000:"*alarm*:android.intent.action.TIME_TICK" + +4m40s268ms (2) 098 -alarm=1000:"*alarm*:android.intent.action.TIME_TICK" + +4m40s268ms (2) 098 -wake_lock_in=1000:"*alarm*:android.intent.action.TIME_TICK" + +4m50s016ms (1) 098 charge=2764 + +4m55s175ms (2) 098 +wake_lock_in=1001:"RILJ" + +4m55s177ms (2) 098 -wake_lock_in=1001:"RILJ" + +4m58s368ms (2) 097 + Details: cpu=8170u+30500s + /proc/stat=36190 usr, 22680 sys, 810 io, 10 irq, 1720 sirq, 812950 idle (7.0% of 2h 25m 43s 600ms), PlatformIdleStat null + +4m58s375ms (2) 097 +wake_lock_in=1001:"RILJ" + +4m58s380ms (2) 097 -wake_lock_in=1001:"RILJ" + +4m58s381ms (2) 097 stats=0:"battery-level" + +5m12s814ms (2) 097 +alarm=u0a84:"*walarm*:*job.delay*" + +5m12s814ms (2) 097 +wake_lock_in=u0a84:"*walarm*:*job.delay*" + +5m12s816ms (2) 097 +alarm=1000:"*walarm*:WifiConnectivityManager Schedule Periodic Scan Timer" + +5m12s816ms (2) 097 +alarm=u0a30:"*walarm*:com.google.android.location.ALARM_WAKEUP_ACTIVITY_DETECTION" + +5m12s817ms (2) 097 -alarm=u0a84:"*walarm*:*job.delay*" + +5m12s817ms (2) 097 -alarm=1000:"*walarm*:WifiConnectivityManager Schedule Periodic Scan Timer" + +5m12s818ms (2) 097 +wake_lock_in=u0a30:"NlpWakeLock" + +5m12s822ms (2) 097 +wake_lock_in=u0a30:"NlpCollectorWakeLock" + +5m12s839ms (2) 097 +proc=u0a84:"com.google.android.apps.tachyon" + +5m12s839ms (2) 097 +wifi_scan +job=u0a84:"com.google.android.apps.tachyon/.jobs.NativeJobSchedulerJobService" + +5m12s844ms (2) 097 -wake_lock_in=u0a30:"NlpWakeLock" + +5m12s851ms (2) 097 -wake_lock_in=u0a84:"*walarm*:*job.delay*" + +5m12s851ms (2) 097 +wake_lock_in=1000:"*alarm*" + +5m12s857ms (2) 097 -alarm=u0a30:"*walarm*:com.google.android.location.ALARM_WAKEUP_ACTIVITY_DETECTION" + +5m12s857ms (2) 097 -wifi_scan -wake_lock_in=1000:"*alarm*" + +5m13s199ms (2) 097 +wake_lock_in=u0a30:"*net_scheduler*" + +5m13s222ms (2) 097 -wake_lock_in=u0a30:"*net_scheduler*" + +5m13s373ms (3) 097 -sensor +wake_lock_in=u0a30:"NlpWakeLock" + +5m13s377ms (2) 097 -wake_lock_in=u0a30:"NlpWakeLock" + +5m13s378ms (2) 097 +wake_lock_in=u0a30:"NlpWakeLock" + +5m13s381ms (2) 097 -wake_lock_in=u0a30:"NlpCollectorWakeLock" + +5m13s395ms (2) 097 -wake_lock_in=u0a30:"NlpWakeLock" + +5m13s401ms (2) 097 +wake_lock_in=u0a30:"NlpWakeLock" + +5m13s402ms (2) 097 +wake_lock_in=u0a84:"*job*/com.google.android.apps.tachyon/.jobs.NativeJobSchedulerJobService" + +5m13s405ms (2) 097 -wake_lock_in=u0a30:"NlpWakeLock" + +5m13s406ms (2) 097 +wake_lock_in=u0a30:"GCoreFlp" + +5m13s409ms (2) 097 -wake_lock_in=u0a30:"GCoreFlp" + +5m13s410ms (2) 097 +wake_lock_in=u0a30:"GCoreFlp" + +5m13s412ms (2) 097 -wake_lock_in=u0a30:"GCoreFlp" + +5m13s420ms (2) 097 -job=u0a84:"com.google.android.apps.tachyon/.jobs.NativeJobSchedulerJobService" + +5m13s420ms (2) 097 -wake_lock_in=u0a84:"*job*/com.google.android.apps.tachyon/.jobs.NativeJobSchedulerJobService" + +5m15s179ms (2) 097 +wake_lock_in=1001:"RILJ" + +5m15s182ms (2) 097 -wake_lock_in=1001:"RILJ" + +5m35s202ms (2) 097 +wake_lock_in=1001:"RILJ" + +5m35s205ms (2) 097 -wake_lock_in=1001:"RILJ" + +5m37s026ms (2) 097 +alarm=1000:"*walarm*:DhcpClient.wlan0.RENEW" + +5m37s026ms (2) 097 +wake_lock_in=1000:"*walarm*:DhcpClient.wlan0.RENEW" + +5m37s027ms (2) 097 -alarm=1000:"*walarm*:DhcpClient.wlan0.RENEW" + +5m37s027ms (2) 097 -wake_lock_in=1000:"*walarm*:DhcpClient.wlan0.RENEW" + +5m40s191ms (2) 097 +alarm=1000:"*alarm*:android.intent.action.TIME_TICK" + +5m40s191ms (2) 097 +wake_lock_in=1000:"*alarm*:android.intent.action.TIME_TICK" + +5m40s208ms (2) 097 -proc=u0a28:"com.android.externalstorage" + +5m40s271ms (2) 097 -alarm=1000:"*alarm*:android.intent.action.TIME_TICK" + +5m40s271ms (2) 097 -wake_lock_in=1000:"*alarm*:android.intent.action.TIME_TICK" + +5m50s316ms (2) 097 volt=4225 charge=2757 + +5m55s212ms (2) 097 +wake_lock_in=1001:"RILJ" + +5m55s214ms (2) 097 -wake_lock_in=1001:"RILJ" + +5m56s493ms (2) 097 +alarm=u0a100:"*walarm*:*job.delay*" + +5m56s493ms (2) 097 +wake_lock_in=u0a100:"*walarm*:*job.delay*" + +5m56s495ms (2) 097 -alarm=u0a100:"*walarm*:*job.delay*" + +5m56s500ms (2) 097 -wake_lock_in=u0a100:"*walarm*:*job.delay*" + +6m15s236ms (2) 097 +wake_lock_in=1001:"RILJ" + +6m15s238ms (2) 097 -wake_lock_in=1001:"RILJ" + +6m35s260ms (2) 097 +wake_lock_in=1001:"RILJ" + +6m35s262ms (2) 097 -wake_lock_in=1001:"RILJ" + +6m40s191ms (2) 097 +alarm=1000:"*alarm*:android.intent.action.TIME_TICK" + +6m40s191ms (2) 097 +wake_lock_in=1000:"*alarm*:android.intent.action.TIME_TICK" + +6m40s271ms (2) 097 -alarm=1000:"*alarm*:android.intent.action.TIME_TICK" + +6m40s271ms (2) 097 -wake_lock_in=1000:"*alarm*:android.intent.action.TIME_TICK" + +6m48s969ms (1) 097 charge=2752 + +6m50s476ms (1) 097 charge=2750 + +6m55s269ms (2) 097 +wake_lock_in=1001:"RILJ" + +6m55s271ms (2) 097 -wake_lock_in=1001:"RILJ" + +7m15s275ms (2) 097 +wake_lock_in=1001:"RILJ" + +7m15s277ms (2) 097 -wake_lock_in=1001:"RILJ" + +7m35s281ms (2) 097 +wake_lock_in=1001:"RILJ" + +7m35s283ms (2) 097 -wake_lock_in=1001:"RILJ" + +7m40s191ms (2) 097 +alarm=1000:"*alarm*:android.intent.action.TIME_TICK" + +7m40s191ms (2) 097 +wake_lock_in=1000:"*alarm*:android.intent.action.TIME_TICK" + +7m40s273ms (2) 097 -alarm=1000:"*alarm*:android.intent.action.TIME_TICK" + +7m40s273ms (2) 097 -wake_lock_in=1000:"*alarm*:android.intent.action.TIME_TICK" + +7m50s537ms (1) 097 charge=2744 + +7m55s290ms (2) 097 +wake_lock_in=1001:"RILJ" + +7m55s293ms (2) 097 -wake_lock_in=1001:"RILJ" + +8m13s411ms (2) 097 +alarm=1000:"*walarm*:WifiConnectivityManager Schedule Periodic Scan Timer" + +8m13s411ms (2) 097 +wake_lock_in=1000:"*walarm*:WifiConnectivityManager Schedule Periodic Scan Timer" + +8m13s411ms (2) 097 +alarm=u0a30:"*walarm*:com.google.android.location.ALARM_WAKEUP_ACTIVITY_DETECTION" + +8m13s413ms (2) 097 -alarm=1000:"*walarm*:WifiConnectivityManager Schedule Periodic Scan Timer" + +8m13s413ms (2) 097 -wake_lock_in=1000:"*walarm*:WifiConnectivityManager Schedule Periodic Scan Timer" + +8m13s413ms (2) 097 +wake_lock_in=1000:"*alarm*" + +8m13s414ms (2) 097 +wifi_scan +wake_lock_in=u0a30:"NlpWakeLock" + +8m13s419ms (3) 097 +sensor +wake_lock_in=u0a30:"NlpCollectorWakeLock" + +8m13s423ms (2) 097 -alarm=u0a30:"*walarm*:com.google.android.location.ALARM_WAKEUP_ACTIVITY_DETECTION" + +8m13s423ms (2) 097 -wake_lock_in=1000:"*alarm*" + +8m13s427ms (3) 097 -sensor -wifi_scan -wake_lock_in=u0a30:"NlpWakeLock" + +8m13s962ms (2) 097 +wake_lock_in=u0a30:"NlpWakeLock" + +8m13s965ms (2) 097 -wake_lock_in=u0a30:"NlpWakeLock" + +8m13s966ms (2) 097 +wake_lock_in=u0a30:"NlpWakeLock" + +8m13s969ms (2) 097 -wake_lock_in=u0a30:"NlpCollectorWakeLock" + +8m13s979ms (2) 097 -wake_lock_in=u0a30:"NlpWakeLock" + +8m13s981ms (2) 097 +wake_lock_in=u0a30:"NlpWakeLock" + +8m13s985ms (2) 097 -wake_lock_in=u0a30:"NlpWakeLock" + +8m13s986ms (2) 097 +wake_lock_in=u0a30:"GCoreFlp" + +8m13s988ms (2) 097 -wake_lock_in=u0a30:"GCoreFlp" + +8m13s988ms (2) 097 +wake_lock_in=u0a30:"GCoreFlp" + +8m13s991ms (2) 097 -wake_lock_in=u0a30:"GCoreFlp" + +8m15s315ms (2) 097 +wake_lock_in=1001:"RILJ" + +8m15s317ms (2) 097 -wake_lock_in=1001:"RILJ" + +8m35s328ms (2) 097 +wake_lock_in=1001:"RILJ" + +8m35s331ms (2) 097 -wake_lock_in=1001:"RILJ" + +8m40s191ms (2) 097 +alarm=1000:"*alarm*:android.intent.action.TIME_TICK" + +8m40s192ms (2) 097 +wake_lock_in=1000:"*alarm*:android.intent.action.TIME_TICK" + +8m40s272ms (2) 097 -alarm=1000:"*alarm*:android.intent.action.TIME_TICK" + +8m40s273ms (2) 097 -wake_lock_in=1000:"*alarm*:android.intent.action.TIME_TICK" + +8m50s697ms (1) 097 charge=2737 + +8m55s336ms (2) 097 +wake_lock_in=1001:"RILJ" + +8m55s339ms (2) 097 -wake_lock_in=1001:"RILJ" + +9m15s360ms (2) 097 +wake_lock_in=1001:"RILJ" + +9m15s362ms (2) 097 -wake_lock_in=1001:"RILJ" + +9m35s383ms (2) 097 +wake_lock_in=1001:"RILJ" + +9m35s388ms (2) 097 -wake_lock_in=1001:"RILJ" + +9m40s191ms (2) 097 +alarm=1000:"*alarm*:android.intent.action.TIME_TICK" + +9m40s192ms (2) 097 +wake_lock_in=1000:"*alarm*:android.intent.action.TIME_TICK" + +9m40s274ms (2) 097 -alarm=1000:"*alarm*:android.intent.action.TIME_TICK" + +9m40s274ms (2) 097 -wake_lock_in=1000:"*alarm*:android.intent.action.TIME_TICK" + +9m50s856ms (1) 097 charge=2730 + +9m55s394ms (2) 097 +wake_lock_in=1001:"RILJ" + +9m55s397ms (2) 097 -wake_lock_in=1001:"RILJ" + +10m15s417ms (2) 097 +wake_lock_in=1001:"RILJ" + +10m15s419ms (2) 097 -wake_lock_in=1001:"RILJ" + +10m35s426ms (2) 097 +wake_lock_in=1001:"RILJ" + +10m35s428ms (2) 097 -wake_lock_in=1001:"RILJ" + +10m37s050ms (2) 097 +alarm=1000:"*walarm*:DhcpClient.wlan0.RENEW" + +10m37s050ms (2) 097 +wake_lock_in=1000:"*walarm*:DhcpClient.wlan0.RENEW" + +10m37s050ms (2) 097 -alarm=1000:"*walarm*:DhcpClient.wlan0.RENEW" + +10m37s050ms (2) 097 -wake_lock_in=1000:"*walarm*:DhcpClient.wlan0.RENEW" + +10m40s191ms (2) 097 +alarm=1000:"*alarm*:android.intent.action.TIME_TICK" + +10m40s191ms (2) 097 +wake_lock_in=1000:"*alarm*:android.intent.action.TIME_TICK" + +10m40s274ms (2) 097 -alarm=1000:"*alarm*:android.intent.action.TIME_TICK" + +10m40s274ms (2) 097 -wake_lock_in=1000:"*alarm*:android.intent.action.TIME_TICK" + +10m50s973ms (2) 096 charge=2725 + Details: cpu=172160u+265060s + /proc/stat=170180 usr, 169370 sys, 1540 io, 20 irq, 12650 sirq, 3844700 idle (8.4% of 11h 39m 44s 600ms), PlatformIdleStat null + +10m50s981ms (2) 096 +wake_lock_in=1001:"RILJ" + +10m50s986ms (2) 096 -wake_lock_in=1001:"RILJ" + +10m50s986ms (2) 096 charge=2724 stats=0:"battery-level" + +10m55s435ms (2) 096 +wake_lock_in=1001:"RILJ" + +10m55s437ms (2) 096 -wake_lock_in=1001:"RILJ" + +11m13s992ms (2) 096 +alarm=1000:"*walarm*:WifiConnectivityManager Schedule Periodic Scan Timer" + +11m13s992ms (2) 096 +wake_lock_in=1000:"*walarm*:WifiConnectivityManager Schedule Periodic Scan Timer" + +11m13s992ms (2) 096 +alarm=u0a30:"*walarm*:com.google.android.location.ALARM_WAKEUP_ACTIVITY_DETECTION" + +11m13s994ms (2) 096 -alarm=1000:"*walarm*:WifiConnectivityManager Schedule Periodic Scan Timer" + +11m13s994ms (2) 096 -wake_lock_in=1000:"*walarm*:WifiConnectivityManager Schedule Periodic Scan Timer" + +11m13s994ms (2) 096 +wake_lock_in=1000:"*alarm*" + +11m13s996ms (2) 096 +wifi_scan +wake_lock_in=u0a30:"NlpWakeLock" + +11m14s001ms (2) 096 +wake_lock_in=u0a30:"NlpCollectorWakeLock" + +11m14s006ms (2) 096 -alarm=u0a30:"*walarm*:com.google.android.location.ALARM_WAKEUP_ACTIVITY_DETECTION" + +11m14s006ms (2) 096 -wake_lock_in=1000:"*alarm*" + +11m14s013ms (2) 096 -wifi_scan -wake_lock_in=u0a30:"NlpWakeLock" + +11m14s543ms (2) 096 +wake_lock_in=u0a30:"NlpWakeLock" + +11m14s546ms (2) 096 -wake_lock_in=u0a30:"NlpWakeLock" + +11m14s547ms (2) 096 +wake_lock_in=u0a30:"NlpWakeLock" + +11m14s549ms (2) 096 -wake_lock_in=u0a30:"NlpCollectorWakeLock" + +11m14s561ms (2) 096 -wake_lock_in=u0a30:"NlpWakeLock" + +11m14s564ms (2) 096 +wake_lock_in=u0a30:"NlpWakeLock" + +11m14s568ms (2) 096 -wake_lock_in=u0a30:"NlpWakeLock" + +11m14s569ms (2) 096 +wake_lock_in=u0a30:"GCoreFlp" + +11m14s570ms (2) 096 -wake_lock_in=u0a30:"GCoreFlp" + +11m14s571ms (2) 096 +wake_lock_in=u0a30:"GCoreFlp" + +11m14s574ms (2) 096 -wake_lock_in=u0a30:"GCoreFlp" + +11m15s459ms (2) 096 +wake_lock_in=1001:"RILJ" + +11m15s461ms (2) 096 -wake_lock_in=1001:"RILJ" + +11m35s464ms (2) 096 +wake_lock_in=1001:"RILJ" + +11m35s466ms (2) 096 -wake_lock_in=1001:"RILJ" + +11m40s191ms (2) 096 +alarm=1000:"*alarm*:android.intent.action.TIME_TICK" + +11m40s192ms (2) 096 +wake_lock_in=1000:"*alarm*:android.intent.action.TIME_TICK" + +11m40s275ms (2) 096 -alarm=1000:"*alarm*:android.intent.action.TIME_TICK" + +11m40s275ms (2) 096 -wake_lock_in=1000:"*alarm*:android.intent.action.TIME_TICK" + +11m51s174ms (1) 096 charge=2717 + +11m55s473ms (2) 096 +wake_lock_in=1001:"RILJ" + +11m55s475ms (2) 096 -wake_lock_in=1001:"RILJ" + +12m15s497ms (2) 096 +wake_lock_in=1001:"RILJ" + +12m15s500ms (2) 096 -wake_lock_in=1001:"RILJ" + +12m35s522ms (2) 096 +wake_lock_in=1001:"RILJ" + +12m35s524ms (2) 096 -wake_lock_in=1001:"RILJ" + +12m40s191ms (2) 096 +alarm=1000:"*alarm*:android.intent.action.TIME_TICK" + +12m40s192ms (2) 096 +wake_lock_in=1000:"*alarm*:android.intent.action.TIME_TICK" + +12m40s277ms (2) 096 -alarm=1000:"*alarm*:android.intent.action.TIME_TICK" + +12m40s277ms (2) 096 -wake_lock_in=1000:"*alarm*:android.intent.action.TIME_TICK" + +12m51s436ms (1) 096 charge=2710 + +12m55s531ms (2) 096 +wake_lock_in=1001:"RILJ" + +12m55s533ms (2) 096 -wake_lock_in=1001:"RILJ" + +13m15s555ms (2) 096 +wake_lock_in=1001:"RILJ" + +13m15s557ms (2) 096 -wake_lock_in=1001:"RILJ" + +13m35s576ms (2) 096 +wake_lock_in=1001:"RILJ" + +13m35s579ms (2) 096 -wake_lock_in=1001:"RILJ" + +13m40s191ms (2) 096 +alarm=1000:"*alarm*:android.intent.action.TIME_TICK" + +13m40s192ms (2) 096 +wake_lock_in=1000:"*alarm*:android.intent.action.TIME_TICK" + +13m40s261ms (2) 096 -alarm=1000:"*alarm*:android.intent.action.TIME_TICK" + +13m40s261ms (2) 096 -wake_lock_in=1000:"*alarm*:android.intent.action.TIME_TICK" + +13m51s480ms (2) 096 volt=4202 charge=2704 + +13m55s584ms (2) 096 +wake_lock_in=1001:"RILJ" + +13m55s587ms (2) 096 -wake_lock_in=1001:"RILJ" + +14m14s572ms (2) 096 +alarm=u0a30:"*walarm*:com.google.android.location.ALARM_WAKEUP_ACTIVITY_DETECTION" + +14m14s573ms (2) 096 +wake_lock_in=1000:"*walarm*:com.google.android.location.ALARM_WAKEUP_ACTIVITY_DETECTION" + +14m14s575ms (2) 096 +wake_lock_in=u0a30:"NlpWakeLock" + +14m14s579ms (2) 096 +wake_lock_in=u0a30:"NlpCollectorWakeLock" + +14m14s584ms (2) 096 -alarm=u0a30:"*walarm*:com.google.android.location.ALARM_WAKEUP_ACTIVITY_DETECTION" + +14m14s584ms (2) 096 -wake_lock_in=1000:"*walarm*:com.google.android.location.ALARM_WAKEUP_ACTIVITY_DETECTION" + +14m14s588ms (2) 096 -wake_lock_in=u0a30:"NlpWakeLock" + +14m15s129ms (2) 096 +wake_lock_in=u0a30:"NlpWakeLock" + +14m15s133ms (2) 096 -wake_lock_in=u0a30:"NlpWakeLock" + +14m15s134ms (2) 096 +wake_lock_in=u0a30:"NlpWakeLock" + +14m15s136ms (2) 096 -wake_lock_in=u0a30:"NlpCollectorWakeLock" + +14m15s146ms (2) 096 -wake_lock_in=u0a30:"NlpWakeLock" + +14m15s152ms (2) 096 +wake_lock_in=u0a30:"NlpWakeLock" + +14m15s156ms (2) 096 -wake_lock_in=u0a30:"NlpWakeLock" + +14m15s156ms (2) 096 +wake_lock_in=u0a30:"GCoreFlp" + +14m15s158ms (2) 096 -wake_lock_in=u0a30:"GCoreFlp" + +14m15s158ms (2) 096 +wake_lock_in=u0a30:"GCoreFlp" + +14m15s161ms (2) 096 -wake_lock_in=u0a30:"GCoreFlp" + +14m15s609ms (2) 096 +wake_lock_in=1001:"RILJ" + +14m15s614ms (2) 096 -wake_lock_in=1001:"RILJ" + +14m35s618ms (2) 096 +wake_lock_in=1001:"RILJ" + +14m35s620ms (2) 096 -wake_lock_in=1001:"RILJ" + +14m40s191ms (2) 096 +alarm=1000:"*alarm*:android.intent.action.TIME_TICK" + +14m40s192ms (2) 096 +wake_lock_in=1000:"*alarm*:android.intent.action.TIME_TICK" + +14m40s261ms (2) 096 -alarm=1000:"*alarm*:android.intent.action.TIME_TICK" + +14m40s261ms (2) 096 -wake_lock_in=1000:"*alarm*:android.intent.action.TIME_TICK" + +14m51s656ms (1) 096 charge=2697 + +14m55s626ms (2) 096 +wake_lock_in=1001:"RILJ" + +14m55s629ms (2) 096 -wake_lock_in=1001:"RILJ" + +15m15s650ms (2) 096 +wake_lock_in=1001:"RILJ" + +15m15s652ms (2) 096 -wake_lock_in=1001:"RILJ" + +15m33s350ms (2) 096 +alarm=1000:"*walarm*:WifiConnectivityManager Schedule Periodic Scan Timer" + +15m33s350ms (2) 096 +wake_lock_in=1000:"*walarm*:WifiConnectivityManager Schedule Periodic Scan Timer" + +15m33s350ms (2) 096 +alarm=1000:"*walarm*:WifiConnectivityManager Schedule Watchdog Timer" + +15m33s352ms (2) 096 -alarm=1000:"*walarm*:WifiConnectivityManager Schedule Periodic Scan Timer" + +15m33s353ms (2) 096 -wake_lock_in=1000:"*walarm*:WifiConnectivityManager Schedule Periodic Scan Timer" + +15m33s353ms (2) 096 +wake_lock_in=1000:"*alarm*" + +15m33s354ms (2) 096 -alarm=1000:"*walarm*:WifiConnectivityManager Schedule Watchdog Timer" + +15m33s354ms (2) 096 +wifi_scan -wake_lock_in=1000:"*alarm*" + +15m33s451ms (1) 096 -wifi_scan + +15m35s674ms (2) 096 +wake_lock_in=1001:"RILJ" + +15m35s676ms (2) 096 -wake_lock_in=1001:"RILJ" + +15m37s074ms (2) 096 +alarm=1000:"*walarm*:DhcpClient.wlan0.RENEW" + +15m37s074ms (2) 096 +wake_lock_in=1000:"*walarm*:DhcpClient.wlan0.RENEW" + +15m37s074ms (2) 096 -alarm=1000:"*walarm*:DhcpClient.wlan0.RENEW" + +15m37s074ms (2) 096 -wake_lock_in=1000:"*walarm*:DhcpClient.wlan0.RENEW" + +15m40s191ms (2) 096 +alarm=1000:"*alarm*:android.intent.action.TIME_TICK" + +15m40s192ms (2) 096 +wake_lock_in=1000:"*alarm*:android.intent.action.TIME_TICK" + +15m40s263ms (2) 096 -alarm=1000:"*alarm*:android.intent.action.TIME_TICK" + +15m40s263ms (2) 096 -wake_lock_in=1000:"*alarm*:android.intent.action.TIME_TICK" + +15m51s839ms (2) 095 charge=2692 + Details: cpu=5260u+8060s + /proc/stat=5220 usr, 4980 sys, 50 io, 0 irq, 410 sirq, 115250 idle (8.5% of 20m 59s 100ms), PlatformIdleStat null + +15m51s846ms (2) 095 +wake_lock_in=1001:"RILJ" + +15m51s851ms (2) 095 -wake_lock_in=1001:"RILJ" + +15m51s852ms (2) 095 charge=2690 stats=0:"battery-level" + +15m55s679ms (2) 095 +wake_lock_in=1001:"RILJ" + +15m55s681ms (2) 095 -wake_lock_in=1001:"RILJ" + +16m15s683ms (2) 095 +wake_lock_in=1001:"RILJ" + +16m15s686ms (2) 095 -wake_lock_in=1001:"RILJ" + +16m35s706ms (2) 095 +wake_lock_in=1001:"RILJ" + +16m35s709ms (2) 095 -wake_lock_in=1001:"RILJ" + +16m40s191ms (2) 095 +alarm=1000:"*alarm*:android.intent.action.TIME_TICK" + +16m40s192ms (2) 095 +wake_lock_in=1000:"*alarm*:android.intent.action.TIME_TICK" + +16m40s263ms (2) 095 -alarm=1000:"*alarm*:android.intent.action.TIME_TICK" + +16m40s263ms (2) 095 -wake_lock_in=1000:"*alarm*:android.intent.action.TIME_TICK" + +16m48s969ms (1) 095 charge=2685 + +16m52s078ms (1) 095 charge=2684 + +16m55s712ms (2) 095 +wake_lock_in=1001:"RILJ" + +16m55s715ms (2) 095 -wake_lock_in=1001:"RILJ" + +17m15s160ms (2) 095 +alarm=u0a30:"*walarm*:com.google.android.location.ALARM_WAKEUP_ACTIVITY_DETECTION" + +17m15s160ms (2) 095 +wake_lock_in=1000:"*walarm*:com.google.android.location.ALARM_WAKEUP_ACTIVITY_DETECTION" + +17m15s163ms (2) 095 +wake_lock_in=u0a30:"NlpWakeLock" + +17m15s167ms (2) 095 +wake_lock_in=u0a30:"NlpCollectorWakeLock" + +17m15s172ms (2) 095 -alarm=u0a30:"*walarm*:com.google.android.location.ALARM_WAKEUP_ACTIVITY_DETECTION" + +17m15s172ms (2) 095 -wake_lock_in=1000:"*walarm*:com.google.android.location.ALARM_WAKEUP_ACTIVITY_DETECTION" + +17m15s177ms (2) 095 -wake_lock_in=u0a30:"NlpWakeLock" + +17m15s711ms (2) 095 +wake_lock_in=u0a30:"NlpWakeLock" + +17m15s715ms (2) 095 -wake_lock_in=u0a30:"NlpWakeLock" + +17m15s716ms (2) 095 +wake_lock_in=u0a30:"NlpWakeLock" + +17m15s718ms (2) 095 -wake_lock_in=u0a30:"NlpCollectorWakeLock" + +17m15s725ms (2) 095 +wake_lock_in=1001:"RILJ" + +17m15s728ms (2) 095 -wake_lock_in=u0a30:"NlpWakeLock" + +17m15s728ms (2) 095 -wake_lock_in=1001:"RILJ" + +17m15s731ms (2) 095 +wake_lock_in=u0a30:"NlpWakeLock" + +17m15s734ms (2) 095 -wake_lock_in=u0a30:"NlpWakeLock" + +17m15s735ms (2) 095 +wake_lock_in=u0a30:"GCoreFlp" + +17m15s737ms (2) 095 -wake_lock_in=u0a30:"GCoreFlp" + +17m15s737ms (2) 095 +wake_lock_in=u0a30:"GCoreFlp" + +17m15s740ms (2) 095 -wake_lock_in=u0a30:"GCoreFlp" + +17m35s749ms (2) 095 +wake_lock_in=1001:"RILJ" + +17m35s751ms (2) 095 -wake_lock_in=1001:"RILJ" + +17m40s191ms (2) 095 +alarm=1000:"*alarm*:android.intent.action.TIME_TICK" + +17m40s192ms (2) 095 +wake_lock_in=1000:"*alarm*:android.intent.action.TIME_TICK" + +17m40s264ms (2) 095 -alarm=1000:"*alarm*:android.intent.action.TIME_TICK" + +17m40s264ms (2) 095 -wake_lock_in=1000:"*alarm*:android.intent.action.TIME_TICK" + +17m52s136ms (1) 095 charge=2677 + +17m55s757ms (2) 095 +wake_lock_in=1001:"RILJ" + +17m55s759ms (2) 095 -wake_lock_in=1001:"RILJ" + +18m15s767ms (2) 095 +wake_lock_in=1001:"RILJ" + +18m15s769ms (2) 095 -wake_lock_in=1001:"RILJ" + +18m23s412ms (2) 094 charge=2675 + Details: cpu=103510u+158550s + /proc/stat=100930 usr, 100340 sys, 1000 io, 10 irq, 7670 sirq, 2192810 idle (8.7% of 6h 40m 27s 600ms), PlatformIdleStat null + +18m23s421ms (2) 094 +wake_lock_in=1001:"RILJ" + +18m23s425ms (2) 094 -wake_lock_in=1001:"RILJ" + +18m23s425ms (2) 094 stats=0:"battery-level" + +18m35s782ms (2) 094 +wake_lock_in=1001:"RILJ" + +18m35s785ms (2) 094 -wake_lock_in=1001:"RILJ" + +18m40s190ms (2) 094 +alarm=1000:"*alarm*:android.intent.action.TIME_TICK" + +18m40s191ms (2) 094 +wake_lock_in=1000:"*alarm*:android.intent.action.TIME_TICK" + +18m40s283ms (2) 094 -alarm=1000:"*alarm*:android.intent.action.TIME_TICK" + +18m40s283ms (2) 094 -wake_lock_in=1000:"*alarm*:android.intent.action.TIME_TICK" + +18m52s396ms (1) 094 charge=2670 + +18m55s791ms (2) 094 +wake_lock_in=1001:"RILJ" + +18m55s793ms (2) 094 -wake_lock_in=1001:"RILJ" + +19m15s803ms (2) 094 +wake_lock_in=1001:"RILJ" + +19m15s805ms (2) 094 -wake_lock_in=1001:"RILJ" + +19m16s941ms (2) 094 +alarm=u0a84:"*walarm*:*job.deadline*" + +19m16s941ms (2) 094 +wake_lock_in=u0a84:"*walarm*:*job.deadline*" + +19m16s941ms (2) 094 +alarm=u0a84:"*walarm*:*job.delay*" + +19m16s942ms (2) 094 +alarm=1000:"*walarm*:WifiConnectivityManager Schedule Periodic Scan Timer" + +19m16s943ms (2) 094 +alarm=u0a30:"*walarm*:com.google.android.gms.matchstick.register_intent_action" + +19m16s949ms (2) 094 -alarm=1000:"*walarm*:WifiConnectivityManager Schedule Periodic Scan Timer" + +19m16s950ms (2) 094 -alarm=u0a84:"*walarm*:*job.deadline*" + +19m16s950ms (2) 094 +wifi_scan -alarm=u0a84:"*walarm*:*job.delay*" + +19m16s963ms (2) 094 +wake_lock_in=u0a30:"*alarm*" + +19m16s963ms (2) 094 -wake_lock_in=u0a84:"*walarm*:*job.deadline*" + +19m16s963ms (2) 094 -alarm=u0a30:"*walarm*:com.google.android.gms.matchstick.register_intent_action" + +19m16s963ms (2) 094 -wake_lock_in=u0a30:"*alarm*" + +19m16s972ms (2) 094 +job=u0a84:"com.google.android.apps.tachyon/.jobs.NativeJobSchedulerJobService" + +19m16s975ms (2) 094 +job=u0a65:"com.google.android.googlequicksearchbox/com.google.android.apps.gsa.tasks.BackgroundTasksJobService" + +19m16s975ms (2) 094 +wake_lock_in=u0a84:"*job*/com.google.android.apps.tachyon/.jobs.NativeJobSchedulerJobService" + +19m16s977ms (2) 094 +wake_lock_in=u0a65:"*job*/com.google.android.googlequicksearchbox/com.google.android.apps.gsa.tasks.BackgroundTasksJobService" + +19m16s979ms (2) 094 -job=u0a84:"com.google.android.apps.tachyon/.jobs.NativeJobSchedulerJobService" + +19m16s979ms (2) 094 -wifi_scan -wake_lock_in=u0a84:"*job*/com.google.android.apps.tachyon/.jobs.NativeJobSchedulerJobService" + +19m17s074ms (2) 094 -job=u0a65:"com.google.android.googlequicksearchbox/com.google.android.apps.gsa.tasks.BackgroundTasksJobService" + +19m17s074ms (2) 094 -wake_lock_in=u0a65:"*job*/com.google.android.googlequicksearchbox/com.google.android.apps.gsa.tasks.BackgroundTasksJobService" + +19m35s827ms (2) 094 +wake_lock_in=1001:"RILJ" + +19m35s830ms (2) 094 -wake_lock_in=1001:"RILJ" + +19m40s190ms (2) 094 +alarm=1000:"*alarm*:android.intent.action.TIME_TICK" + +19m40s190ms (2) 094 +wake_lock_in=1000:"*alarm*:android.intent.action.TIME_TICK" + +19m40s267ms (2) 094 -alarm=1000:"*alarm*:android.intent.action.TIME_TICK" + +19m40s267ms (2) 094 -wake_lock_in=1000:"*alarm*:android.intent.action.TIME_TICK" + +19m52s453ms (1) 094 charge=2663 + +19m55s836ms (2) 094 +wake_lock_in=1001:"RILJ" + +19m55s839ms (2) 094 -wake_lock_in=1001:"RILJ" + +20m15s740ms (2) 094 +alarm=u0a30:"*walarm*:com.google.android.location.ALARM_WAKEUP_ACTIVITY_DETECTION" + +20m15s741ms (2) 094 +wake_lock_in=1000:"*walarm*:com.google.android.location.ALARM_WAKEUP_ACTIVITY_DETECTION" + +20m15s742ms (2) 094 +wake_lock_in=u0a30:"NlpWakeLock" + +20m15s747ms (2) 094 +wake_lock_in=u0a30:"NlpCollectorWakeLock" + +20m15s751ms (2) 094 -alarm=u0a30:"*walarm*:com.google.android.location.ALARM_WAKEUP_ACTIVITY_DETECTION" + +20m15s752ms (2) 094 -wake_lock_in=1000:"*walarm*:com.google.android.location.ALARM_WAKEUP_ACTIVITY_DETECTION" + +20m15s757ms (2) 094 -wake_lock_in=u0a30:"NlpWakeLock" + +20m15s859ms (2) 094 +wake_lock_in=1001:"RILJ" + +20m15s862ms (2) 094 -wake_lock_in=1001:"RILJ" + +20m16s290ms (2) 094 +wake_lock_in=u0a30:"NlpWakeLock" + +20m16s295ms (2) 094 -wake_lock_in=u0a30:"NlpCollectorWakeLock" + +20m16s304ms (2) 094 -wake_lock_in=u0a30:"NlpWakeLock" + +20m16s308ms (2) 094 +wake_lock_in=u0a30:"NlpWakeLock" + +20m16s312ms (2) 094 -wake_lock_in=u0a30:"NlpWakeLock" + +20m16s313ms (2) 094 +wake_lock_in=u0a30:"GCoreFlp" + +20m16s314ms (2) 094 -wake_lock_in=u0a30:"GCoreFlp" + +20m16s315ms (2) 094 +wake_lock_in=u0a30:"GCoreFlp" + +20m16s318ms (2) 094 -wake_lock_in=u0a30:"GCoreFlp" + +20m35s883ms (2) 094 +wake_lock_in=1001:"RILJ" + +20m35s886ms (2) 094 -wake_lock_in=1001:"RILJ" + +20m37s105ms (2) 094 +alarm=1000:"*walarm*:DhcpClient.wlan0.RENEW" + +20m37s105ms (2) 094 +wake_lock_in=1000:"*walarm*:DhcpClient.wlan0.RENEW" + +20m37s105ms (2) 094 -alarm=1000:"*walarm*:DhcpClient.wlan0.RENEW" + +20m37s105ms (2) 094 -wake_lock_in=1000:"*walarm*:DhcpClient.wlan0.RENEW" + +20m40s191ms (2) 094 +alarm=1000:"*alarm*:android.intent.action.TIME_TICK" + +20m40s191ms (2) 094 +wake_lock_in=1000:"*alarm*:android.intent.action.TIME_TICK" + +20m40s267ms (2) 094 -alarm=1000:"*alarm*:android.intent.action.TIME_TICK" + +20m40s267ms (2) 094 -wake_lock_in=1000:"*alarm*:android.intent.action.TIME_TICK" + +20m47s522ms (2) 094 -proc=u0a118:"org.mozilla.geckoview_example" + +20m47s547ms (2) 094 -top=u0a118:"org.mozilla.geckoview_example" + +20m47s547ms (2) 094 +top=u0a6:"com.motorola.ccc.ota" + +20m47s554ms (2) 094 -proc=u0a118:"org.mozilla.geckoview_example:tab" + +20m49s613ms (3) 094 brightness=dark +wake_lock_in=1001:"RILJ" + +20m49s861ms (2) 094 -wake_lock_in=1001:"RILJ" + +20m49s861ms (2) 094 stats=0:"dump" + +20m50s057ms (2) 094 +wake_lock_in=1001:"RILJ" + +20m50s061ms (2) 094 -wake_lock_in=1001:"RILJ" + +20m50s061ms (2) 094 stats=0:"dump" + +Per-PID Stats: + PID 1544 wake time: +1s795ms + PID 2014 wake time: +203ms + PID 1544 wake time: +5ms + PID 1985 wake time: +5s434ms + PID 2366 wake time: +29ms + PID 1544 wake time: +101ms + PID 1544 wake time: +81ms + PID 1544 wake time: +7ms + PID 1544 wake time: +752ms + PID 2366 wake time: +51ms + +Discharge step durations: + #0: +2m31s573ms to 94 (screen-on, power-save-off, device-idle-off) + #1: +5m0s866ms to 95 (screen-on, power-save-off, device-idle-off) + #2: +5m52s605ms to 96 (screen-on, power-save-off, device-idle-off) + #3: +3m8s791ms to 97 (screen-on, power-save-off, device-idle-off) + Estimated discharge time remaining: +6h29m15s52ms + Estimated screen on time: 6h 54m 5s 800ms + +Daily stats: + Current start time: 2019-07-03-01-01-55 + Next min deadline: 2019-07-04-01-00-00 + Next max deadline: 2019-07-04-03-00-00 + Current daily discharge step durations: + #0: +2m31s573ms to 94 (screen-on, power-save-off, device-idle-off) + #1: +5m0s866ms to 95 (screen-on, power-save-off, device-idle-off) + #2: +5m52s605ms to 96 (screen-on, power-save-off, device-idle-off) + #3: +3m8s791ms to 97 (screen-on, power-save-off, device-idle-off) + #4: +4m0s541ms to 98 (screen-on, power-save-off, device-idle-off) + #5: +5m28s940ms to 93 (screen-on, power-save-off, device-idle-off) + #6: +3m32s400ms to 94 (screen-on, power-save-off, device-idle-off) + #7: +6m8s472ms to 96 (screen-on, power-save-off, device-idle-off) + #8: +3m51s528ms to 97 (screen-on, power-save-off, device-idle-off) + #9: +5m59s536ms to 98 (screen-on, power-save-off, device-idle-off) + #10: +3m29s129ms to 98 (screen-on, power-save-off, device-idle-off) + #11: +3m0s480ms to 98 (screen-on, power-save-off, device-idle-off) + #12: +4m49s722ms to 96 (screen-on, power-save-off, device-idle-off) + #13: +4m0s489ms to 97 (screen-on, power-save-off, device-idle-off) + #14: +3m0s579ms to 98 (screen-on, power-save-off, device-idle-off) + #15: +3m1s987ms to 98 (screen-on, power-save-off, device-idle-off) + Discharge total time: 6h 58m 30s 200ms (from 16 steps) + Discharge screen on time: 6h 58m 30s 200ms (from 16 steps) + Package changes: + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Daily from 2019-07-02-03-38-21 to 2019-07-03-01-01-55: + Discharge step durations: + #0: +4m22s262ms to 96 (screen-on, power-save-off, device-idle-off) + #1: +2m40s260ms to 97 (screen-on, power-save-off, device-idle-off) + #2: +3m59s182ms to 98 (screen-on, power-save-off, device-idle-off) + #3: +2m36s315ms to 92 (screen-on, power-save-off, device-idle-off) + #4: +5m0s800ms to 93 (screen-on, power-save-off, device-idle-off) + #5: +5m0s859ms to 94 (screen-on, power-save-off, device-idle-off) + #6: +6m0s801ms to 95 (screen-on, power-save-off, device-idle-off) + #7: +5m38s645ms to 96 (screen-on, power-save-off, device-idle-off) + #8: +3m0s593ms to 97 (screen-on, power-save-off, device-idle-off) + #9: +4m0s638ms to 98 (screen-on, power-save-off, device-idle-off) + #10: +3m11s195ms to 98 (screen-on, power-save-off, device-idle-off) + #11: +2m54s941ms to 97 (screen-on, power-save-off, device-idle-off) + #12: +3m30s455ms to 97 (screen-on, power-save-off, device-idle-off) + #13: +4m0s491ms to 98 (screen-on, power-save-off, device-idle-off) + #14: +5m0s801ms to 95 (screen-on, power-save-off, device-idle-off) + #15: +3m36s165ms to 97 (screen-on, power-save-off, device-idle-off) + #16: +5m0s681ms to 98 (screen-on, power-save-off, device-idle-off) + #17: +3m3s300ms to 97 (screen-on, power-save-off, device-idle-off) + #18: +3m19s93ms to 98 (screen-on, power-save-off, device-idle-off) + Discharge total time: 6h 39m 46s 700ms (from 19 steps) + Discharge screen on time: 6h 39m 46s 700ms (from 19 steps) + Package changes: + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Daily from 2019-07-01-03-39-30 to 2019-07-02-03-38-21: + Discharge step durations: + #0: +5m0s700ms to 95 (screen-on, power-save-off, device-idle-off) + #1: +4m43s724ms to 96 (screen-on, power-save-off, device-idle-off) + #2: +3m27s57ms to 97 (screen-on, power-save-off, device-idle-off) + #3: +3m13s450ms to 98 (screen-on, power-save-off, device-idle-off) + #4: +4m2s156ms to 94 (screen-on, power-save-off, device-idle-off) + #5: +5m51s857ms to 95 (screen-on, power-save-off, device-idle-off) + #6: +3m0s351ms to 97 (screen-on, power-save-off, device-idle-off) + #7: +3m53s414ms to 98 (screen-on, power-save-off, device-idle-off) + #8: +3m37s832ms to 97 (screen-on, power-save-off, device-idle-off) + #9: +3m0s378ms to 98 (screen-on, power-save-off, device-idle-off) + Discharge total time: 6h 38m 29s 100ms (from 10 steps) + Discharge screen on time: 6h 38m 29s 100ms (from 10 steps) + Package changes: + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update com.google.android.videos vers=41200081 + Update com.google.android.youtube vers=1425573340 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Daily from 2019-06-30-03-49-43 to 2019-07-01-03-39-30: + Discharge step durations: + #0: +5m22s623ms to 96 (screen-on, power-save-off, device-idle-off) + #1: +3m20s32ms to 97 (screen-on, power-save-off, device-idle-off) + #2: +3m38s58ms to 98 (screen-on, power-save-off, device-idle-off) + #3: +3m0s479ms to 97 (screen-on, power-save-off, device-idle-off) + #4: +5m0s92ms to 96 (screen-on, power-save-off, device-idle-off) + #5: +2m58s381ms to 97 (screen-on, power-save-off, device-idle-off) + #6: +3m0s481ms to 98 (screen-on, power-save-off, device-idle-off) + Discharge total time: 6h 16m 13s 500ms (from 7 steps) + Discharge screen on time: 6h 16m 13s 500ms (from 7 steps) + Package changes: + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update com.google.android.apps.maps vers=1019101144 + Update org.mozilla.geckoview_example vers=1 + Daily from 2019-06-29-03-53-51 to 2019-06-30-03-49-43: + Discharge step durations: + #0: +3m22s443ms to 97 (screen-on, power-save-off, device-idle-off) + #1: +3m0s480ms to 98 (screen-on, power-save-off, device-idle-off) + #2: +4m0s492ms to 97 (screen-on, power-save-off, device-idle-off) + #3: +4m22s122ms to 98 (screen-on, power-save-off, device-idle-off) + #4: +4m22s516ms to 97 (screen-on, power-save-off, device-idle-off) + #5: +3m38s784ms to 98 (screen-on, power-save-off, device-idle-off) + Discharge total time: 6h 19m 40s 600ms (from 6 steps) + Discharge screen on time: 6h 19m 40s 600ms (from 6 steps) + Package changes: + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Daily from 2019-06-28-03-18-43 to 2019-06-29-03-53-51: + Discharge step durations: + #0: +4m0s696ms to 94 (screen-on, power-save-off, device-idle-off) + #1: +5m0s780ms to 95 (screen-on, power-save-off, device-idle-off) + #2: +6m0s950ms to 96 (screen-on, power-save-off, device-idle-off) + #3: +4m0s515ms to 97 (screen-on, power-save-off, device-idle-off) + #4: +6m0s965ms to 93 (screen-on, power-save-off, device-idle-off) + #5: +4m0s630ms to 94 (screen-on, power-save-off, device-idle-off) + #6: +8m1s278ms to 95 (screen-on, power-save-off, device-idle-off) + #7: +3m41s781ms to 97 (screen-on, power-save-off, device-idle-off) + #8: +4m0s666ms to 98 (screen-on, power-save-off, device-idle-off) + #9: +5m3s804ms to 96 (screen-on, power-save-off, device-idle-off) + #10: +4m0s739ms to 97 (screen-on, power-save-off, device-idle-off) + #11: +3m0s481ms to 98 (screen-on, power-save-off, device-idle-off) + #12: +6m10s883ms to 96 (screen-on, power-save-off, device-idle-off) + #13: +3m47s465ms to 97 (screen-on, power-save-off, device-idle-off) + #14: +3m0s583ms to 98 (screen-on, power-save-off, device-idle-off) + Discharge total time: 7h 45m 48s 100ms (from 15 steps) + Discharge screen on time: 7h 45m 48s 100ms (from 15 steps) + Package changes: + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.reference.browser.raptor vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Daily from 2019-06-27-04-47-23 to 2019-06-28-03-18-43: + Discharge step durations: + #0: +5m29s877ms to 95 (screen-on, power-save-off, device-idle-off) + #1: +5m3s762ms to 96 (screen-on, power-save-off, device-idle-off) + #2: +4m2s174ms to 97 (screen-on, power-save-off, device-idle-off) + #3: +2m58s895ms to 98 (screen-on, power-save-off, device-idle-off) + #4: +5m0s585ms to 96 (screen-on, power-save-off, device-idle-off) + #5: +4m0s599ms to 97 (screen-on, power-save-off, device-idle-off) + #6: +3m0s480ms to 98 (screen-on, power-save-off, device-idle-off) + #7: +3m36s286ms to 91 (screen-on, power-save-off, device-idle-off) + #8: +4m0s698ms to 92 (screen-on, power-save-off, device-idle-off) + #9: +6m19s775ms to 93 (screen-on, power-save-off, device-idle-off) + #10: +3m41s767ms to 94 (screen-on, power-save-off, device-idle-off) + #11: +8m1s180ms to 96 (screen-on, power-save-off, device-idle-off) + #12: +3m47s800ms to 97 (screen-on, power-save-off, device-idle-off) + #13: +4m0s545ms to 98 (screen-on, power-save-off, device-idle-off) + #14: +4m17s216ms to 93 (screen-on, power-save-off, device-idle-off) + #15: +3m0s324ms to 94 (screen-on, power-save-off, device-idle-off) + #16: +5m35s824ms to 95 (screen-on, power-save-off, device-idle-off) + #17: +5m0s112ms to 96 (screen-on, power-save-off, device-idle-off) + #18: +3m26s403ms to 97 (screen-on, power-save-off, device-idle-off) + #19: +3m0s504ms to 98 (screen-on, power-save-off, device-idle-off) + Discharge total time: 7h 17m 4s 0ms (from 20 steps) + Discharge screen on time: 7h 17m 4s 0ms (from 20 steps) + Package changes: + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.reference.browser.raptor vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.reference.browser.raptor vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=0 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update com.google.android.apps.photos vers=4846836 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Daily from 2019-06-26-04-43-23 to 2019-06-27-04-47-23: + Discharge step durations: + #0: +3m50s915ms to 91 (screen-on, power-save-off, device-idle-off) + #1: +4m5s396ms to 92 (screen-on, power-save-off, device-idle-off) + #2: +4m56s85ms to 93 (screen-on, power-save-off, device-idle-off) + #3: +5m0s759ms to 94 (screen-on, power-save-off, device-idle-off) + #4: +5m0s700ms to 95 (screen-on, power-save-off, device-idle-off) + #5: +5m14s8ms to 96 (screen-on, power-save-off, device-idle-off) + #6: +3m0s480ms to 97 (screen-on, power-save-off, device-idle-off) + #7: +4m2s92ms to 98 (screen-on, power-save-off, device-idle-off) + #8: +5m6s217ms to 95 (screen-on, power-save-off, device-idle-off) + #9: +5m28s275ms to 96 (screen-on, power-save-off, device-idle-off) + #10: +3m26s946ms to 97 (screen-on, power-save-off, device-idle-off) + #11: +2m59s11ms to 98 (screen-on, power-save-off, device-idle-off) + Discharge total time: 7h 14m 50s 700ms (from 12 steps) + Discharge screen on time: 7h 14m 50s 700ms (from 12 steps) + Package changes: + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Daily from 2019-06-25-01-43-11 to 2019-06-26-04-43-23: + Discharge step durations: + #0: +6m1s66ms to 95 (screen-on, power-save-off, device-idle-off) + #1: +5m0s816ms to 96 (screen-on, power-save-off, device-idle-off) + #2: +5m0s776ms to 97 (screen-on, power-save-off, device-idle-off) + #3: +5m0s700ms to 98 (screen-on, power-save-off, device-idle-off) + #4: +4m0s745ms to 98 (screen-on, power-save-off, device-idle-off) + #5: +5m0s900ms to 97 (screen-on, power-save-off, device-idle-off) + #6: +5m0s800ms to 98 (screen-on, power-save-off, device-idle-off) + #7: +4m33s841ms to 96 (screen-on, power-save-off, device-idle-off) + #8: +4m0s700ms to 97 (screen-on, power-save-off, device-idle-off) + #9: +4m0s579ms to 98 (screen-on, power-save-off, device-idle-off) + #10: +3m0s500ms to 98 (screen-on, power-save-off, device-idle-off) + #11: +5m1s817ms to 95 (screen-on, power-save-off, device-idle-off) + #12: +5m16s527ms to 96 (screen-on, power-save-off, device-idle-off) + #13: +3m23s971ms to 97 (screen-on, power-save-off, device-idle-off) + #14: +3m0s527ms to 98 (screen-on, power-save-off, device-idle-off) + Discharge total time: 7h 29m 21s 700ms (from 15 steps) + Discharge screen on time: 7h 29m 21s 700ms (from 15 steps) + Package changes: + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Daily from 2019-06-24-04-30-21 to 2019-06-25-01-43-11: + Discharge step durations: + #0: +3m0s606ms to 61 (screen-on, power-save-off, device-idle-off) + #1: +5m2s437ms to 62 (screen-on, power-save-off, device-idle-off) + #2: +2m58s844ms to 63 (screen-on, power-save-off, device-idle-off) + #3: +5m0s809ms to 64 (screen-on, power-save-off, device-idle-off) + #4: +6m0s827ms to 65 (screen-on, power-save-off, device-idle-off) + #5: +3m38s33ms to 65 (screen-on, power-save-off, device-idle-off) + #6: +2m0s222ms to 66 (screen-on, power-save-off, device-idle-off) + #7: +4m0s639ms to 66 (screen-on, power-save-off, device-idle-off) + #8: +5m1s499ms to 67 (screen-on, power-save-off, device-idle-off) + #9: +2m59s789ms to 68 (screen-on, power-save-off, device-idle-off) + #10: +7m1s111ms to 69 (screen-on, power-save-off, device-idle-off) + #11: +3m51s135ms to 69 (screen-on, power-save-off, device-idle-off) + #12: +3m3s399ms to 68 (screen-on, power-save-off, device-idle-off) + #13: +4m57s849ms to 69 (screen-on, power-save-off, device-idle-off) + #14: +6m0s972ms to 70 (screen-on, power-save-off, device-idle-off) + #15: +4m0s607ms to 71 (screen-on, power-save-off, device-idle-off) + #16: +5m0s822ms to 72 (screen-on, power-save-off, device-idle-off) + #17: +3m50s52ms to 73 (screen-on, power-save-off, device-idle-off) + #18: +5m11s366ms to 74 (screen-on, power-save-off, device-idle-off) + #19: +4m0s694ms to 75 (screen-on, power-save-off, device-idle-off) + #20: +5m53s967ms to 76 (screen-on, power-save-off, device-idle-off) + #21: +7m1s118ms to 77 (screen-on, power-save-off, device-idle-off) + #22: +5m0s702ms to 78 (screen-on, power-save-off, device-idle-off) + #23: +3m59s265ms to 75 (screen-on, power-save-off, device-idle-off) + #24: +5m38s306ms to 76 (screen-on, power-save-off, device-idle-off) + #25: +5m23s455ms to 77 (screen-on, power-save-off, device-idle-off) + #26: +3m59s161ms to 78 (screen-on, power-save-off, device-idle-off) + #27: +5m0s772ms to 79 (screen-on, power-save-off, device-idle-off) + #28: +5m57s968ms to 79 (screen-on, power-save-off, device-idle-off) + #29: +5m0s651ms to 80 (screen-on, power-save-off, device-idle-off) + #30: +4m0s678ms to 80 (screen-on, power-save-off, device-idle-off) + #31: +7m0s984ms to 81 (screen-on, power-save-off, device-idle-off) + #32: +5m0s809ms to 80 (screen-on, power-save-off, device-idle-off) + #33: +6m1s11ms to 81 (screen-on, power-save-off, device-idle-off) + #34: +4m0s486ms to 82 (screen-on, power-save-off, device-idle-off) + #35: +4m30s253ms to 82 (screen-on, power-save-off, device-idle-off) + #36: +7m17s812ms to 83 (screen-on, power-save-off, device-idle-off) + #37: +6m0s960ms to 84 (screen-on, power-save-off, device-idle-off) + #38: +4m0s538ms to 85 (screen-on, power-save-off, device-idle-off) + #39: +3m25s548ms to 85 (screen-on, power-save-off, device-idle-off) + #40: +7m1s22ms to 86 (screen-on, power-save-off, device-idle-off) + #41: +4m59s237ms to 87 (screen-on, power-save-off, device-idle-off) + #42: +6m1s46ms to 88 (screen-on, power-save-off, device-idle-off) + #43: +7m1s33ms to 89 (screen-on, power-save-off, device-idle-off) + #44: +4m59s258ms to 90 (screen-on, power-save-off, device-idle-off) + #45: +5m0s701ms to 92 (screen-on, power-save-off, device-idle-off) + #46: +4m57s120ms to 93 (screen-on, power-save-off, device-idle-off) + #47: +4m0s693ms to 94 (screen-on, power-save-off, device-idle-off) + #48: +6m53s570ms to 96 (screen-on, power-save-off, device-idle-off) + #49: +7m1s141ms to 97 (screen-on, power-save-off, device-idle-off) + #50: +4m49s156ms to 94 (screen-on, power-save-off, device-idle-off) + #51: +9m59s602ms to 95 (screen-on, power-save-off, device-idle-off) + #52: +3m27s559ms to 92 (screen-on, power-save-off, device-idle-off) + #53: +5m0s799ms to 93 (screen-on, power-save-off, device-idle-off) + #54: +4m0s640ms to 94 (screen-on, power-save-off, device-idle-off) + #55: +6m0s981ms to 95 (screen-on, power-save-off, device-idle-off) + #56: +7m1s64ms to 96 (screen-on, power-save-off, device-idle-off) + #57: +4m0s542ms to 97 (screen-on, power-save-off, device-idle-off) + #58: +3m56s802ms to 98 (screen-on, power-save-off, device-idle-off) + #59: +2m28s39ms to 97 (screen-on, power-save-off, device-idle-off) + #60: +2m21s945ms to 98 (screen-on, power-save-off, device-idle-off) + Discharge total time: 8h 15m 1s 800ms (from 61 steps) + Discharge screen on time: 8h 15m 1s 800ms (from 61 steps) + Charge step durations: + #0: +1m46s170ms to 89 (screen-on, power-save-off, device-idle-off) + #1: +2m11s820ms to 88 (screen-on, power-save-off, device-idle-off) + #2: +2m0s318ms to 87 (screen-on, power-save-off, device-idle-off) + #3: +2m0s321ms to 86 (screen-on, power-save-off, device-idle-off) + #4: +2m0s321ms to 85 (screen-on, power-save-off, device-idle-off) + #5: +1m51s751ms to 84 (screen-on, power-save-off, device-idle-off) + #6: +1m51s261ms to 83 (screen-on, power-save-off, device-idle-off) + #7: +1m4s208ms to 82 (screen-on, power-save-off, device-idle-off) + #8: +1m13s741ms to 81 (screen-on, power-save-off, device-idle-off) + #9: +1m6s904ms to 80 (screen-on, power-save-off, device-idle-off) + #10: +1m53s574ms to 79 (screen-on, power-save-off, device-idle-off) + #11: +1m7s743ms to 78 (screen-on, power-save-off, device-idle-off) + #12: +1m26s229ms to 77 (screen-on, power-save-off, device-idle-off) + #13: +1m27s717ms to 76 (screen-on, power-save-off, device-idle-off) + #14: +44s92ms to 75 (screen-on, power-save-off, device-idle-off) + #15: +1m5s153ms to 74 (screen-on, power-save-off, device-idle-off) + #16: +1m1s741ms to 73 (screen-on, power-save-off, device-idle-off) + #17: +1m53s108ms to 72 (screen-on, power-save-off, device-idle-off) + #18: +1m5s790ms to 71 (screen-on, power-save-off, device-idle-off) + #19: +1m19s852ms to 70 (screen-on, power-save-off, device-idle-off) + #20: +1m27s758ms to 69 (screen-on, power-save-off, device-idle-off) + #21: +12s871ms to 68 (screen-on, power-save-off, device-idle-off) + #22: +2m0s321ms to 67 (screen-on, power-save-off, device-idle-off) + #23: +2m0s318ms to 66 (screen-on, power-save-off, device-idle-off) + #24: +2m0s320ms to 65 (screen-on, power-save-off, device-idle-off) + Charge total time: 2h 31m 33s 600ms (from 25 steps) + Charge screen on time: 2h 31m 33s 600ms (from 25 steps) + Package changes: + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.fenix.raptor vers=1 + Update org.mozilla.reference.browser vers=11281213 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.fennec_aurora vers=2015636401 + Update org.mozilla.fenix.raptor vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.fenix.raptor vers=1 + Update org.mozilla.fenix.raptor vers=1 + Update org.mozilla.reference.browser vers=11281213 + Update org.mozilla.reference.browser vers=11281213 + Update org.mozilla.reference.browser vers=11281213 + Update org.mozilla.fennec_aurora vers=2015636401 + Update org.mozilla.fenix.raptor vers=1 + Update org.mozilla.fennec_aurora vers=2015636401 + Update org.mozilla.fenix.raptor vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.reference.browser vers=11281213 + Update org.mozilla.fennec_aurora vers=2015636401 + Update org.mozilla.fennec_aurora vers=2015636401 + Update org.mozilla.fennec_aurora vers=2015636401 + Update org.mozilla.reference.browser.raptor vers=1 + Update org.mozilla.fennec_aurora vers=2015636441 + Update org.mozilla.fenix.raptor vers=1 + Update org.mozilla.reference.browser.raptor vers=1 + Update com.google.android.gms vers=17785018 + Update com.google.android.youtube vers=1424573340 + Update org.mozilla.fennec_aurora vers=2015636457 + Update org.mozilla.fenix.performancetest vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.fennec_aurora vers=2015636457 + Daily from 2019-06-23-04-45-16 to 2019-06-24-04-30-21: + Discharge step durations: + #0: +3m23s357ms to 92 (screen-on, power-save-off, device-idle-off) + #1: +3m37s763ms to 93 (screen-on, power-save-off, device-idle-off) + #2: +3m3s603ms to 94 (screen-on, power-save-off, device-idle-off) + #3: +5m10s816ms to 95 (screen-on, power-save-off, device-idle-off) + #4: +4m49s185ms to 96 (screen-on, power-save-off, device-idle-off) + #5: +2m58s972ms to 97 (screen-on, power-save-off, device-idle-off) + #6: +4m0s497ms to 98 (screen-on, power-save-off, device-idle-off) + #7: +3m0s632ms to 94 (screen-on, power-save-off, device-idle-off) + #8: +5m41s798ms to 95 (screen-on, power-save-off, device-idle-off) + #9: +5m19s809ms to 96 (screen-on, power-save-off, device-idle-off) + #10: +3m0s630ms to 97 (screen-on, power-save-off, device-idle-off) + #11: +4m0s490ms to 98 (screen-on, power-save-off, device-idle-off) + Discharge total time: 6h 41m 2s 900ms (from 12 steps) + Discharge screen on time: 6h 41m 2s 900ms (from 12 steps) + Package changes: + Update org.mozilla.geckoview_example vers=1 + Update com.android.chrome vers=377010107 + Update com.google.android.apps.tachyon vers=2980701 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + +Statistics since last charge: + System starts: 0, currently on battery: true + Estimated battery capacity: 2876 mAh + Time on battery: 20m 50s 94ms (100.0%) realtime, 20m 50s 94ms (100.0%) uptime + Time on battery screen off: 0ms (0.0%) realtime, 0ms (0.0%) uptime + Total run time: 20m 50s 103ms realtime, 20m 50s 103ms uptime + Battery time remaining: 6h 29m 15s 52ms + Discharge: 132 mAh + Screen off discharge: 0 mAh + Screen on discharge: 132 mAh + Start clock time: 2019-07-03-23-07-19 + Screen on: 20m 50s 94ms (100.0%) 0x, Interactive: 20m 50s 93ms (100.0%) + Screen brightnesses: + dark 481ms (0.0%) + medium 20m 49s 613ms (100.0%) + Total full wakelock time: 20m 50s 86ms + Mobile total received: 0B, sent: 0B (packets received 0, sent 0) + Phone signal levels: + poor 20m 50s 93ms (100.0%) 0x + Signal scanning time: 0ms + Radio types: + none 20m 50s 93ms (100.0%) 0x + Mobile radio active time: 0ms (0.0%) 0x + Radio Idle time: 0ms (--%) + Radio Rx time: 0ms (--%) + Radio Tx time: 0ms (--%) + [0] 0ms (--%) + [1] 0ms (--%) + [2] 0ms (--%) + [3] 0ms (--%) + [4] 0ms (--%) + Radio Power drain: 0mAh + Wi-Fi total received: 0B, sent: 0B (packets received 0, sent 0) + Wifi on: 20m 50s 93ms (100.0%), Wifi running: 20m 50s 93ms (100.0%) + Wifi states: (no activity) + Wifi supplicant states: + completed 20m 50s 93ms (100.0%) 0x + Wifi signal levels: + level(4) 20m 50s 93ms (100.0%) 0x + WiFi Idle time: 0ms (--%) + WiFi Rx time: 0ms (--%) + WiFi Tx time: 0ms (--%) + WiFi Power drain: 0mAh + Bluetooth total received: 0B, sent: 0B + Bluetooth scan time: 0ms + Bluetooth Idle time: 0ms (--%) + Bluetooth Rx time: 0ms (--%) + Bluetooth Tx time: 0ms (--%) + Bluetooth Power drain: 0mAh + + Device battery use since last full charge + Amount discharged (lower bound): 4 + Amount discharged (upper bound): 5 + Amount discharged while screen on: 5 + Amount discharged while screen off: 0 + + Estimated power use (mAh): + Capacity: 2876, Computed drain: 128, actual drain: 115-144 + Screen: 70.7 + Uid u0a118: 14.5 ( cpu=14.5 ) + Uid 2000: 13.9 ( cpu=13.9 ) + Uid 0: 9.90 ( cpu=9.90 ) + Cell standby: 6.94 ( radio=6.94 ) + Uid 1036: 5.55 ( cpu=5.55 ) + Idle: 4.97 + Uid 1000: 0.700 ( cpu=0.526 sensor=0.174 ) + Uid u0a46: 0.529 ( sensor=0.529 ) + Uid u0a84: 0.233 ( cpu=0.233 ) + Uid 1001: 0.146 ( cpu=0.0159 sensor=0.130 ) + Wifi: 0.132 ( cpu=0.0261 wifi=0.106 ) + Uid u0a30: 0.0694 ( cpu=0.0634 wifi=0.00540 sensor=0.000578 ) + Uid 9999: 0.0657 ( cpu=0.0657 ) + Uid u0a64: 0.0603 ( cpu=0.0522 sensor=0.00811 ) + Uid u0a119: 0.0349 ( cpu=0.0349 ) + Uid u0a47: 0.0241 ( cpu=0.0241 ) + Uid u0a65: 0.00967 ( cpu=0.00967 ) + Uid 1041: 0.00841 ( cpu=0.000299 sensor=0.00811 ) + Uid u0a40: 0.00766 ( cpu=0.00766 ) + Uid u0a11: 0.00642 ( cpu=0.00642 ) + Uid u0a54: 0.00501 ( cpu=0.00501 ) + Uid u0a109: 0.00355 ( cpu=0.00355 ) + Uid u0a6: 0.00257 ( cpu=0.00257 ) + Uid u0a31: 0.00158 ( cpu=0.00158 ) + Uid u0a13: 0.00103 ( cpu=0.00103 ) + Uid u0a21: 0.000984 ( cpu=0.000984 ) + Uid 1013: 0.000856 ( cpu=0.000856 ) + Uid u0a102: 0.000599 ( cpu=0.000599 ) + Uid u0a98: 0.000428 ( cpu=0.000428 ) + Uid u0a107: 0.000257 ( cpu=0.000257 ) + Uid u0a24: 0.000171 ( cpu=0.000171 ) + Uid u0a83: 0.000171 ( cpu=0.000171 ) + Uid u0a77: 0.000128 ( cpu=0.000128 ) + Uid u0a74: 0.000128 ( cpu=0.000128 ) + + 0: + Total cpu time: u=6s 670ms s=3m 44s 643ms p=0mAh + Proc VosWDThread: + CPU: 0ms usr + 30ms krn ; 0ms fg + Proc kworker/u16:7: + CPU: 0ms usr + 1s 610ms krn ; 0ms fg + Proc kworker/u16:6: + CPU: 0ms usr + 2s 340ms krn ; 0ms fg + Proc kworker/u16:2: + CPU: 0ms usr + 4s 300ms krn ; 0ms fg + Proc kworker/5:1H: + CPU: 0ms usr + 40ms krn ; 0ms fg + Proc kworker/0:1H: + CPU: 0ms usr + 10ms krn ; 0ms fg + Proc irq/45-408000.q: + CPU: 0ms usr + 40ms krn ; 0ms fg + Proc VosTXThread: + CPU: 0ms usr + 1m 31s 780ms krn ; 0ms fg + Proc wpa_supplicant: + CPU: 290ms usr + 270ms krn ; 0ms fg + Proc ksoftirqd/7: + CPU: 0ms usr + 840ms krn ; 0ms fg + Proc ksoftirqd/6: + CPU: 0ms usr + 4s 610ms krn ; 0ms fg + Proc ksoftirqd/5: + CPU: 0ms usr + 300ms krn ; 0ms fg + Proc ksoftirqd/4: + CPU: 0ms usr + 280ms krn ; 0ms fg + Proc ksoftirqd/3: + CPU: 0ms usr + 20ms krn ; 0ms fg + Proc ksoftirqd/1: + CPU: 0ms usr + 10ms krn ; 0ms fg + Proc ksoftirqd/0: + CPU: 0ms usr + 10ms krn ; 0ms fg + Proc msm_watchdog: + CPU: 0ms usr + 110ms krn ; 0ms fg + Proc rcuop/7: + CPU: 0ms usr + 100ms krn ; 0ms fg + Proc rcuop/6: + CPU: 0ms usr + 410ms krn ; 0ms fg + Proc rcuop/5: + CPU: 0ms usr + 130ms krn ; 0ms fg + Proc rcuop/4: + CPU: 0ms usr + 450ms krn ; 0ms fg + Proc rcuop/3: + CPU: 0ms usr + 20ms krn ; 0ms fg + Proc rcuop/2: + CPU: 0ms usr + 60ms krn ; 0ms fg + Proc rcuop/1: + CPU: 0ms usr + 40ms krn ; 0ms fg + Proc rcuop/0: + CPU: 0ms usr + 150ms krn ; 0ms fg + Proc VosMCThread: + CPU: 0ms usr + 610ms krn ; 0ms fg + Proc mmc-cmdqd/0: + CPU: 0ms usr + 860ms krn ; 0ms fg + Proc magiskd: + CPU: 10ms usr + 50ms krn ; 0ms fg + Proc healthd: + CPU: 10ms usr + 30ms krn ; 0ms fg + Proc kworker/7:1: + CPU: 0ms usr + 20ms krn ; 0ms fg + Proc kworker/6:1: + CPU: 0ms usr + 90ms krn ; 0ms fg + Proc kworker/5:2: + CPU: 0ms usr + 330ms krn ; 0ms fg + Proc kworker/5:1: + CPU: 0ms usr + 190ms krn ; 0ms fg + Proc kworker/4:3: + CPU: 0ms usr + 60ms krn ; 0ms fg + Proc kworker/4:1: + CPU: 0ms usr + 20ms krn ; 0ms fg + Proc kworker/4:0: + CPU: 0ms usr + 10ms krn ; 0ms fg + Proc kworker/3:1: + CPU: 0ms usr + 50ms krn ; 0ms fg + Proc kworker/2:1: + CPU: 0ms usr + 100ms krn ; 0ms fg + Proc kworker/1:0: + CPU: 0ms usr + 200ms krn ; 0ms fg + Proc kworker/0:3: + CPU: 0ms usr + 170ms krn ; 0ms fg + Proc wlan_logging_th: + CPU: 0ms usr + 130ms krn ; 0ms fg + Proc installd: + CPU: 0ms usr + 10ms krn ; 0ms fg + Proc irq/32-spdm_bw_: + CPU: 0ms usr + 10ms krn ; 0ms fg + Proc hwrng: + CPU: 0ms usr + 40ms krn ; 0ms fg + Proc rcu_preempt: + CPU: 0ms usr + 900ms krn ; 0ms fg + Proc vold: + CPU: 0ms usr + 30ms krn ; 0ms fg + Proc spi4: + CPU: 0ms usr + 180ms krn ; 0ms fg + Proc rild: + CPU: 60ms usr + 50ms krn ; 0ms fg + Proc netd: + CPU: 20ms usr + 20ms krn ; 0ms fg + Proc logd: + CPU: 33s 380ms usr + 1m 21s 40ms krn ; 0ms fg + Proc lmkd: + CPU: 10ms usr + 30ms krn ; 0ms fg + Proc adbd: + CPU: 38s 800ms usr + 3m 9s 370ms krn ; 0ms fg + Proc cnd: + CPU: 30ms usr + 10ms krn ; 0ms fg + Proc surfaceflinger: + CPU: 590ms usr + 580ms krn ; 0ms fg + Proc kswapd0: + CPU: 0ms usr + 760ms krn ; 0ms fg + Proc ueventd: + CPU: 150ms usr + 30ms krn ; 0ms fg + Proc zygote: + CPU: 30ms usr + 50ms krn ; 0ms fg + Proc msm_irqbalance: + CPU: 840ms usr + 600ms krn ; 0ms fg + Proc cnss-daemon: + CPU: 10ms usr + 100ms krn ; 0ms fg + Proc system: + CPU: 0ms usr + 30ms krn ; 0ms fg + Proc logcat: + CPU: 5s 710ms usr + 10s 470ms krn ; 0ms fg + Proc irq/57-7824900.: + CPU: 0ms usr + 40ms krn ; 0ms fg + Proc mdss_fb0: + CPU: 0ms usr + 110ms krn ; 0ms fg + Proc f2fs_gc-259:22: + CPU: 0ms usr + 20ms krn ; 0ms fg + Proc VosRXThread: + CPU: 0ms usr + 1m 15s 670ms krn ; 0ms fg + 1000: + Wake lock LocationManagerService realtime + Wake lock *alarm* realtime + Wake lock GnssLocationProvider realtime + Sensor 13: 10m 25s 46ms realtime (0 times) + Cached for: 20m 50s 91ms + Total running: 20m 50s 91ms + Total cpu time: u=6s 944ms s=5s 360ms p=0mAh + Proc .dataservices: + CPU: 10ms usr + 20ms krn ; 0ms fg + Proc servicemanager: + CPU: 30ms usr + 90ms krn ; 0ms fg + Proc com.motorola.process.system: + CPU: 10ms usr + 0ms krn ; 0ms fg + Proc com.motorola.modemservice: + CPU: 30ms usr + 10ms krn ; 0ms fg + Proc system: + CPU: 5s 880ms usr + 4s 390ms krn ; 0ms fg + Proc com.qualcomm.telephony: + CPU: 10ms usr + 10ms krn ; 0ms fg + Apk android: + Wakeup alarm *walarm*:WifiConnectivityManager Schedule Periodic Scan Timer: 6 times + Wakeup alarm *walarm*:WifiConnectivityManager Schedule Watchdog Timer: 1 times + Wakeup alarm *walarm*:DhcpClient.wlan0.RENEW: 5 times + 1001: + Wake lock RILJ realtime + Sensor 8: 10m 25s 46ms realtime (0 times) + Sensor 21: 20m 50s 91ms realtime (0 times) + Fg Service for: 20m 50s 91ms + Total running: 20m 50s 91ms + Total cpu time: u=224ms s=147ms p=0mAh + Proc com.android.phone: + CPU: 180ms usr + 110ms krn ; 0ms fg + 1010: + Wifi Running: 0ms (0.0%) + Full Wifi Lock: 0ms (0.0%) + Wifi Scan: 621ms (0.0%) 6x + Total cpu time: u=330ms s=280ms p=0mAh + (nothing executed) + 1013: + Total cpu time: u=10ms s=10ms p=0mAh + Proc mediaserver: + CPU: 10ms usr + 10ms krn ; 0ms fg + 1036: + Total cpu time: u=41s 497ms s=1m 28s 180ms p=0mAh + (nothing executed) + 1037: + Cached for: 20m 50s 90ms + Total running: 20m 50s 90ms + 1041: + Sensor 8: 10m 25s 45ms realtime (0 times) + Total cpu time: u=4ms s=3ms p=0mAh + Proc audioserver: + CPU: 0ms usr + 10ms krn ; 0ms fg + 2000: + Total cpu time: u=1m 23s 217ms s=4m 0s 700ms p=0mAh + Proc logcat: + CPU: 24s 200ms usr + 19s 540ms krn ; 0ms fg + 9999: + Total cpu time: u=890ms s=647ms p=0mAh + (nothing executed) + u0a6: + Foreground activities: 13s 139ms realtime (1 times) + Top for: 13s 963ms + Foreground for: 20m 36s 126ms + Total running: 20m 50s 89ms + Total cpu time: u=43ms s=17ms p=0mAh + Proc com.motorola.ccc: + CPU: 10ms usr + 10ms krn ; 0ms fg + Proc com.motorola.ccc.ota: + CPU: 0ms usr + 0ms krn ; 1s 490ms fg + u0a11: + Foreground for: 3m 33s 123ms + Cached for: 17m 16s 965ms + Total running: 20m 50s 88ms + Total cpu time: u=140ms s=10ms p=0mAh + Proc com.android.providers.calendar: + CPU: 170ms usr + 10ms krn ; 0ms fg + u0a13: + Background for: 20m 50s 88ms + Total running: 20m 50s 88ms + Total cpu time: u=20ms s=4ms p=0mAh + Proc com.google.android.ims: + CPU: 10ms usr + 0ms krn ; 0ms fg + u0a21: + Foreground for: 16ms + Cached for: 20m 50s 72ms + Total running: 20m 50s 88ms + Total cpu time: u=20ms s=3ms p=0mAh + Proc com.android.defcontainer: + CPU: 50ms usr + 10ms krn ; 0ms fg + Apk com.android.defcontainer: + Service com.android.defcontainer.DefaultContainerService: + Created for: 0ms uptime + Starts: 0, launches: 1 + u0a24: + Background for: 4ms + Cached for: 20m 50s 84ms + Total running: 20m 50s 88ms + Total cpu time: u=0ms s=4ms p=0mAh + Proc com.android.documentsui: + CPU: 20ms usr + 20ms krn ; 0ms fg + u0a25: + Cached for: 20m 50s 88ms + Total running: 20m 50s 88ms + Proc android.process.media: + CPU: 20ms usr + 10ms krn ; 0ms fg + Proc com.android.mtp: + CPU: 30ms usr + 0ms krn ; 0ms fg + u0a28: + Cached for: 5m 40s 202ms + Total running: 5m 40s 202ms + Proc com.android.externalstorage: + CPU: 20ms usr + 0ms krn ; 0ms fg + u0a30: + Wifi Running: 0ms (0.0%) + Full Wifi Lock: 1s 876ms (0.2%) + Wifi Scan: 1s 690ms (0.1%) 1x + Wake lock *net_scheduler* realtime + Wake lock GCoreFlp realtime + Wake lock LocationManagerService realtime + Wake lock NlpWakeLock realtime + Wake lock *alarm* realtime + Wake lock Wakeful StateMachine: GeofencerStateMachine realtime + Wake lock NlpCollectorWakeLock realtime + Wake lock Icing realtime + Sensor 1: 3s 786ms realtime (7 times) + Fg Service for: 55s 287ms + Foreground for: 19m 54s 801ms + Total running: 20m 50s 88ms + Total cpu time: u=1s 123ms s=360ms p=0mAh + Proc com.google.process.gapps: + CPU: 130ms usr + 60ms krn ; 0ms fg + Proc com.google.android.gms: + CPU: 130ms usr + 70ms krn ; 0ms fg + Proc com.google.android.gms.persistent: + CPU: 1s 110ms usr + 250ms krn ; 0ms fg + Apk com.google.android.gms: + Wakeup alarm *walarm*:com.google.android.location.ALARM_WAKEUP_LOCATOR: 1 times + Wakeup alarm *walarm*:com.google.android.location.ALARM_WAKEUP_ACTIVITY_DETECTION: 7 times + Wakeup alarm *walarm*:com.google.android.gms.matchstick.register_intent_action: 1 times + Service com.google.android.gms.ads.identifier.service.AdvertisingIdService: + Created for: 0ms uptime + Starts: 0, launches: 1 + Service com.google.android.gms.wearable.service.WearableControlService: + Created for: 18ms uptime + Starts: 1, launches: 1 + Service com.google.android.gms.config.ConfigService: + Created for: 21ms uptime + Starts: 1, launches: 1 + Service com.google.android.gms.chimera.GmsInternalBoundBrokerService: + Created for: 0ms uptime + Starts: 0, launches: 4 + Service com.google.android.gms.icing.service.IndexWorkerService: + Created for: 73ms uptime + Starts: 1, launches: 1 + Service com.google.android.gms.chimera.PersistentIntentOperationService: + Created for: 7s 17ms uptime + Starts: 1, launches: 1 + Service com.google.android.gms.chimera.GmsIntentOperationService: + Created for: 14s 223ms uptime + Starts: 2, launches: 2 + Service com.google.android.gms.chimera.GmsBoundBrokerService: + Created for: 0ms uptime + Starts: 0, launches: 1 + Service com.google.android.gms.measurement.service.MeasurementBrokerService: + Created for: 0ms uptime + Starts: 0, launches: 1 + u0a31: + Fg Service for: 20m 50s 88ms + Total running: 20m 50s 88ms + Total cpu time: u=20ms s=17ms p=0mAh + Proc com.google.android.ext.services: + CPU: 30ms usr + 10ms krn ; 0ms fg + u0a40: + Background for: 207ms + Cached for: 20m 49s 880ms + Total running: 20m 50s 87ms + Total cpu time: u=133ms s=46ms p=0mAh + Proc com.android.launcher3: + CPU: 120ms usr + 40ms krn ; 0ms fg + u0a46: + Sensor 12: 10m 25s 44ms realtime (0 times) + Sensor 13: 10m 25s 44ms realtime (0 times) + Sensor 20: 20m 50s 87ms realtime (0 times) + Fg Service for: 20m 50s 87ms + Total running: 20m 50s 87ms + u0a47: + Foreground for: 20m 50s 87ms + Total running: 20m 50s 87ms + Total cpu time: u=367ms s=197ms p=0mAh + Proc com.motorola.motocare: + CPU: 330ms usr + 160ms krn ; 0ms fg + u0a50: + Background for: 20m 50s 87ms + Total running: 20m 50s 87ms + u0a54: + Cached for: 1m 11s 252ms + Total running: 1m 11s 252ms + Total cpu time: u=100ms s=17ms p=0mAh + Proc com.android.vending: + CPU: 10ms usr + 0ms krn ; 0ms fg + u0a61: + Fg Service for: 20m 50s 87ms + Total running: 20m 50s 87ms + u0a62: + Foreground for: 20m 50s 87ms + Total running: 20m 50s 87ms + u0a64: + Sensor 12: 10m 25s 44ms realtime (0 times) + Fg Service for: 20m 50s 87ms + Total running: 20m 50s 87ms + Total cpu time: u=883ms s=337ms p=0mAh + Proc com.android.systemui: + CPU: 820ms usr + 310ms krn ; 0ms fg + u0a65: + Wake lock *alarm* realtime + Wake lock *job*/com.google.android.googlequicksearchbox/com.google.android.apps.gsa.tasks.BackgroundTasksJobService realtime + Job com.google.android.googlequicksearchbox/com.google.android.apps.gsa.tasks.BackgroundTasksJobService: 99ms realtime (1 times) + Fg Service for: 20m 50s 87ms + Total running: 20m 50s 87ms + Total cpu time: u=176ms s=50ms p=0mAh + Proc com.google.android.googlequicksearchbox:search: + CPU: 20ms usr + 20ms krn ; 0ms fg + Apk com.google.android.googlequicksearchbox: + Service com.google.android.apps.gsa.tasks.BackgroundTasksJobService: + Created for: 0ms uptime + Starts: 0, launches: 1 + Service com.google.android.apps.gsa.shared.util.keepalive.StandaloneKeepAlive$KeepAliveService: + Created for: 10s 35ms uptime + Starts: 1, launches: 1 + Service com.google.android.apps.gsa.search.core.BroadcastListenerService: + Created for: 7ms uptime + Starts: 1, launches: 1 + u0a69: + Background for: 20m 50s 87ms + Total running: 20m 50s 87ms + u0a74: + Cached for: 20m 50s 86ms + Total running: 20m 50s 86ms + Total cpu time: u=3ms s=0ms p=0mAh + Proc com.google.android.calendar: + CPU: 0ms usr + 10ms krn ; 0ms fg + u0a77: + Background for: 5ms + Cached for: 20m 50s 81ms + Total running: 20m 50s 86ms + Total cpu time: u=3ms s=0ms p=0mAh + Proc com.android.chrome: + CPU: 20ms usr + 10ms krn ; 0ms fg + u0a83: + Cached for: 20m 50s 86ms + Total running: 20m 50s 86ms + Total cpu time: u=0ms s=4ms p=0mAh + Proc com.google.android.apps.docs: + CPU: 150ms usr + 30ms krn ; 0ms fg + u0a84: + Wake lock *job*/com.google.android.apps.tachyon/.jobs.NativeJobSchedulerJobService realtime + Wake lock *alarm* realtime + Job com.google.android.apps.tachyon/.jobs.NativeJobSchedulerJobService: 588ms realtime (2 times) + Background for: 575ms + Cached for: 15m 36s 697ms + Total running: 15m 37s 272ms + Total cpu time: u=3s 623ms s=1s 833ms p=0mAh + Proc com.google.android.apps.tachyon: + CPU: 1s 260ms usr + 850ms krn ; 0ms fg + 1 starts + Apk com.google.android.apps.tachyon: + Service com.google.android.apps.tachyon.jobs.NativeJobSchedulerJobService: + Created for: 0ms uptime + Starts: 0, launches: 2 + Apk android: + Wakeup alarm *walarm*:*job.delay*: 42 times + Wakeup alarm *walarm*:*job.deadline*: 1 times + u0a98: + Background for: 20m 50s 86ms + Total running: 20m 50s 86ms + Total cpu time: u=6ms s=4ms p=0mAh + Proc com.google.android.inputmethod.latin: + CPU: 10ms usr + 0ms krn ; 0ms fg + u0a100: + Wake lock *alarm* realtime + Apk android: + Wakeup alarm *walarm*:*job.delay*: 21 times + u0a102: + Background for: 20m 50s 86ms + Total running: 20m 50s 86ms + Total cpu time: u=10ms s=4ms p=0mAh + u0a107: + Background for: 4ms + Cached for: 20m 50s 82ms + Total running: 20m 50s 86ms + Total cpu time: u=6ms s=0ms p=0mAh + u0a109: + Background for: 20m 50s 86ms + Total running: 20m 50s 86ms + Total cpu time: u=53ms s=30ms p=0mAh + Proc com.motorola.timeweatherwidget: + CPU: 50ms usr + 40ms krn ; 0ms fg + Apk com.motorola.timeweatherwidget: + Service com.motorola.commandcenter.row2.RowWidgetUpdater2: + Created for: 1s 362ms uptime + Starts: 6, launches: 6 + u0a118: + Wake lock *launch* realtime + Wake lock Icing realtime + Foreground activities: 20m 36s 840ms realtime (1 times) + Top for: 20m 36s 860ms + Background for: 5ms + Cached for: 80ms + Total running: 20m 36s 945ms + Total cpu time: u=4m 32s 276ms s=1m 7s 593ms p=0mAh + Proc org.mozilla.geckoview_example: + CPU: 3m 45s 780ms usr + 48s 840ms krn ; 0ms fg + 1 starts + Proc org.mozilla.geckoview_example:tab: + CPU: 40ms usr + 10ms krn ; 0ms fg + 1 starts + Apk org.mozilla.geckoview_example: + Service org.mozilla.gecko.process.GeckoServiceChildProcess$tab: + Created for: 0ms uptime + Starts: 0, launches: 1 + u0a119: + Wake lock TestdroidDeviceService: 20m 50s 86ms full (0 times) realtime + Background for: 20m 50s 86ms + Total running: 20m 50s 86ms + Total cpu time: u=530ms s=287ms p=0mAh + Proc com.bitbar.testdroid.monitor:monitor: + CPU: 300ms usr + 200ms krn ; 0ms fg + Proc com.bitbar.testdroid.monitor:data: + CPU: 270ms usr + 100ms krn ; 0ms fg
\ No newline at end of file diff --git a/testing/raptor/test/files/batterystats-android-8.txt b/testing/raptor/test/files/batterystats-android-8.txt new file mode 100644 index 0000000000..f4aa694bab --- /dev/null +++ b/testing/raptor/test/files/batterystats-android-8.txt @@ -0,0 +1,1669 @@ +Battery History (1% used, 3908 used of 256KB, 70 strings using 4868): + 0 (10) RESET:TIME: 2019-07-03-23-26-43 + 0 (1) 100 status=discharging health=good plug=none temp=310 volt=4298 charge=-3764 +running +wake_lock +sensor +wifi_radio +phone_scanning +screen phone_state=out +wifi_running +wifi wifi_signal_strength=4 wifi_suppl=completed + 0 (2) 100 proc=u0a9:"com.google.android.apps.turbo" + 0 (2) 100 proc=u0a130:"com.bitbar.testdroid.monitor:monitor" + 0 (2) 100 proc=1000:"WebViewLoader-arm64-v8a" + 0 (2) 100 proc=1000:"com.google.SSRestartDetector" + 0 (2) 100 proc=u0a43:"com.android.defcontainer" + 0 (2) 100 proc=u0a17:"com.google.process.gapps" + 0 (2) 100 proc=u0a104:"com.breel.wallpapers" + 0 (2) 100 proc=1001:"com.android.phone" + 0 (2) 100 proc=u0a28:"com.android.externalstorage" + 0 (2) 100 proc=u0a8:"com.android.vending" + 0 (2) 100 proc=u0a3:"android.process.media" + 0 (2) 100 proc=u0a73:"com.android.chrome" + 0 (2) 100 proc=u0a98:"com.google.android.apps.photos" + 0 (2) 100 proc=1001:"com.google.modemservice" + 0 (2) 100 proc=u0a11:"com.google.android.googlequicksearchbox:interactor" + 0 (2) 100 proc=u0a3:"com.android.mtp" + 0 (2) 100 proc=u0a5:"com.android.documentsui" + 0 (2) 100 proc=1027:"com.android.nfc" + 0 (2) 100 proc=1000:".dataservices" + 0 (2) 100 proc=u0a97:"com.google.android.apps.docs" + 0 (2) 100 proc=u0a17:"com.google.android.gms" + 0 (2) 100 proc=u0a44:"com.google.android.apps.messaging" + 0 (2) 100 proc=u0a57:"com.google.android.apps.gcs" + 0 (2) 100 proc=u0a51:"com.google.android.contacts" + 0 (2) 100 proc=u0a38:"com.android.systemui" + 0 (2) 100 proc=u0a17:"com.google.android.gms.persistent" + 0 (2) 100 proc=u0a130:"com.bitbar.testdroid.monitor:data" + 0 (2) 100 proc=u0a11:"com.google.android.googlequicksearchbox:search" + 0 (2) 100 proc=1001:"com.android.ims.rcsservice" + 0 (2) 100 proc=u0a40:"com.google.intelligence.sense" + 0 (2) 100 proc=u0a130:"com.bitbar.testdroid.monitor" + 0 (2) 100 proc=1001:"com.qualcomm.qcrilmsgtunnel" + 0 (2) 100 proc=u0a73:"com.android.chrome:webview_service" + 0 (2) 100 proc=1002:"com.android.bluetooth" + 0 (2) 100 proc=u0a32:"com.google.android.apps.nexuslauncher" + 0 (2) 100 proc=u0a91:"com.google.android.inputmethod.latin" + 0 (2) 100 proc=1000:"WebViewLoader-armeabi-v7a" + 0 (2) 100 proc=u0a55:"com.verizon.mips.services" + 0 (2) 100 proc=1001:"com.qualcomm.qti.telephonyservice" + 0 (2) 100 top=u0a130:"com.bitbar.testdroid.monitor" + 0 (2) 100 user=0:"0" + 0 (2) 100 userfg=0:"0" + +8ms (2) 100 +wake_lock_in=1002:"bluetooth_timer" + +8ms (2) 100 -wake_lock_in=1002:"bluetooth_timer" + +10ms (2) 100 +wake_lock_in=1001:"RILJ" + +11ms (2) 100 -wake_lock_in=1001:"RILJ" + +11ms (2) 100 stats=0:"dump" + +86ms (2) 100 +proc=u0a9:"com.google.android.apps.turbo" + +86ms (2) 100 +proc=u0a130:"com.bitbar.testdroid.monitor:monitor" + +86ms (2) 100 +proc=1000:"WebViewLoader-arm64-v8a" + +86ms (2) 100 +proc=1000:"com.google.SSRestartDetector" + +86ms (2) 100 +proc=u0a43:"com.android.defcontainer" + +86ms (2) 100 +proc=u0a17:"com.google.process.gapps" + +86ms (2) 100 +proc=u0a104:"com.breel.wallpapers" + +86ms (2) 100 +proc=1001:"com.android.phone" + +86ms (2) 100 +proc=u0a28:"com.android.externalstorage" + +86ms (2) 100 +proc=u0a8:"com.android.vending" + +86ms (2) 100 +proc=u0a3:"android.process.media" + +86ms (2) 100 +proc=u0a73:"com.android.chrome" + +86ms (2) 100 +proc=u0a98:"com.google.android.apps.photos" + +86ms (2) 100 +proc=1001:"com.google.modemservice" + +86ms (2) 100 +proc=u0a11:"com.google.android.googlequicksearchbox:interactor" + +86ms (2) 100 +proc=u0a3:"com.android.mtp" + +86ms (2) 100 +proc=u0a5:"com.android.documentsui" + +86ms (2) 100 +proc=1027:"com.android.nfc" + +86ms (2) 100 +proc=1000:".dataservices" + +86ms (2) 100 +proc=u0a97:"com.google.android.apps.docs" + +86ms (2) 100 +proc=u0a17:"com.google.android.gms" + +86ms (2) 100 +proc=u0a44:"com.google.android.apps.messaging" + +86ms (2) 100 +proc=u0a57:"com.google.android.apps.gcs" + +86ms (2) 100 +proc=u0a51:"com.google.android.contacts" + +86ms (2) 100 +proc=u0a38:"com.android.systemui" + +86ms (2) 100 +proc=u0a17:"com.google.android.gms.persistent" + +86ms (2) 100 +proc=u0a130:"com.bitbar.testdroid.monitor:data" + +86ms (2) 100 +proc=u0a11:"com.google.android.googlequicksearchbox:search" + +86ms (2) 100 +proc=1001:"com.android.ims.rcsservice" + +86ms (2) 100 +proc=u0a40:"com.google.intelligence.sense" + +86ms (2) 100 +proc=u0a130:"com.bitbar.testdroid.monitor" + +86ms (2) 100 +proc=1001:"com.qualcomm.qcrilmsgtunnel" + +86ms (2) 100 +proc=u0a73:"com.android.chrome:webview_service" + +86ms (2) 100 +proc=1002:"com.android.bluetooth" + +86ms (2) 100 +proc=u0a32:"com.google.android.apps.nexuslauncher" + +86ms (2) 100 +proc=u0a91:"com.google.android.inputmethod.latin" + +86ms (2) 100 +proc=1000:"WebViewLoader-armeabi-v7a" + +86ms (2) 100 +proc=u0a55:"com.verizon.mips.services" + +86ms (2) 100 +proc=1001:"com.qualcomm.qti.telephonyservice" + +550ms (2) 100 +wake_lock_in=u0a17:"Wakeful StateMachine: GeofencerStateMachine" + +551ms (2) 100 -wake_lock_in=u0a17:"Wakeful StateMachine: GeofencerStateMachine" + +573ms (2) 100 +wake_lock_in=u0a17:"Icing" + +577ms (2) 100 +wake_lock_in=u0a120:"Icing" + +577ms (2) 100 -wake_lock_in=u0a17:"Icing" + +578ms (2) 100 -wake_lock_in=u0a120:"Icing" + +599ms (2) 100 +wake_lock_in=u0a17:"Icing" + +599ms (2) 100 +wake_lock_in=u0a120:"Icing" + +599ms (2) 100 -wake_lock_in=u0a120:"Icing" + +615ms (2) 100 -wake_lock_in=u0a17:"Icing" + +1s531ms (2) 100 +wake_lock_in=1000:"GnssLocationProvider" + +1s532ms (2) 100 -wake_lock_in=1000:"GnssLocationProvider" + +1s536ms (2) 100 +wake_lock_in=u0a17:"NlpWakeLock" + +1s537ms (2) 100 -wake_lock_in=u0a17:"NlpWakeLock" + +1s537ms (2) 100 +wake_lock_in=u0a17:"NlpWakeLock" + +1s538ms (2) 100 -wake_lock_in=u0a17:"NlpWakeLock" + +1s538ms (2) 100 +wake_lock_in=u0a17:"NlpWakeLock" + +1s538ms (2) 100 -wake_lock_in=u0a17:"NlpWakeLock" + +1s539ms (2) 100 +wake_lock_in=u0a17:"GCoreFlp" + +1s539ms (2) 100 -wake_lock_in=u0a17:"GCoreFlp" + +1s539ms (2) 100 +wake_lock_in=u0a17:"GCoreFlp" + +1s540ms (2) 100 -wake_lock_in=u0a17:"GCoreFlp" + +1s821ms (2) 100 +wake_lock_in=1000:"GnssLocationProvider" + +1s822ms (2) 100 -wake_lock_in=1000:"GnssLocationProvider" + +1s824ms (2) 100 +wake_lock_in=u0a17:"NlpWakeLock" + +1s825ms (2) 100 -wake_lock_in=u0a17:"NlpWakeLock" + +1s825ms (2) 100 +wake_lock_in=u0a17:"NlpWakeLock" + +1s825ms (2) 100 -wake_lock_in=u0a17:"NlpWakeLock" + +1s825ms (2) 100 +wake_lock_in=u0a17:"NlpWakeLock" + +1s826ms (2) 100 -wake_lock_in=u0a17:"NlpWakeLock" + +1s827ms (2) 100 +wake_lock_in=u0a17:"GCoreFlp" + +1s827ms (2) 100 -wake_lock_in=u0a17:"GCoreFlp" + +1s827ms (2) 100 +wake_lock_in=u0a17:"GCoreFlp" + +1s828ms (2) 100 -wake_lock_in=u0a17:"GCoreFlp" + +2s136ms (2) 100 +wake_lock_in=1000:"GnssLocationProvider" + +2s137ms (2) 100 -wake_lock_in=1000:"GnssLocationProvider" + +2s139ms (2) 100 +wake_lock_in=u0a17:"NlpWakeLock" + +2s140ms (2) 100 -wake_lock_in=u0a17:"NlpWakeLock" + +2s140ms (2) 100 +wake_lock_in=u0a17:"NlpWakeLock" + +2s141ms (2) 100 -wake_lock_in=u0a17:"NlpWakeLock" + +2s142ms (2) 100 +wake_lock_in=u0a17:"NlpWakeLock" + +2s142ms (2) 100 -wake_lock_in=u0a17:"NlpWakeLock" + +2s143ms (2) 100 +wake_lock_in=u0a17:"GCoreFlp" + +2s144ms (2) 100 -wake_lock_in=u0a17:"GCoreFlp" + +2s144ms (2) 100 +wake_lock_in=u0a17:"GCoreFlp" + +2s144ms (2) 100 -wake_lock_in=u0a17:"GCoreFlp" + +2s144ms (2) 100 +wake_lock_in=u0a17:"GCoreFlp" + +2s145ms (2) 100 -wake_lock_in=u0a17:"GCoreFlp" + +2s145ms (2) 100 +wake_lock_in=u0a17:"GCoreFlp" + +2s145ms (2) 100 -wake_lock_in=u0a17:"GCoreFlp" + +2s428ms (2) 100 +wake_lock_in=1000:"GnssLocationProvider" + +2s428ms (2) 100 -wake_lock_in=1000:"GnssLocationProvider" + +2s430ms (2) 100 +wake_lock_in=u0a17:"NlpWakeLock" + +2s431ms (2) 100 -wake_lock_in=u0a17:"NlpWakeLock" + +2s431ms (2) 100 +wake_lock_in=u0a17:"NlpWakeLock" + +2s432ms (2) 100 -wake_lock_in=u0a17:"NlpWakeLock" + +2s432ms (2) 100 +wake_lock_in=u0a17:"NlpWakeLock" + +2s434ms (2) 100 -wake_lock_in=u0a17:"NlpWakeLock" + +2s434ms (2) 100 +wake_lock_in=u0a17:"GCoreFlp" + +2s435ms (2) 100 -wake_lock_in=u0a17:"GCoreFlp" + +2s435ms (2) 100 +wake_lock_in=u0a17:"GCoreFlp" + +2s435ms (2) 100 -wake_lock_in=u0a17:"GCoreFlp" + +2s846ms (2) 100 +wake_lock_in=1000:"GnssLocationProvider" + +2s846ms (2) 100 -wake_lock_in=1000:"GnssLocationProvider" + +2s848ms (2) 100 +wake_lock_in=u0a17:"NlpWakeLock" + +2s849ms (2) 100 -wake_lock_in=u0a17:"NlpWakeLock" + +2s850ms (2) 100 +wake_lock_in=u0a17:"NlpWakeLock" + +2s851ms (2) 100 -wake_lock_in=u0a17:"NlpWakeLock" + +2s851ms (2) 100 +wake_lock_in=u0a17:"NlpWakeLock" + +2s852ms (2) 100 -wake_lock_in=u0a17:"NlpWakeLock" + +2s853ms (2) 100 +wake_lock_in=u0a17:"GCoreFlp" + +2s853ms (2) 100 -wake_lock_in=u0a17:"GCoreFlp" + +2s853ms (2) 100 +wake_lock_in=u0a17:"GCoreFlp" + +2s854ms (2) 100 -wake_lock_in=u0a17:"GCoreFlp" + +2s854ms (2) 100 +wake_lock_in=u0a17:"GCoreFlp" + +2s854ms (2) 100 -wake_lock_in=u0a17:"GCoreFlp" + +2s855ms (2) 100 +wake_lock_in=u0a17:"GCoreFlp" + +2s855ms (2) 100 -wake_lock_in=u0a17:"GCoreFlp" + +3s130ms (2) 100 +wake_lock_in=1000:"GnssLocationProvider" + +3s131ms (2) 100 -wake_lock_in=1000:"GnssLocationProvider" + +3s133ms (2) 100 +wake_lock_in=u0a17:"NlpWakeLock" + +3s134ms (2) 100 -wake_lock_in=u0a17:"NlpWakeLock" + +3s134ms (2) 100 +wake_lock_in=u0a17:"NlpWakeLock" + +3s135ms (2) 100 -wake_lock_in=u0a17:"NlpWakeLock" + +3s135ms (2) 100 +wake_lock_in=u0a17:"NlpWakeLock" + +3s136ms (2) 100 -wake_lock_in=u0a17:"NlpWakeLock" + +3s136ms (2) 100 +wake_lock_in=u0a17:"GCoreFlp" + +3s137ms (2) 100 -wake_lock_in=u0a17:"GCoreFlp" + +3s137ms (2) 100 +wake_lock_in=u0a17:"GCoreFlp" + +3s138ms (2) 100 -wake_lock_in=u0a17:"GCoreFlp" + +3s138ms (2) 100 +wake_lock_in=u0a17:"GCoreFlp" + +3s138ms (2) 100 -wake_lock_in=u0a17:"GCoreFlp" + +3s139ms (2) 100 +wake_lock_in=u0a17:"GCoreFlp" + +3s139ms (2) 100 -wake_lock_in=u0a17:"GCoreFlp" + +3s281ms (2) 100 +wake_lock_in=u0a120:"*launch*" + +3s294ms (2) 100 +proc=u0a120:"org.mozilla.geckoview_example" + +3s296ms (2) 100 -top=u0a130:"com.bitbar.testdroid.monitor" + +3s296ms (2) 100 +top=u0a120:"org.mozilla.geckoview_example" + +3s521ms (2) 100 +proc=u0a120:"org.mozilla.geckoview_example:tab" + +3s671ms (2) 100 -wake_lock_in=u0a120:"*launch*" + +16s988ms (2) 100 +alarm=1000:"*alarm*:android.intent.action.TIME_TICK" + +16s988ms (2) 100 +wake_lock_in=1000:"*alarm*:android.intent.action.TIME_TICK" + +16s993ms (2) 100 -alarm=1000:"*alarm*:android.intent.action.TIME_TICK" + +16s993ms (2) 100 -wake_lock_in=1000:"*alarm*:android.intent.action.TIME_TICK" + +51s283ms (2) 100 temp=320 volt=4253 charge=-3769 + +1m00s378ms (2) 100 +wake_lock_in=1000:"NetworkStats" + +1m00s394ms (2) 100 -wake_lock_in=1000:"NetworkStats" + +1m04s070ms (2) 100 -proc=u0a130:"com.bitbar.testdroid.monitor:monitor" + +1m16s988ms (2) 100 +alarm=1000:"*alarm*:android.intent.action.TIME_TICK" + +1m16s988ms (2) 100 +wake_lock_in=1000:"*alarm*:android.intent.action.TIME_TICK" + +1m16s992ms (2) 100 -alarm=1000:"*alarm*:android.intent.action.TIME_TICK" + +1m16s992ms (2) 100 -wake_lock_in=1000:"*alarm*:android.intent.action.TIME_TICK" + +1m29s680ms (2) 100 +wake_lock_in=1000:"NetworkStats" + +1m29s703ms (2) 100 -wake_lock_in=1000:"NetworkStats" + +1m54s059ms (2) 100 +alarm=1000:"*walarm*:WifiConnectivityManager Schedule Periodic Scan Timer" + +1m54s059ms (2) 100 +wake_lock_in=1000:"*walarm*:WifiConnectivityManager Schedule Periodic Scan Timer" + +1m54s059ms (2) 100 -alarm=1000:"*walarm*:WifiConnectivityManager Schedule Periodic Scan Timer" + +1m54s059ms (2) 100 +wifi_scan -wake_lock_in=1000:"*walarm*:WifiConnectivityManager Schedule Periodic Scan Timer" + +1m54s422ms (1) 100 -wifi_scan + +1m56s873ms (2) 100 +wake_lock_in=1000:"NetworkStats" + +1m56s895ms (2) 100 -wake_lock_in=1000:"NetworkStats" + +2m16s988ms (2) 100 +alarm=1000:"*alarm*:android.intent.action.TIME_TICK" + +2m16s988ms (2) 100 +wake_lock_in=1000:"*alarm*:android.intent.action.TIME_TICK" + +2m16s995ms (2) 100 -alarm=1000:"*alarm*:android.intent.action.TIME_TICK" + +2m16s995ms (2) 100 -wake_lock_in=1000:"*alarm*:android.intent.action.TIME_TICK" + +2m17s001ms (2) 100 -proc=u0a51:"com.google.android.contacts" + +2m26s300ms (2) 100 +wake_lock_in=1000:"NetworkStats" + +2m26s323ms (2) 100 -wake_lock_in=1000:"NetworkStats" + +2m58s481ms (2) 100 +wake_lock_in=1000:"NetworkStats" + +2m58s502ms (2) 100 -wake_lock_in=1000:"NetworkStats" + +3m16s988ms (2) 100 +alarm=1000:"*alarm*:android.intent.action.TIME_TICK" + +3m16s988ms (2) 100 +wake_lock_in=1000:"*alarm*:android.intent.action.TIME_TICK" + +3m16s988ms (2) 100 +alarm=1000:"*walarm*:ScheduleConditionProvider.EVALUATE" + +3m16s992ms (2) 100 -alarm=1000:"*alarm*:android.intent.action.TIME_TICK" + +3m16s994ms (2) 100 -alarm=1000:"*walarm*:ScheduleConditionProvider.EVALUATE" + +3m16s994ms (2) 100 -wake_lock_in=1000:"*alarm*:android.intent.action.TIME_TICK" + +3m29s593ms (2) 100 +wake_lock_in=1000:"NetworkStats" + +3m29s614ms (2) 100 -wake_lock_in=1000:"NetworkStats" + +3m59s622ms (2) 100 +wake_lock_in=1000:"NetworkStats" + +3m59s644ms (2) 100 -wake_lock_in=1000:"NetworkStats" + +4m09s040ms (2) 100 +alarm=1000:"*walarm*:DhcpClient.wlan0.RENEW" + +4m09s040ms (2) 100 +wake_lock_in=1000:"*walarm*:DhcpClient.wlan0.RENEW" + +4m09s040ms (2) 100 -alarm=1000:"*walarm*:DhcpClient.wlan0.RENEW" + +4m09s040ms (2) 100 -wake_lock_in=1000:"*walarm*:DhcpClient.wlan0.RENEW" + +4m16s988ms (2) 100 +alarm=1000:"*alarm*:android.intent.action.TIME_TICK" + +4m16s988ms (2) 100 +wake_lock_in=1000:"*alarm*:android.intent.action.TIME_TICK" + +4m16s992ms (2) 100 -alarm=1000:"*alarm*:android.intent.action.TIME_TICK" + +4m16s992ms (2) 100 -wake_lock_in=1000:"*alarm*:android.intent.action.TIME_TICK" + +4m29s890ms (2) 100 +wake_lock_in=1000:"NetworkStats" + +4m29s912ms (2) 100 -wake_lock_in=1000:"NetworkStats" + +5m00s949ms (2) 100 +wake_lock_in=1000:"NetworkStats" + +5m00s972ms (2) 100 -wake_lock_in=1000:"NetworkStats" + +5m16s988ms (2) 100 +alarm=1000:"*alarm*:android.intent.action.TIME_TICK" + +5m16s988ms (2) 100 +wake_lock_in=1000:"*alarm*:android.intent.action.TIME_TICK" + +5m16s992ms (2) 100 -alarm=1000:"*alarm*:android.intent.action.TIME_TICK" + +5m16s992ms (2) 100 -wake_lock_in=1000:"*alarm*:android.intent.action.TIME_TICK" + +5m30s885ms (2) 100 +wake_lock_in=1000:"NetworkStats" + +5m30s906ms (2) 100 -wake_lock_in=1000:"NetworkStats" + +5m44s809ms (2) 100 +alarm=1000:"*walarm*:WifiConnectivityManager Schedule Periodic Scan Timer" + +5m44s809ms (2) 100 +wake_lock_in=1000:"*walarm*:WifiConnectivityManager Schedule Periodic Scan Timer" + +5m44s809ms (2) 100 +alarm=u0a11:"*walarm*:*job.delay*" + +5m44s809ms (2) 100 +alarm=u0a17:"*walarm*:com.google.android.gms.gcm.ACTION_CHECK_QUEUE" + +5m44s810ms (2) 100 -alarm=u0a11:"*walarm*:*job.delay*" + +5m44s810ms (2) 100 -wake_lock_in=1000:"*walarm*:WifiConnectivityManager Schedule Periodic Scan Timer" + +5m44s810ms (2) 100 +wake_lock_in=1000:"*alarm*" + +5m44s810ms (2) 100 -alarm=1000:"*walarm*:WifiConnectivityManager Schedule Periodic Scan Timer" + +5m44s810ms (2) 100 -wake_lock_in=1000:"*alarm*" + +5m44s811ms (2) 100 +wake_lock_in=u0a17:"*alarm*" + +5m44s811ms (2) 100 +wake_lock_in=u0a17:"*net_scheduler*" + +5m44s815ms (2) 100 +job=u0a11:"com.google.android.googlequicksearchbox/com.google.android.apps.gsa.tasks.BackgroundTasksJobService" + +5m44s815ms (2) 100 -alarm=u0a17:"*walarm*:com.google.android.gms.gcm.ACTION_CHECK_QUEUE" + +5m44s815ms (2) 100 +wifi_scan -wake_lock_in=u0a17:"*alarm*" + +5m44s815ms (2) 100 +wake_lock_in=u0a11:"*job*/com.google.android.googlequicksearchbox/com.google.android.apps.gsa.tasks.BackgroundTasksJobService" + +5m44s821ms (2) 100 +wake_lock_in=u0a57:"*net_scheduler*" + +5m44s829ms (2) 100 -wake_lock_in=u0a17:"*net_scheduler*" + +5m44s838ms (2) 100 -job=u0a11:"com.google.android.googlequicksearchbox/com.google.android.apps.gsa.tasks.BackgroundTasksJobService" + +5m44s838ms (2) 100 -wake_lock_in=u0a11:"*job*/com.google.android.googlequicksearchbox/com.google.android.apps.gsa.tasks.BackgroundTasksJobService" + +5m44s845ms (2) 100 -wake_lock_in=u0a57:"*net_scheduler*" + +5m44s846ms (2) 100 +wake_lock_in=u0a17:"*net_scheduler*" + +5m44s848ms (2) 100 -wake_lock_in=u0a17:"*net_scheduler*" + +5m44s849ms (2) 100 +wake_lock_in=u0a17:"*net_scheduler*" + +5m44s858ms (2) 100 +wake_lock_in=u0a17:"*gms_scheduler*/com.google.android.gms/.icing.service.IcingGcmTaskService" + +5m44s869ms (2) 100 +wake_lock_in=u0a17:"*gms_scheduler*/com.google.android.gms/.stats.PlatformStatsCollectorService" + +5m44s874ms (2) 100 +wake_lock_in=u0a17:"Icing" + +5m44s878ms (2) 100 -wake_lock_in=u0a17:"*net_scheduler*" + +5m44s878ms (2) 100 -wake_lock_in=u0a17:"*gms_scheduler*/com.google.android.gms/.icing.service.IcingGcmTaskService" + +5m44s878ms (2) 100 +wake_lock_in=u0a17:"*net_scheduler*" + +5m44s880ms (2) 100 -wake_lock_in=u0a17:"*net_scheduler*" + +5m44s905ms (2) 100 -wake_lock_in=u0a17:"Icing" + +5m44s935ms (2) 100 -wake_lock_in=u0a17:"*gms_scheduler*/com.google.android.gms/.stats.PlatformStatsCollectorService" + +5m44s935ms (2) 100 +wake_lock_in=u0a17:"*net_scheduler*" + +5m44s953ms (2) 100 -wifi_scan -wake_lock_in=u0a17:"*net_scheduler*" + +6m00s025ms (2) 100 +wake_lock_in=1000:"NetworkStats" + +6m00s041ms (2) 100 -wake_lock_in=1000:"NetworkStats" + +6m16s988ms (2) 100 +alarm=1000:"*alarm*:android.intent.action.TIME_TICK" + +6m16s988ms (2) 100 +wake_lock_in=1000:"*alarm*:android.intent.action.TIME_TICK" + +6m16s992ms (2) 100 -alarm=1000:"*alarm*:android.intent.action.TIME_TICK" + +6m16s992ms (2) 100 -wake_lock_in=1000:"*alarm*:android.intent.action.TIME_TICK" + +6m30s246ms (2) 100 +wake_lock_in=1000:"NetworkStats" + +6m30s268ms (2) 100 -wake_lock_in=1000:"NetworkStats" + +6m59s525ms (2) 100 +wake_lock_in=1000:"NetworkStats" + +6m59s546ms (2) 100 -wake_lock_in=1000:"NetworkStats" + +7m16s988ms (2) 100 +alarm=1000:"*alarm*:android.intent.action.TIME_TICK" + +7m16s988ms (2) 100 +wake_lock_in=1000:"*alarm*:android.intent.action.TIME_TICK" + +7m16s992ms (2) 100 -alarm=1000:"*alarm*:android.intent.action.TIME_TICK" + +7m16s992ms (2) 100 -wake_lock_in=1000:"*alarm*:android.intent.action.TIME_TICK" + +7m29s644ms (2) 100 +wake_lock_in=1000:"NetworkStats" + +7m29s666ms (2) 100 -wake_lock_in=1000:"NetworkStats" + +7m32s266ms (2) 100 temp=340 volt=4228 charge=-3810 + +8m00s046ms (2) 100 +wake_lock_in=1000:"NetworkStats" + +8m00s068ms (2) 100 -wake_lock_in=1000:"NetworkStats" + +8m16s988ms (2) 100 +alarm=1000:"*alarm*:android.intent.action.TIME_TICK" + +8m16s988ms (2) 100 +wake_lock_in=1000:"*alarm*:android.intent.action.TIME_TICK" + +8m16s992ms (2) 100 -alarm=1000:"*alarm*:android.intent.action.TIME_TICK" + +8m16s992ms (2) 100 -wake_lock_in=1000:"*alarm*:android.intent.action.TIME_TICK" + +8m30s401ms (2) 100 +wake_lock_in=1000:"NetworkStats" + +8m30s424ms (2) 100 -wake_lock_in=1000:"NetworkStats" + +8m59s175ms (2) 100 +wake_lock_in=1000:"NetworkStats" + +8m59s197ms (2) 100 -wake_lock_in=1000:"NetworkStats" + +9m09s056ms (2) 100 +alarm=1000:"*walarm*:DhcpClient.wlan0.RENEW" + +9m09s056ms (2) 100 +wake_lock_in=1000:"*walarm*:DhcpClient.wlan0.RENEW" + +9m09s056ms (2) 100 -alarm=1000:"*walarm*:DhcpClient.wlan0.RENEW" + +9m09s056ms (2) 100 -wake_lock_in=1000:"*walarm*:DhcpClient.wlan0.RENEW" + +9m16s988ms (2) 100 +alarm=1000:"*alarm*:android.intent.action.TIME_TICK" + +9m16s988ms (2) 100 +wake_lock_in=1000:"*alarm*:android.intent.action.TIME_TICK" + +9m16s992ms (2) 100 -alarm=1000:"*alarm*:android.intent.action.TIME_TICK" + +9m16s992ms (2) 100 -wake_lock_in=1000:"*alarm*:android.intent.action.TIME_TICK" + +9m18s804ms (2) 100 +alarm=1000:"*walarm*:WifiConnectivityManager Schedule Periodic Scan Timer" + +9m18s804ms (2) 100 +wake_lock_in=1000:"*walarm*:WifiConnectivityManager Schedule Periodic Scan Timer" + +9m18s804ms (2) 100 +alarm=u0a11:"*walarm*:*job.delay*" + +9m18s804ms (2) 100 -alarm=u0a11:"*walarm*:*job.delay*" + +9m18s804ms (2) 100 -wake_lock_in=1000:"*walarm*:WifiConnectivityManager Schedule Periodic Scan Timer" + +9m18s804ms (2) 100 +wake_lock_in=1000:"*alarm*" + +9m18s805ms (2) 100 -alarm=1000:"*walarm*:WifiConnectivityManager Schedule Periodic Scan Timer" + +9m18s805ms (2) 100 -wake_lock_in=1000:"*alarm*" + +9m18s808ms (2) 100 +wifi_scan +job=u0a11:"com.google.android.googlequicksearchbox/com.google.android.apps.gsa.tasks.BackgroundTasksJobService" + +9m18s809ms (2) 100 +wake_lock_in=u0a11:"*job*/com.google.android.googlequicksearchbox/com.google.android.apps.gsa.tasks.BackgroundTasksJobService" + +9m18s811ms (2) 100 -job=u0a11:"com.google.android.googlequicksearchbox/com.google.android.apps.gsa.tasks.BackgroundTasksJobService" + +9m18s811ms (2) 100 -wake_lock_in=u0a11:"*job*/com.google.android.googlequicksearchbox/com.google.android.apps.gsa.tasks.BackgroundTasksJobService" + +9m18s821ms (2) 100 +alarm=u0a11:"*walarm*:*job.delay*" + +9m18s821ms (2) 100 +wake_lock_in=u0a11:"*walarm*:*job.delay*" + +9m18s821ms (2) 100 -alarm=u0a11:"*walarm*:*job.delay*" + +9m18s821ms (2) 100 -wake_lock_in=u0a11:"*walarm*:*job.delay*" + +9m18s824ms (2) 100 +job=u0a11:"com.google.android.googlequicksearchbox/com.google.android.apps.gsa.tasks.BackgroundTasksJobService" + +9m18s825ms (2) 100 +wake_lock_in=u0a11:"*job*/com.google.android.googlequicksearchbox/com.google.android.apps.gsa.tasks.BackgroundTasksJobService" + +9m18s828ms (2) 100 -job=u0a11:"com.google.android.googlequicksearchbox/com.google.android.apps.gsa.tasks.BackgroundTasksJobService" + +9m18s828ms (2) 100 -wifi_scan -wake_lock_in=u0a11:"*job*/com.google.android.googlequicksearchbox/com.google.android.apps.gsa.tasks.BackgroundTasksJobService" + +9m30s121ms (2) 100 +wake_lock_in=1000:"NetworkStats" + +9m30s135ms (2) 100 -wake_lock_in=1000:"NetworkStats" + +10m01s059ms (2) 100 +wake_lock_in=1000:"NetworkStats" + +10m01s082ms (2) 100 -wake_lock_in=1000:"NetworkStats" + +10m16s988ms (2) 100 +alarm=1000:"*alarm*:android.intent.action.TIME_TICK" + +10m16s988ms (2) 100 +wake_lock_in=1000:"*alarm*:android.intent.action.TIME_TICK" + +10m16s992ms (2) 100 -alarm=1000:"*alarm*:android.intent.action.TIME_TICK" + +10m16s992ms (2) 100 -wake_lock_in=1000:"*alarm*:android.intent.action.TIME_TICK" + +10m33s762ms (2) 100 +wake_lock_in=1000:"NetworkStats" + +10m33s783ms (2) 100 -wake_lock_in=1000:"NetworkStats" + +11m05s912ms (2) 100 +wake_lock_in=1000:"NetworkStats" + +11m05s932ms (2) 100 -wake_lock_in=1000:"NetworkStats" + +11m07s265ms (2) 100 temp=350 charge=-3831 + +11m16s988ms (2) 100 +alarm=1000:"*alarm*:android.intent.action.TIME_TICK" + +11m16s988ms (2) 100 +wake_lock_in=1000:"*alarm*:android.intent.action.TIME_TICK" + +11m16s992ms (2) 100 -alarm=1000:"*alarm*:android.intent.action.TIME_TICK" + +11m16s993ms (2) 100 -wake_lock_in=1000:"*alarm*:android.intent.action.TIME_TICK" + +11m38s129ms (2) 100 +wake_lock_in=1000:"NetworkStats" + +11m38s153ms (2) 100 -wake_lock_in=1000:"NetworkStats" + +11m58s804ms (2) 100 +alarm=1000:"*walarm*:WifiConnectivityManager Schedule Periodic Scan Timer" + +11m58s804ms (2) 100 +wake_lock_in=1000:"*walarm*:WifiConnectivityManager Schedule Periodic Scan Timer" + +11m58s804ms (2) 100 -alarm=1000:"*walarm*:WifiConnectivityManager Schedule Periodic Scan Timer" + +11m58s804ms (2) 100 +wifi_scan -wake_lock_in=1000:"*walarm*:WifiConnectivityManager Schedule Periodic Scan Timer" + +11m59s028ms (1) 100 -wifi_scan + +12m09s853ms (2) 100 +wake_lock_in=1000:"NetworkStats" + +12m09s867ms (2) 100 -wake_lock_in=1000:"NetworkStats" + +12m16s988ms (2) 100 +alarm=1000:"*alarm*:android.intent.action.TIME_TICK" + +12m16s988ms (2) 100 +wake_lock_in=1000:"*alarm*:android.intent.action.TIME_TICK" + +12m16s992ms (2) 100 -alarm=1000:"*alarm*:android.intent.action.TIME_TICK" + +12m16s992ms (2) 100 -wake_lock_in=1000:"*alarm*:android.intent.action.TIME_TICK" + +12m41s590ms (2) 100 +wake_lock_in=1000:"NetworkStats" + +12m41s612ms (2) 100 -wake_lock_in=1000:"NetworkStats" + +13m14s287ms (2) 100 +wake_lock_in=1000:"NetworkStats" + +13m14s301ms (2) 100 -wake_lock_in=1000:"NetworkStats" + +13m16s988ms (2) 100 +alarm=1000:"*alarm*:android.intent.action.TIME_TICK" + +13m16s988ms (2) 100 +wake_lock_in=1000:"*alarm*:android.intent.action.TIME_TICK" + +13m16s995ms (2) 100 -alarm=1000:"*alarm*:android.intent.action.TIME_TICK" + +13m16s995ms (2) 100 -wake_lock_in=1000:"*alarm*:android.intent.action.TIME_TICK" + +13m16s999ms (2) 100 -proc=u0a73:"com.android.chrome:webview_service" + +13m46s367ms (2) 100 +wake_lock_in=1000:"NetworkStats" + +13m46s388ms (2) 100 -wake_lock_in=1000:"NetworkStats" + +14m03s019ms (2) 100 +wake_lock_in=1002:"bluetooth_timer" + +14m03s023ms (2) 100 -wake_lock_in=1002:"bluetooth_timer" + +14m03s024ms (2) 100 +wake_lock_in=1002:"bluetooth_timer" + +14m03s025ms (2) 100 -wake_lock_in=1002:"bluetooth_timer" + +14m03s026ms (2) 100 +wake_lock_in=1002:"bluetooth_timer" + +14m03s026ms (2) 100 -wake_lock_in=1002:"bluetooth_timer" + +14m03s027ms (2) 100 +wake_lock_in=1002:"bluetooth_timer" + +14m03s027ms (2) 100 -wake_lock_in=1002:"bluetooth_timer" + +14m03s028ms (2) 100 +wake_lock_in=1002:"bluetooth_timer" + +14m03s028ms (2) 100 -wake_lock_in=1002:"bluetooth_timer" + +14m09s090ms (2) 100 +alarm=1000:"*walarm*:DhcpClient.wlan0.RENEW" + +14m09s090ms (2) 100 +wake_lock_in=1000:"*walarm*:DhcpClient.wlan0.RENEW" + +14m09s090ms (2) 100 -alarm=1000:"*walarm*:DhcpClient.wlan0.RENEW" + +14m09s090ms (2) 100 -wake_lock_in=1000:"*walarm*:DhcpClient.wlan0.RENEW" + +14m16s988ms (2) 100 +alarm=1000:"*alarm*:android.intent.action.TIME_TICK" + +14m16s988ms (2) 100 +wake_lock_in=1000:"*alarm*:android.intent.action.TIME_TICK" + +14m16s991ms (2) 100 -alarm=1000:"*alarm*:android.intent.action.TIME_TICK" + +14m16s991ms (2) 100 -wake_lock_in=1000:"*alarm*:android.intent.action.TIME_TICK" + +14m18s313ms (2) 100 +wake_lock_in=1000:"NetworkStats" + +14m18s335ms (2) 100 -wake_lock_in=1000:"NetworkStats" + +14m50s541ms (2) 100 +wake_lock_in=1000:"NetworkStats" + +14m50s564ms (2) 100 -wake_lock_in=1000:"NetworkStats" + +15m16s988ms (2) 100 +alarm=1000:"*alarm*:android.intent.action.TIME_TICK" + +15m16s988ms (2) 100 +wake_lock_in=1000:"*alarm*:android.intent.action.TIME_TICK" + +15m16s992ms (2) 100 -alarm=1000:"*alarm*:android.intent.action.TIME_TICK" + +15m16s992ms (2) 100 -wake_lock_in=1000:"*alarm*:android.intent.action.TIME_TICK" + +15m22s173ms (2) 100 +wake_lock_in=1000:"NetworkStats" + +15m22s187ms (2) 100 -wake_lock_in=1000:"NetworkStats" + +15m50s769ms (2) 100 +wake_lock_in=1000:"NetworkStats" + +15m50s792ms (2) 100 -wake_lock_in=1000:"NetworkStats" + +16m01s608ms (1) 100 wifi_suppl=group-handshake + +16m01s609ms (2) 100 wifi_suppl=completed +wake_lock_in=1000:"IpReachabilityMonitor.wlan0" + +16m05s113ms (2) 100 -wake_lock_in=1000:"IpReachabilityMonitor.wlan0" + +16m16s988ms (2) 100 +alarm=1000:"*alarm*:android.intent.action.TIME_TICK" + +16m16s988ms (2) 100 +wake_lock_in=1000:"*alarm*:android.intent.action.TIME_TICK" + +16m16s992ms (2) 100 -alarm=1000:"*alarm*:android.intent.action.TIME_TICK" + +16m16s992ms (2) 100 -wake_lock_in=1000:"*alarm*:android.intent.action.TIME_TICK" + +16m19s710ms (2) 100 +wake_lock_in=1000:"NetworkStats" + +16m19s732ms (2) 100 -wake_lock_in=1000:"NetworkStats" + +16m24s795ms (2) 100 +alarm=1000:"*walarm*:WifiConnectivityManager Schedule Periodic Scan Timer" + +16m24s795ms (2) 100 +wake_lock_in=1000:"*walarm*:WifiConnectivityManager Schedule Periodic Scan Timer" + +16m24s795ms (2) 100 +alarm=u0a11:"*walarm*:*job.deadline*" + +16m24s795ms (2) 100 +alarm=u0a11:"*walarm*:*job.delay*" + +16m24s795ms (2) 100 -alarm=u0a11:"*walarm*:*job.deadline*" + +16m24s795ms (2) 100 -wake_lock_in=1000:"*walarm*:WifiConnectivityManager Schedule Periodic Scan Timer" + +16m24s795ms (2) 100 +wake_lock_in=1000:"*alarm*" + +16m24s796ms (2) 100 -alarm=1000:"*walarm*:WifiConnectivityManager Schedule Periodic Scan Timer" + +16m24s796ms (2) 100 -wake_lock_in=1000:"*alarm*" + +16m24s796ms (2) 100 +wake_lock_in=u0a11:"*alarm*" + +16m24s796ms (2) 100 -alarm=u0a11:"*walarm*:*job.delay*" + +16m24s796ms (2) 100 -wake_lock_in=u0a11:"*alarm*" + +16m24s799ms (2) 100 +job=u0a11:"com.google.android.googlequicksearchbox/com.google.android.apps.gsa.tasks.BackgroundTasksJobService" + +16m24s800ms (2) 100 +wifi_scan +wake_lock_in=u0a11:"*job*/com.google.android.googlequicksearchbox/com.google.android.apps.gsa.tasks.BackgroundTasksJobService" + +16m24s814ms (2) 100 -job=u0a11:"com.google.android.googlequicksearchbox/com.google.android.apps.gsa.tasks.BackgroundTasksJobService" + +16m24s814ms (2) 100 -wifi_scan -wake_lock_in=u0a11:"*job*/com.google.android.googlequicksearchbox/com.google.android.apps.gsa.tasks.BackgroundTasksJobService" + +16m46s860ms (2) 100 +wake_lock_in=1000:"NetworkStats" + +16m46s883ms (2) 100 -wake_lock_in=1000:"NetworkStats" + +17m15s487ms (2) 100 +wake_lock_in=1000:"NetworkStats" + +17m15s509ms (2) 100 -wake_lock_in=1000:"NetworkStats" + +17m16s988ms (2) 100 +alarm=1000:"*alarm*:android.intent.action.TIME_TICK" + +17m16s988ms (2) 100 +wake_lock_in=1000:"*alarm*:android.intent.action.TIME_TICK" + +17m16s993ms (2) 100 -alarm=1000:"*alarm*:android.intent.action.TIME_TICK" + +17m16s993ms (2) 100 -wake_lock_in=1000:"*alarm*:android.intent.action.TIME_TICK" + +17m32s266ms (2) 100 volt=4198 charge=-3871 + +17m42s713ms (2) 100 +wake_lock_in=1000:"NetworkStats" + +17m42s735ms (2) 100 -wake_lock_in=1000:"NetworkStats" + +18m10s985ms (2) 100 +wake_lock_in=1000:"NetworkStats" + +18m11s007ms (2) 100 -wake_lock_in=1000:"NetworkStats" + +18m11s015ms (2) 100 temp=360 charge=-3875 + +18m16s988ms (2) 100 +alarm=1000:"*alarm*:android.intent.action.TIME_TICK" + +18m16s988ms (2) 100 +wake_lock_in=1000:"*alarm*:android.intent.action.TIME_TICK" + +18m16s992ms (2) 100 -alarm=1000:"*alarm*:android.intent.action.TIME_TICK" + +18m16s992ms (2) 100 -wake_lock_in=1000:"*alarm*:android.intent.action.TIME_TICK" + +18m41s948ms (2) 100 +wake_lock_in=1000:"NetworkStats" + +18m41s971ms (2) 100 -wake_lock_in=1000:"NetworkStats" + +19m09s103ms (2) 100 +alarm=1000:"*walarm*:DhcpClient.wlan0.RENEW" + +19m09s103ms (2) 100 +wake_lock_in=1000:"*walarm*:DhcpClient.wlan0.RENEW" + +19m09s103ms (2) 100 -alarm=1000:"*walarm*:DhcpClient.wlan0.RENEW" + +19m09s103ms (2) 100 -wake_lock_in=1000:"*walarm*:DhcpClient.wlan0.RENEW" + +19m10s212ms (2) 100 +wake_lock_in=1000:"NetworkStats" + +19m10s236ms (2) 100 -wake_lock_in=1000:"NetworkStats" + +19m16s988ms (2) 100 +alarm=1000:"*alarm*:android.intent.action.TIME_TICK" + +19m16s988ms (2) 100 +wake_lock_in=1000:"*alarm*:android.intent.action.TIME_TICK" + +19m16s992ms (2) 100 -alarm=1000:"*alarm*:android.intent.action.TIME_TICK" + +19m16s992ms (2) 100 -wake_lock_in=1000:"*alarm*:android.intent.action.TIME_TICK" + +19m38s467ms (2) 100 +wake_lock_in=1000:"NetworkStats" + +19m38s489ms (2) 100 -wake_lock_in=1000:"NetworkStats" + +20m07s414ms (2) 100 +wake_lock_in=1000:"NetworkStats" + +20m07s436ms (2) 100 -wake_lock_in=1000:"NetworkStats" + +20m16s988ms (2) 100 +alarm=1000:"*alarm*:android.intent.action.TIME_TICK" + +20m16s988ms (2) 100 +wake_lock_in=1000:"*alarm*:android.intent.action.TIME_TICK" + +20m16s992ms (2) 100 -alarm=1000:"*alarm*:android.intent.action.TIME_TICK" + +20m16s992ms (2) 100 -wake_lock_in=1000:"*alarm*:android.intent.action.TIME_TICK" + +20m36s876ms (2) 100 -proc=u0a120:"org.mozilla.geckoview_example:tab" + +20m36s885ms (2) 100 -proc=u0a120:"org.mozilla.geckoview_example" + +20m36s890ms (2) 100 -top=u0a120:"org.mozilla.geckoview_example" + +20m36s890ms (2) 100 +top=u0a130:"com.bitbar.testdroid.monitor" + +20m37s324ms (2) 100 +wake_lock_in=1002:"bluetooth_timer" + +20m37s325ms (2) 100 +wake_lock_in=1001:"RILJ" + +20m37s326ms (2) 100 -wake_lock_in=1001:"RILJ" + +20m37s328ms (2) 100 -wake_lock_in=1002:"bluetooth_timer" + +20m37s329ms (2) 100 stats=0:"dump" + +20m37s426ms (2) 100 +wake_lock_in=1001:"RILJ" + +20m37s427ms (2) 100 +wake_lock_in=1002:"bluetooth_timer" + +20m37s427ms (2) 100 -wake_lock_in=1002:"bluetooth_timer" + +20m37s427ms (2) 100 -wake_lock_in=1001:"RILJ" + +20m37s428ms (2) 100 stats=0:"dump" + +Per-PID Stats: + PID 1195 wake time: +4s424ms + PID 1636 wake time: +3ms + PID 1358 wake time: +9ms + PID 1195 wake time: +42ms + PID 1195 wake time: +4ms + PID 1697 wake time: +151ms + PID 2580 wake time: +56ms + PID 1697 wake time: +24ms + PID 1195 wake time: +390ms + PID 2580 wake time: +38ms + +Daily stats: + Current start time: 2019-07-03-01-04-33 + Next min deadline: 2019-07-04-01-00-00 + Next max deadline: 2019-07-04-03-00-00 + Current daily discharge step durations: + #0: +17m18s990ms to 98 (screen-on, power-save-off, device-idle-off) + #1: +5m46s407ms to 96 (screen-on, power-save-off, device-idle-off) + #2: +10m34s926ms to 97 (screen-on, power-save-off, device-idle-off) + #3: +3m34s468ms to 98 (screen-on, power-save-off, device-idle-off) + #4: +2m23s419ms to 91 (screen-on, power-save-off, device-idle-off) + #5: +3m22s332ms to 92 (screen-on, power-save-off, device-idle-off) + #6: +4m14s249ms to 93 (screen-on, power-save-off, device-idle-off) + #7: +2m55s259ms to 94 (screen-on, power-save-off, device-idle-off) + #8: +1m2s155ms to 95 (screen-on, power-save-off, device-idle-off) + #9: +3m39s810ms to 95 (screen-on, power-save-off, device-idle-off) + #10: +3m26s392ms to 96 (screen-on, power-save-off, device-idle-off) + #11: +4m8s330ms to 97 (screen-on, power-save-off, device-idle-off) + #12: +2m50s719ms to 98 (screen-on, power-save-off, device-idle-off) + #13: +2m45s46ms to 97 (screen-on, power-save-off, device-idle-off) + #14: +2m42s993ms to 98 (screen-on, power-save-off, device-idle-off) + #15: +3m31s821ms to 95 (screen-on, power-save-off, device-idle-off) + #16: +4m52s748ms to 96 (screen-on, power-save-off, device-idle-off) + #17: +4m14s919ms to 97 (screen-on, power-save-off, device-idle-off) + #18: +4m16s376ms to 98 (screen-on, power-save-off, device-idle-off) + Discharge total time: 7h 41m 31s 300ms (from 19 steps) + Discharge screen on time: 7h 41m 31s 300ms (from 19 steps) + Package changes: + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.fennec_aurora vers=2015638179 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.fenix.performancetest vers=1 + Update org.mozilla.fenix.performancetest vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Daily from 2019-07-02-03-23-11 to 2019-07-03-01-04-33: + Discharge step durations: + #0: +2m45s319ms to 93 (screen-on, power-save-off, device-idle-off) + #1: +2m24s418ms to 94 (screen-on, power-save-off, device-idle-off) + #2: +3m41s447ms to 95 (screen-on, power-save-off, device-idle-off) + #3: +3m54s135ms to 96 (screen-on, power-save-off, device-idle-off) + #4: +3m41s862ms to 97 (screen-on, power-save-off, device-idle-off) + #5: +1m34s316ms to 98 (screen-on, power-save-off, device-idle-off) + #6: +3m57s43ms to 98 (screen-on, power-save-off, device-idle-off) + #7: +3m17s714ms to 97 (screen-on, power-save-off, device-idle-off) + #8: +9m32s931ms to 98 (screen-on, power-save-off, device-idle-off) + #9: +5m12s597ms to 98 (screen-on, power-save-off, device-idle-off) + #10: +2m29s58ms to 96 (screen-on, power-save-off, device-idle-off) + #11: +3m48s85ms to 97 (screen-on, power-save-off, device-idle-off) + #12: +2m34s604ms to 93 (screen-on, power-save-off, device-idle-off) + #13: +3m34s318ms to 94 (screen-on, power-save-off, device-idle-off) + #14: +7m8s85ms to 95 (screen-on, power-save-off, device-idle-off) + #15: +3m32s299ms to 96 (screen-on, power-save-off, device-idle-off) + #16: +3m32s298ms to 96 (screen-on, power-save-off, device-idle-off) + #17: +4m31s208ms to 97 (screen-on, power-save-off, device-idle-off) + #18: +3m13s603ms to 98 (screen-on, power-save-off, device-idle-off) + #19: +2m46s110ms to 98 (screen-on, power-save-off, device-idle-off) + #20: +4m52s196ms to 98 (screen-on, power-save-off, device-idle-off) + Discharge total time: 6h 30m 45s 900ms (from 21 steps) + Discharge screen on time: 6h 30m 45s 900ms (from 21 steps) + Package changes: + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Daily from 2019-07-01-03-35-11 to 2019-07-02-03-23-11: + Discharge step durations: + #0: +6m2s535ms to 97 (screen-on, power-save-off, device-idle-off) + #1: +3m4s836ms to 98 (screen-on, power-save-off, device-idle-off) + #2: +1m13s52ms to 68 (screen-on, power-save-off, device-idle-off) + #3: +2m4s712ms to 69 (screen-on, power-save-off, device-idle-off) + #4: +3m37s26ms to 70 (screen-on, power-save-off, device-idle-off) + #5: +4m18s262ms to 71 (screen-on, power-save-off, device-idle-off) + #6: +1m16s800ms to 72 (screen-on, power-save-off, device-idle-off) + #7: +3m22s458ms to 73 (screen-on, power-save-off, device-idle-off) + #8: +2m47s281ms to 73 (screen-on, power-save-off, device-idle-off) + #9: +4m10s509ms to 74 (screen-on, power-save-off, device-idle-off) + #10: +3m4s765ms to 75 (screen-on, power-save-off, device-idle-off) + #11: +2m44s726ms to 76 (screen-on, power-save-off, device-idle-off) + #12: +1m56s818ms to 77 (screen-on, power-save-off, device-idle-off) + #13: +5m40s82ms to 78 (screen-on, power-save-off, device-idle-off) + #14: +4m8s691ms to 78 (screen-on, power-save-off, device-idle-off) + #15: +6m43s275ms to 79 (screen-on, power-save-off, device-idle-off) + #16: +4m46s223ms to 80 (screen-on, power-save-off, device-idle-off) + #17: +4m28s985ms to 80 (screen-on, power-save-off, device-idle-off) + #18: +6m56s180ms to 81 (screen-on, power-save-off, device-idle-off) + #19: +4m58s871ms to 82 (screen-on, power-save-off, device-idle-off) + #20: +6m16s664ms to 83 (screen-on, power-save-off, device-idle-off) + #21: +7m11s674ms to 84 (screen-on, power-save-off, device-idle-off) + #22: +4m14s758ms to 85 (screen-on, power-save-off, device-idle-off) + #23: +1m27s936ms to 85 (screen-on, power-save-off, device-idle-off) + #24: +3m35s269ms to 86 (screen-on, power-save-off, device-idle-off) + #25: +3m23s134ms to 87 (screen-on, power-save-off, device-idle-off) + #26: +3m23s133ms to 87 (screen-on, power-save-off, device-idle-off) + #27: +5m57s139ms to 90 (screen-on, power-save-off, device-idle-off) + #28: +4m18s563ms to 91 (screen-on, power-save-off, device-idle-off) + #29: +4m18s563ms to 91 (screen-on, power-save-off, device-idle-off) + #30: +2m0s21ms to 93 (screen-on, power-save-off, device-idle-off) + #31: +2m5s829ms to 94 (screen-on, power-save-off, device-idle-off) + #32: +2m5s788ms to 95 (screen-on, power-save-off, device-idle-off) + #33: +2m11s596ms to 96 (screen-on, power-save-off, device-idle-off) + #34: +4m49s265ms to 97 (screen-on, power-save-off, device-idle-off) + #35: +1m59s680ms to 98 (screen-on, power-save-off, device-idle-off) + #36: +7m42s296ms to 98 (screen-on, power-save-off, device-idle-off) + #37: +1m43s441ms to 93 (screen-on, power-save-off, device-idle-off) + #38: +4m4s515ms to 94 (screen-on, power-save-off, device-idle-off) + #39: +3m55s401ms to 95 (screen-on, power-save-off, device-idle-off) + #40: +2m0s111ms to 96 (screen-on, power-save-off, device-idle-off) + #41: +2m46s11ms to 97 (screen-on, power-save-off, device-idle-off) + #42: +4m27s285ms to 96 (screen-on, power-save-off, device-idle-off) + #43: +4m34s315ms to 97 (screen-on, power-save-off, device-idle-off) + #44: +4m11s312ms to 98 (screen-on, power-save-off, device-idle-off) + Discharge total time: 6h 22m 35s 0ms (from 45 steps) + Discharge screen on time: 6h 22m 35s 0ms (from 45 steps) + Charge step durations: + #0: +1m29s593ms to 89 (screen-on, power-save-off, device-idle-off) + #1: +1m28s198ms to 88 (screen-on, power-save-off, device-idle-off) + #2: +1m14s166ms to 87 (screen-on, power-save-off, device-idle-off) + #3: +1m17s637ms to 86 (screen-on, power-save-off, device-idle-off) + #4: +1m5s8ms to 85 (screen-on, power-save-off, device-idle-off) + #5: +55s529ms to 84 (screen-on, power-save-off, device-idle-off) + #6: +1m1s213ms to 83 (screen-on, power-save-off, device-idle-off) + #7: +1m1s214ms to 82 (screen-on, power-save-off, device-idle-off) + #8: +59s763ms to 81 (screen-on, power-save-off, device-idle-off) + #9: +59s762ms to 80 (screen-on, power-save-off, device-idle-off) + #10: +58s311ms to 79 (screen-on, power-save-off, device-idle-off) + #11: +59s201ms to 78 (screen-on, power-save-off, device-idle-off) + #12: +59s999ms to 77 (screen-on, power-save-off, device-idle-off) + #13: +59s999ms to 76 (screen-on, power-save-off, device-idle-off) + #14: +33s637ms to 75 (screen-on, power-save-off, device-idle-off) + #15: +46s660ms to 74 (screen-on, power-save-off, device-idle-off) + #16: +48s129ms to 73 (screen-on, power-save-off, device-idle-off) + #17: +51s574ms to 72 (screen-on, power-save-off, device-idle-off) + Charge total time: 1h 42m 44s 400ms (from 18 steps) + Charge screen on time: 1h 42m 44s 400ms (from 18 steps) + Package changes: + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.fenix.performancetest vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.fenix.performancetest vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.fenix.performancetest vers=1 + Update org.mozilla.fenix.performancetest vers=1 + Update org.mozilla.fenix.performancetest vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update com.android.vending vers=81532000 + Update com.android.vending vers=81532000 + Update com.google.android.videos vers=41200081 + Update com.google.android.videos vers=41200081 + Update com.google.android.youtube vers=1425573400 + Update com.google.android.youtube vers=1425573400 + Daily from 2019-06-30-03-51-31 to 2019-07-01-03-35-11: + Discharge step durations: + #0: +3m13s628ms to 98 (screen-on, power-save-off, device-idle-off) + #1: +2m47s696ms to 96 (screen-on, power-save-off, device-idle-off) + #2: +5m12s215ms to 97 (screen-on, power-save-off, device-idle-off) + #3: +4m12s528ms to 98 (screen-on, power-save-off, device-idle-off) + Discharge total time: 6h 25m 51s 600ms (from 4 steps) + Discharge screen on time: 6h 25m 51s 600ms (from 4 steps) + Package changes: + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update com.google.android.apps.maps vers=1019101040 + Update com.google.android.apps.maps vers=1019101040 + Daily from 2019-06-29-03-53-42 to 2019-06-30-03-51-31: + Discharge step durations: + #0: +3m42s886ms to 89 (screen-on, power-save-off, device-idle-off) + #1: +3m47s101ms to 90 (screen-on, power-save-off, device-idle-off) + #2: +2m42s264ms to 91 (screen-on, power-save-off, device-idle-off) + #3: +3m16s737ms to 92 (screen-on, power-save-off, device-idle-off) + #4: +1m31s889ms to 93 (screen-on, power-save-off, device-idle-off) + #5: +2m52s192ms to 93 (screen-on, power-save-off, device-idle-off) + #6: +3m32s783ms to 94 (screen-on, power-save-off, device-idle-off) + #7: +4m26s509ms to 96 (screen-on, power-save-off, device-idle-off) + #8: +4m9s160ms to 97 (screen-on, power-save-off, device-idle-off) + #9: +2m1s431ms to 98 (screen-on, power-save-off, device-idle-off) + #10: +7m36s775ms to 98 (screen-on, power-save-off, device-idle-off) + #11: +6m48s278ms to 98 (screen-on, power-save-off, device-idle-off) + Discharge total time: 6h 27m 13s 300ms (from 12 steps) + Discharge screen on time: 6h 27m 13s 300ms (from 12 steps) + Package changes: + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Daily from 2019-06-28-04-56-52 to 2019-06-29-03-53-42: + Discharge step durations: + #0: +5m36s159ms to 98 (screen-on, power-save-off, device-idle-off) + #1: +7m12s201ms to 91 (screen-on, power-save-off, device-idle-off) + #2: +6m34s890ms to 92 (screen-on, power-save-off, device-idle-off) + #3: +3m29s703ms to 93 (screen-on, power-save-off, device-idle-off) + #4: +3m29s722ms to 94 (screen-on, power-save-off, device-idle-off) + #5: +4m26s849ms to 95 (screen-on, power-save-off, device-idle-off) + #6: +1m41s616ms to 96 (screen-on, power-save-off, device-idle-off) + #7: +3m10s764ms to 97 (screen-on, power-save-off, device-idle-off) + #8: +4m36s578ms to 95 (screen-on, power-save-off, device-idle-off) + #9: +5m2s511ms to 96 (screen-on, power-save-off, device-idle-off) + #10: +3m26s952ms to 97 (screen-on, power-save-off, device-idle-off) + #11: +3m59s764ms to 96 (screen-on, power-save-off, device-idle-off) + #12: +6m41s143ms to 97 (screen-on, power-save-off, device-idle-off) + #13: +5m0s149ms to 98 (screen-on, power-save-off, device-idle-off) + #14: +1m13s423ms to 98 (screen-on, power-save-off, device-idle-off) + #15: +2m53s851ms to 96 (screen-on, power-save-off, device-idle-off) + #16: +5m5s897ms to 97 (screen-on, power-save-off, device-idle-off) + #17: +5m18s969ms to 98 (screen-on, power-save-off, device-idle-off) + #18: +12m43s982ms to 98 (screen-on, power-save-off, device-idle-off) + Discharge total time: 8h 2m 54s 300ms (from 19 steps) + Discharge screen on time: 8h 2m 54s 300ms (from 19 steps) + Package changes: + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.reference.browser.raptor vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.fenix.performancetest vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.reference.browser.raptor vers=1 + Update org.mozilla.fenix.performancetest vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.fenix.performancetest vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.fenix.performancetest vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update com.google.android.apps.photos vers=4846885 + Update com.google.android.apps.photos vers=4846885 + Daily from 2019-06-27-04-41-44 to 2019-06-28-04-56-52: + Discharge step durations: + #0: +2m48s271ms to 98 (screen-on, power-save-off, device-idle-off) + #1: +6m23s489ms to 92 (screen-on, power-save-off, device-idle-off) + #2: +5m36s548ms to 93 (screen-on, power-save-off, device-idle-off) + #3: +5m35s928ms to 94 (screen-on, power-save-off, device-idle-off) + #4: +6m14s349ms to 95 (screen-on, power-save-off, device-idle-off) + #5: +3m1s825ms to 96 (screen-on, power-save-off, device-idle-off) + #6: +4m0s433ms to 97 (screen-on, power-save-off, device-idle-off) + #7: +3m57s910ms to 98 (screen-on, power-save-off, device-idle-off) + #8: +4m19s103ms to 98 (screen-on, power-save-off, device-idle-off) + #9: +2m9s929ms to 96 (screen-on, power-save-off, device-idle-off) + #10: +4m13s353ms to 97 (screen-on, power-save-off, device-idle-off) + #11: +6m9s802ms to 98 (screen-on, power-save-off, device-idle-off) + Discharge total time: 7h 34m 17s 800ms (from 12 steps) + Discharge screen on time: 7h 34m 17s 800ms (from 12 steps) + Package changes: + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.fenix.raptor vers=1 + Update org.mozilla.reference.browser.raptor vers=1 + Update org.mozilla.fennec_aurora vers=2015636979 + Update org.mozilla.reference.browser.raptor vers=1 + Update org.mozilla.fenix.performancetest vers=1 + Update org.mozilla.fenix.performancetest vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Daily from 2019-06-26-04-29-41 to 2019-06-27-04-41-44: + Discharge step durations: + #0: +3m14s655ms to 90 (screen-on, power-save-off, device-idle-off) + #1: +3m37s34ms to 91 (screen-on, power-save-off, device-idle-off) + #2: +3m37s61ms to 92 (screen-on, power-save-off, device-idle-off) + #3: +3m42s890ms to 93 (screen-on, power-save-off, device-idle-off) + #4: +3m38s537ms to 94 (screen-on, power-save-off, device-idle-off) + #5: +2m22s824ms to 95 (screen-on, power-save-off, device-idle-off) + #6: +3m37s54ms to 96 (screen-on, power-save-off, device-idle-off) + #7: +4m19s284ms to 97 (screen-on, power-save-off, device-idle-off) + #8: +5m8s753ms to 98 (screen-on, power-save-off, device-idle-off) + #9: +4m44s170ms to 98 (screen-on, power-save-off, device-idle-off) + #10: +3m53s227ms to 98 (screen-on, power-save-off, device-idle-off) + Discharge total time: 6h 21m 8s 0ms (from 11 steps) + Discharge screen on time: 6h 21m 8s 0ms (from 11 steps) + Package changes: + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update com.google.android.gms vers=17785028 + Update com.google.android.gms vers=17785028 + Daily from 2019-06-25-01-29-55 to 2019-06-26-04-29-41: + Discharge step durations: + #0: +5m4s365ms to 95 (screen-on, power-save-off, device-idle-off) + #1: +6m38s468ms to 96 (screen-on, power-save-off, device-idle-off) + #2: +4m27s81ms to 97 (screen-on, power-save-off, device-idle-off) + #3: +7m5s799ms to 98 (screen-on, power-save-off, device-idle-off) + #4: +6m59s932ms to 98 (screen-on, power-save-off, device-idle-off) + #5: +3m44s382ms to 96 (screen-on, power-save-off, device-idle-off) + #6: +5m21s802ms to 97 (screen-on, power-save-off, device-idle-off) + #7: +5m31s978ms to 98 (screen-on, power-save-off, device-idle-off) + Discharge total time: 9h 21m 12s 500ms (from 8 steps) + Discharge screen on time: 9h 21m 12s 500ms (from 8 steps) + Package changes: + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update com.google.android.youtube vers=1424573400 + Update com.google.android.youtube vers=1424573400 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Daily from 2019-06-24-04-15-14 to 2019-06-25-01-29-55: + Discharge step durations: + #0: +7m38s303ms to 30 (screen-on, power-save-off, device-idle-off) + #1: +5m0s60ms to 31 (screen-on, power-save-off, device-idle-off) + #2: +6m43s705ms to 32 (screen-on, power-save-off, device-idle-off) + #3: +2m44s678ms to 29 (screen-on, power-save-off, device-idle-off) + #4: +4m59s474ms to 30 (screen-on, power-save-off, device-idle-off) + #5: +2m0s119ms to 31 (screen-on, power-save-off, device-idle-off) + #6: +5m23s389ms to 32 (screen-on, power-save-off, device-idle-off) + #7: +5m0s221ms to 31 (screen-on, power-save-off, device-idle-off) + #8: +7m41s899ms to 32 (screen-on, power-save-off, device-idle-off) + #9: +4m0s137ms to 33 (screen-on, power-save-off, device-idle-off) + #10: +4m1s434ms to 30 (screen-on, power-save-off, device-idle-off) + #11: +3m46s629ms to 31 (screen-on, power-save-off, device-idle-off) + #12: +5m52s478ms to 32 (screen-on, power-save-off, device-idle-off) + #13: +4m42s916ms to 33 (screen-on, power-save-off, device-idle-off) + #14: +1m13s100ms to 34 (screen-on, power-save-off, device-idle-off) + #15: +3m47s79ms to 35 (screen-on, power-save-off, device-idle-off) + #16: +3m2s287ms to 36 (screen-on, power-save-off, device-idle-off) + #17: +3m31s244ms to 37 (screen-on, power-save-off, device-idle-off) + #18: +3m21s24ms to 38 (screen-on, power-save-off, device-idle-off) + #19: +3m43s831ms to 39 (screen-on, power-save-off, device-idle-off) + #20: +5m0s99ms to 40 (screen-on, power-save-off, device-idle-off) + #21: +6m3s936ms to 41 (screen-on, power-save-off, device-idle-off) + #22: +3m57s614ms to 42 (screen-on, power-save-off, device-idle-off) + #23: +3m41s455ms to 43 (screen-on, power-save-off, device-idle-off) + #24: +3m53s96ms to 44 (screen-on, power-save-off, device-idle-off) + #25: +3m39s995ms to 45 (screen-on, power-save-off, device-idle-off) + #26: +3m42s902ms to 46 (screen-on, power-save-off, device-idle-off) + #27: +3m8s680ms to 47 (screen-on, power-save-off, device-idle-off) + #28: +1m55s475ms to 47 (screen-on, power-save-off, device-idle-off) + #29: +2m46s252ms to 49 (screen-on, power-save-off, device-idle-off) + #30: +3m32s704ms to 50 (screen-on, power-save-off, device-idle-off) + #31: +3m41s454ms to 51 (screen-on, power-save-off, device-idle-off) + #32: +3m35s630ms to 52 (screen-on, power-save-off, device-idle-off) + #33: +3m49s170ms to 53 (screen-on, power-save-off, device-idle-off) + #34: +1m23s33ms to 53 (screen-on, power-save-off, device-idle-off) + #35: +2m50s144ms to 54 (screen-on, power-save-off, device-idle-off) + #36: +2m57s597ms to 55 (screen-on, power-save-off, device-idle-off) + #37: +3m12s291ms to 56 (screen-on, power-save-off, device-idle-off) + #38: +3m50s111ms to 57 (screen-on, power-save-off, device-idle-off) + #39: +2m35s951ms to 58 (screen-on, power-save-off, device-idle-off) + #40: +5m2s243ms to 59 (screen-on, power-save-off, device-idle-off) + #41: +3m30s969ms to 60 (screen-on, power-save-off, device-idle-off) + #42: +5m11s982ms to 61 (screen-on, power-save-off, device-idle-off) + #43: +3m36s661ms to 62 (screen-on, power-save-off, device-idle-off) + #44: +5m0s199ms to 63 (screen-on, power-save-off, device-idle-off) + #45: +1m23s997ms to 62 (screen-on, power-save-off, device-idle-off) + #46: +4m13s108ms to 50 (screen-on, power-save-off, device-idle-off) + #47: +2m43s605ms to 51 (screen-on, power-save-off, device-idle-off) + #48: +3m4s644ms to 52 (screen-on, power-save-off, device-idle-off) + #49: +2m24s227ms to 53 (screen-on, power-save-off, device-idle-off) + #50: +4m11s71ms to 54 (screen-on, power-save-off, device-idle-off) + #51: +5m12s975ms to 55 (screen-on, power-save-off, device-idle-off) + #52: +2m25s611ms to 56 (screen-on, power-save-off, device-idle-off) + #53: +4m19s83ms to 57 (screen-on, power-save-off, device-idle-off) + #54: +2m17s31ms to 58 (screen-on, power-save-off, device-idle-off) + #55: +1m56s610ms to 58 (screen-on, power-save-off, device-idle-off) + #56: +4m1s267ms to 59 (screen-on, power-save-off, device-idle-off) + #57: +1m20s78ms to 60 (screen-on, power-save-off, device-idle-off) + #58: +2m20s87ms to 61 (screen-on, power-save-off, device-idle-off) + #59: +3m49s576ms to 61 (screen-on, power-save-off, device-idle-off) + #60: +1m51s913ms to 62 (screen-on, power-save-off, device-idle-off) + #61: +3m24s48ms to 63 (screen-on, power-save-off, device-idle-off) + #62: +4m57s644ms to 64 (screen-on, power-save-off, device-idle-off) + #63: +3m37s998ms to 65 (screen-on, power-save-off, device-idle-off) + #64: +3m24s273ms to 65 (screen-on, power-save-off, device-idle-off) + #65: +5m10s852ms to 66 (screen-on, power-save-off, device-idle-off) + #66: +3m23s224ms to 67 (screen-on, power-save-off, device-idle-off) + #67: +2m39s990ms to 67 (screen-on, power-save-off, device-idle-off) + #68: +4m48s291ms to 68 (screen-on, power-save-off, device-idle-off) + #69: +2m49s16ms to 69 (screen-on, power-save-off, device-idle-off) + #70: +3m48s746ms to 70 (screen-on, power-save-off, device-idle-off) + #71: +2m22s174ms to 71 (screen-on, power-save-off, device-idle-off) + #72: +3m3s623ms to 71 (screen-on, power-save-off, device-idle-off) + #73: +2m32s594ms to 72 (screen-on, power-save-off, device-idle-off) + #74: +2m51s738ms to 73 (screen-on, power-save-off, device-idle-off) + #75: +2m38s626ms to 73 (screen-on, power-save-off, device-idle-off) + #76: +3m48s313ms to 74 (screen-on, power-save-off, device-idle-off) + #77: +4m54s489ms to 75 (screen-on, power-save-off, device-idle-off) + #78: +3m53s181ms to 75 (screen-on, power-save-off, device-idle-off) + #79: +1m30s807ms to 76 (screen-on, power-save-off, device-idle-off) + #80: +3m37s439ms to 77 (screen-on, power-save-off, device-idle-off) + #81: +1m32s353ms to 78 (screen-on, power-save-off, device-idle-off) + #82: +3m6s72ms to 78 (screen-on, power-save-off, device-idle-off) + #83: +5m18s82ms to 79 (screen-on, power-save-off, device-idle-off) + #84: +1m40s559ms to 80 (screen-on, power-save-off, device-idle-off) + #85: +4m51s163ms to 81 (screen-on, power-save-off, device-idle-off) + #86: +3m9s384ms to 82 (screen-on, power-save-off, device-idle-off) + #87: +3m54s805ms to 83 (screen-on, power-save-off, device-idle-off) + #88: +4m38s236ms to 84 (screen-on, power-save-off, device-idle-off) + #89: +3m0s160ms to 85 (screen-on, power-save-off, device-idle-off) + #90: +3m7s81ms to 85 (screen-on, power-save-off, device-idle-off) + #91: +3m57s420ms to 86 (screen-on, power-save-off, device-idle-off) + #92: +4m0s362ms to 87 (screen-on, power-save-off, device-idle-off) + #93: +3m55s976ms to 88 (screen-on, power-save-off, device-idle-off) + #94: +3m47s223ms to 89 (screen-on, power-save-off, device-idle-off) + #95: +3m47s296ms to 88 (screen-on, power-save-off, device-idle-off) + #96: +4m1s850ms to 89 (screen-on, power-save-off, device-idle-off) + #97: +4m0s439ms to 90 (screen-on, power-save-off, device-idle-off) + #98: +3m48s662ms to 86 (screen-on, power-save-off, device-idle-off) + #99: +4m4s644ms to 87 (screen-on, power-save-off, device-idle-off) + #100: +4m40s388ms to 88 (screen-on, power-save-off, device-idle-off) + #101: +3m52s207ms to 89 (screen-on, power-save-off, device-idle-off) + #102: +3m46s553ms to 90 (screen-on, power-save-off, device-idle-off) + #103: +2m11s105ms to 85 (screen-on, power-save-off, device-idle-off) + #104: +3m9s753ms to 86 (screen-on, power-save-off, device-idle-off) + #105: +2m57s666ms to 87 (screen-on, power-save-off, device-idle-off) + #106: +5m10s775ms to 88 (screen-on, power-save-off, device-idle-off) + #107: +3m21s421ms to 89 (screen-on, power-save-off, device-idle-off) + #108: +4m11s832ms to 90 (screen-on, power-save-off, device-idle-off) + #109: +2m34s418ms to 91 (screen-on, power-save-off, device-idle-off) + #110: +2m27s672ms to 92 (screen-on, power-save-off, device-idle-off) + #111: +4m32s561ms to 92 (screen-on, power-save-off, device-idle-off) + #112: +6m0s366ms to 93 (screen-on, power-save-off, device-idle-off) + #113: +3m0s55ms to 93 (screen-on, power-save-off, device-idle-off) + #114: +5m49s976ms to 94 (screen-on, power-save-off, device-idle-off) + #115: +6m46s380ms to 95 (screen-on, power-save-off, device-idle-off) + #116: +2m0s123ms to 96 (screen-on, power-save-off, device-idle-off) + #117: +3m27s490ms to 96 (screen-on, power-save-off, device-idle-off) + #118: +4m43s851ms to 97 (screen-on, power-save-off, device-idle-off) + #119: +4m32s477ms to 98 (screen-on, power-save-off, device-idle-off) + #120: +3m20s908ms to 98 (screen-on, power-save-off, device-idle-off) + Discharge total time: 6h 11m 8s 800ms (from 121 steps) + Discharge screen on time: 6h 11m 8s 800ms (from 121 steps) + Charge step durations: + #0: +2m0s3ms to 89 (screen-on, power-save-off, device-idle-off) + #1: +1m0s0ms to 88 (screen-on, power-save-off, device-idle-off) + #2: +1m3s876ms to 87 (screen-on, power-save-off, device-idle-off) + #3: +1m50s500ms to 86 (screen-on, power-save-off, device-idle-off) + #4: +1m5s621ms to 85 (screen-on, power-save-off, device-idle-off) + #5: +1m0s4ms to 84 (screen-on, power-save-off, device-idle-off) + #6: +59s997ms to 83 (screen-on, power-save-off, device-idle-off) + #7: +1m0s0ms to 82 (screen-on, power-save-off, device-idle-off) + #8: +59s999ms to 81 (screen-on, power-save-off, device-idle-off) + #9: +1m0s1ms to 80 (screen-on, power-save-off, device-idle-off) + #10: +1m0s1ms to 79 (screen-on, power-save-off, device-idle-off) + #11: +59s998ms to 78 (screen-on, power-save-off, device-idle-off) + #12: +1m0s0ms to 77 (screen-on, power-save-off, device-idle-off) + #13: +1m54s445ms to 76 (screen-on, power-save-off, device-idle-off) + #14: +5s558ms to 75 (screen-on, power-save-off, device-idle-off) + #15: +59s999ms to 74 (screen-on, power-save-off, device-idle-off) + #16: +59s999ms to 73 (screen-on, power-save-off, device-idle-off) + #17: +1m0s1ms to 72 (screen-on, power-save-off, device-idle-off) + #18: +59s998ms to 71 (screen-on, power-save-off, device-idle-off) + #19: +1m0s1ms to 70 (screen-on, power-save-off, device-idle-off) + #20: +1m0s1ms to 69 (screen-on, power-save-off, device-idle-off) + #21: +59s998ms to 68 (screen-on, power-save-off, device-idle-off) + #22: +1m0s3ms to 67 (screen-on, power-save-off, device-idle-off) + #23: +59s999ms to 66 (screen-on, power-save-off, device-idle-off) + #24: +1m0s432ms to 65 (screen-on, power-save-off, device-idle-off) + #25: +48s85ms to 64 (screen-on, power-save-off, device-idle-off) + #26: +48s84ms to 64 (screen-on, power-save-off, device-idle-off) + #27: +23s397ms to 62 (screen-on, power-save-off, device-idle-off) + #28: +1m12s771ms to 61 (screen-on, power-save-off, device-idle-off) + #29: +23s616ms to 60 (screen-on, power-save-off, device-idle-off) + #30: +23s615ms to 60 (screen-on, power-save-off, device-idle-off) + #31: +59s999ms to 58 (screen-on, power-save-off, device-idle-off) + #32: +1m0s1ms to 57 (screen-on, power-save-off, device-idle-off) + #33: +1m10s288ms to 56 (screen-on, power-save-off, device-idle-off) + #34: +45s167ms to 55 (screen-on, power-save-off, device-idle-off) + #35: +45s167ms to 55 (screen-on, power-save-off, device-idle-off) + #36: +19s378ms to 53 (screen-on, power-save-off, device-idle-off) + #37: +1m10s945ms to 52 (screen-on, power-save-off, device-idle-off) + #38: +45s892ms to 51 (screen-on, power-save-off, device-idle-off) + #39: +45s892ms to 51 (screen-on, power-save-off, device-idle-off) + #40: +17s271ms to 49 (screen-on, power-save-off, device-idle-off) + #41: +59s997ms to 48 (screen-on, power-save-off, device-idle-off) + #42: +15s985ms to 47 (screen-on, power-save-off, device-idle-off) + #43: +46s622ms to 46 (screen-on, power-save-off, device-idle-off) + #44: +1m33s232ms to 45 (screen-on, power-save-off, device-idle-off) + #45: +24s164ms to 44 (screen-on, power-save-off, device-idle-off) + #46: +1m9s84ms to 43 (screen-on, power-save-off, device-idle-off) + #47: +51s725ms to 42 (screen-on, power-save-off, device-idle-off) + #48: +51s725ms to 42 (screen-on, power-save-off, device-idle-off) + #49: +7s467ms to 40 (screen-on, power-save-off, device-idle-off) + #50: +1m33s81ms to 39 (screen-on, power-save-off, device-idle-off) + #51: +26s917ms to 38 (screen-on, power-save-off, device-idle-off) + #52: +1m12s176ms to 37 (screen-on, power-save-off, device-idle-off) + #53: +49s552ms to 36 (screen-on, power-save-off, device-idle-off) + #54: +46s645ms to 35 (screen-on, power-save-off, device-idle-off) + #55: +46s644ms to 35 (screen-on, power-save-off, device-idle-off) + #56: +24s985ms to 33 (screen-on, power-save-off, device-idle-off) + #57: +12s917ms to 35 (screen-on, power-save-off, device-idle-off) + #58: +1m42s262ms to 34 (screen-on, power-save-off, device-idle-off) + #59: +17s731ms to 33 (screen-on, power-save-off, device-idle-off) + #60: +1m28s779ms to 32 (screen-on, power-save-off, device-idle-off) + #61: +53s939ms to 35 (screen-on, power-save-off, device-idle-off) + #62: +53s938ms to 35 (screen-on, power-save-off, device-idle-off) + #63: +11s957ms to 33 (screen-on, power-save-off, device-idle-off) + #64: +25s481ms to 64 (screen-on, power-save-off, device-idle-off) + #65: +50s240ms to 61 (screen-on, power-save-off, device-idle-off) + #66: +35s794ms to 60 (screen-on, power-save-off, device-idle-off) + #67: +45s241ms to 59 (screen-on, power-save-off, device-idle-off) + #68: +48s163ms to 56 (screen-on, power-save-off, device-idle-off) + #69: +1m18s660ms to 89 (screen-on, power-save-off, device-idle-off) + #70: +28s146ms to 89 (screen-on, power-save-off, device-idle-off) + Charge total time: 1h 28m 34s 400ms (from 71 steps) + Charge screen on time: 1h 28m 34s 400ms (from 71 steps) + Package changes: + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.fennec_aurora vers=2015636403 + Update org.mozilla.fenix.raptor vers=1 + Update org.mozilla.reference.browser vers=11281213 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.reference.browser vers=11281214 + Update org.mozilla.fennec_aurora vers=2015636403 + Update org.mozilla.reference.browser vers=11281213 + Update org.mozilla.fennec_aurora vers=2015636401 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.fenix.raptor vers=1 + Update org.mozilla.reference.browser vers=11281213 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.fenix.raptor vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.fennec_aurora vers=2015636401 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.fenix.raptor vers=1 + Update org.mozilla.fenix.raptor vers=1 + Update org.mozilla.fenix.raptor vers=1 + Update org.mozilla.fennec_aurora vers=2015636403 + Update org.mozilla.fennec_aurora vers=2015636403 + Update org.mozilla.fenix.raptor vers=1 + Update org.mozilla.fenix.raptor vers=1 + Update org.mozilla.fenix.raptor vers=1 + Update org.mozilla.fennec_aurora vers=2015636401 + Update org.mozilla.reference.browser.raptor vers=1 + Update org.mozilla.fennec_aurora vers=2015636441 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.reference.browser.raptor vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.fennec_aurora vers=2015636457 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.fenix.performancetest vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.fenix.performancetest vers=1 + Update org.mozilla.reference.browser.raptor vers=1 + Update org.mozilla.geckoview_example vers=1 + Update com.android.chrome vers=377010137 + Update com.android.chrome vers=377010137 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.reference.browser.raptor vers=1 + Daily from 2019-06-23-04-45-02 to 2019-06-24-04-15-14: + Discharge step durations: + #0: +2m47s180ms to 91 (screen-on, power-save-off, device-idle-off) + #1: +3m45s842ms to 92 (screen-on, power-save-off, device-idle-off) + #2: +3m44s384ms to 93 (screen-on, power-save-off, device-idle-off) + #3: +3m37s80ms to 94 (screen-on, power-save-off, device-idle-off) + #4: +2m50s673ms to 95 (screen-on, power-save-off, device-idle-off) + #5: +2m44s669ms to 96 (screen-on, power-save-off, device-idle-off) + #6: +4m35s281ms to 97 (screen-on, power-save-off, device-idle-off) + #7: +2m40s50ms to 98 (screen-on, power-save-off, device-idle-off) + #8: +3m10s7ms to 97 (screen-on, power-save-off, device-idle-off) + #9: +4m38s222ms to 98 (screen-on, power-save-off, device-idle-off) + Discharge total time: 5h 45m 33s 800ms (from 10 steps) + Discharge screen on time: 5h 45m 33s 800ms (from 10 steps) + Package changes: + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + Update org.mozilla.geckoview_example vers=1 + +Statistics since last charge: + System starts: 0, currently on battery: true + Estimated battery capacity: 2700 mAh + Min learned battery capacity: 2606 mAh + Max learned battery capacity: 2606 mAh + Time on battery: 20m 37s 454ms (100.0%) realtime, 20m 37s 454ms (100.0%) uptime + Time on battery screen off: 0ms (0.0%) realtime, 0ms (0.0%) uptime + Total run time: 20m 37s 471ms realtime, 20m 37s 471ms uptime + Discharge: 111 mAh + Screen off discharge: 0 mAh + Screen on discharge: 111 mAh + Start clock time: 2019-07-03-23-26-42 + Screen on: 20m 37s 454ms (100.0%) 0x, Interactive: 20m 37s 454ms (100.0%) + Screen brightnesses: + dark 20m 37s 454ms (100.0%) + Total full wakelock time: 1m 4s 45ms + Mobile total received: 0B, sent: 0B (packets received 0, sent 0) + Phone signal levels: + none 20m 37s 454ms (100.0%) 0x + Signal scanning time: 20m 37s 454ms + Radio types: (no activity) + Mobile radio active time: 0ms (0.0%) 0x + Radio Idle time: 40m 21s 575ms (99.3%) + Radio Rx time: 16s 999ms (0.7%) + Radio Tx time: 0ms (0.0%) + [0] 0ms (--%) + [1] 0ms (--%) + [2] 0ms (--%) + [3] 0ms (--%) + [4] 0ms (--%) + Radio Power drain: 0mAh + Wi-Fi total received: 19.20MB, sent: 60.76MB (packets received 264638, sent 265477) + Wifi on: 20m 37s 454ms (100.0%), Wifi running: 20m 37s 454ms (100.0%) + Wifi states: + sta 20m 37s 454ms (100.0%) 0x + Wifi supplicant states: + group-handshake 1ms (0.0%) 1x + completed 20m 37s 453ms (100.0%) 1x + Wifi signal levels: + level(4) 20m 37s 454ms (100.0%) 0x + WiFi Idle time: 18m 37s 949ms (96.8%) + WiFi Rx time: 5s 410ms (0.5%) + WiFi Tx time: 32s 105ms (2.8%) + WiFi Power drain: 2.69mAh + Bluetooth total received: 0B, sent: 0B + Bluetooth scan time: 0ms + Bluetooth Idle time: 707ms (2.7%) + Bluetooth Rx time: 25s 583ms (97.3%) + Bluetooth Tx time: 0ms (0.0%) + Bluetooth Power drain: 0.157mAh + + Device battery use since last full charge + Amount discharged (lower bound): 0 + Amount discharged (upper bound): 0 + Amount discharged while screen on: 0 + Amount discharged while screen off: 0 + + Estimated power use (mAh): + Capacity: 2700, Computed drain: 71.8, actual drain: 0 + Screen: 51.7 Excluded from smearing + Uid 2000: 5.01 ( cpu=2.75 wifi=2.26 ) Excluded from smearing + Uid u0a120: 4.70 ( cpu=4.70 wifi=0.000556 ) Including smearing: 67.4 ( screen=51.5 proportional=11.2 ) + Cell standby: 3.78 ( radio=3.78 ) Excluded from smearing + Idle: 2.48 Excluded from smearing + Uid 1036: 1.80 ( cpu=1.80 ) Excluded from smearing + Uid 0: 0.937 ( cpu=0.936 wifi=0.000556 ) Excluded from smearing + Wifi: 0.501 ( cpu=0.0673 wifi=0.433 ) Including smearing: 0.600 ( proportional=0.0998 ) + Uid u0a55: 0.303 ( cpu=0.303 wifi=0.000208 ) Including smearing: 0.363 ( proportional=0.0604 ) + Uid 1000: 0.278 ( cpu=0.244 sensor=0.0344 ) Excluded from smearing + Bluetooth: 0.157 ( cpu=0.000645 bt=0.157 ) Including smearing: 0.189 ( proportional=0.0314 ) + Uid u0a17: 0.107 ( cpu=0.0215 sensor=0.0859 ) Excluded from smearing + Uid u0a38: 0.0136 ( cpu=0.0129 sensor=0.000687 ) Excluded from smearing + Uid u0a130: 0.00669 ( cpu=0.00669 ) Including smearing: 0.200 ( screen=0.160 proportional=0.0332 ) + Uid 1001: 0.00411 ( cpu=0.00411 ) Excluded from smearing + Uid u0a11: 0.00276 ( cpu=0.00276 ) Including smearing: 0.00331 ( proportional=0.000550 ) + Uid 1027: 0.00106 ( cpu=0.00106 ) Excluded from smearing + Uid u0a98: 0.000821 ( cpu=0.000821 ) Including smearing: 0.000985 ( proportional=0.000164 ) + Uid u0a8: 0.000762 ( cpu=0.000762 ) Including smearing: 0.000915 ( proportional=0.000152 ) + Uid u0a9: 0.000762 ( cpu=0.000762 ) Including smearing: 0.000915 ( proportional=0.000152 ) + Uid 1017: 0.000411 ( cpu=0.000411 ) Excluded from smearing + Uid 1021: 0.000411 ( cpu=0.000411 ) Excluded from smearing + Uid u0a91: 0.000411 ( cpu=0.000411 ) Including smearing: 0.000492 ( proportional=0.0000819 ) + Uid 1013: 0.000352 ( cpu=0.000352 ) Excluded from smearing + Uid u0a43: 0.000352 ( cpu=0.000352 ) Including smearing: 0.000422 ( proportional=0.0000702 ) + Uid u0a57: 0.000176 ( cpu=0.000176 ) Excluded from smearing + Uid u0a97: 0.000176 ( cpu=0.000176 ) Including smearing: 0.000211 ( proportional=0.0000351 ) + +Memory Stats + 0: + Wi-Fi network: 1.36KB received, 5.87KB sent (packets 19 received, 71 sent) + WiFi Idle time: 0ms (0.0%) + WiFi Rx time: 0ms (0.0%) + WiFi Tx time: 8ms (100.0%) + WiFi Power drain: 0mAh + Total cpu time: u=1s 186ms s=14s 774ms + Proc android.hardware.wifi@1.0-service: + CPU: 340ms usr + 370ms krn ; 0ms fg + Proc kworker/u16:7: + CPU: 0ms usr + 1s 340ms krn ; 0ms fg + Proc kworker/u16:5: + CPU: 0ms usr + 90ms krn ; 0ms fg + Proc kworker/u16:3: + CPU: 0ms usr + 70ms krn ; 0ms fg + Proc kworker/u16:0: + CPU: 0ms usr + 1s 0ms krn ; 0ms fg + Proc wpa_supplicant: + CPU: 10ms usr + 10ms krn ; 0ms fg + Proc ksoftirqd/5: + CPU: 0ms usr + 100ms krn ; 0ms fg + Proc ksoftirqd/4: + CPU: 0ms usr + 460ms krn ; 0ms fg + Proc ksoftirqd/3: + CPU: 0ms usr + 610ms krn ; 0ms fg + Proc ksoftirqd/2: + CPU: 0ms usr + 40ms krn ; 0ms fg + Proc ksoftirqd/1: + CPU: 0ms usr + 580ms krn ; 0ms fg + Proc ksoftirqd/0: + CPU: 0ms usr + 160ms krn ; 0ms fg + Proc msm_watchdog: + CPU: 0ms usr + 10ms krn ; 0ms fg + Proc rcuop/7: + CPU: 0ms usr + 10ms krn ; 0ms fg + Proc rcuop/6: + CPU: 0ms usr + 10ms krn ; 0ms fg + Proc rcuop/5: + CPU: 0ms usr + 10ms krn ; 0ms fg + Proc rcuop/4: + CPU: 0ms usr + 10ms krn ; 0ms fg + Proc rcuop/2: + CPU: 0ms usr + 10ms krn ; 0ms fg + Proc rcuop/1: + CPU: 0ms usr + 10ms krn ; 0ms fg + Proc rcuop/0: + CPU: 0ms usr + 100ms krn ; 0ms fg + Proc kworker/7:0: + CPU: 0ms usr + 10ms krn ; 0ms fg + Proc kworker/5:3: + CPU: 0ms usr + 10ms krn ; 0ms fg + Proc kworker/4:2: + CPU: 0ms usr + 10ms krn ; 0ms fg + Proc kworker/4:0: + CPU: 0ms usr + 10ms krn ; 0ms fg + Proc kworker/3:2: + CPU: 0ms usr + 30ms krn ; 0ms fg + Proc kworker/3:0: + CPU: 0ms usr + 10ms krn ; 0ms fg + Proc kworker/2:4: + CPU: 0ms usr + 10ms krn ; 0ms fg + Proc kworker/2:3: + CPU: 0ms usr + 480ms krn ; 0ms fg + Proc kworker/1:2: + CPU: 0ms usr + 80ms krn ; 0ms fg + Proc kworker/0:4: + CPU: 0ms usr + 330ms krn ; 0ms fg + Proc kworker/0:2: + CPU: 0ms usr + 10ms krn ; 0ms fg + Proc wlan_logging_th: + CPU: 0ms usr + 60ms krn ; 0ms fg + Proc jbd2/sda45-8: + CPU: 0ms usr + 50ms krn ; 0ms fg + Proc hwrng: + CPU: 0ms usr + 10ms krn ; 0ms fg + Proc rcu_preempt: + CPU: 0ms usr + 70ms krn ; 0ms fg + Proc vold: + CPU: 10ms usr + 20ms krn ; 0ms fg + Proc netd: + CPU: 10ms usr + 10ms krn ; 0ms fg + Proc logd: + CPU: 5s 20ms usr + 15s 630ms krn ; 0ms fg + Proc lmkd: + CPU: 0ms usr + 10ms krn ; 0ms fg + Proc adbd: + CPU: 2s 660ms usr + 18s 650ms krn ; 0ms fg + Proc surfaceflinger: + CPU: 70ms usr + 80ms krn ; 0ms fg + Proc magisk_daemon: + CPU: 10ms usr + 0ms krn ; 0ms fg + Proc smem_native_rpm: + CPU: 0ms usr + 110ms krn ; 0ms fg + Proc ueventd: + CPU: 10ms usr + 0ms krn ; 0ms fg + Proc zygote: + CPU: 0ms usr + 10ms krn ; 0ms fg + Proc msm_irqbalance: + CPU: 40ms usr + 630ms krn ; 0ms fg + Proc system: + CPU: 0ms usr + 10ms krn ; 0ms fg + Proc logcat: + CPU: 730ms usr + 1s 890ms krn ; 0ms fg + Proc mdss_fb0: + CPU: 0ms usr + 90ms krn ; 0ms fg + Proc android.hardware.usb@1.1-service.wahoo: + CPU: 10ms usr + 0ms krn ; 0ms fg + Proc irq/35-1008000.: + CPU: 0ms usr + 10ms krn ; 0ms fg + Proc cds_mc_thread: + CPU: 0ms usr + 410ms krn ; 0ms fg + 1000: + Wi-Fi network: 1.28KB received, 1.24KB sent (packets 4 received, 4 sent) + Wake lock *alarm* realtime + Wake lock NetworkStats realtime + Wake lock GnssLocationProvider realtime + Wake lock IpReachabilityMonitor.wlan0 realtime + Sensor 7: 20m 37s 450ms realtime (0 times) + Fg Service for: 20m 37s 450ms + Total running: 20m 37s 450ms + Total cpu time: u=2s 133ms s=2s 20ms + Proc android.hardware.memtrack@1.0-service: + CPU: 20ms usr + 10ms krn ; 0ms fg + Proc .dataservices: + CPU: 0ms usr + 10ms krn ; 0ms fg + Proc servicemanager: + CPU: 10ms usr + 20ms krn ; 0ms fg + Proc sensors.qcom: + CPU: 0ms usr + 20ms krn ; 0ms fg + Proc android.hardware.graphics.composer@2.1-service: + CPU: 50ms usr + 90ms krn ; 0ms fg + Proc system: + CPU: 1s 570ms usr + 1s 350ms krn ; 0ms fg + Proc android.hardware.power@1.1-service.wahoo: + CPU: 0ms usr + 20ms krn ; 0ms fg + Proc android.hardware.sensors@1.0-service: + CPU: 30ms usr + 90ms krn ; 0ms fg + Apk android: + Wakeup alarm *walarm*:ScheduleConditionProvider.EVALUATE: 1 times + Wakeup alarm *walarm*:WifiConnectivityManager Schedule Periodic Scan Timer: 5 times + Wakeup alarm *walarm*:DhcpClient.wlan0.RENEW: 4 times + 1001: + Wake lock RILJ realtime + Fg Service for: 20m 37s 449ms + Total running: 20m 37s 449ms + Total cpu time: u=50ms s=20ms + Proc com.android.phone: + CPU: 10ms usr + 10ms krn ; 0ms fg + Proc ipacm: + CPU: 0ms usr + 10ms krn ; 0ms fg + Proc rild: + CPU: 10ms usr + 10ms krn ; 0ms fg + 1002: + Wake lock bluetooth_timer realtime + Foreground for: 20m 37s 449ms + Total running: 20m 37s 449ms + Total cpu time: u=7ms s=4ms + Proc com.android.bluetooth: + CPU: 10ms usr + 0ms krn ; 0ms fg + 1010: + Wifi Running: 0ms (0.0%) + Full Wifi Lock: 0ms (0.0%) + Wifi Scan (blamed): 1s 259ms (0.1%) 5x + Wifi Scan (actual): 1s 259ms (0.1%) 5x + Background Wifi Scan: 0ms (0.0%) 0x + WiFi Idle time: 0ms (0.0%) + WiFi Rx time: 1s 259ms (50.0%) + WiFi Tx time: 1s 259ms (50.0%) + WiFi Power drain: 0mAh + Total cpu time: u=537ms s=610ms + Proc wificond: + CPU: 10ms usr + 70ms krn ; 0ms fg + 1013: + Total cpu time: u=0ms s=6ms + Proc mediaserver: + CPU: 0ms usr + 10ms krn ; 0ms fg + 1017: + Total cpu time: u=0ms s=7ms + (nothing executed) + 1021: + Total cpu time: u=3ms s=4ms + Proc lowi-server: + CPU: 0ms usr + 10ms krn ; 0ms fg + 1027: + Fg Service for: 20m 37s 445ms + Total running: 20m 37s 445ms + Total cpu time: u=14ms s=4ms + Proc com.android.nfc: + CPU: 10ms usr + 0ms krn ; 0ms fg + 1036: + Total cpu time: u=7s 726ms s=22s 987ms + (nothing executed) + 2000: + Wi-Fi network: 19.19MB received, 60.74MB sent (packets 264527 received, 265285 sent) + WiFi Idle time: 0ms (0.0%) + WiFi Rx time: 4s 149ms (11.9%) + WiFi Tx time: 30s 823ms (88.1%) + WiFi Power drain: 0mAh + Total cpu time: u=12s 40ms s=34s 910ms + Proc logcat: + CPU: 4s 100ms usr + 3s 950ms krn ; 0ms fg + u0a3: + Cached for: 20m 37s 441ms + Total running: 20m 37s 441ms + u0a5: + Background for: 1ms + Cached for: 20m 37s 440ms + Total running: 20m 37s 441ms + u0a8: + Wi-Fi network: 104B received, 374B sent (packets 2 received, 6 sent) + Cached for: 20m 37s 440ms + Total running: 20m 37s 440ms + Total cpu time: u=13ms s=0ms + Proc com.android.vending: + CPU: 40ms usr + 10ms krn ; 0ms fg + u0a9: + Background for: 20m 37s 440ms + Total running: 20m 37s 440ms + Total cpu time: u=10ms s=3ms + Proc com.google.android.apps.turbo: + CPU: 10ms usr + 20ms krn ; 0ms fg + Apk com.google.android.apps.turbo: + Service com.google.android.apps.turbo.database.BatteryHistoryLoggerService: + Created for: 16ms uptime + Starts: 5, launches: 5 + u0a11: + Wake lock *alarm* realtime + Wake lock *job*/com.google.android.googlequicksearchbox/com.google.android.apps.gsa.tasks.BackgroundTasksJobService realtime + Job com.google.android.googlequicksearchbox/com.google.android.apps.gsa.tasks.BackgroundTasksJobService: 45ms realtime (4 times) + Fg Service for: 20m 37s 440ms + Total running: 20m 37s 440ms + Total cpu time: u=37ms s=10ms + Proc com.google.android.googlequicksearchbox:search: + CPU: 40ms usr + 20ms krn ; 0ms fg + Apk android: + Wakeup alarm *walarm*:*job.delay*: 4 times + Wakeup alarm *walarm*:*job.deadline*: 1 times + Apk com.google.android.googlequicksearchbox: + Service com.google.android.apps.gsa.tasks.BackgroundTasksJobService: + Created for: 0ms uptime + Starts: 0, launches: 4 + u0a17: + Wi-Fi network: 52B received, 187B sent (packets 1 received, 3 sent) + Wake lock *net_scheduler* realtime + Wake lock GCoreFlp realtime + Wake lock NlpWakeLock realtime + Wake lock *gms_scheduler*/com.google.android.gms/.stats.PlatformStatsCollectorService realtime + Wake lock *alarm* realtime + Wake lock Wakeful StateMachine: GeofencerStateMachine realtime + Wake lock Icing realtime + Wake lock *gms_scheduler*/com.google.android.gms/.icing.service.IcingGcmTaskService realtime + Sensor 17: 20m 37s 440ms realtime (0 times) + Fg Service for: 20m 37s 440ms + Total running: 20m 37s 440ms + Total cpu time: u=303ms s=63ms + Proc com.google.process.gapps: + CPU: 30ms usr + 10ms krn ; 0ms fg + Proc com.google.android.gms: + CPU: 140ms usr + 20ms krn ; 0ms fg + Proc com.google.android.gms.persistent: + CPU: 290ms usr + 70ms krn ; 0ms fg + Apk com.google.android.gms: + Wakeup alarm *walarm*:com.google.android.gms.gcm.ACTION_CHECK_QUEUE: 1 times + Service com.google.android.gms.wearable.service.WearableControlService: + Created for: 16ms uptime + Starts: 1, launches: 1 + Service com.google.android.gms.config.ConfigService: + Created for: 16ms uptime + Starts: 1, launches: 1 + Service com.google.android.gms.stats.PlatformStatsCollectorService: + Created for: 0ms uptime + Starts: 0, launches: 1 + Service com.google.android.gms.icing.service.IndexWorkerService: + Created for: 84ms uptime + Starts: 2, launches: 2 + Service com.google.android.gms.chimera.PersistentIntentOperationService: + Created for: 7s 11ms uptime + Starts: 1, launches: 1 + Service com.google.android.gms.chimera.GmsIntentOperationService: + Created for: 7s 53ms uptime + Starts: 1, launches: 1 + Service com.google.android.gms.icing.service.IcingGcmTaskService: + Created for: 0ms uptime + Starts: 0, launches: 1 + u0a28: + Cached for: 20m 37s 438ms + Total running: 20m 37s 438ms + u0a32: + Fg Service for: 20m 37s 438ms + Total running: 20m 37s 438ms + u0a38: + Sensor 24: 20m 37s 438ms realtime (0 times) + Sensor 28: 20m 37s 438ms realtime (0 times) + Fg Service for: 20m 37s 438ms + Total running: 20m 37s 438ms + Total cpu time: u=203ms s=17ms + Proc com.android.systemui: + CPU: 150ms usr + 20ms krn ; 0ms fg + u0a40: + Fg Service for: 20m 37s 438ms + Total running: 20m 37s 438ms + u0a43: + Foreground for: 5ms + Cached for: 20m 37s 433ms + Total running: 20m 37s 438ms + Total cpu time: u=6ms s=0ms + Proc com.android.defcontainer: + CPU: 10ms usr + 0ms krn ; 0ms fg + Apk com.android.defcontainer: + Service com.android.defcontainer.DefaultContainerService: + Created for: 0ms uptime + Starts: 0, launches: 1 + u0a44: + Cached for: 20m 37s 438ms + Total running: 20m 37s 438ms + u0a51: + Cached for: 2m 16s 985ms + Total running: 2m 16s 985ms + u0a55: + Wi-Fi network: 2.31KB received, 3.42KB sent (packets 24 received, 33 sent) + WiFi Idle time: 0ms (0.0%) + WiFi Rx time: 0ms (0.0%) + WiFi Tx time: 3ms (100.0%) + WiFi Power drain: 0mAh + Background for: 8s 857ms + Cached for: 20m 28s 581ms + Total running: 20m 37s 438ms + Total cpu time: u=1s 676ms s=3s 483ms + Proc com.verizon.mips.services: + CPU: 1s 330ms usr + 2s 760ms krn ; 0ms fg + Apk com.verizon.mips.services: + Service com.vzw.hss.myverizon.rdd.wifidata.WifiThroughputAlarmService: + Created for: 15ms uptime + Starts: 4, launches: 4 + Service com.vzw.hss.myverizon.rdd.wifidata.WifiScanAlarmService: + Created for: 28ms uptime + Starts: 4, launches: 4 + Service com.vzw.hss.myverizon.rdd.wifidata.RDDWifiDataStateService: + Created for: 8s 789ms uptime + Starts: 4, launches: 4 + u0a57: + Wake lock *net_scheduler* realtime + Background for: 22ms + Cached for: 20m 37s 416ms + Total running: 20m 37s 438ms + Total cpu time: u=0ms s=3ms + Apk com.google.android.apps.gcs: + Service com.google.android.apps.gcs.service.WfaEligibilityRefreshingGcmTaskService: + Created for: 0ms uptime + Starts: 0, launches: 1 + u0a73: + Background for: 2ms + Cached for: 20m 37s 436ms + Total running: 20m 37s 438ms + Proc com.android.chrome: + CPU: 10ms usr + 0ms krn ; 0ms fg + u0a91: + Background for: 20m 37s 438ms + Total running: 20m 37s 438ms + Total cpu time: u=7ms s=0ms + Proc com.google.android.inputmethod.latin: + CPU: 10ms usr + 0ms krn ; 0ms fg + u0a97: + Cached for: 20m 37s 438ms + Total running: 20m 37s 438ms + Total cpu time: u=3ms s=0ms + Proc com.google.android.apps.docs: + CPU: 30ms usr + 10ms krn ; 0ms fg + u0a98: + Background for: 7ms + Cached for: 20m 37s 431ms + Total running: 20m 37s 438ms + Total cpu time: u=14ms s=0ms + Proc com.google.android.apps.photos: + CPU: 50ms usr + 10ms krn ; 0ms fg + u0a104: + Fg Service for: 20m 37s 438ms + Total running: 20m 37s 438ms + u0a120: + Wi-Fi network: 5.70KB received, 8.29KB sent (packets 61 received, 75 sent) + WiFi Idle time: 0ms (0.0%) + WiFi Rx time: 0ms (0.0%) + WiFi Tx time: 8ms (100.0%) + WiFi Power drain: 0mAh + Wake lock *launch* realtime + Wake lock Icing realtime + Foreground activities: 20m 33s 557ms realtime (1 times) + Top for: 20m 33s 557ms + Cached for: 44ms + Total running: 20m 33s 601ms + Total cpu time: u=54s 450ms s=25s 646ms + Proc org.mozilla.geckoview_example: + CPU: 0ms usr + 0ms krn ; 0ms fg + 1 starts + Proc org.mozilla.geckoview_example:tab: + CPU: 0ms usr + 0ms krn ; 0ms fg + 1 starts + Apk org.mozilla.geckoview_example: + Service org.mozilla.gecko.process.GeckoServiceChildProcess$tab: + Created for: 0ms uptime + Starts: 0, launches: 1 + u0a130: + User activity: 1 other + Wake lock TestdroidDeviceService: 1m 4s 45ms full (0 times) realtime + Foreground activities: 3s 827ms realtime (1 times) (running) + Top for: 4s 603ms + Background for: 1m 0s 5ms + Cached for: 19m 32s 830ms + Total running: 20m 37s 438ms + Total cpu time: u=107ms s=7ms + Proc com.bitbar.testdroid.monitor: + CPU: 50ms usr + 10ms krn ; 2s 370ms fg + Proc com.bitbar.testdroid.monitor:monitor: + CPU: 20ms usr + 10ms krn ; 0ms fg + Proc com.bitbar.testdroid.monitor:data: + CPU: 30ms usr + 0ms krn ; 0ms fg
\ No newline at end of file diff --git a/testing/raptor/test/files/fake_binary.exe b/testing/raptor/test/files/fake_binary.exe new file mode 100755 index 0000000000..e69de29bb2 --- /dev/null +++ b/testing/raptor/test/files/fake_binary.exe diff --git a/testing/raptor/test/files/top-info.txt b/testing/raptor/test/files/top-info.txt new file mode 100644 index 0000000000..91e413ccbe --- /dev/null +++ b/testing/raptor/test/files/top-info.txt @@ -0,0 +1,41 @@ +Tasks: 142 total, 1 running, 140 sleeping, 0 stopped, 1 zombie +Mem: 1548824k total, 1234756k used, 314068k free, 37080k buffers +Swap: 0k total, 0k used, 0k free, 552360k cached +200%cpu 122%user 9%nice 50%sys 13%idle 0%iow 0%irq 6%sirq 0%host + PID USER [%CPU]%CPU %MEM TIME+ ARGS +17504 u0_a83 93.7 93.7 14.2 0:12.12 org.mozilla.geckoview_example +17529 u0_a83 43.7 43.7 19.3 0:11.80 org.mozilla.geckoview_example:tab + 7030 u0_a54 28.1 28.1 5.6 0:05.47 com.google.android.tts + 1598 root 9.3 9.3 0.1 0:13.73 dhcpclient -i eth0 + 1667 system 6.2 6.2 9.6 16:10.78 system_server + 1400 system 6.2 6.2 0.2 8:15.20 android.hardware.sensors@1.0-service +17729 shell 3.1 3.1 0.1 0:00.02 top -O %CPU -n 1 + 1411 system 3.1 3.1 0.7 23:06.11 surfaceflinger +17497 shell 0.0 0.0 0.1 0:00.01 sh - +17321 root 0.0 0.0 0.0 0:00.13 [kworker/0:1] +17320 root 0.0 0.0 0.0 0:00.15 [kworker/u4:1] +17306 root 0.0 0.0 0.0 0:00.21 [kworker/u5:1] +16545 root 0.0 0.0 0.0 0:00.17 [kworker/0:0] +16543 root 0.0 0.0 0.0 0:00.15 [kworker/u4:2] +16411 root 0.0 0.0 0.0 0:00.41 [kworker/u5:2] +15827 root 0.0 0.0 0.0 0:00.04 [kworker/1:2] +14998 root 0.0 0.0 0.0 0:00.03 [kworker/1:1] +14996 root 0.0 0.0 0.0 0:00.38 [kworker/0:2] +14790 root 0.0 0.0 0.0 0:01.04 [kworker/u5:0] +14167 root 0.0 0.0 0.0 0:01.32 [kworker/u4:0] +11922 u0_a50 0.0 0.0 6.9 0:00.80 com.google.android.apps.docs +11906 u0_a67 0.0 0.0 5.0 0:00.25 com.google.android.apps.photos +11887 u0_a11 0.0 0.0 4.3 0:00.25 com.android.documentsui +11864 u0_a6 0.0 0.0 3.3 0:00.19 com.android.defcontainer +10866 u0_a15 0.0 0.0 3.3 0:00.04 com.google.android.partnersetup + 8956 u0_a1 0.0 0.0 3.7 0:00.40 com.android.providers.calendar + 8070 u0_a10 0.0 0.0 6.7 0:01.21 com.google.android.gms.unstable + 6638 u0_a10 0.0 0.0 7.4 0:12.89 com.google.android.gms + 2291 u0_a30 0.0 0.0 9.0 5:45.93 com.google.android.googlequicksearchbox:search + 2230 u0_a10 0.0 0.0 3.9 0:02.00 com.google.process.gapps + 2213 u0_a22 0.0 0.0 7.2 4:12.95 com.google.android.apps.nexuslauncher + 2195 u0_a30 0.0 0.0 4.1 0:00.37 com.google.android.googlequicksearchbox:interactor + 2163 u0_a10 0.0 0.0 8.2 1:49.32 com.google.android.gms.persistent + 1882 radio 0.0 0.0 5.1 0:53.61 com.android.phone + 1875 wifi 0.0 0.0 0.4 0:02.25 wpa_supplicant -Dnl80211 -iwlan0 -c/vendor/etc/wifi/wpa_supplicant.conf -g@android:wpa_wla+ + 1828 webview_zyg+ 0.0 0.0 3.0 0:00.45 webview_zygote32
\ No newline at end of file diff --git a/testing/raptor/test/geckoProfileTest.tar b/testing/raptor/test/geckoProfileTest.tar Binary files differnew file mode 100644 index 0000000000..b197606ffd --- /dev/null +++ b/testing/raptor/test/geckoProfileTest.tar diff --git a/testing/raptor/test/python.ini b/testing/raptor/test/python.ini new file mode 100644 index 0000000000..a9f8202cc2 --- /dev/null +++ b/testing/raptor/test/python.ini @@ -0,0 +1,13 @@ +[DEFAULT] +subsuite = raptor + +[test_cmdline.py] +[test_manifest.py] +[test_control_server.py] +[test_utils.py] +[test_playback.py] +[test_print_tests.py] +[test_raptor.py] +[test_cpu.py] +[test_power.py] +[test_gecko_profile.py] diff --git a/testing/raptor/test/test_cmdline.py b/testing/raptor/test/test_cmdline.py new file mode 100644 index 0000000000..60ab84b07a --- /dev/null +++ b/testing/raptor/test/test_cmdline.py @@ -0,0 +1,159 @@ +import os +import sys + +import mozunit +import pytest + +# need this so the raptor unit tests can find raptor/raptor classes +here = os.path.abspath(os.path.dirname(__file__)) +raptor_dir = os.path.join(os.path.dirname(here), "raptor") +sys.path.insert(0, raptor_dir) + +from argparse import ArgumentParser, Namespace + +from cmdline import verify_options + + +def test_verify_options(filedir): + args = Namespace( + app="firefox", + binary="invalid/path", + gecko_profile="False", + page_cycles=1, + page_timeout=60000, + debug="True", + power_test=False, + cpu_test=False, + memory_test=False, + chimera=False, + browsertime_video=False, + browsertime_visualmetrics=False, + fission=True, + fission_mobile=False, + test_bytecode_cache=False, + webext=False, + extra_prefs=[], + ) + parser = ArgumentParser() + + with pytest.raises(SystemExit): + verify_options(parser, args) + + args.binary = os.path.join(filedir, "fake_binary.exe") + verify_options(parser, args) # assert no exception + + args = Namespace( + app="geckoview", + binary="org.mozilla.geckoview_example", + activity="org.mozilla.geckoview_example.GeckoViewActivity", + intent="android.intent.action.MAIN", + gecko_profile="False", + is_release_build=False, + host="sophie", + power_test=False, + cpu_test=False, + memory_test=False, + chimera=False, + browsertime_video=False, + browsertime_visualmetrics=False, + fission=True, + fission_mobile=False, + test_bytecode_cache=False, + webext=False, + extra_prefs=[], + ) + verify_options(parser, args) # assert no exception + + args = Namespace( + app="refbrow", + binary="org.mozilla.reference.browser", + activity="org.mozilla.reference.browser.BrowserTestActivity", + intent="android.intent.action.MAIN", + gecko_profile="False", + is_release_build=False, + host="sophie", + power_test=False, + cpu_test=False, + memory_test=False, + chimera=False, + browsertime_video=False, + browsertime_visualmetrics=False, + fission=True, + fission_mobile=False, + test_bytecode_cache=False, + webext=False, + extra_prefs=[], + ) + verify_options(parser, args) # assert no exception + + args = Namespace( + app="fenix", + binary="org.mozilla.fenix.browser", + activity="org.mozilla.fenix.browser.BrowserPerformanceTestActivity", + intent="android.intent.action.VIEW", + gecko_profile="False", + is_release_build=False, + host="sophie", + power_test=False, + cpu_test=False, + memory_test=False, + chimera=False, + browsertime_video=False, + browsertime_visualmetrics=False, + fission=True, + fission_mobile=False, + test_bytecode_cache=False, + webext=False, + extra_prefs=[], + ) + verify_options(parser, args) # assert no exception + + args = Namespace( + app="geckoview", + binary="org.mozilla.geckoview_example", + activity="org.mozilla.geckoview_example.GeckoViewActivity", + intent="android.intent.action.MAIN", + gecko_profile="False", + is_release_build=False, + host="sophie", + power_test=False, + cpu_test=True, + memory_test=False, + chimera=False, + browsertime_video=False, + browsertime_visualmetrics=False, + fission=True, + fission_mobile=False, + test_bytecode_cache=False, + webext=False, + extra_prefs=[], + ) + verify_options(parser, args) # assert no exception + + args = Namespace( + app="refbrow", + binary="org.mozilla.reference.browser", + activity=None, + intent="android.intent.action.MAIN", + gecko_profile="False", + is_release_build=False, + host="sophie", + power_test=False, + cpu_test=False, + memory_test=False, + chimera=False, + browsertime_video=False, + browsertime_visualmetrics=False, + fission=True, + fission_mobile=False, + test_bytecode_cache=False, + webext=False, + extra_prefs=[], + ) + parser = ArgumentParser() + + verify_options(parser, args) # also will work as uses default activity + + +if __name__ == "__main__": + mozunit.main() diff --git a/testing/raptor/test/test_control_server.py b/testing/raptor/test/test_control_server.py new file mode 100644 index 0000000000..83d47a4d3f --- /dev/null +++ b/testing/raptor/test/test_control_server.py @@ -0,0 +1,189 @@ +import os +import shutil +import sys +from unittest import mock + +import mozunit +import requests + +try: + from http.server import HTTPServer # py3 +except ImportError: + from BaseHTTPServer import HTTPServer # py2 + +from mozlog.structuredlog import StructuredLogger, set_default_logger +from raptor.control_server import RaptorControlServer + +# need this so the raptor unit tests can find output & filter classes +here = os.path.abspath(os.path.dirname(__file__)) +raptor_dir = os.path.join(os.path.dirname(here), "raptor") +sys.path.insert(0, raptor_dir) + +from raptor.results import RaptorResultsHandler + +set_default_logger(StructuredLogger("test_control_server")) + + +def clear_cache(): + # remove the condprof download cache + from condprof.client import CONDPROF_CACHE # noqa + + if os.path.exists(CONDPROF_CACHE): + shutil.rmtree(CONDPROF_CACHE) + + +clear_cache() + + +def test_start_and_stop(): + + results_handler = RaptorResultsHandler() + control = RaptorControlServer(results_handler) + + assert control.server is None + control.start() + assert isinstance(control.server, HTTPServer) + assert control.server.fileno() + assert control._server_thread.is_alive() + + control.stop() + assert not control._server_thread.is_alive() + + +def test_server_get_timeout(raptor): + test_name = "test-name" + url = "test-url" + metrics = {"metric1": False, "metric2": True, "metric3": True} + page_cycle = 1 + + def post_state(): + requests.post( + "http://127.0.0.1:%s/" % raptor.control_server.port, + json={ + "type": "webext_raptor-page-timeout", + "data": [test_name, url, page_cycle, metrics], + }, + ) + + assert len(raptor.results_handler.page_timeout_list) == 0 + + post_state() + + assert len(raptor.results_handler.page_timeout_list) == 1 + + timeout_details = raptor.results_handler.page_timeout_list[0] + assert timeout_details["test_name"] == test_name + assert timeout_details["url"] == url + + pending_metrics = [k for k, v in metrics.items() if v] + assert len(timeout_details["pending_metrics"].split(", ")) == len(pending_metrics) + + +def test_server_android_app_backgrounding(): + # Mock the background and foreground functions + with mock.patch.object( + RaptorControlServer, "background_app", return_value=True + ) as _, mock.patch.object( + RaptorControlServer, "foreground_app", return_value=True + ) as _: + + results_handler = RaptorResultsHandler() + control = RaptorControlServer(results_handler) + control.backgrounded = False + + control.start() + assert control._server_thread.is_alive() + + def post_start_background(): + requests.post( + "http://127.0.0.1:%s/" % control.port, + json={ + "type": "webext_start_background", + "data": "starting background app", + }, + ) + + def post_end_background(): + requests.post( + "http://127.0.0.1:%s/" % control.port, + json={"type": "webext_end_background", "data": "ending background app"}, + ) + + # Test that app is backgrounded + post_start_background() + control.background_app.assert_called() + + # Test that app is returned to foreground + post_end_background() + control.foreground_app.assert_called() + + # Make sure the control server stops after these requests + control.stop() + assert not control._server_thread.is_alive() + + +def test_server_wait_states(raptor): + import datetime + + def post_state(): + requests.post( + "http://127.0.0.1:%s/" % raptor.control_server.port, + json={"type": "webext_status", "data": "test status"}, + ) + + wait_time = 5 + message_state = "webext_status/test status" + rhc = raptor.control_server.server.RequestHandlerClass + + # Test initial state + assert rhc.wait_after_messages == {} + assert rhc.waiting_in_state is None + assert rhc.wait_timeout == 60 + assert raptor.control_server_wait_get() == "None" + + # Test setting a state + assert raptor.control_server_wait_set(message_state) == "" + assert message_state in rhc.wait_after_messages + assert rhc.wait_after_messages[message_state] + + # Test clearing a non-existent state + assert raptor.control_server_wait_clear("nothing") == "" + assert message_state in rhc.wait_after_messages + + # Test clearing a state + assert raptor.control_server_wait_clear(message_state) == "" + assert message_state not in rhc.wait_after_messages + + # Test clearing all states + assert raptor.control_server_wait_set(message_state) == "" + assert message_state in rhc.wait_after_messages + assert raptor.control_server_wait_clear("all") == "" + assert rhc.wait_after_messages == {} + + # Test wait timeout + # Block on post request + assert raptor.control_server_wait_set(message_state) == "" + assert rhc.wait_after_messages[message_state] + assert raptor.control_server_wait_timeout(wait_time) == "" + assert rhc.wait_timeout == wait_time + start = datetime.datetime.now() + post_state() + assert datetime.datetime.now() - start < datetime.timedelta(seconds=wait_time + 2) + assert raptor.control_server_wait_get() == "None" + assert message_state not in rhc.wait_after_messages + + raptor.clean_up() + assert not raptor.control_server._server_thread.is_alive() + + +def test_clean_up_stop_server(raptor): + assert raptor.control_server._server_thread.is_alive() + assert raptor.control_server.port is not None + assert raptor.control_server.server is not None + + raptor.clean_up() + assert not raptor.control_server._server_thread.is_alive() + + +if __name__ == "__main__": + mozunit.main() diff --git a/testing/raptor/test/test_cpu.py b/testing/raptor/test/test_cpu.py new file mode 100644 index 0000000000..c60d5c6577 --- /dev/null +++ b/testing/raptor/test/test_cpu.py @@ -0,0 +1,196 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +import os +import sys +from unittest import mock + +import mozunit + +# need this so the raptor unit tests can find output & filter classes +here = os.path.abspath(os.path.dirname(__file__)) +raptor_dir = os.path.join(os.path.dirname(here), "raptor") +sys.path.insert(0, raptor_dir) + +import cpu +from webextension import WebExtensionAndroid + + +def test_no_device(): + raptor = WebExtensionAndroid( + "geckoview", + "org.mozilla.org.mozilla.geckoview_example", + cpu_test=True, + ) + raptor.device = None + resp = cpu.start_android_cpu_profiler(raptor) + assert resp is None + + +def test_usage_with_invalid_data_returns_zero(): + with mock.patch("mozdevice.adb.ADBDevice") as device: + with mock.patch("control_server.RaptorControlServer") as control_server: + # Create a device that returns invalid data + device.shell_output.side_effect = ["8.0.0", "geckoview"] + device._verbose = True + + # Create a control server + control_server.cpu_test = True + control_server.device = device + raptor = WebExtensionAndroid("geckoview", "org.mozilla.geckoview_example") + raptor.config["cpu_test"] = True + raptor.control_server = control_server + raptor.device = device + + cpu_profiler = cpu.AndroidCPUProfiler(raptor) + cpu_profiler.get_app_cpu_usage() + + # Verify the call to submit data was made + avg_cpuinfo_data = { + "type": "cpu", + "test": "usage_with_invalid_data_returns_zero-avg", + "unit": "%", + "values": {"avg": 0}, + } + min_cpuinfo_data = { + "type": "cpu", + "test": "usage_with_invalid_data_returns_zero-min", + "unit": "%", + "values": {"min": 0}, + } + max_cpuinfo_data = { + "type": "cpu", + "test": "usage_with_invalid_data_returns_zero-max", + "unit": "%", + "values": {"max": 0}, + } + + cpu_profiler.generate_android_cpu_profile( + "usage_with_invalid_data_returns_zero" + ) + control_server.submit_supporting_data.assert_has_calls( + [ + mock.call(avg_cpuinfo_data), + mock.call(min_cpuinfo_data), + mock.call(max_cpuinfo_data), + ] + ) + + +def test_usage_with_output(): + with mock.patch("mozdevice.adb.ADBDevice") as device: + with mock.patch("control_server.RaptorControlServer") as control_server: + # Override the shell output with sample CPU usage details + filepath = os.path.abspath(os.path.dirname(__file__)) + "/files/" + with open(filepath + "top-info.txt", "r") as f: + test_data = f.read() + device.shell_output.side_effect = ["8.0.0", test_data] + device._verbose = True + + # Create a control server + control_server.cpu_test = True + control_server.test_name = "cpuunittest" + control_server.device = device + control_server.app_name = "org.mozilla.geckoview_example" + raptor = WebExtensionAndroid("geckoview", "org.mozilla.geckoview_example") + raptor.device = device + raptor.config["cpu_test"] = True + raptor.control_server = control_server + + cpu_profiler = cpu.AndroidCPUProfiler(raptor) + cpu_profiler.polls.append(cpu_profiler.get_app_cpu_usage()) + cpu_profiler.polls.append(0) + + # Verify the response contains our expected CPU % of 93.7 + avg_cpuinfo_data = { + "type": "cpu", + "test": "usage_with_integer_cpu_info_output-avg", + "unit": "%", + "values": {"avg": 93.7 / 2}, + } + min_cpuinfo_data = { + "type": "cpu", + "test": "usage_with_integer_cpu_info_output-min", + "unit": "%", + "values": {"min": 0}, + } + max_cpuinfo_data = { + "type": "cpu", + "test": "usage_with_integer_cpu_info_output-max", + "unit": "%", + "values": {"max": 93.7}, + } + + cpu_profiler.generate_android_cpu_profile( + "usage_with_integer_cpu_info_output" + ) + control_server.submit_supporting_data.assert_has_calls( + [ + mock.call(avg_cpuinfo_data), + mock.call(min_cpuinfo_data), + mock.call(max_cpuinfo_data), + ] + ) + + +def test_usage_with_fallback(): + with mock.patch("mozdevice.adb.ADBDevice") as device: + with mock.patch("control_server.RaptorControlServer") as control_server: + device._verbose = True + + # Return what our shell call to dumpsys would give us + shell_output = ( + " 31093 u0_a196 10 -10 8% S " + + "66 1392100K 137012K fg org.mozilla.geckoview_example" + ) + + # We set the version to be less than Android 8 + device.shell_output.side_effect = ["7.0.0", shell_output] + + # Create a control server + control_server.cpu_test = True + control_server.test_name = "cpuunittest" + control_server.device = device + control_server.app_name = "org.mozilla.geckoview_example" + raptor = WebExtensionAndroid("geckoview", "org.mozilla.geckoview_example") + raptor.device = device + raptor.config["cpu_test"] = True + raptor.control_server = control_server + + cpu_profiler = cpu.AndroidCPUProfiler(raptor) + cpu_profiler.polls.append(cpu_profiler.get_app_cpu_usage()) + cpu_profiler.polls.append(0) + + # Verify the response contains our expected CPU % of 8 + # pylint --py3k W1619 + avg_cpuinfo_data = { + "type": "cpu", + "test": "usage_with_fallback-avg", + "unit": "%", + "values": {"avg": 8 / 2}, + } + min_cpuinfo_data = { + "type": "cpu", + "test": "usage_with_fallback-min", + "unit": "%", + "values": {"min": 0}, + } + max_cpuinfo_data = { + "type": "cpu", + "test": "usage_with_fallback-max", + "unit": "%", + "values": {"max": 8}, + } + + cpu_profiler.generate_android_cpu_profile("usage_with_fallback") + control_server.submit_supporting_data.assert_has_calls( + [ + mock.call(avg_cpuinfo_data), + mock.call(min_cpuinfo_data), + mock.call(max_cpuinfo_data), + ] + ) + + +if __name__ == "__main__": + mozunit.main() diff --git a/testing/raptor/test/test_gecko_profile.py b/testing/raptor/test/test_gecko_profile.py new file mode 100644 index 0000000000..a62a399ce1 --- /dev/null +++ b/testing/raptor/test/test_gecko_profile.py @@ -0,0 +1,54 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import os +import shutil +import sys +import tarfile +import tempfile + +import mozunit + +# need this so raptor imports work both from /raptor and via mach +here = os.path.abspath(os.path.dirname(__file__)) + +raptor_dir = os.path.join(os.path.dirname(here), "raptor") +sys.path.insert(0, raptor_dir) + +from gecko_profile import GeckoProfile + + +def test_browsertime_profiling(): + result_dir = tempfile.mkdtemp() + # untar geckoProfile.tar + with tarfile.open(os.path.join(here, "geckoProfileTest.tar")) as f: + f.extractall(path=result_dir) + + # Makes sure we can run the profile process against a browsertime-generated + # profile (geckoProfile-1.json in this test dir) + upload_dir = tempfile.mkdtemp() + symbols_path = tempfile.mkdtemp() + raptor_config = { + "symbols_path": symbols_path, + "browsertime": True, + "browsertime_result_dir": os.path.join(result_dir, "amazon"), + } + test_config = {"name": "tp6"} + try: + profile = GeckoProfile(upload_dir, raptor_config, test_config) + profile.symbolicate() + profile.clean() + arcname = os.environ["RAPTOR_LATEST_GECKO_PROFILE_ARCHIVE"] + assert os.stat(arcname).st_size > 1000000, "We got a 1mb+ zip" + except Exception: + assert False, "Symbolication failed!" + raise + finally: + shutil.rmtree(upload_dir) + shutil.rmtree(symbols_path) + shutil.rmtree(result_dir) + + +if __name__ == "__main__": + mozunit.main() diff --git a/testing/raptor/test/test_manifest.py b/testing/raptor/test/test_manifest.py new file mode 100644 index 0000000000..6f18270ffd --- /dev/null +++ b/testing/raptor/test/test_manifest.py @@ -0,0 +1,441 @@ +import os +import sys + +import mozinfo +import mozunit +import pytest +from six.moves.urllib.parse import parse_qs, urlsplit + +# need this so raptor imports work both from /raptor and via mach +here = os.path.abspath(os.path.dirname(__file__)) + +raptor_dir = os.path.join(os.path.dirname(here), "raptor") +sys.path.insert(0, raptor_dir) + +from manifest import ( + add_test_url_params, + get_browser_test_list, + get_raptor_test_list, + validate_test_ini, +) + +# some test details (test INIs) +VALID_MANIFESTS = [ + { + # page load test with local playback + "alert_on": "fcp", + "alert_threshold": 2.0, + "apps": "firefox", + "lower_is_better": True, + "manifest": "valid_details_0", + "measure": ["fnbpaint", "fcp"], + "page_cycles": 25, + "playback": "mitmproxy", + "playback_pageset_manifest": "pageset.manifest", + "test_url": "http://www.test-url/goes/here", + "type": "pageload", + "unit": "ms", + }, + { + # test optional settings with None + "alert_threshold": 2.0, + "apps": "firefox", + "lower_is_better": True, + "manifest": "valid_details_1", + "measure": "fnbpaint, fcb", + "page_cycles": 25, + "test_url": "http://www.test-url/goes/here", + "type": "pageload", + "unit": "ms", + "alert_change_type": None, + "alert_on": None, + "playback": None, + }, + { + # page load test for geckoview + "alert_threshold": 2.0, + "apps": "geckoview", + "browser_cycles": 10, + "lower_is_better": False, + "manifest": "valid_details_2", + "measure": "fcp", + "page_cycles": 1, + "test_url": "http://www.test-url/goes/here", + "type": "pageload", + "unit": "score", + }, + { + # benchmark test for chrome + "alert_threshold": 2.0, + "apps": "chrome", + "lower_is_better": False, + "manifest": "valid_details_1", + "measure": "fcp", + "page_cycles": 5, + "test_url": "http://www.test-url/goes/here", + "type": "benchmark", + "unit": "score", + }, + { + # benchmark test for chromium + "alert_threshold": 2.0, + "apps": "chromium", + "lower_is_better": False, + "manifest": "valid_details_1", + "measure": "fcp", + "page_cycles": 5, + "test_url": "http://www.test-url/goes/here", + "type": "benchmark", + "unit": "score", + }, +] + +INVALID_MANIFESTS = [ + { + "alert_threshold": 2.0, + "apps": "firefox", + "lower_is_better": True, + "manifest": "invalid_details_0", + "page_cycles": 25, + "playback": "mitmproxy", + "playback_pageset_manifest": "pageset.manifest", + "test_url": "http://www.test-url/goes/here", + "type": "pageload", + "unit": "ms", + }, + { + "alert_threshold": 2.0, + "apps": "chrome", + "lower_is_better": True, + "manifest": "invalid_details_1", + "measure": "fnbpaint, fcp", + "page_cycles": 25, + "playback": "mitmproxy", + "test_url": "http://www.test-url/goes/here", + "type": "pageload", + "unit": "ms", + }, + { + "alert_threshold": 2.0, + "apps": "chromium", + "lower_is_better": True, + "manifest": "invalid_details_1", + "measure": "fnbpaint, fcp", + "page_cycles": 25, + "playback": "mitmproxy", + "test_url": "http://www.test-url/goes/here", + "type": "pageload", + "unit": "ms", + }, + { + "alert_on": "nope", + "alert_threshold": 2.0, + "apps": "firefox", + "lower_is_better": True, + "manifest": "invalid_details_2", + "measure": "fnbpaint, fcp", + "page_cycles": 25, + "playback": "mitmproxy", + "playback_pageset_manifest": "pageset.manifest", + "test_url": "http://www.test-url/goes/here", + "type": "pageload", + "unit": "ms", + }, +] + + +@pytest.mark.parametrize( + "app", ["firefox", "chrome", "chromium", "geckoview", "refbrow", "fenix"] +) +def test_get_browser_test_list(app): + test_list = get_browser_test_list(app, run_local=True) + assert len(test_list) > 0 + + +@pytest.mark.parametrize("test_details", VALID_MANIFESTS) +def test_validate_test_ini_valid(test_details): + assert validate_test_ini(test_details) + + +@pytest.mark.parametrize("test_details", INVALID_MANIFESTS) +def test_validate_test_ini_invalid(test_details): + assert not (validate_test_ini(test_details)) + + +def test_get_raptor_test_list_firefox(create_args): + args = create_args(browser_cycles=1) + + test_list = get_raptor_test_list(args, mozinfo.os) + assert len(test_list) == 4 + + subtests = [ + "raptor-tp6-unittest-google-firefox", + "raptor-tp6-unittest-amazon-firefox", + "raptor-tp6-unittest-facebook-firefox", + "raptor-tp6-unittest-youtube-firefox", + ] + + for next_subtest in test_list: + assert next_subtest["name"] in subtests + + +def test_get_raptor_test_list_chrome(create_args): + args = create_args(app="chrome", test="speedometer", browser_cycles=1) + + test_list = get_raptor_test_list(args, mozinfo.os) + assert len(test_list) == 1 + assert test_list[0]["name"] == "speedometer" + + +def test_get_raptor_test_list_geckoview(create_args): + args = create_args(app="geckoview", test="unity-webgl", browser_cycles=1) + + test_list = get_raptor_test_list(args, mozinfo.os) + assert len(test_list) == 1 + assert test_list[0]["name"] == "unity-webgl" + + +def test_get_raptor_test_list_gecko_profiling_enabled(create_args): + args = create_args(test="amazon", gecko_profile=True, browser_cycles=1) + + test_list = get_raptor_test_list(args, mozinfo.os) + assert len(test_list) == 1 + assert test_list[0]["name"] == "amazon" + assert test_list[0]["gecko_profile"] is True + assert test_list[0].get("gecko_profile_entries") == "14000000" + assert test_list[0].get("gecko_profile_interval") == "1" + assert test_list[0].get("gecko_profile_threads") is None + assert test_list[0].get("gecko_profile_features") is None + + +def test_get_raptor_test_list_gecko_profiling_enabled_args_override(create_args): + args = create_args( + test="amazon", + gecko_profile=True, + gecko_profile_entries=42, + gecko_profile_interval=100, + gecko_profile_threads="Foo", + gecko_profile_features="Mood,UserNetWorth", + browser_cycles=1, + ) + + test_list = get_raptor_test_list(args, mozinfo.os) + assert len(test_list) == 1 + assert test_list[0]["name"] == "amazon" + assert test_list[0]["gecko_profile"] is True + assert test_list[0]["gecko_profile_entries"] == "42" + assert test_list[0]["gecko_profile_interval"] == "100" + assert test_list[0]["gecko_profile_threads"] == "Foo" + assert test_list[0]["gecko_profile_features"] == "Mood,UserNetWorth" + + +def test_get_raptor_test_list_gecko_profiling_enabled_extra_args_override(create_args): + args = create_args( + test="amazon", + gecko_profile=True, + gecko_profile_entries=42, + gecko_profile_interval=100, + gecko_profile_extra_threads=["Foo", "Oof"], + gecko_profile_threads="String,Rope", + browser_cycles=1, + ) + + test_list = get_raptor_test_list(args, mozinfo.os) + assert len(test_list) == 1 + assert test_list[0]["name"] == "amazon" + assert test_list[0]["gecko_profile"] is True + assert test_list[0]["gecko_profile_entries"] == "42" + assert test_list[0]["gecko_profile_interval"] == "100" + assert test_list[0]["gecko_profile_threads"] == "String,Rope,Foo,Oof" + + +def test_get_raptor_test_list_gecko_profiling_disabled(create_args): + args = create_args( + test="amazon", + gecko_profile=False, + gecko_profile_entries=42, + gecko_profile_interval=100, + gecko_profile_threads=["Foo"], + gecko_profile_features=["Temperature"], + browser_cycles=1, + ) + + test_list = get_raptor_test_list(args, mozinfo.os) + assert len(test_list) == 1 + assert test_list[0]["name"] == "amazon" + assert test_list[0].get("gecko_profile") is None + assert test_list[0].get("gecko_profile_entries") is None + assert test_list[0].get("gecko_profile_interval") is None + assert test_list[0].get("gecko_profile_threads") is None + assert test_list[0].get("gecko_profile_features") is None + + +def test_get_raptor_test_list_gecko_profiling_disabled_args_override(create_args): + args = create_args( + test="amazon", + gecko_profile=False, + gecko_profile_entries=42, + gecko_profile_interval=100, + gecko_profile_threads=["Foo"], + gecko_profile_features=["Temperature"], + browser_cycles=1, + ) + + test_list = get_raptor_test_list(args, mozinfo.os) + assert len(test_list) == 1 + assert test_list[0]["name"] == "amazon" + assert test_list[0].get("gecko_profile") is None + assert test_list[0].get("gecko_profile_entries") is None + assert test_list[0].get("gecko_profile_interval") is None + assert test_list[0].get("gecko_profile_threads") is None + assert test_list[0].get("gecko_profile_features") is None + + +def test_get_raptor_test_list_extra_profiler_run_enabled(create_args): + args = create_args(test="amazon", extra_profiler_run=True, browser_cycles=1) + + test_list = get_raptor_test_list(args, mozinfo.os) + assert len(test_list) == 1 + assert test_list[0]["name"] == "amazon" + assert test_list[0]["extra_profiler_run"] is True + + +def test_get_raptor_test_list_extra_profiler_run_disabled(create_args): + args = create_args(test="amazon", browser_cycles=1) + + test_list = get_raptor_test_list(args, mozinfo.os) + assert len(test_list) == 1 + assert test_list[0]["name"] == "amazon" + assert test_list[0].get("extra_profiler_run") is None + + +def test_get_raptor_test_list_debug_mode(create_args): + args = create_args(test="amazon", debug_mode=True, browser_cycles=1) + + test_list = get_raptor_test_list(args, mozinfo.os) + assert len(test_list) == 1 + assert test_list[0]["name"] == "amazon" + assert test_list[0]["debug_mode"] is True + assert test_list[0]["page_cycles"] == 2 + + +def test_get_raptor_test_list_using_live_sites(create_args): + args = create_args(test="amazon", live_sites=True, browser_cycles=1) + + test_list = get_raptor_test_list(args, mozinfo.os) + assert len(test_list) == 1 + assert test_list[0]["name"] == "amazon" + assert test_list[0]["use_live_sites"] == "true" + assert test_list[0]["playback"] is None + + +def test_get_raptor_test_list_using_collect_perfstats(create_args): + args = create_args(test="amazon", collect_perfstats=True, browser_cycles=1) + + test_list = get_raptor_test_list(args, mozinfo.os) + assert len(test_list) == 1 + assert test_list[0]["name"] == "amazon" + assert test_list[0]["perfstats"] == "true" + + +def test_get_raptor_test_list_override_page_cycles(create_args): + args = create_args(test="amazon", page_cycles=99, browser_cycles=1) + + test_list = get_raptor_test_list(args, mozinfo.os) + assert len(test_list) == 1 + assert test_list[0]["name"] == "amazon" + assert test_list[0]["page_cycles"] == 99 + + +def test_get_raptor_test_list_override_page_timeout(create_args): + args = create_args(test="amazon", page_timeout=9999, browser_cycles=1) + + test_list = get_raptor_test_list(args, mozinfo.os) + assert len(test_list) == 1 + assert test_list[0]["name"] == "amazon" + assert test_list[0]["page_timeout"] == 9999 + + +def test_get_raptor_test_list_add_test_url_params(create_args): + args = create_args(test="amazon", test_url_params="c=4", browser_cycles=1) + + test_list = get_raptor_test_list(args, mozinfo.os) + assert len(test_list) == 1 + assert test_list[0]["name"] == "amazon" + query_params = parse_qs(urlsplit(test_list[0]["test_url"]).query) + assert query_params.get("c") == ["4"] + + +def test_get_raptor_test_list_refbrow(create_args): + args = create_args(app="refbrow", test="speedometer", browser_cycles=1) + + test_list = get_raptor_test_list(args, mozinfo.os) + assert len(test_list) == 1 + assert test_list[0]["name"] == "speedometer" + + +def test_get_raptor_test_list_fenix(create_args): + args = create_args(app="fenix", test="speedometer", browser_cycles=1) + + test_list = get_raptor_test_list(args, mozinfo.os) + assert len(test_list) == 1 + assert test_list[0]["name"] == "speedometer" + + +def test_add_test_url_params_with_single_extra_param(): + initial_test_url = "http://test.com?a=1&b=2" + extra_params = "c=3" + + result = add_test_url_params(initial_test_url, extra_params) + + expected_params = {"a": ["1"], "b": ["2"], "c": ["3"]} + actual_params = parse_qs(urlsplit(result).query) + assert actual_params == expected_params + + +def test_add_test_url_params_with_multiple_extra_param(): + initial_test_url = "http://test.com?a=1&b=2" + extra_params = "c=3&d=4" + + result = add_test_url_params(initial_test_url, extra_params) + + expected_params = {"a": ["1"], "b": ["2"], "c": ["3"], "d": ["4"]} + actual_params = parse_qs(urlsplit(result).query) + assert actual_params == expected_params + + +def test_add_test_url_params_without_params_in_url(): + initial_test_url = "http://test.com" + extra_params = "c=3" + + result = add_test_url_params(initial_test_url, extra_params) + + expected_params = {"c": ["3"]} + actual_params = parse_qs(urlsplit(result).query) + assert actual_params == expected_params + + +def test_add_test_url_params_overwrites_single_param(): + initial_test_url = "http://test.com?a=1&b=2" + extra_params = "b=3" + + result = add_test_url_params(initial_test_url, extra_params) + + expected_params = {"a": ["1"], "b": ["3"]} + actual_params = parse_qs(urlsplit(result).query) + assert actual_params == expected_params + + +def test_add_test_url_params_overwrites_multiple_param(): + initial_test_url = "http://test.com?a=1&b=2&c=3" + extra_params = "c=4&b=5" + + result = add_test_url_params(initial_test_url, extra_params) + + expected_params = {"a": ["1"], "b": ["5"], "c": ["4"]} + actual_params = parse_qs(urlsplit(result).query) + assert actual_params == expected_params + + +if __name__ == "__main__": + mozunit.main() diff --git a/testing/raptor/test/test_playback.py b/testing/raptor/test/test_playback.py new file mode 100644 index 0000000000..c01065ea83 --- /dev/null +++ b/testing/raptor/test/test_playback.py @@ -0,0 +1,65 @@ +import os +import time + +import mozinfo +import mozunit +from mozlog.structuredlog import StructuredLogger, set_default_logger + +# need this so raptor imports work both from /raptor and via mach +here = os.path.abspath(os.path.dirname(__file__)) + +set_default_logger(StructuredLogger("test_playback")) + +from mozproxy import get_playback +from mozproxy.backends.mitm.desktop import MitmproxyDesktop + +config = {} + +run_local = True +if os.environ.get("TOOLTOOLCACHE") is None: + run_local = False + + +def test_get_playback(get_binary): + config["platform"] = mozinfo.os + if "win" in config["platform"]: + # this test is not yet supported on windows + assert True + return + config["obj_path"] = os.path.dirname(get_binary("firefox")) + config["playback_tool"] = "mitmproxy" + config["playback_version"] = "7.0.4" + config["playback_files"] = [ + os.path.join( + os.path.dirname(os.path.abspath(os.path.dirname(__file__))), + "raptor", + "tooltool-manifests", + "playback", + "mitm7-linux-firefox-amazon.manifest", + ) + ] + config["binary"] = get_binary("firefox") + config["run_local"] = run_local + config["app"] = "firefox" + config["host"] = "127.0.0.1" + + playback = get_playback(config) + assert isinstance(playback, MitmproxyDesktop) + playback.start() + time.sleep(1) + playback.stop() + + +def test_get_unsupported_playback(): + config["playback_tool"] = "unsupported" + playback = get_playback(config) + assert playback is None + + +def test_get_playback_missing_tool_name(): + playback = get_playback(config) + assert playback is None + + +if __name__ == "__main__": + mozunit.main() diff --git a/testing/raptor/test/test_power.py b/testing/raptor/test/test_power.py new file mode 100644 index 0000000000..52b87f0446 --- /dev/null +++ b/testing/raptor/test/test_power.py @@ -0,0 +1,248 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +import os +import sys +import tempfile +from unittest import mock + +import mozunit + +# need this so raptor imports work both from /raptor and via mach +here = os.path.abspath(os.path.dirname(__file__)) + +raptor_dir = os.path.join(os.path.dirname(here), "raptor") +sys.path.insert(0, raptor_dir) + +import power +from webextension import WebExtensionAndroid + + +def test_android7_power(): + if not os.getenv("MOZ_UPLOAD_DIR"): + os.environ["MOZ_UPLOAD_DIR"] = tempfile.mkdtemp() + + with mock.patch("mozdevice.adb.ADBDevice") as device: + with mock.patch("control_server.RaptorControlServer") as control_server: + # Override the shell output with sample CPU usage details + filepath = os.path.abspath(os.path.dirname(__file__)) + "/files/" + f = open(filepath + "batterystats-android-7.txt", "r") + batterystats_return_value = f.read() + + # Multiple shell output calls are performed + # and only those with non-None output are required + device.shell_output.return_value = None + device.shell_output.side_effect = [ + None, + None, + "Test value", + "Test value", + batterystats_return_value, + "7.0.0", + ] + + device._verbose = True + device.version = 7 + + # Create a control server + control_server.power_test = True + control_server.test_name = "gve-pytest" + control_server.device = device + control_server.app_name = "org.mozilla.geckoview_example" + web_extension = WebExtensionAndroid( + "geckoview", "org.mozilla.geckoview_example", power_test=True + ) + web_extension.device = device + web_extension.config["power_test"] = True + web_extension.control_server = control_server + web_extension.power_test_time = 20 # minutes + web_extension.os_baseline_data = { + "type": "power", + "test": "gve-pytest", + "unit": "mAh", + "values": {"cpu": float(5), "wifi": float(5), "screen": float(5)}, + } + + # Verify the response contains our expected calculations + # (no proportional measure on android 7) + power_data = { + "type": "power", + "test": "gve-pytest", + "unit": "mAh", + "values": { + "cpu": float(14.5), + "wifi": float(0.132), + "screen": float(70.7), + }, + } + + pc_data = { + "type": "power", + "test": "gve-pytest-%change", + "unit": "%", + "values": { + "cpu": float(14.5), + "wifi": float(0.132000000000005), + "screen": float(70.70000000000002), + }, + } + + power.finish_android_power_test(web_extension, "gve-pytest") + + control_server.submit_supporting_data.assert_has_calls( + [ + mock.call(power_data), + mock.call(pc_data), + mock.call(web_extension.os_baseline_data), + ] + ) + + +def test_android8_power(): + if not os.getenv("MOZ_UPLOAD_DIR"): + os.environ["MOZ_UPLOAD_DIR"] = tempfile.mkdtemp() + + with mock.patch("mozdevice.adb.ADBDevice") as device: + with mock.patch("control_server.RaptorControlServer") as control_server: + # Override the shell output with sample CPU usage details + filepath = os.path.abspath(os.path.dirname(__file__)) + "/files/" + f = open(filepath + "batterystats-android-8.txt", "r") + batterystats_return_value = f.read() + print(type(batterystats_return_value)) + + # Multiple shell output calls are performed + # and only those with non-None output are required + device.shell_output.return_value = None + device.shell_output.side_effect = [ + None, + None, + "Test value", + "Test value", + batterystats_return_value, + "8.0.0", + ] + + device._verbose = True + device.version = 8 + + # Create a control server + control_server.power_test = True + control_server.test_name = "gve-pytest" + control_server.device = device + control_server.app_name = "org.mozilla.geckoview_example" + web_extension = WebExtensionAndroid( + "geckoview", "org.mozilla.geckoview_example", power_test=True + ) + web_extension.device = device + web_extension.config["power_test"] = True + web_extension.control_server = control_server + web_extension.power_test_time = 20 # minutes + web_extension.os_baseline_data = { + "type": "power", + "test": "gve-pytest", + "unit": "mAh", + "values": { + "cpu": float(5), + "wifi": float(5), + "screen": float(5), + "proportional": float(5), + }, + } + + # Verify the response contains our expected calculations + power_data = { + "type": "power", + "test": "gve-pytest", + "unit": "mAh", + "values": { + "cpu": float(4.7), + "wifi": float(0.000556), + "screen": float(51.5), + "proportional": float(11.2), + }, + } + + pc_data = { + "type": "power", + "test": "gve-pytest-%change", + "unit": "%", + "values": { + "cpu": float(4.700000000000017), + "wifi": float(0.0005559999999888987), + "screen": float(51.5), + "proportional": float(11.199999999999989), + }, + } + + power.finish_android_power_test(web_extension, "gve-pytest") + + control_server.submit_supporting_data.assert_has_calls( + [ + mock.call(power_data), + mock.call(pc_data), + mock.call(web_extension.os_baseline_data), + ] + ) + + +def test_androidos_baseline_power(): + if not os.getenv("MOZ_UPLOAD_DIR"): + os.environ["MOZ_UPLOAD_DIR"] = tempfile.mkdtemp() + + with mock.patch("mozdevice.adb.ADBDevice") as device: + with mock.patch("control_server.RaptorControlServer") as control_server: + # Override the shell output with sample CPU usage details + filepath = os.path.abspath(os.path.dirname(__file__)) + "/files/" + f = open(filepath + "batterystats-android-8.txt", "r") + batterystats_return_value = f.read() + + # Multiple shell output calls are performed + # and only those with non-None output are required + device.shell_output.return_value = None + device.shell_output.side_effect = [ + None, + None, + "Test value", + "Test value", + batterystats_return_value, + "8.0.0", + ] + + device._verbose = True + device.version = 8 + + # Create a control server + control_server.power_test = True + control_server.test_name = "gve-pytest" + control_server.device = device + control_server.app_name = "org.mozilla.geckoview_example" + web_extension = WebExtensionAndroid( + "geckoview", "org.mozilla.geckoview_example", power_test=True + ) + web_extension.device = device + web_extension.config["power_test"] = True + web_extension.control_server = control_server + + # Expected OS baseline calculation result + os_baseline_data = { + "type": "power", + "test": "gve-pytest", + "unit": "mAh", + "values": { + "cpu": float(10.786654), + "wifi": float(2.26132), + "screen": float(51.66), + "proportional": float(11.294805199999999), + }, + } + + # Verify the response contains our expected calculations + power.finish_android_power_test( + web_extension, "gve-pytest", os_baseline=True + ) + + assert web_extension.os_baseline_data == os_baseline_data + + +if __name__ == "__main__": + mozunit.main() diff --git a/testing/raptor/test/test_print_tests.py b/testing/raptor/test/test_print_tests.py new file mode 100644 index 0000000000..739d3b0d6c --- /dev/null +++ b/testing/raptor/test/test_print_tests.py @@ -0,0 +1,53 @@ +import os + +import mozunit +import pytest + +# need this so raptor imports work both from /raptor and via mach +here = os.path.abspath(os.path.dirname(__file__)) +from raptor import cmdline + + +def test_pageload_subtests(capsys, monkeypatch, tmpdir): + # cmdline.py is hard-coded to use raptor.ini from the same directory so we need + # to monkey patch os.dirname, which is not ideal. If we could make --print-tests + # respect the --test path that would be much better. + def mock(path): + return str(tmpdir) + + monkeypatch.setattr(os.path, "dirname", mock) + manifest = tmpdir.join("raptor.ini") + manifest.write( + """ +[DEFAULT] +type = pageload +apps = firefox + +[raptor-subtest-1] +measure = foo, bar + +[raptor-subtest-2] +""" + ) + with pytest.raises(SystemExit): + cmdline.parse_args(["--print-tests"]) + captured = capsys.readouterr() + assert ( + captured.out + == """ +Raptor Tests Available for Firefox Desktop +========================================== + +raptor + type: pageload + subtests: + raptor-subtest-1 (foo, bar) + raptor-subtest-2 + +Done. +""" + ) + + +if __name__ == "__main__": + mozunit.main() diff --git a/testing/raptor/test/test_raptor.py b/testing/raptor/test/test_raptor.py new file mode 100644 index 0000000000..ef1a5a73cb --- /dev/null +++ b/testing/raptor/test/test_raptor.py @@ -0,0 +1,386 @@ +import os +import sys +import threading +import time +import traceback +from unittest import mock +from unittest.mock import Mock + +import mozunit +import pytest +from mozprofile import BaseProfile +from mozrunner.errors import RunnerNotStartedError +from six import reraise + +# need this so the raptor unit tests can find output & filter classes +here = os.path.abspath(os.path.dirname(__file__)) +raptor_dir = os.path.join(os.path.dirname(here), "raptor") +sys.path.insert(0, raptor_dir) + + +from browsertime import BrowsertimeAndroid, BrowsertimeDesktop +from webextension import ( + WebExtensionAndroid, + WebExtensionDesktopChrome, + WebExtensionFirefox, +) + +DEFAULT_TIMEOUT = 125 + + +class TestBrowserThread(threading.Thread): + def __init__(self, raptor_instance, tests, names): + super(TestBrowserThread, self).__init__() + self.raptor_instance = raptor_instance + self.tests = tests + self.names = names + self.exc = None + + def print_error(self): + if self.exc is None: + return + type, value, tb = self.exc + traceback.print_exception(type, value, tb, None, sys.stderr) + + def run(self): + try: + self.raptor_instance.run_tests(self.tests, self.names) + except BaseException: + self.exc = sys.exc_info() + + +# Perftest tests +@pytest.mark.parametrize( + "perftest_class, app_name", + [ + [WebExtensionFirefox, "firefox"], + [WebExtensionDesktopChrome, "chrome"], + [WebExtensionDesktopChrome, "chromium"], + [WebExtensionAndroid, "geckoview"], + [BrowsertimeDesktop, "firefox"], + [BrowsertimeDesktop, "chrome"], + [BrowsertimeDesktop, "chromium"], + [BrowsertimeAndroid, "geckoview"], + ], +) +def test_build_profile(options, perftest_class, app_name, get_prefs): + options["app"] = app_name + perftest_instance = perftest_class(**options) + assert isinstance(perftest_instance.profile, BaseProfile) + if app_name != "firefox": + return + + # These prefs are set in mozprofile + firefox_prefs = [ + 'user_pref("app.update.checkInstallTime", false);', + 'user_pref("app.update.disabledForTesting", true);', + 'user_pref("' + 'security.turn_off_all_security_so_that_viruses_can_take_over_this_computer", true);', + ] + # This pref is set in raptor + raptor_pref = 'user_pref("security.enable_java", false);' + + prefs_file = os.path.join(perftest_instance.profile.profile, "user.js") + with open(prefs_file, "r") as fh: + prefs = fh.read() + for firefox_pref in firefox_prefs: + assert firefox_pref in prefs + assert raptor_pref in prefs + + +def test_perftest_host_ip(ConcretePerftest, options, get_prefs): + os.environ["HOST_IP"] = "some_dummy_host_ip" + options["host"] = "HOST_IP" + + perftest = ConcretePerftest(**options) + + assert perftest.config["host"] == os.environ["HOST_IP"] + + +@pytest.mark.parametrize( + "app_name, expected_e10s_flag", + [["firefox", True], ["geckoview", True]], +) +def test_e10s_enabling(ConcretePerftest, options, app_name, expected_e10s_flag): + options["app"] = app_name + perftest = ConcretePerftest(profile_class="firefox", **options) + assert perftest.config["e10s"] == expected_e10s_flag + + +def test_profile_was_provided_locally(ConcretePerftest, options): + perftest = ConcretePerftest(**options) + assert os.path.isdir(perftest.config["local_profile_dir"]) + + +@pytest.mark.parametrize( + "profile_class, app, expected_profile", + [ + ["firefox", "firefox", "firefox"], + [None, "firefox", "firefox"], + ["firefox", None, "firefox"], + ], +) +def test_profile_class_assignation( + ConcretePerftest, options, profile_class, app, expected_profile +): + options["app"] = app + perftest = ConcretePerftest(profile_class=profile_class, **options) + assert perftest.profile_class == expected_profile + + +def test_raptor_venv(ConcretePerftest, options): + perftest = ConcretePerftest(**options) + assert perftest.raptor_venv.endswith("raptor-venv") + + +@pytest.mark.parametrize( + "run_local," + "debug_mode," + "post_startup_delay," + "expected_post_startup_delay," + "expected_debug_mode", + [ + [True, True, 1234, 1234, True], + [True, True, 12345, 3000, True], + [False, False, 1234, 1234, False], + [False, False, 12345, 12345, False], + [True, False, 1234, 1234, False], + [True, False, 12345, 12345, False], + [False, True, 1234, 1234, False], + [False, True, 12345, 12345, False], + ], +) +def test_post_startup_delay( + ConcretePerftest, + options, + run_local, + debug_mode, + post_startup_delay, + expected_post_startup_delay, + expected_debug_mode, +): + perftest = ConcretePerftest( + run_local=run_local, + debug_mode=debug_mode, + post_startup_delay=post_startup_delay, + **options + ) + assert perftest.post_startup_delay == expected_post_startup_delay + assert perftest.debug_mode == expected_debug_mode + + +@pytest.mark.parametrize( + "alert, expected_alert", [["test_to_alert_on", "test_to_alert_on"], [None, None]] +) +def test_perftest_run_test_setup( + ConcretePerftest, options, mock_test, alert, expected_alert +): + perftest = ConcretePerftest(**options) + mock_test["alert_on"] = alert + + perftest.run_test_setup(mock_test) + + assert perftest.config["subtest_alert_on"] == expected_alert + + +# WebExtension tests +@pytest.mark.parametrize( + "app", ["firefox", pytest.mark.xfail("chrome"), pytest.mark.xfail("chromium")] +) +def test_start_browser(get_binary, app): + binary = get_binary(app) + assert binary + + raptor = WebExtensionFirefox(app, binary, post_startup_delay=0) + + tests = [{"name": "raptor-{}-tp6".format(app), "page_timeout": 1000}] + test_names = [test["name"] for test in tests] + + thread = TestBrowserThread(raptor, tests, test_names) + thread.start() + + timeout = time.time() + 5 # seconds + while time.time() < timeout: + try: + is_running = raptor.runner.is_running() + assert is_running + break + except RunnerNotStartedError: + time.sleep(0.1) + else: + # browser didn't start + # if the thread had an error, display it here + thread.print_error() + assert False + + raptor.clean_up() + thread.join(5) + + if thread.exc is not None: + exc, value, tb = thread.exc + reraise(exc, value, tb) + + assert not raptor.runner.is_running() + assert raptor.runner.returncode is not None + + +# Browsertime tests +def test_cmd_arguments(ConcreteBrowsertime, browsertime_options, mock_test): + expected_cmd = { + browsertime_options["browsertime_node"], + browsertime_options["browsertime_browsertimejs"], + "--firefox.geckodriverPath", + browsertime_options["browsertime_geckodriver"], + "--browsertime.page_cycles", + "1", + "--browsertime.url", + mock_test["test_url"], + "--browsertime.secondary_url", + mock_test["secondary_url"], + "--browsertime.page_cycle_delay", + "1000", + "--browsertime.post_startup_delay", + str(DEFAULT_TIMEOUT), + "--firefox.profileTemplate", + "--skipHar", + "--video", + "true", + "--visualMetrics", + "false", + "--timeouts.pageLoad", + str(DEFAULT_TIMEOUT), + "--timeouts.script", + str(DEFAULT_TIMEOUT), + "--resultDir", + "--iterations", + "1", + } + if browsertime_options.get("app") in ["chrome", "chrome-m"]: + expected_cmd.add( + "--chrome.chromedriverPath", browsertime_options["browsertime_chromedriver"] + ) + + browsertime = ConcreteBrowsertime( + post_startup_delay=DEFAULT_TIMEOUT, **browsertime_options + ) + browsertime.run_test_setup(mock_test) + cmd = browsertime._compose_cmd(mock_test, DEFAULT_TIMEOUT) + + assert expected_cmd.issubset(set(cmd)) + + +def extract_arg_value(cmd, arg): + param_index = cmd.index(arg) + 1 + return cmd[param_index] + + +@pytest.mark.parametrize( + "arg_to_test, expected, test_patch, options_patch", + [ + ["--iterations", "1", {}, {"browser_cycles": None}], + ["--iterations", "123", {"browser_cycles": 123}, {}], + ["--video", "false", {}, {"browsertime_video": None}], + ["--video", "true", {}, {"browsertime_video": "dummy_value"}], + ["--timeouts.script", str(DEFAULT_TIMEOUT), {}, {"page_cycles": None}], + ["--timeouts.script", str(123 * DEFAULT_TIMEOUT), {"page_cycles": 123}, {}], + ["--browsertime.page_cycles", "1", {}, {"page_cycles": None}], + ["--browsertime.page_cycles", "123", {"page_cycles": 123}, {}], + ], +) +def test_browsertime_arguments( + ConcreteBrowsertime, + browsertime_options, + mock_test, + arg_to_test, + expected, + test_patch, + options_patch, +): + mock_test.update(test_patch) + browsertime_options.update(options_patch) + browsertime = ConcreteBrowsertime( + post_startup_delay=DEFAULT_TIMEOUT, **browsertime_options + ) + browsertime.run_test_setup(mock_test) + cmd = browsertime._compose_cmd(mock_test, DEFAULT_TIMEOUT) + + param_value = extract_arg_value(cmd, arg_to_test) + assert expected == param_value + + +@pytest.mark.parametrize( + "timeout, expected_timeout, test_patch, options_patch", + [ + [0, 80, {}, {}], + [0, 80, {}, {"gecko_profile": False}], + [1000, 381, {}, {"gecko_profile": True}], + ], +) +def test_compute_process_timeout( + ConcreteBrowsertime, + browsertime_options, + mock_test, + timeout, + expected_timeout, + test_patch, + options_patch, +): + mock_test.update(test_patch) + browsertime_options.update(options_patch) + browsertime = ConcreteBrowsertime( + post_startup_delay=DEFAULT_TIMEOUT, **browsertime_options + ) + bt_timeout = browsertime._compute_process_timeout(mock_test, timeout) + assert bt_timeout == expected_timeout + + +@pytest.mark.parametrize( + "host, playback, benchmark", + [["127.0.0.1", True, False], ["localhost", False, True]], +) +def test_android_reverse_ports(host, playback, benchmark): + raptor = WebExtensionAndroid( + "geckoview", + "org.mozilla.geckoview_example", + host=host, + extra_prefs={}, + ) + if benchmark: + benchmark_mock = mock.patch("raptor.raptor.benchmark.Benchmark") + raptor.benchmark = benchmark_mock + raptor.benchmark.port = 1234 + + if playback: + playback_mock = mock.patch( + "mozbase.mozproxy.mozproxy.backends.mitm.mitm.MitmproxyAndroid" + ) + playback_mock.port = 4321 + raptor.playback = playback_mock + + raptor.set_reverse_port = Mock() + raptor.set_reverse_ports() + + raptor.set_reverse_port.assert_any_call(raptor.control_server.port) + if benchmark: + raptor.set_reverse_port.assert_any_call(1234) + + if playback: + raptor.set_reverse_port.assert_any_call(4321) + + +def test_android_reverse_ports_non_local_host(): + raptor = WebExtensionAndroid( + "geckoview", + "org.mozilla.geckoview_example", + host="192.168.100.10", + extra_prefs={}, + ) + + raptor.set_reverse_port = Mock() + raptor.set_reverse_ports() + + raptor.set_reverse_port.assert_not_called() + + +if __name__ == "__main__": + mozunit.main() diff --git a/testing/raptor/test/test_utils.py b/testing/raptor/test/test_utils.py new file mode 100644 index 0000000000..1fcf755e0b --- /dev/null +++ b/testing/raptor/test/test_utils.py @@ -0,0 +1,79 @@ +import os +import sys +import tempfile + +import mozunit +import pytest +import yaml + +# need this so raptor imports work both from /raptor and via mach +here = os.path.abspath(os.path.dirname(__file__)) + +raptor_dir = os.path.join(os.path.dirname(here), "raptor") +sys.path.insert(0, raptor_dir) + +from utils import bool_from_str, transform_platform, write_yml_file + + +@pytest.mark.parametrize("platform", ["win", "mac", "linux64"]) +def test_transform_platform(platform): + transformed = transform_platform("mitmproxy-rel-bin-{platform}.manifest", platform) + assert "{platform}" not in transformed + if platform == "mac": + assert "osx" in transformed + else: + assert platform in transformed + + +def test_transform_platform_no_change(): + starting_string = "nothing-in-here-to-transform" + assert transform_platform(starting_string, "mac") == starting_string + + +@pytest.mark.parametrize("processor", ["x86_64", "other"]) +def test_transform_platform_processor(processor): + transformed = transform_platform( + "string-with-processor-{x64}.manifest", "win", processor + ) + assert "{x64}" not in transformed + if processor == "x86_64": + assert "_x64" in transformed + + +def test_write_yml_file(): + yml_file = os.path.join(tempfile.mkdtemp(), "utils.yaml") + + yml_data = dict(args=["--a", "apple", "--b", "banana"], env=dict(LOG_VERBOSE=1)) + + assert not os.path.exists(yml_file) + write_yml_file(yml_file, yml_data) + assert os.path.exists(yml_file) + + with open(yml_file, "r") as yml_in: + yml_loaded = yaml.unsafe_load(yml_in) + assert yml_loaded == yml_data + + +@pytest.mark.parametrize( + "value, expected_result", + [ + ("true", True), + ("TRUE", True), + ("True", True), + ("false", False), + ("FALSE", False), + ("False", False), + ], +) +def test_bool_from_str(value, expected_result): + assert expected_result == bool_from_str(value) + + +@pytest.mark.parametrize("invalid_value", ["invalid_str", ""]) +def test_bool_from_str_with_invalid_values(invalid_value): + with pytest.raises(ValueError): + bool_from_str(invalid_value) + + +if __name__ == "__main__": + mozunit.main() diff --git a/testing/raptor/webext/raptor/benchmark.js b/testing/raptor/webext/raptor/benchmark.js new file mode 100644 index 0000000000..277ceeb710 --- /dev/null +++ b/testing/raptor/webext/raptor/benchmark.js @@ -0,0 +1,49 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +// receives result from benchmark and relays onto our background runner + +async function receiveMessage(event) { + raptorLog("raptor benchmark received message"); + raptorLog(event.data); + + // raptor benchmark message data [0] is raptor tag, [1] is benchmark + // name, and the rest is actual benchmark results that we want to fw + if (event.data[0] == "raptor-benchmark") { + await sendResult(event.data[1], event.data.slice(2)); + } +} + +/** + * Send result back to background runner script + */ +async function sendResult(type, value) { + raptorLog(`sending result back to runner: ${type} ${value}`); + + let response; + if (typeof browser !== "undefined") { + response = await browser.runtime.sendMessage({ type, value }); + } else { + response = await new Promise(resolve => { + chrome.runtime.sendMessage({ type, value }, resolve); + }); + } + + if (response) { + raptorLog(`Response: ${response.text}`); + } +} + +function raptorLog(text, level = "info") { + let prefix = ""; + + if (level == "error") { + prefix = "ERROR: "; + } + + console[level](`${prefix}[raptor-benchmarkjs] ${text}`); +} + +raptorLog("raptor benchmark content loaded"); +window.addEventListener("message", receiveMessage); diff --git a/testing/raptor/webext/raptor/icon.png b/testing/raptor/webext/raptor/icon.png Binary files differnew file mode 100644 index 0000000000..253851bc46 --- /dev/null +++ b/testing/raptor/webext/raptor/icon.png diff --git a/testing/raptor/webext/raptor/manifest.json b/testing/raptor/webext/raptor/manifest.json new file mode 100644 index 0000000000..1bc408dfc0 --- /dev/null +++ b/testing/raptor/webext/raptor/manifest.json @@ -0,0 +1,89 @@ +{ + "browser_specific_settings": { + "gecko": { + "id": "raptor@mozilla.org" + } + }, + "manifest_version": 2, + "name": "Raptor", + "version": "0.1", + "description": "Performance measurement framework prototype", + "background": { + "scripts": ["auto_gen_test_config.js", "runner.js"] + }, + "content_scripts": [ + { + "matches": [ + "*://*.allrecipes.com/*", + "*://*.apple.com/*", + "*://*.amazon.com/*", + "*://*.bing.com/*", + "*://*.booking.com/*", + "*://*.cnn.com/*", + "*://*.dailymail.co.uk/*", + "*://*.ebay.com/*", + "*://*.ebay-kleinanzeigen.de/*", + "*://*.espn.com/*", + "*://*.facebook.com/*", + "*://*.fandom.com/*", + "*://*.google.com/*", + "*://*.imdb.com/*", + "*://*.imgur.com/*", + "*://*.instagram.com/*", + "*://*.linkedin.com/*", + "*://*.live.com/*", + "*://*.microsoft.com/*", + "*://*.netflix.com/*", + "*://*.office.com/*", + "*://*.paypal.com/*", + "*://*.pinterest.com/*", + "*://*.reddit.com/*", + "*://*.stackoverflow.com/*", + "*://*.sina.com.cn/*", + "*://*.tumblr.com/*", + "*://*.twitch.tv/*", + "*://*.twitter.com/*", + "*://*.vice.com/*", + "*://*.web.de/*", + "*://*.wikia.com/*", + "*://*.wikipedia.org/*", + "*://*.yahoo.com/*", + "*://*.youtube.com/*", + "*://*.yandex.ru/*" + ], + "js": ["pageload.js"], + "run_at": "document_end" + }, + { + "matches": [ + "*://*/Speedometer/index.html*", + "*://*/StyleBench/*", + "*://*/MotionMark/*", + "*://*/SunSpider/*", + "*://*/webaudio/*", + "*://*/unity-webgl/index.html*", + "*://*/wasm-misc/index.html*", + "*://*/wasm-godot/index.html*", + "*://*/assorted-dom/assorted/results.html*", + "*://*.mozaws.net/*", + "*://*/ARES-6/index.html*", + "*://*/JetStream2/index.html*", + "*://*.amazonaws.com/*" + ], + "js": ["benchmark.js"], + "run_at": "document_end" + } + ], + "browser_action": { + "browser_style": true, + "default_icon": "icon.png", + "default_title": "Raptor LOADED" + }, + "permissions": [ + "<all_urls>", + "tabs", + "storage", + "alarms", + "geckoProfiler" + ] +} diff --git a/testing/raptor/webext/raptor/pageload.js b/testing/raptor/webext/raptor/pageload.js new file mode 100644 index 0000000000..0ed35da309 --- /dev/null +++ b/testing/raptor/webext/raptor/pageload.js @@ -0,0 +1,396 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +// Supported test types +const TEST_BENCHMARK = "benchmark"; +const TEST_PAGE_LOAD = "pageload"; + +// content script for use with pageload tests +var perfData = window.performance; +var gRetryCounter = 0; + +// measure hero element; must exist inside test page; +// supported on: Firefox, Chromium, Geckoview +// default only; this is set via control server settings json +var getHero = false; +var heroesToCapture = []; + +// measure time-to-first-non-blank-paint +// supported on: Firefox, Geckoview +// note: this browser pref must be enabled: +// dom.performance.time_to_non_blank_paint.enabled = True +// default only; this is set via control server settings json +var getFNBPaint = false; + +// measure time-to-first-contentful-paint +// supported on: Firefox, Chromium, Geckoview +// note: this browser pref must be enabled: +// dom.performance.time_to_contentful_paint.enabled = True +// default only; this is set via control server settings json +var getFCP = false; + +// measure domContentFlushed +// supported on: Firefox, Geckoview +// note: this browser pref must be enabled: +// dom.performance.time_to_dom_content_flushed.enabled = True +// default only; this is set via control server settings json +var getDCF = false; + +// measure TTFI +// supported on: Firefox, Geckoview +// note: this browser pref must be enabled: +// dom.performance.time_to_first_interactive.enabled = True +// default only; this is set via control server settings json +var getTTFI = false; + +// supported on: Firefox, Chromium, Geckoview +// default only; this is set via control server settings json +var getLoadTime = false; + +// performance.timing measurement used as 'starttime' +var startMeasure = "fetchStart"; + +async function raptorContentHandler() { + raptorLog("pageloadjs raptorContentHandler!"); + // let the main raptor runner know that we (pageloadjs) is ready! + await sendPageloadReady(); + + // retrieve test settings from local ext storage + let settings; + if (typeof browser !== "undefined") { + ({ settings } = await browser.storage.local.get("settings")); + } else { + ({ settings } = await new Promise(resolve => { + chrome.storage.local.get("settings", resolve); + })); + } + setup(settings); +} + +function setup(settings) { + if (settings.type != TEST_PAGE_LOAD) { + return; + } + + if (settings.measure == undefined) { + raptorLog("abort: 'measure' key not found in test settings"); + return; + } + + if (settings.measure.fnbpaint !== undefined) { + getFNBPaint = settings.measure.fnbpaint; + if (getFNBPaint) { + raptorLog("will be measuring fnbpaint"); + measureFNBPaint(); + } + } + + if (settings.measure.dcf !== undefined) { + getDCF = settings.measure.dcf; + if (getDCF) { + raptorLog("will be measuring dcf"); + measureDCF(); + } + } + + if (settings.measure.fcp !== undefined) { + getFCP = settings.measure.fcp; + if (getFCP) { + raptorLog("will be measuring first-contentful-paint"); + measureFCP(); + } + } + + if (settings.measure.hero !== undefined) { + if (settings.measure.hero.length !== 0) { + getHero = true; + heroesToCapture = settings.measure.hero; + raptorLog(`hero elements to measure: ${heroesToCapture}`); + measureHero(); + } + } + + if (settings.measure.ttfi !== undefined) { + getTTFI = settings.measure.ttfi; + if (getTTFI) { + raptorLog("will be measuring ttfi"); + measureTTFI(); + } + } + + if (settings.measure.loadtime !== undefined) { + getLoadTime = settings.measure.loadtime; + if (getLoadTime) { + raptorLog("will be measuring loadtime"); + measureLoadTime(); + } + } +} + +function measureHero() { + let obs; + + const heroElementsFound = window.document.querySelectorAll("[elementtiming]"); + raptorLog(`found ${heroElementsFound.length} hero elements in the page`); + + if (heroElementsFound) { + async function callbackHero(entries, observer) { + for (const entry in entries) { + const heroFound = entry.target.getAttribute("elementtiming"); + // mark the time now as when hero element received + perfData.mark(heroFound); + const resultType = `hero:${heroFound}`; + raptorLog(`found ${resultType}`); + // calculcate result: performance.timing.fetchStart - time when we got hero element + perfData.measure( + (name = resultType), + (startMark = startMeasure), + (endMark = heroFound) + ); + const perfResult = perfData.getEntriesByName(resultType); + const _result = Math.round(perfResult[0].duration); + await sendResult(resultType, _result); + perfData.clearMarks(); + perfData.clearMeasures(); + obs.disconnect(); + } + } + // we want the element 100% visible on the viewport + const options = { root: null, rootMargin: "0px", threshold: [1] }; + try { + obs = new window.IntersectionObserver(callbackHero, options); + heroElementsFound.forEach(function(el) { + // if hero element is one we want to measure, add it to the observer + if (heroesToCapture.indexOf(el.getAttribute("elementtiming")) > -1) { + obs.observe(el); + } + }); + } catch (err) { + raptorLog(err); + } + } else { + raptorLog("couldn't find hero element"); + } +} + +async function measureFNBPaint() { + const x = window.performance.timing.timeToNonBlankPaint; + + if (typeof x == "undefined") { + raptorLog( + "timeToNonBlankPaint is undefined; ensure the pref is enabled", + "error" + ); + return; + } + if (x > 0) { + raptorLog("got fnbpaint"); + gRetryCounter = 0; + const startTime = perfData.timing.fetchStart; + await sendResult("fnbpaint", x - startTime); + } else { + gRetryCounter += 1; + if (gRetryCounter <= 10) { + raptorLog( + `fnbpaint is not yet available, retry number ${gRetryCounter}...` + ); + window.setTimeout(measureFNBPaint, 100); + } else { + raptorLog( + `unable to get a value for fnbpaint after ${gRetryCounter} retries` + ); + } + } +} + +async function measureDCF() { + const x = window.performance.timing.timeToDOMContentFlushed; + + if (typeof x == "undefined") { + raptorLog( + "domContentFlushed is undefined; ensure the pref is enabled", + "error" + ); + return; + } + if (x > 0) { + raptorLog(`got domContentFlushed: ${x}`); + gRetryCounter = 0; + const startTime = perfData.timing.fetchStart; + await sendResult("dcf", x - startTime); + } else { + gRetryCounter += 1; + if (gRetryCounter <= 10) { + raptorLog( + `dcf is not yet available (0), retry number ${gRetryCounter}...` + ); + window.setTimeout(measureDCF, 100); + } else { + raptorLog(`unable to get a value for dcf after ${gRetryCounter} retries`); + } + } +} + +async function measureTTFI() { + const x = window.performance.timing.timeToFirstInteractive; + + if (typeof x == "undefined") { + raptorLog( + "timeToFirstInteractive is undefined; ensure the pref is enabled", + "error" + ); + return; + } + if (x > 0) { + raptorLog(`got timeToFirstInteractive: ${x}`); + gRetryCounter = 0; + const startTime = perfData.timing.fetchStart; + await sendResult("ttfi", x - startTime); + } else { + gRetryCounter += 1; + // NOTE: currently the gecko implementation doesn't look at network + // requests, so this is closer to TimeToFirstInteractive than + // TimeToInteractive. TTFI/TTI requires running at least 5 seconds + // past last "busy" point, give 25 seconds here (overall the harness + // times out at 30 seconds). Some pages will never get 5 seconds + // without a busy period! + if (gRetryCounter <= 25 * (1000 / 200)) { + raptorLog( + `TTFI is not yet available (0), retry number ${gRetryCounter}...` + ); + window.setTimeout(measureTTFI, 200); + } else { + // unable to get a value for TTFI - negative value will be filtered out later + raptorLog("TTFI was not available for this pageload"); + await sendResult("ttfi", -1); + } + } +} + +async function measureFCP() { + // see https://developer.mozilla.org/en-US/docs/Web/API/PerformancePaintTiming + let result = window.performance.timing.timeToContentfulPaint; + + // Firefox implementation of FCP is not yet spec-compliant (see Bug 1519410) + if (typeof result == "undefined") { + // we're on chromium + result = 0; + + const perfEntries = perfData.getEntriesByType("paint"); + if (perfEntries.length >= 2) { + if ( + perfEntries[1].name == "first-contentful-paint" && + perfEntries[1].startTime != undefined + ) { + // this value is actually the final measurement / time to get the FCP event in MS + result = perfEntries[1].startTime; + } + } + } + + if (result > 0) { + raptorLog("got time to first-contentful-paint"); + if (typeof browser !== "undefined") { + // Firefox returns a timestamp, not the actual measurement in MS; need to calculate result + const startTime = perfData.timing.fetchStart; + result = result - startTime; + } + await sendResult("fcp", result); + perfData.clearMarks(); + perfData.clearMeasures(); + } else { + gRetryCounter += 1; + if (gRetryCounter <= 10) { + raptorLog( + `time to first-contentful-paint is not yet available (0), retry number ${gRetryCounter}...` + ); + window.setTimeout(measureFCP, 100); + } else { + raptorLog( + `unable to get a value for time-to-fcp after ${gRetryCounter} retries` + ); + } + } +} + +async function measureLoadTime() { + const x = window.performance.timing.loadEventStart; + + if (typeof x == "undefined") { + raptorLog("loadEventStart is undefined", "error"); + return; + } + if (x > 0) { + raptorLog(`got loadEventStart: ${x}`); + gRetryCounter = 0; + const startTime = perfData.timing.fetchStart; + await sendResult("loadtime", x - startTime); + } else { + gRetryCounter += 1; + if (gRetryCounter <= 40 * (1000 / 200)) { + raptorLog( + `loadEventStart is not yet available (0), retry number ${gRetryCounter}...` + ); + window.setTimeout(measureLoadTime, 100); + } else { + raptorLog( + `unable to get a value for loadEventStart after ${gRetryCounter} retries` + ); + } + } +} + +/** + * Send message to runnerjs indicating pageloadjs is ready to start getting measures + */ +async function sendPageloadReady() { + raptorLog("sending pageloadjs-ready message to runnerjs"); + + let response; + if (typeof browser !== "undefined") { + response = await browser.runtime.sendMessage({ type: "pageloadjs-ready" }); + } else { + response = await new Promise(resolve => { + chrome.runtime.sendMessage({ type: "pageloadjs-ready" }, resolve); + }); + } + + if (response) { + raptorLog(`Response: ${response.text}`); + } +} + +/** + * Send result back to background runner script + */ +async function sendResult(type, value) { + raptorLog(`sending result back to runner: ${type} ${value}`); + + let response; + if (typeof browser !== "undefined") { + response = await browser.runtime.sendMessage({ type, value }); + } else { + response = await new Promise(resolve => { + chrome.runtime.sendMessage({ type, value }, resolve); + }); + } + + if (response) { + raptorLog(`Response: ${response.text}`); + } +} + +function raptorLog(text, level = "info") { + let prefix = ""; + + if (level == "error") { + prefix = "ERROR: "; + } + + console[level](`${prefix}[raptor-pageloadjs] ${text}`); +} + +if (window.addEventListener) { + window.addEventListener("load", raptorContentHandler); +} diff --git a/testing/raptor/webext/raptor/runner.js b/testing/raptor/webext/raptor/runner.js new file mode 100644 index 0000000000..643bb68ab5 --- /dev/null +++ b/testing/raptor/webext/raptor/runner.js @@ -0,0 +1,797 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +// this extension requires a 'control server' to be running on port 8000 +// (see raptor prototype framework). It will provide the test options, as +// well as receive test results + +// note: currently the prototype assumes the test page(s) are +// already available somewhere independently; so for now locally +// inside the 'talos-pagesets' dir or 'heroes' dir (tarek's github +// repo) or 'webkit/PerformanceTests' dir (for benchmarks) first run: +// 'python -m SimpleHTTPServer 8081' +// to serve out the pages that we want to prototype with. Also +// update the manifest content 'matches' accordingly + +// Supported test types +const TEST_BENCHMARK = "benchmark"; +const TEST_PAGE_LOAD = "pageload"; +const TEST_SCENARIO = "scenario"; + +const ANDROID_BROWSERS = ["fenix", "geckoview", "refbrow"]; + +// when the browser starts this webext runner will start automatically; we +// want to give the browser some time (ms) to settle before starting tests +var postStartupDelay; + +// delay (ms) between pageload cycles +var pageCycleDelay = 1000; + +var newTabPerCycle = false; + +// delay (ms) for foregrounding app +var foregroundDelay = 5000; + +var isGecko = false; +var isGeckoAndroid = false; +var ext; +var testName = null; +var settingsURL = null; +var csPort = null; +var host = null; +var benchmarkPort = null; +var testType; +var browserCycle = 0; +var pageCycles = 0; +var pageCycle = 0; +var testURL; +var testTabId; +var scenarioTestTime = 60000; +var getHero = false; +var getFNBPaint = false; +var getFCP = false; +var getDCF = false; +var getTTFI = false; +var getLoadTime = false; +var isHeroPending = false; +var pendingHeroes = []; +var settings = {}; +var isFNBPaintPending = false; +var isFCPPending = false; +var isDCFPending = false; +var isTTFIPending = false; +var isLoadTimePending = false; +var isScenarioPending = false; +var isBenchmarkPending = false; +var isBackgroundTest = false; +var pageTimeout = 10000; // default pageload timeout +var geckoProfiling = false; +var geckoInterval = 1; +var geckoEntries = 1000000; +var geckoThreads = []; +var geckoFeatures = null; +var debugMode = 0; +var screenCapture = false; + +var results = { + name: "", + page: "", + type: "", + browser_cycle: 0, + expected_browser_cycles: 0, + cold: false, + lower_is_better: true, + alert_change_type: "relative", + alert_threshold: 2.0, + measurements: {}, +}; + +async function getTestSettings() { + raptorLog("getting test settings from control server"); + const response = await fetch(settingsURL); + const data = await response.text(); + raptorLog(`test settings received: ${data}`); + + // parse the test settings + settings = JSON.parse(data)["raptor-options"]; + testType = settings.type; + pageCycles = settings.page_cycles; + testURL = settings.test_url; + scenarioTestTime = settings.scenario_time; + isBackgroundTest = settings.background_test; + + // for pageload type tests, the testURL is fine as is - we don't have + // to add a port as it's accessed via proxy and the playback tool + // however for benchmark tests, their source is served out on a local + // webserver, so we need to swap in the webserver port into the testURL + if (testType == TEST_BENCHMARK) { + // just replace the '<port>' keyword in the URL with actual benchmarkPort + testURL = testURL.replace("<port>", benchmarkPort); + } + + if (host) { + // just replace the '<host>' keyword in the URL with actual host + testURL = testURL.replace("<host>", host); + } + + raptorLog(`test URL: ${testURL}`); + + results.alert_change_type = settings.alert_change_type; + results.alert_threshold = settings.alert_threshold; + results.browser_cycle = browserCycle; + results.cold = settings.cold; + results.expected_browser_cycles = settings.expected_browser_cycles; + results.lower_is_better = settings.lower_is_better === true; + results.name = testName; + results.page = testURL; + results.type = testType; + results.unit = settings.unit; + results.subtest_unit = settings.subtest_unit; + results.subtest_lower_is_better = settings.subtest_lower_is_better === true; + + if (settings.gecko_profile === true) { + results.extra_options = ["gecko-profile"]; + + geckoProfiling = true; + geckoEntries = settings.gecko_profile_entries; + geckoInterval = settings.gecko_profile_interval; + geckoThreads = settings.gecko_profile_threads; + geckoFeatures = settings.gecko_profile_features; + } + + if (settings.screen_capture !== undefined) { + screenCapture = settings.screen_capture; + } + + if (settings.newtab_per_cycle !== undefined) { + newTabPerCycle = settings.newtab_per_cycle; + } + + if (settings.page_timeout !== undefined) { + pageTimeout = settings.page_timeout; + } + raptorLog(`using page timeout: ${pageTimeout}ms`); + + switch (testType) { + case TEST_PAGE_LOAD: + if (settings.measure !== undefined) { + if (settings.measure.fnbpaint !== undefined) { + getFNBPaint = settings.measure.fnbpaint; + } + if (settings.measure.dcf !== undefined) { + getDCF = settings.measure.dcf; + } + if (settings.measure.fcp !== undefined) { + getFCP = settings.measure.fcp; + } + if (settings.measure.hero !== undefined) { + if (settings.measure.hero.length !== 0) { + getHero = true; + } + } + if (settings.measure.ttfi !== undefined) { + getTTFI = settings.measure.ttfi; + } + if (settings.measure.loadtime !== undefined) { + getLoadTime = settings.measure.loadtime; + } + } else { + raptorLog("abort: 'measure' key not found in test settings"); + await cleanUp(); + } + break; + } + + // write options to storage that our content script needs to know + if (isGecko) { + await ext.storage.local.clear(); + await ext.storage.local.set({ settings }); + } else { + await new Promise(resolve => { + ext.storage.local.clear(() => { + ext.storage.local.set({ settings }, resolve); + }); + }); + } + raptorLog("wrote settings to ext local storage"); +} + +async function sleep(delay) { + return new Promise(resolve => setTimeout(resolve, delay)); +} + +async function startScenarioTimer() { + setTimeout(function() { + isScenarioPending = false; + results.measurements.scenario = [1]; + }, scenarioTestTime); + + await postToControlServer("status", `started scenario test timer`); +} + +async function closeTab(tabId) { + // Don't close the last tab which would close the window or application + const tabs = await queryForTabs({ currentWindow: true }); + if (tabs.length == 1) { + await postToControlServer("status", `Not closing last Tab: ${tabs[0].id}`); + return; + } + + await postToControlServer("status", `closing Tab: ${tabId}`); + + if (isGecko) { + await ext.tabs.remove(tabId); + } else { + await new Promise(resolve => { + ext.tabs.remove(tabId, resolve); + }); + } + + await postToControlServer("status", `closed tab: ${tabId}`); +} + +async function getCurrentTabId() { + const tabs = await queryForTabs({ currentWindow: true, active: true }); + if (!tabs.length) { + throw new Error("No active tab has been found."); + } + + await postToControlServer("status", "found active tab with id " + tabs[0].id); + return tabs[0].id; +} + +async function openTab() { + await postToControlServer("status", "opening new tab"); + + let tab; + if (isGecko) { + tab = await ext.tabs.create({ url: "about:blank" }); + } else { + tab = await new Promise(resolve => { + ext.tabs.create({ url: "about:blank" }, resolve); + }); + } + + await postToControlServer("status", `opened new empty tab: ${tab.id}`); + + return tab.id; +} + +async function queryForTabs(options = {}) { + let tabs; + + if (isGecko) { + tabs = await ext.tabs.query(options); + } else { + tabs = await new Promise(resolve => { + ext.tabs.query(options, resolve); + }); + } + + return tabs; +} + +/** + * Update the given tab by navigating to the test URL + */ +async function updateTab(tabId, url) { + await postToControlServer("status", `update tab ${tabId} for ${url}`); + + // "null" = active tab + if (isGecko) { + await ext.tabs.update(tabId, { url }); + } else { + await new Promise(resolve => { + ext.tabs.update(tabId, { url }, resolve); + }); + } + + await postToControlServer("status", `tab ${tabId} updated`); +} + +async function collectResults() { + // now we can set the page timeout timer and wait for pageload test result from content + raptorLog("ready to poll for results; turning on page-timeout timer"); + setTimeoutAlarm("raptor-page-timeout", pageTimeout); + + // wait for pageload test result from content + await waitForResults(); + + // move on to next cycle (or test complete) + await nextCycle(); +} + +function checkForTestFinished() { + let finished = false; + + switch (testType) { + case TEST_BENCHMARK: + finished = !isBenchmarkPending; + break; + case TEST_PAGE_LOAD: + if ( + !isHeroPending && + !isFNBPaintPending && + !isFCPPending && + !isDCFPending && + !isTTFIPending && + !isLoadTimePending + ) { + finished = true; + } + break; + + case TEST_SCENARIO: + finished = !isScenarioPending; + break; + } + + return finished; +} + +async function waitForResults() { + raptorLog("waiting for results..."); + + while (!checkForTestFinished()) { + raptorLog("results pending..."); + await sleep(250); + } + + await cancelTimeoutAlarm("raptor-page-timeout"); + + await postToControlServer("status", "results received"); + + if (geckoProfiling) { + await getGeckoProfile(); + } + + if (screenCapture) { + await getScreenCapture(); + } +} + +async function getScreenCapture() { + raptorLog("capturing screenshot"); + + try { + let screenshotUri; + + if (isGecko) { + screenshotUri = await ext.tabs.captureVisibleTab(); + } else { + screenshotUri = await new Promise(resolve => + ext.tabs.captureVisibleTab(resolve) + ); + } + + await postToControlServer("screenshot", [ + screenshotUri, + testName, + pageCycle, + ]); + } catch (e) { + raptorLog(`failed to capture screenshot: ${e}`); + } +} + +async function startGeckoProfiling() { + await postToControlServer( + "status", + `starting Gecko profiling for threads: ${geckoThreads}` + ); + const features = geckoFeatures + ? geckoFeatures.split(",") + : ["js", "leaf", "stackwalk", "cpu", "responsiveness"]; + await ext.geckoProfiler.start({ + bufferSize: geckoEntries, + interval: geckoInterval, + features, + threads: geckoThreads.split(","), + }); +} + +async function stopGeckoProfiling() { + await postToControlServer("status", "stopping gecko profiling"); + await ext.geckoProfiler.stop(); +} + +async function getGeckoProfile() { + // trigger saving the gecko profile, and send the file name to the control server + const fileName = `${testName}_pagecycle_${pageCycle}.profile`; + + await postToControlServer("status", `saving gecko profile ${fileName}`); + await ext.geckoProfiler.dumpProfileToFile(fileName); + await postToControlServer("gecko_profile", fileName); + + // must stop the profiler so it clears the buffer before next cycle + await stopGeckoProfiling(); + + // resume if we have more pagecycles left + if (pageCycle + 1 <= pageCycles) { + await startGeckoProfiling(); + } +} + +async function nextCycle() { + pageCycle++; + if (isBackgroundTest) { + await postToControlServer( + "end_background", + `bringing app to foreground, pausing for ${foregroundDelay / + 1000} seconds` + ); + // wait a bit to be sure the app is in foreground before starting + // new test, or finishing test + await sleep(foregroundDelay); + } + if (pageCycle == 1) { + const text = `running ${pageCycles} pagecycles of ${testURL}`; + await postToControlServer("status", text); + // start the profiler if enabled + if (geckoProfiling) { + await startGeckoProfiling(); + } + } + if (pageCycle <= pageCycles) { + if (isBackgroundTest) { + await postToControlServer( + "start_background", + `bringing app to background` + ); + } + + await sleep(pageCycleDelay); + + await postToControlServer("status", `begin page cycle ${pageCycle}`); + + switch (testType) { + case TEST_BENCHMARK: + isBenchmarkPending = true; + break; + + case TEST_PAGE_LOAD: + if (getHero) { + isHeroPending = true; + pendingHeroes = Array.from(settings.measure.hero); + } + if (getFNBPaint) { + isFNBPaintPending = true; + } + if (getFCP) { + isFCPPending = true; + } + if (getDCF) { + isDCFPending = true; + } + if (getTTFI) { + isTTFIPending = true; + } + if (getLoadTime) { + isLoadTimePending = true; + } + break; + + case TEST_SCENARIO: + isScenarioPending = true; + break; + } + + if (newTabPerCycle) { + // close previous test tab and open a new one + await closeTab(testTabId); + testTabId = await openTab(); + } + + await updateTab(testTabId, testURL); + + if (testType == TEST_SCENARIO) { + await startScenarioTimer(); + } + + // For benchmark or scenario type tests we can proceed directly to + // waitForResult. However for page-load tests we must first wait until + // we hear back from pageloaderjs that it has been successfully loaded + // in the test page and has been invoked; and only then start looking + // for measurements. + if (testType != TEST_PAGE_LOAD) { + await collectResults(); + } + + await postToControlServer("status", `ended page cycle ${pageCycle}`); + } else { + await verifyResults(); + } +} + +async function timeoutAlarmListener() { + raptorLog(`raptor-page-timeout on ${testURL}`, "error"); + + const pendingMetrics = { + hero: isHeroPending, + "fnb paint": isFNBPaintPending, + fcp: isFCPPending, + dcf: isDCFPending, + ttfi: isTTFIPending, + "load time": isLoadTimePending, + }; + + let msgData = [testName, testURL, pageCycle]; + if (testType == TEST_PAGE_LOAD) { + msgData.push(pendingMetrics); + } + + await postToControlServer("raptor-page-timeout", msgData); + await getScreenCapture(); + + // call clean-up to shutdown gracefully + await cleanUp(); +} + +function setTimeoutAlarm(timeoutName, timeoutMS) { + // webext alarms require date.now NOT performance.now + const now = Date.now(); // eslint-disable-line mozilla/avoid-Date-timing + const timeout_when = now + timeoutMS; + ext.alarms.create(timeoutName, { when: timeout_when }); + + raptorLog( + `now is ${now}, set raptor alarm ${timeoutName} to expire ` + + `at ${timeout_when}` + ); +} + +async function cancelTimeoutAlarm(timeoutName) { + let cleared = false; + + if (isGecko) { + cleared = await ext.alarms.clear(timeoutName); + } else { + cleared = await new Promise(resolve => { + chrome.alarms.clear(timeoutName, resolve); + }); + } + + if (cleared) { + raptorLog(`cancelled raptor alarm ${timeoutName}`); + } else { + raptorLog(`failed to clear raptor alarm ${timeoutName}`, "error"); + } +} + +function resultListener(request, sender, sendResponse) { + raptorLog(`received message from ${sender.tab.url}`); + + // check if this is a message from pageloaderjs indicating it is ready to start + if (request.type == "pageloadjs-ready") { + raptorLog("received pageloadjs-ready!"); + + sendResponse({ text: "pageloadjs-ready-response" }); + collectResults(); + return; + } + + if (request.type && request.value) { + raptorLog(`result: ${request.type} ${request.value}`); + sendResponse({ text: `confirmed ${request.type}` }); + + if (!(request.type in results.measurements)) { + results.measurements[request.type] = []; + } + + switch (testType) { + case TEST_BENCHMARK: + // benchmark results received (all results for that complete benchmark run) + raptorLog("received results from benchmark"); + results.measurements[request.type].push(request.value); + isBenchmarkPending = false; + break; + + case TEST_PAGE_LOAD: + // a single pageload measurement was received + if (request.type.indexOf("hero") > -1) { + results.measurements[request.type].push(request.value); + const _found = request.type.split("hero:")[1]; + const index = pendingHeroes.indexOf(_found); + if (index > -1) { + pendingHeroes.splice(index, 1); + if (!pendingHeroes.length) { + raptorLog("measured all expected hero elements"); + isHeroPending = false; + } + } + } else if (request.type == "fnbpaint") { + results.measurements.fnbpaint.push(request.value); + isFNBPaintPending = false; + } else if (request.type == "dcf") { + results.measurements.dcf.push(request.value); + isDCFPending = false; + } else if (request.type == "ttfi") { + results.measurements.ttfi.push(request.value); + isTTFIPending = false; + } else if (request.type == "fcp") { + results.measurements.fcp.push(request.value); + isFCPPending = false; + } else if (request.type == "loadtime") { + results.measurements.loadtime.push(request.value); + isLoadTimePending = false; + } + break; + } + } else { + raptorLog(`unknown message received from content: ${request}`); + } +} + +async function verifyResults() { + raptorLog("Verifying results:"); + raptorLog(results); + + for (var x in results.measurements) { + const count = results.measurements[x].length; + if (count == pageCycles) { + raptorLog(`have ${count} results for ${x}, as expected`); + } else { + raptorLog( + `expected ${pageCycles} results for ${x} but only have ${count}`, + "error" + ); + } + } + + await postToControlServer("results", results); + + // we're finished, move to cleanup + await cleanUp(); +} + +async function postToControlServer(msgType, msgData = "") { + await new Promise(resolve => { + // requires 'control server' running at port 8000 to receive results + const xhr = new XMLHttpRequest(); + xhr.open("POST", `http://${host}:${csPort}/`, true); + xhr.setRequestHeader("Content-Type", "application/json"); + + xhr.onreadystatechange = () => { + if (xhr.readyState == XMLHttpRequest.DONE) { + if (xhr.status != 200) { + // Failed to send the message. At least add a console error. + let msg = msgType; + if (msgType != "screenshot") { + msg += ` with '${msgData}'`; + } + raptorLog(`failed to post ${msg} to control server`, "error"); + } + + resolve(); + } + }; + + xhr.send( + JSON.stringify({ + type: `webext_${msgType}`, + data: msgData, + }) + ); + }); +} + +async function cleanUp() { + // close tab unless raptor debug-mode is enabled + if (debugMode == 1) { + raptorLog("debug-mode enabled, leaving tab open"); + } else { + await closeTab(testTabId); + } + + if (testType == TEST_PAGE_LOAD) { + // remove listeners + ext.alarms.onAlarm.removeListener(timeoutAlarmListener); + ext.runtime.onMessage.removeListener(resultListener); + } + raptorLog(`${testType} test finished`); + + // if profiling was enabled, stop the profiler - may have already + // been stopped but stop again here in cleanup in case of timeout + if (geckoProfiling) { + await stopGeckoProfiling(); + } + + // tell the control server we are done and the browser can be shutdown + await postToControlServer("shutdownBrowser"); +} + +async function raptorRunner() { + await postToControlServer("status", "starting raptorRunner"); + + if (isBackgroundTest) { + await postToControlServer( + "status", + "raptor test will be backgrounding the app" + ); + } + + await getTestSettings(); + + raptorLog(`${testType} test start`); + + ext.alarms.onAlarm.addListener(timeoutAlarmListener); + ext.runtime.onMessage.addListener(resultListener); + + // create new empty tab, which starts the test; we want to + // wait some time for the browser to settle before beginning + const text = `* pausing ${postStartupDelay / + 1000} seconds to let browser settle... *`; + await postToControlServer("status", text); + await sleep(postStartupDelay); + + if (!isGeckoAndroid) { + await openTab(); + } + + testTabId = await getCurrentTabId(); + + await nextCycle(); +} + +function raptorLog(text, level = "info") { + let prefix = ""; + + if (level == "error") { + prefix = "ERROR: "; + } + + console[level](`${prefix}[raptor-runnerjs] ${text}`); +} + +async function init() { + const config = getTestConfig(); + testName = config.test_name; + settingsURL = config.test_settings_url; + csPort = config.cs_port; + benchmarkPort = config.benchmark_port; + postStartupDelay = config.post_startup_delay; + host = config.host; + debugMode = config.debug_mode; + browserCycle = config.browser_cycle; + + try { + // Chromium based browsers do not support the "browser" namespace and + // raise an exception when accessing it. + const info = await browser.runtime.getBrowserInfo(); + results.browser = `${info.name} ${info.version} ${info.buildID}`; + + ext = browser; + isGecko = true; + isGeckoAndroid = ANDROID_BROWSERS.includes(info.name.toLowerCase); + } catch (e) { + const regex = /(Chrome)\/([\w\.]+)/; + const userAgent = window.navigator.userAgent; + results.browser = regex + .exec(userAgent) + .splice(1, 2) + .join(" "); + + ext = chrome; + } + + await postToControlServer("loaded"); + await postToControlServer("status", `testing on ${results.browser}`); + await postToControlServer("status", `test name is: ${testName}`); + await postToControlServer("status", `test settings url is: ${settingsURL}`); + + try { + if (window.document.readyState != "complete") { + await new Promise(resolve => { + window.addEventListener("load", resolve); + raptorLog("Waiting for load event..."); + }); + } + + await raptorRunner(); + } catch (e) { + await postToControlServer("error", [e.message, e.stack]); + await postToControlServer("shutdownBrowser"); + } +} + +init(); |
