/* This Source Code Form is subject to the terms of the Mozilla Public * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ // this extension requires a 'control server' to be running on port 8000 // (see raptor prototype framework). It will provide the test options, as // well as receive test results // note: currently the prototype assumes the test page(s) are // already available somewhere independently; so for now locally // inside the 'talos-pagesets' dir or 'heroes' dir (tarek's github // repo) or 'webkit/PerformanceTests' dir (for benchmarks) first run: // 'python -m SimpleHTTPServer 8081' // to serve out the pages that we want to prototype with. Also // update the manifest content 'matches' accordingly // Supported test types const TEST_BENCHMARK = "benchmark"; const TEST_PAGE_LOAD = "pageload"; const TEST_SCENARIO = "scenario"; const ANDROID_BROWSERS = ["fenix", "geckoview", "refbrow"]; // when the browser starts this webext runner will start automatically; we // want to give the browser some time (ms) to settle before starting tests var postStartupDelay; // delay (ms) between pageload cycles var pageCycleDelay = 1000; var newTabPerCycle = false; // delay (ms) for foregrounding app var foregroundDelay = 5000; var isGecko = false; var isGeckoAndroid = false; var ext; var testName = null; var settingsURL = null; var csPort = null; var host = null; var benchmarkPort = null; var testType; var browserCycle = 0; var pageCycles = 0; var pageCycle = 0; var testURL; var testTabId; var scenarioTestTime = 60000; var getHero = false; var getFNBPaint = false; var getFCP = false; var getDCF = false; var getTTFI = false; var getLoadTime = false; var isHeroPending = false; var pendingHeroes = []; var settings = {}; var isFNBPaintPending = false; var isFCPPending = false; var isDCFPending = false; var isTTFIPending = false; var isLoadTimePending = false; var isScenarioPending = false; var isBenchmarkPending = false; var isBackgroundTest = false; var pageTimeout = 10000; // default pageload timeout var geckoProfiling = false; var geckoInterval = 1; var geckoEntries = 1000000; var geckoThreads = []; var geckoFeatures = null; var debugMode = 0; var screenCapture = false; var results = { name: "", page: "", type: "", browser_cycle: 0, expected_browser_cycles: 0, cold: false, lower_is_better: true, alert_change_type: "relative", alert_threshold: 2.0, measurements: {}, }; async function getTestSettings() { raptorLog("getting test settings from control server"); const response = await fetch(settingsURL); const data = await response.text(); raptorLog(`test settings received: ${data}`); // parse the test settings settings = JSON.parse(data)["raptor-options"]; testType = settings.type; pageCycles = settings.page_cycles; testURL = settings.test_url; scenarioTestTime = settings.scenario_time; isBackgroundTest = settings.background_test; // for pageload type tests, the testURL is fine as is - we don't have // to add a port as it's accessed via proxy and the playback tool // however for benchmark tests, their source is served out on a local // webserver, so we need to swap in the webserver port into the testURL if (testType == TEST_BENCHMARK) { // just replace the '' keyword in the URL with actual benchmarkPort testURL = testURL.replace("", benchmarkPort); } if (host) { // just replace the '' keyword in the URL with actual host testURL = testURL.replace("", host); } raptorLog(`test URL: ${testURL}`); results.alert_change_type = settings.alert_change_type; results.alert_threshold = settings.alert_threshold; results.browser_cycle = browserCycle; results.cold = settings.cold; results.expected_browser_cycles = settings.expected_browser_cycles; results.lower_is_better = settings.lower_is_better === true; results.name = testName; results.page = testURL; results.type = testType; results.unit = settings.unit; results.subtest_unit = settings.subtest_unit; results.subtest_lower_is_better = settings.subtest_lower_is_better === true; if (settings.gecko_profile === true) { results.extra_options = ["gecko-profile"]; geckoProfiling = true; geckoEntries = settings.gecko_profile_entries; geckoInterval = settings.gecko_profile_interval; geckoThreads = settings.gecko_profile_threads; geckoFeatures = settings.gecko_profile_features; } if (settings.screen_capture !== undefined) { screenCapture = settings.screen_capture; } if (settings.newtab_per_cycle !== undefined) { newTabPerCycle = settings.newtab_per_cycle; } if (settings.page_timeout !== undefined) { pageTimeout = settings.page_timeout; } raptorLog(`using page timeout: ${pageTimeout}ms`); switch (testType) { case TEST_PAGE_LOAD: if (settings.measure !== undefined) { if (settings.measure.fnbpaint !== undefined) { getFNBPaint = settings.measure.fnbpaint; } if (settings.measure.dcf !== undefined) { getDCF = settings.measure.dcf; } if (settings.measure.fcp !== undefined) { getFCP = settings.measure.fcp; } if (settings.measure.hero !== undefined) { if (settings.measure.hero.length !== 0) { getHero = true; } } if (settings.measure.ttfi !== undefined) { getTTFI = settings.measure.ttfi; } if (settings.measure.loadtime !== undefined) { getLoadTime = settings.measure.loadtime; } } else { raptorLog("abort: 'measure' key not found in test settings"); await cleanUp(); } break; } // write options to storage that our content script needs to know if (isGecko) { await ext.storage.local.clear(); await ext.storage.local.set({ settings }); } else { await new Promise(resolve => { ext.storage.local.clear(() => { ext.storage.local.set({ settings }, resolve); }); }); } raptorLog("wrote settings to ext local storage"); } async function sleep(delay) { return new Promise(resolve => setTimeout(resolve, delay)); } async function startScenarioTimer() { setTimeout(function () { isScenarioPending = false; results.measurements.scenario = [1]; }, scenarioTestTime); await postToControlServer("status", `started scenario test timer`); } async function closeTab(tabId) { // Don't close the last tab which would close the window or application const tabs = await queryForTabs({ currentWindow: true }); if (tabs.length == 1) { await postToControlServer("status", `Not closing last Tab: ${tabs[0].id}`); return; } await postToControlServer("status", `closing Tab: ${tabId}`); if (isGecko) { await ext.tabs.remove(tabId); } else { await new Promise(resolve => { ext.tabs.remove(tabId, resolve); }); } await postToControlServer("status", `closed tab: ${tabId}`); } async function getCurrentTabId() { const tabs = await queryForTabs({ currentWindow: true, active: true }); if (!tabs.length) { throw new Error("No active tab has been found."); } await postToControlServer("status", "found active tab with id " + tabs[0].id); return tabs[0].id; } async function openTab() { await postToControlServer("status", "opening new tab"); let tab; if (isGecko) { tab = await ext.tabs.create({ url: "about:blank" }); } else { tab = await new Promise(resolve => { ext.tabs.create({ url: "about:blank" }, resolve); }); } await postToControlServer("status", `opened new empty tab: ${tab.id}`); return tab.id; } async function queryForTabs(options = {}) { let tabs; if (isGecko) { tabs = await ext.tabs.query(options); } else { tabs = await new Promise(resolve => { ext.tabs.query(options, resolve); }); } return tabs; } /** * Update the given tab by navigating to the test URL */ async function updateTab(tabId, url) { await postToControlServer("status", `update tab ${tabId} for ${url}`); // "null" = active tab if (isGecko) { await ext.tabs.update(tabId, { url }); } else { await new Promise(resolve => { ext.tabs.update(tabId, { url }, resolve); }); } await postToControlServer("status", `tab ${tabId} updated`); } async function collectResults() { // now we can set the page timeout timer and wait for pageload test result from content raptorLog("ready to poll for results; turning on page-timeout timer"); setTimeoutAlarm("raptor-page-timeout", pageTimeout); // wait for pageload test result from content await waitForResults(); // move on to next cycle (or test complete) await nextCycle(); } function checkForTestFinished() { let finished = false; switch (testType) { case TEST_BENCHMARK: finished = !isBenchmarkPending; break; case TEST_PAGE_LOAD: if ( !isHeroPending && !isFNBPaintPending && !isFCPPending && !isDCFPending && !isTTFIPending && !isLoadTimePending ) { finished = true; } break; case TEST_SCENARIO: finished = !isScenarioPending; break; } return finished; } async function waitForResults() { raptorLog("waiting for results..."); while (!checkForTestFinished()) { raptorLog("results pending..."); await sleep(250); } await cancelTimeoutAlarm("raptor-page-timeout"); await postToControlServer("status", "results received"); if (geckoProfiling) { await getGeckoProfile(); } if (screenCapture) { await getScreenCapture(); } } async function getScreenCapture() { raptorLog("capturing screenshot"); try { let screenshotUri; if (isGecko) { screenshotUri = await ext.tabs.captureVisibleTab(); } else { screenshotUri = await new Promise(resolve => ext.tabs.captureVisibleTab(resolve) ); } await postToControlServer("screenshot", [ screenshotUri, testName, pageCycle, ]); } catch (e) { raptorLog(`failed to capture screenshot: ${e}`); } } async function startGeckoProfiling() { await postToControlServer( "status", `starting Gecko profiling for threads: ${geckoThreads}` ); const features = geckoFeatures ? geckoFeatures.split(",") : ["js", "leaf", "stackwalk", "cpu", "responsiveness"]; await ext.geckoProfiler.start({ bufferSize: geckoEntries, interval: geckoInterval, features, threads: geckoThreads.split(","), }); } async function stopGeckoProfiling() { await postToControlServer("status", "stopping gecko profiling"); await ext.geckoProfiler.stop(); } async function getGeckoProfile() { // trigger saving the gecko profile, and send the file name to the control server const fileName = `${testName}_pagecycle_${pageCycle}.profile`; await postToControlServer("status", `saving gecko profile ${fileName}`); await ext.geckoProfiler.dumpProfileToFile(fileName); await postToControlServer("gecko_profile", fileName); // must stop the profiler so it clears the buffer before next cycle await stopGeckoProfiling(); // resume if we have more pagecycles left if (pageCycle + 1 <= pageCycles) { await startGeckoProfiling(); } } async function nextCycle() { pageCycle++; if (isBackgroundTest) { await postToControlServer( "end_background", `bringing app to foreground, pausing for ${ foregroundDelay / 1000 } seconds` ); // wait a bit to be sure the app is in foreground before starting // new test, or finishing test await sleep(foregroundDelay); } if (pageCycle == 1) { const text = `running ${pageCycles} pagecycles of ${testURL}`; await postToControlServer("status", text); // start the profiler if enabled if (geckoProfiling) { await startGeckoProfiling(); } } if (pageCycle <= pageCycles) { if (isBackgroundTest) { await postToControlServer( "start_background", `bringing app to background` ); } await sleep(pageCycleDelay); await postToControlServer("status", `begin page cycle ${pageCycle}`); switch (testType) { case TEST_BENCHMARK: isBenchmarkPending = true; break; case TEST_PAGE_LOAD: if (getHero) { isHeroPending = true; pendingHeroes = Array.from(settings.measure.hero); } if (getFNBPaint) { isFNBPaintPending = true; } if (getFCP) { isFCPPending = true; } if (getDCF) { isDCFPending = true; } if (getTTFI) { isTTFIPending = true; } if (getLoadTime) { isLoadTimePending = true; } break; case TEST_SCENARIO: isScenarioPending = true; break; } if (newTabPerCycle) { // close previous test tab and open a new one await closeTab(testTabId); testTabId = await openTab(); } await updateTab(testTabId, testURL); if (testType == TEST_SCENARIO) { await startScenarioTimer(); } // For benchmark or scenario type tests we can proceed directly to // waitForResult. However for page-load tests we must first wait until // we hear back from pageloaderjs that it has been successfully loaded // in the test page and has been invoked; and only then start looking // for measurements. if (testType != TEST_PAGE_LOAD) { await collectResults(); } await postToControlServer("status", `ended page cycle ${pageCycle}`); } else { await verifyResults(); } } async function timeoutAlarmListener() { raptorLog(`raptor-page-timeout on ${testURL}`, "error"); const pendingMetrics = { hero: isHeroPending, "fnb paint": isFNBPaintPending, fcp: isFCPPending, dcf: isDCFPending, ttfi: isTTFIPending, "load time": isLoadTimePending, }; let msgData = [testName, testURL, pageCycle]; if (testType == TEST_PAGE_LOAD) { msgData.push(pendingMetrics); } await postToControlServer("raptor-page-timeout", msgData); await getScreenCapture(); // call clean-up to shutdown gracefully await cleanUp(); } function setTimeoutAlarm(timeoutName, timeoutMS) { // webext alarms require date.now NOT performance.now const now = Date.now(); // eslint-disable-line mozilla/avoid-Date-timing const timeout_when = now + timeoutMS; ext.alarms.create(timeoutName, { when: timeout_when }); raptorLog( `now is ${now}, set raptor alarm ${timeoutName} to expire ` + `at ${timeout_when}` ); } async function cancelTimeoutAlarm(timeoutName) { let cleared = false; if (isGecko) { cleared = await ext.alarms.clear(timeoutName); } else { cleared = await new Promise(resolve => { chrome.alarms.clear(timeoutName, resolve); }); } if (cleared) { raptorLog(`cancelled raptor alarm ${timeoutName}`); } else { raptorLog(`failed to clear raptor alarm ${timeoutName}`, "error"); } } function resultListener(request, sender, sendResponse) { raptorLog(`received message from ${sender.tab.url}`); // check if this is a message from pageloaderjs indicating it is ready to start if (request.type == "pageloadjs-ready") { raptorLog("received pageloadjs-ready!"); sendResponse({ text: "pageloadjs-ready-response" }); collectResults(); return; } if (request.type && request.value) { raptorLog(`result: ${request.type} ${request.value}`); sendResponse({ text: `confirmed ${request.type}` }); if (!(request.type in results.measurements)) { results.measurements[request.type] = []; } switch (testType) { case TEST_BENCHMARK: // benchmark results received (all results for that complete benchmark run) raptorLog("received results from benchmark"); results.measurements[request.type].push(request.value); isBenchmarkPending = false; break; case TEST_PAGE_LOAD: // a single pageload measurement was received if (request.type.indexOf("hero") > -1) { results.measurements[request.type].push(request.value); const _found = request.type.split("hero:")[1]; const index = pendingHeroes.indexOf(_found); if (index > -1) { pendingHeroes.splice(index, 1); if (!pendingHeroes.length) { raptorLog("measured all expected hero elements"); isHeroPending = false; } } } else if (request.type == "fnbpaint") { results.measurements.fnbpaint.push(request.value); isFNBPaintPending = false; } else if (request.type == "dcf") { results.measurements.dcf.push(request.value); isDCFPending = false; } else if (request.type == "ttfi") { results.measurements.ttfi.push(request.value); isTTFIPending = false; } else if (request.type == "fcp") { results.measurements.fcp.push(request.value); isFCPPending = false; } else if (request.type == "loadtime") { results.measurements.loadtime.push(request.value); isLoadTimePending = false; } break; } } else { raptorLog(`unknown message received from content: ${request}`); } } async function verifyResults() { raptorLog("Verifying results:"); raptorLog(results); for (var x in results.measurements) { const count = results.measurements[x].length; if (count == pageCycles) { raptorLog(`have ${count} results for ${x}, as expected`); } else { raptorLog( `expected ${pageCycles} results for ${x} but only have ${count}`, "error" ); } } await postToControlServer("results", results); // we're finished, move to cleanup await cleanUp(); } async function postToControlServer(msgType, msgData = "") { await new Promise(resolve => { // requires 'control server' running at port 8000 to receive results const xhr = new XMLHttpRequest(); xhr.open("POST", `http://${host}:${csPort}/`, true); xhr.setRequestHeader("Content-Type", "application/json"); xhr.onreadystatechange = () => { if (xhr.readyState == XMLHttpRequest.DONE) { if (xhr.status != 200) { // Failed to send the message. At least add a console error. let msg = msgType; if (msgType != "screenshot") { msg += ` with '${msgData}'`; } raptorLog(`failed to post ${msg} to control server`, "error"); } resolve(); } }; xhr.send( JSON.stringify({ type: `webext_${msgType}`, data: msgData, }) ); }); } async function cleanUp() { // close tab unless raptor debug-mode is enabled if (debugMode == 1) { raptorLog("debug-mode enabled, leaving tab open"); } else { await closeTab(testTabId); } if (testType == TEST_PAGE_LOAD) { // remove listeners ext.alarms.onAlarm.removeListener(timeoutAlarmListener); ext.runtime.onMessage.removeListener(resultListener); } raptorLog(`${testType} test finished`); // if profiling was enabled, stop the profiler - may have already // been stopped but stop again here in cleanup in case of timeout if (geckoProfiling) { await stopGeckoProfiling(); } // tell the control server we are done and the browser can be shutdown await postToControlServer("shutdownBrowser"); } async function raptorRunner() { await postToControlServer("status", "starting raptorRunner"); if (isBackgroundTest) { await postToControlServer( "status", "raptor test will be backgrounding the app" ); } await getTestSettings(); raptorLog(`${testType} test start`); ext.alarms.onAlarm.addListener(timeoutAlarmListener); ext.runtime.onMessage.addListener(resultListener); // create new empty tab, which starts the test; we want to // wait some time for the browser to settle before beginning const text = `* pausing ${ postStartupDelay / 1000 } seconds to let browser settle... *`; await postToControlServer("status", text); await sleep(postStartupDelay); if (!isGeckoAndroid) { await openTab(); } testTabId = await getCurrentTabId(); await nextCycle(); } function raptorLog(text, level = "info") { let prefix = ""; if (level == "error") { prefix = "ERROR: "; } console[level](`${prefix}[raptor-runnerjs] ${text}`); } async function init() { const config = getTestConfig(); testName = config.test_name; settingsURL = config.test_settings_url; csPort = config.cs_port; benchmarkPort = config.benchmark_port; postStartupDelay = config.post_startup_delay; host = config.host; debugMode = config.debug_mode; browserCycle = config.browser_cycle; try { // Chromium based browsers do not support the "browser" namespace and // raise an exception when accessing it. const info = await browser.runtime.getBrowserInfo(); results.browser = `${info.name} ${info.version} ${info.buildID}`; ext = browser; isGecko = true; isGeckoAndroid = ANDROID_BROWSERS.includes(info.name.toLowerCase); } catch (e) { const regex = /(Chrome)\/([\w\.]+)/; const userAgent = window.navigator.userAgent; results.browser = regex.exec(userAgent).splice(1, 2).join(" "); ext = chrome; } await postToControlServer("loaded"); await postToControlServer("status", `testing on ${results.browser}`); await postToControlServer("status", `test name is: ${testName}`); await postToControlServer("status", `test settings url is: ${settingsURL}`); try { if (window.document.readyState != "complete") { await new Promise(resolve => { window.addEventListener("load", resolve); raptorLog("Waiting for load event..."); }); } await raptorRunner(); } catch (e) { await postToControlServer("error", [e.message, e.stack]); await postToControlServer("shutdownBrowser"); } } init();