Adding upstream version 115.7.0esr.upstream/115.7.0esr

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

5 files changed, 1324 insertions, 0 deletions

diff --git a/testing/raptor/webext/raptor/benchmark.js b/testing/raptor/webext/raptor/benchmark.js
new file mode 100644
index 0000000000..277ceeb710
--- /dev/null
+++ b/testing/raptor/webext/raptor/benchmark.js
@@ -0,0 +1,49 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+// receives result from benchmark and relays onto our background runner
+
+async function receiveMessage(event) {
+  raptorLog("raptor benchmark received message");
+  raptorLog(event.data);
+
+  // raptor benchmark message data [0] is raptor tag, [1] is benchmark
+  // name, and the rest is actual benchmark results that we want to fw
+  if (event.data[0] == "raptor-benchmark") {
+    await sendResult(event.data[1], event.data.slice(2));
+  }
+}
+
+/**
+ * Send result back to background runner script
+ */
+async function sendResult(type, value) {
+  raptorLog(`sending result back to runner: ${type} ${value}`);
+
+  let response;
+  if (typeof browser !== "undefined") {
+    response = await browser.runtime.sendMessage({ type, value });
+  } else {
+    response = await new Promise(resolve => {
+      chrome.runtime.sendMessage({ type, value }, resolve);
+    });
+  }
+
+  if (response) {
+    raptorLog(`Response: ${response.text}`);
+  }
+}
+
+function raptorLog(text, level = "info") {
+  let prefix = "";
+
+  if (level == "error") {
+    prefix = "ERROR: ";
+  }
+
+  console[level](`${prefix}[raptor-benchmarkjs] ${text}`);
+}
+
+raptorLog("raptor benchmark content loaded");
+window.addEventListener("message", receiveMessage);
diff --git a/testing/raptor/webext/raptor/icon.png b/testing/raptor/webext/raptor/icon.png
new file mode 100644
index 0000000000..253851bc46
--- /dev/null
+++ b/testing/raptor/webext/raptor/icon.png
diff --git a/testing/raptor/webext/raptor/manifest.json b/testing/raptor/webext/raptor/manifest.json
new file mode 100644
index 0000000000..744c36f471
--- /dev/null
+++ b/testing/raptor/webext/raptor/manifest.json
@@ -0,0 +1,83 @@
+{
+  "browser_specific_settings": {
+    "gecko": {
+      "id": "raptor@mozilla.org"
+    }
+  },
+  "manifest_version": 2,
+  "name": "Raptor",
+  "version": "0.1",
+  "description": "Performance measurement framework prototype",
+  "background": {
+    "scripts": ["auto_gen_test_config.js", "runner.js"]
+  },
+  "content_scripts": [
+    {
+      "matches": [
+        "*://*.allrecipes.com/*",
+        "*://*.apple.com/*",
+        "*://*.amazon.com/*",
+        "*://*.bing.com/*",
+        "*://*.booking.com/*",
+        "*://*.cnn.com/*",
+        "*://*.dailymail.co.uk/*",
+        "*://*.ebay.com/*",
+        "*://*.ebay-kleinanzeigen.de/*",
+        "*://*.espn.com/*",
+        "*://*.facebook.com/*",
+        "*://*.fandom.com/*",
+        "*://*.google.com/*",
+        "*://*.imdb.com/*",
+        "*://*.imgur.com/*",
+        "*://*.instagram.com/*",
+        "*://*.linkedin.com/*",
+        "*://*.live.com/*",
+        "*://*.microsoft.com/*",
+        "*://*.netflix.com/*",
+        "*://*.office.com/*",
+        "*://*.paypal.com/*",
+        "*://*.pinterest.com/*",
+        "*://*.reddit.com/*",
+        "*://*.stackoverflow.com/*",
+        "*://*.sina.com.cn/*",
+        "*://*.tumblr.com/*",
+        "*://*.twitch.tv/*",
+        "*://*.twitter.com/*",
+        "*://*.vice.com/*",
+        "*://*.web.de/*",
+        "*://*.wikia.com/*",
+        "*://*.wikipedia.org/*",
+        "*://*.yahoo.com/*",
+        "*://*.youtube.com/*",
+        "*://*.yandex.ru/*"
+      ],
+      "js": ["pageload.js"],
+      "run_at": "document_end"
+    },
+    {
+      "matches": [
+        "*://*/Speedometer/index.html*",
+        "*://*/StyleBench/*",
+        "*://*/MotionMark/*",
+        "*://*/SunSpider/*",
+        "*://*/webaudio/*",
+        "*://*/unity-webgl/index.html*",
+        "*://*/wasm-misc/index.html*",
+        "*://*/wasm-godot/index.html*",
+        "*://*/assorted-dom/assorted/results.html*",
+        "*://*.mozaws.net/*",
+        "*://*/ARES-6/index.html*",
+        "*://*/JetStream2/index.html*",
+        "*://*.amazonaws.com/*"
+      ],
+      "js": ["benchmark.js"],
+      "run_at": "document_end"
+    }
+  ],
+  "browser_action": {
+    "browser_style": true,
+    "default_icon": "icon.png",
+    "default_title": "Raptor LOADED"
+  },
+  "permissions": ["<all_urls>", "tabs", "storage", "alarms", "geckoProfiler"]
+}
diff --git a/testing/raptor/webext/raptor/pageload.js b/testing/raptor/webext/raptor/pageload.js
new file mode 100644
index 0000000000..dff2d56b8b
--- /dev/null
+++ b/testing/raptor/webext/raptor/pageload.js
@@ -0,0 +1,396 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+// Supported test types
+const TEST_BENCHMARK = "benchmark";
+const TEST_PAGE_LOAD = "pageload";
+
+// content script for use with pageload tests
+var perfData = window.performance;
+var gRetryCounter = 0;
+
+// measure hero element; must exist inside test page;
+// supported on: Firefox, Chromium, Geckoview
+// default only; this is set via control server settings json
+var getHero = false;
+var heroesToCapture = [];
+
+// measure time-to-first-non-blank-paint
+// supported on: Firefox, Geckoview
+// note: this browser pref must be enabled:
+// dom.performance.time_to_non_blank_paint.enabled = True
+// default only; this is set via control server settings json
+var getFNBPaint = false;
+
+// measure time-to-first-contentful-paint
+// supported on: Firefox, Chromium, Geckoview
+// note: this browser pref must be enabled:
+// dom.performance.time_to_contentful_paint.enabled = True
+// default only; this is set via control server settings json
+var getFCP = false;
+
+// measure domContentFlushed
+// supported on: Firefox, Geckoview
+// note: this browser pref must be enabled:
+// dom.performance.time_to_dom_content_flushed.enabled = True
+// default only; this is set via control server settings json
+var getDCF = false;
+
+// measure TTFI
+// supported on: Firefox, Geckoview
+// note: this browser pref must be enabled:
+// dom.performance.time_to_first_interactive.enabled = True
+// default only; this is set via control server settings json
+var getTTFI = false;
+
+// supported on: Firefox, Chromium, Geckoview
+// default only; this is set via control server settings json
+var getLoadTime = false;
+
+// performance.timing measurement used as 'starttime'
+var startMeasure = "fetchStart";
+
+async function raptorContentHandler() {
+  raptorLog("pageloadjs raptorContentHandler!");
+  // let the main raptor runner know that we (pageloadjs) is ready!
+  await sendPageloadReady();
+
+  // retrieve test settings from local ext storage
+  let settings;
+  if (typeof browser !== "undefined") {
+    ({ settings } = await browser.storage.local.get("settings"));
+  } else {
+    ({ settings } = await new Promise(resolve => {
+      chrome.storage.local.get("settings", resolve);
+    }));
+  }
+  setup(settings);
+}
+
+function setup(settings) {
+  if (settings.type != TEST_PAGE_LOAD) {
+    return;
+  }
+
+  if (settings.measure == undefined) {
+    raptorLog("abort: 'measure' key not found in test settings");
+    return;
+  }
+
+  if (settings.measure.fnbpaint !== undefined) {
+    getFNBPaint = settings.measure.fnbpaint;
+    if (getFNBPaint) {
+      raptorLog("will be measuring fnbpaint");
+      measureFNBPaint();
+    }
+  }
+
+  if (settings.measure.dcf !== undefined) {
+    getDCF = settings.measure.dcf;
+    if (getDCF) {
+      raptorLog("will be measuring dcf");
+      measureDCF();
+    }
+  }
+
+  if (settings.measure.fcp !== undefined) {
+    getFCP = settings.measure.fcp;
+    if (getFCP) {
+      raptorLog("will be measuring first-contentful-paint");
+      measureFCP();
+    }
+  }
+
+  if (settings.measure.hero !== undefined) {
+    if (settings.measure.hero.length !== 0) {
+      getHero = true;
+      heroesToCapture = settings.measure.hero;
+      raptorLog(`hero elements to measure: ${heroesToCapture}`);
+      measureHero();
+    }
+  }
+
+  if (settings.measure.ttfi !== undefined) {
+    getTTFI = settings.measure.ttfi;
+    if (getTTFI) {
+      raptorLog("will be measuring ttfi");
+      measureTTFI();
+    }
+  }
+
+  if (settings.measure.loadtime !== undefined) {
+    getLoadTime = settings.measure.loadtime;
+    if (getLoadTime) {
+      raptorLog("will be measuring loadtime");
+      measureLoadTime();
+    }
+  }
+}
+
+function measureHero() {
+  let obs;
+
+  const heroElementsFound = window.document.querySelectorAll("[elementtiming]");
+  raptorLog(`found ${heroElementsFound.length} hero elements in the page`);
+
+  if (heroElementsFound) {
+    async function callbackHero(entries, observer) {
+      for (const entry in entries) {
+        const heroFound = entry.target.getAttribute("elementtiming");
+        // mark the time now as when hero element received
+        perfData.mark(heroFound);
+        const resultType = `hero:${heroFound}`;
+        raptorLog(`found ${resultType}`);
+        // calculcate result: performance.timing.fetchStart - time when we got hero element
+        perfData.measure(
+          (name = resultType),
+          (startMark = startMeasure),
+          (endMark = heroFound)
+        );
+        const perfResult = perfData.getEntriesByName(resultType);
+        const _result = Math.round(perfResult[0].duration);
+        await sendResult(resultType, _result);
+        perfData.clearMarks();
+        perfData.clearMeasures();
+        obs.disconnect();
+      }
+    }
+    // we want the element 100% visible on the viewport
+    const options = { root: null, rootMargin: "0px", threshold: [1] };
+    try {
+      obs = new window.IntersectionObserver(callbackHero, options);
+      heroElementsFound.forEach(function (el) {
+        // if hero element is one we want to measure, add it to the observer
+        if (heroesToCapture.indexOf(el.getAttribute("elementtiming")) > -1) {
+          obs.observe(el);
+        }
+      });
+    } catch (err) {
+      raptorLog(err);
+    }
+  } else {
+    raptorLog("couldn't find hero element");
+  }
+}
+
+async function measureFNBPaint() {
+  const x = window.performance.timing.timeToNonBlankPaint;
+
+  if (typeof x == "undefined") {
+    raptorLog(
+      "timeToNonBlankPaint is undefined; ensure the pref is enabled",
+      "error"
+    );
+    return;
+  }
+  if (x > 0) {
+    raptorLog("got fnbpaint");
+    gRetryCounter = 0;
+    const startTime = perfData.timing.fetchStart;
+    await sendResult("fnbpaint", x - startTime);
+  } else {
+    gRetryCounter += 1;
+    if (gRetryCounter <= 10) {
+      raptorLog(
+        `fnbpaint is not yet available, retry number ${gRetryCounter}...`
+      );
+      window.setTimeout(measureFNBPaint, 100);
+    } else {
+      raptorLog(
+        `unable to get a value for fnbpaint after ${gRetryCounter} retries`
+      );
+    }
+  }
+}
+
+async function measureDCF() {
+  const x = window.performance.timing.timeToDOMContentFlushed;
+
+  if (typeof x == "undefined") {
+    raptorLog(
+      "domContentFlushed is undefined; ensure the pref is enabled",
+      "error"
+    );
+    return;
+  }
+  if (x > 0) {
+    raptorLog(`got domContentFlushed: ${x}`);
+    gRetryCounter = 0;
+    const startTime = perfData.timing.fetchStart;
+    await sendResult("dcf", x - startTime);
+  } else {
+    gRetryCounter += 1;
+    if (gRetryCounter <= 10) {
+      raptorLog(
+        `dcf is not yet available (0), retry number ${gRetryCounter}...`
+      );
+      window.setTimeout(measureDCF, 100);
+    } else {
+      raptorLog(`unable to get a value for dcf after ${gRetryCounter} retries`);
+    }
+  }
+}
+
+async function measureTTFI() {
+  const x = window.performance.timing.timeToFirstInteractive;
+
+  if (typeof x == "undefined") {
+    raptorLog(
+      "timeToFirstInteractive is undefined; ensure the pref is enabled",
+      "error"
+    );
+    return;
+  }
+  if (x > 0) {
+    raptorLog(`got timeToFirstInteractive: ${x}`);
+    gRetryCounter = 0;
+    const startTime = perfData.timing.fetchStart;
+    await sendResult("ttfi", x - startTime);
+  } else {
+    gRetryCounter += 1;
+    // NOTE: currently the gecko implementation doesn't look at network
+    // requests, so this is closer to TimeToFirstInteractive than
+    // TimeToInteractive.  TTFI/TTI requires running at least 5 seconds
+    // past last "busy" point, give 25 seconds here (overall the harness
+    // times out at 30 seconds).  Some pages will never get 5 seconds
+    // without a busy period!
+    if (gRetryCounter <= 25 * (1000 / 200)) {
+      raptorLog(
+        `TTFI is not yet available (0), retry number ${gRetryCounter}...`
+      );
+      window.setTimeout(measureTTFI, 200);
+    } else {
+      // unable to get a value for TTFI - negative value will be filtered out later
+      raptorLog("TTFI was not available for this pageload");
+      await sendResult("ttfi", -1);
+    }
+  }
+}
+
+async function measureFCP() {
+  // see https://developer.mozilla.org/en-US/docs/Web/API/PerformancePaintTiming
+  let result = window.performance.timing.timeToContentfulPaint;
+
+  // Firefox implementation of FCP is not yet spec-compliant (see Bug 1519410)
+  if (typeof result == "undefined") {
+    // we're on chromium
+    result = 0;
+
+    const perfEntries = perfData.getEntriesByType("paint");
+    if (perfEntries.length >= 2) {
+      if (
+        perfEntries[1].name == "first-contentful-paint" &&
+        perfEntries[1].startTime != undefined
+      ) {
+        // this value is actually the final measurement / time to get the FCP event in MS
+        result = perfEntries[1].startTime;
+      }
+    }
+  }
+
+  if (result > 0) {
+    raptorLog("got time to first-contentful-paint");
+    if (typeof browser !== "undefined") {
+      // Firefox returns a timestamp, not the actual measurement in MS; need to calculate result
+      const startTime = perfData.timing.fetchStart;
+      result = result - startTime;
+    }
+    await sendResult("fcp", result);
+    perfData.clearMarks();
+    perfData.clearMeasures();
+  } else {
+    gRetryCounter += 1;
+    if (gRetryCounter <= 10) {
+      raptorLog(
+        `time to first-contentful-paint is not yet available (0), retry number ${gRetryCounter}...`
+      );
+      window.setTimeout(measureFCP, 100);
+    } else {
+      raptorLog(
+        `unable to get a value for time-to-fcp after ${gRetryCounter} retries`
+      );
+    }
+  }
+}
+
+async function measureLoadTime() {
+  const x = window.performance.timing.loadEventStart;
+
+  if (typeof x == "undefined") {
+    raptorLog("loadEventStart is undefined", "error");
+    return;
+  }
+  if (x > 0) {
+    raptorLog(`got loadEventStart: ${x}`);
+    gRetryCounter = 0;
+    const startTime = perfData.timing.fetchStart;
+    await sendResult("loadtime", x - startTime);
+  } else {
+    gRetryCounter += 1;
+    if (gRetryCounter <= 40 * (1000 / 200)) {
+      raptorLog(
+        `loadEventStart is not yet available (0), retry number ${gRetryCounter}...`
+      );
+      window.setTimeout(measureLoadTime, 100);
+    } else {
+      raptorLog(
+        `unable to get a value for loadEventStart after ${gRetryCounter} retries`
+      );
+    }
+  }
+}
+
+/**
+ * Send message to runnerjs indicating pageloadjs is ready to start getting measures
+ */
+async function sendPageloadReady() {
+  raptorLog("sending pageloadjs-ready message to runnerjs");
+
+  let response;
+  if (typeof browser !== "undefined") {
+    response = await browser.runtime.sendMessage({ type: "pageloadjs-ready" });
+  } else {
+    response = await new Promise(resolve => {
+      chrome.runtime.sendMessage({ type: "pageloadjs-ready" }, resolve);
+    });
+  }
+
+  if (response) {
+    raptorLog(`Response: ${response.text}`);
+  }
+}
+
+/**
+ * Send result back to background runner script
+ */
+async function sendResult(type, value) {
+  raptorLog(`sending result back to runner: ${type} ${value}`);
+
+  let response;
+  if (typeof browser !== "undefined") {
+    response = await browser.runtime.sendMessage({ type, value });
+  } else {
+    response = await new Promise(resolve => {
+      chrome.runtime.sendMessage({ type, value }, resolve);
+    });
+  }
+
+  if (response) {
+    raptorLog(`Response: ${response.text}`);
+  }
+}
+
+function raptorLog(text, level = "info") {
+  let prefix = "";
+
+  if (level == "error") {
+    prefix = "ERROR: ";
+  }
+
+  console[level](`${prefix}[raptor-pageloadjs] ${text}`);
+}
+
+if (window.addEventListener) {
+  window.addEventListener("load", raptorContentHandler);
+}
diff --git a/testing/raptor/webext/raptor/runner.js b/testing/raptor/webext/raptor/runner.js
new file mode 100644
index 0000000000..7129ec64c1
--- /dev/null
+++ b/testing/raptor/webext/raptor/runner.js
@@ -0,0 +1,796 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+// this extension requires a 'control server' to be running on port 8000
+// (see raptor prototype framework). It will provide the test options, as
+// well as receive test results
+
+// note: currently the prototype assumes the test page(s) are
+// already available somewhere independently; so for now locally
+// inside the 'talos-pagesets' dir or 'heroes' dir (tarek's github
+// repo) or 'webkit/PerformanceTests' dir (for benchmarks) first run:
+// 'python -m SimpleHTTPServer 8081'
+// to serve out the pages that we want to prototype with. Also
+// update the manifest content 'matches' accordingly
+
+// Supported test types
+const TEST_BENCHMARK = "benchmark";
+const TEST_PAGE_LOAD = "pageload";
+const TEST_SCENARIO = "scenario";
+
+const ANDROID_BROWSERS = ["fenix", "geckoview", "refbrow"];
+
+// when the browser starts this webext runner will start automatically; we
+// want to give the browser some time (ms) to settle before starting tests
+var postStartupDelay;
+
+// delay (ms) between pageload cycles
+var pageCycleDelay = 1000;
+
+var newTabPerCycle = false;
+
+// delay (ms) for foregrounding app
+var foregroundDelay = 5000;
+
+var isGecko = false;
+var isGeckoAndroid = false;
+var ext;
+var testName = null;
+var settingsURL = null;
+var csPort = null;
+var host = null;
+var benchmarkPort = null;
+var testType;
+var browserCycle = 0;
+var pageCycles = 0;
+var pageCycle = 0;
+var testURL;
+var testTabId;
+var scenarioTestTime = 60000;
+var getHero = false;
+var getFNBPaint = false;
+var getFCP = false;
+var getDCF = false;
+var getTTFI = false;
+var getLoadTime = false;
+var isHeroPending = false;
+var pendingHeroes = [];
+var settings = {};
+var isFNBPaintPending = false;
+var isFCPPending = false;
+var isDCFPending = false;
+var isTTFIPending = false;
+var isLoadTimePending = false;
+var isScenarioPending = false;
+var isBenchmarkPending = false;
+var isBackgroundTest = false;
+var pageTimeout = 10000; // default pageload timeout
+var geckoProfiling = false;
+var geckoInterval = 1;
+var geckoEntries = 1000000;
+var geckoThreads = [];
+var geckoFeatures = null;
+var debugMode = 0;
+var screenCapture = false;
+
+var results = {
+  name: "",
+  page: "",
+  type: "",
+  browser_cycle: 0,
+  expected_browser_cycles: 0,
+  cold: false,
+  lower_is_better: true,
+  alert_change_type: "relative",
+  alert_threshold: 2.0,
+  measurements: {},
+};
+
+async function getTestSettings() {
+  raptorLog("getting test settings from control server");
+  const response = await fetch(settingsURL);
+  const data = await response.text();
+  raptorLog(`test settings received: ${data}`);
+
+  // parse the test settings
+  settings = JSON.parse(data)["raptor-options"];
+  testType = settings.type;
+  pageCycles = settings.page_cycles;
+  testURL = settings.test_url;
+  scenarioTestTime = settings.scenario_time;
+  isBackgroundTest = settings.background_test;
+
+  // for pageload type tests, the testURL is fine as is - we don't have
+  // to add a port as it's accessed via proxy and the playback tool
+  // however for benchmark tests, their source is served out on a local
+  // webserver, so we need to swap in the webserver port into the testURL
+  if (testType == TEST_BENCHMARK) {
+    // just replace the '<port>' keyword in the URL with actual benchmarkPort
+    testURL = testURL.replace("<port>", benchmarkPort);
+  }
+
+  if (host) {
+    // just replace the '<host>' keyword in the URL with actual host
+    testURL = testURL.replace("<host>", host);
+  }
+
+  raptorLog(`test URL: ${testURL}`);
+
+  results.alert_change_type = settings.alert_change_type;
+  results.alert_threshold = settings.alert_threshold;
+  results.browser_cycle = browserCycle;
+  results.cold = settings.cold;
+  results.expected_browser_cycles = settings.expected_browser_cycles;
+  results.lower_is_better = settings.lower_is_better === true;
+  results.name = testName;
+  results.page = testURL;
+  results.type = testType;
+  results.unit = settings.unit;
+  results.subtest_unit = settings.subtest_unit;
+  results.subtest_lower_is_better = settings.subtest_lower_is_better === true;
+
+  if (settings.gecko_profile === true) {
+    results.extra_options = ["gecko-profile"];
+
+    geckoProfiling = true;
+    geckoEntries = settings.gecko_profile_entries;
+    geckoInterval = settings.gecko_profile_interval;
+    geckoThreads = settings.gecko_profile_threads;
+    geckoFeatures = settings.gecko_profile_features;
+  }
+
+  if (settings.screen_capture !== undefined) {
+    screenCapture = settings.screen_capture;
+  }
+
+  if (settings.newtab_per_cycle !== undefined) {
+    newTabPerCycle = settings.newtab_per_cycle;
+  }
+
+  if (settings.page_timeout !== undefined) {
+    pageTimeout = settings.page_timeout;
+  }
+  raptorLog(`using page timeout: ${pageTimeout}ms`);
+
+  switch (testType) {
+    case TEST_PAGE_LOAD:
+      if (settings.measure !== undefined) {
+        if (settings.measure.fnbpaint !== undefined) {
+          getFNBPaint = settings.measure.fnbpaint;
+        }
+        if (settings.measure.dcf !== undefined) {
+          getDCF = settings.measure.dcf;
+        }
+        if (settings.measure.fcp !== undefined) {
+          getFCP = settings.measure.fcp;
+        }
+        if (settings.measure.hero !== undefined) {
+          if (settings.measure.hero.length !== 0) {
+            getHero = true;
+          }
+        }
+        if (settings.measure.ttfi !== undefined) {
+          getTTFI = settings.measure.ttfi;
+        }
+        if (settings.measure.loadtime !== undefined) {
+          getLoadTime = settings.measure.loadtime;
+        }
+      } else {
+        raptorLog("abort: 'measure' key not found in test settings");
+        await cleanUp();
+      }
+      break;
+  }
+
+  // write options to storage that our content script needs to know
+  if (isGecko) {
+    await ext.storage.local.clear();
+    await ext.storage.local.set({ settings });
+  } else {
+    await new Promise(resolve => {
+      ext.storage.local.clear(() => {
+        ext.storage.local.set({ settings }, resolve);
+      });
+    });
+  }
+  raptorLog("wrote settings to ext local storage");
+}
+
+async function sleep(delay) {
+  return new Promise(resolve => setTimeout(resolve, delay));
+}
+
+async function startScenarioTimer() {
+  setTimeout(function () {
+    isScenarioPending = false;
+    results.measurements.scenario = [1];
+  }, scenarioTestTime);
+
+  await postToControlServer("status", `started scenario test timer`);
+}
+
+async function closeTab(tabId) {
+  // Don't close the last tab which would close the window or application
+  const tabs = await queryForTabs({ currentWindow: true });
+  if (tabs.length == 1) {
+    await postToControlServer("status", `Not closing last Tab: ${tabs[0].id}`);
+    return;
+  }
+
+  await postToControlServer("status", `closing Tab: ${tabId}`);
+
+  if (isGecko) {
+    await ext.tabs.remove(tabId);
+  } else {
+    await new Promise(resolve => {
+      ext.tabs.remove(tabId, resolve);
+    });
+  }
+
+  await postToControlServer("status", `closed tab: ${tabId}`);
+}
+
+async function getCurrentTabId() {
+  const tabs = await queryForTabs({ currentWindow: true, active: true });
+  if (!tabs.length) {
+    throw new Error("No active tab has been found.");
+  }
+
+  await postToControlServer("status", "found active tab with id " + tabs[0].id);
+  return tabs[0].id;
+}
+
+async function openTab() {
+  await postToControlServer("status", "opening new tab");
+
+  let tab;
+  if (isGecko) {
+    tab = await ext.tabs.create({ url: "about:blank" });
+  } else {
+    tab = await new Promise(resolve => {
+      ext.tabs.create({ url: "about:blank" }, resolve);
+    });
+  }
+
+  await postToControlServer("status", `opened new empty tab: ${tab.id}`);
+
+  return tab.id;
+}
+
+async function queryForTabs(options = {}) {
+  let tabs;
+
+  if (isGecko) {
+    tabs = await ext.tabs.query(options);
+  } else {
+    tabs = await new Promise(resolve => {
+      ext.tabs.query(options, resolve);
+    });
+  }
+
+  return tabs;
+}
+
+/**
+ * Update the given tab by navigating to the test URL
+ */
+async function updateTab(tabId, url) {
+  await postToControlServer("status", `update tab ${tabId} for ${url}`);
+
+  // "null" = active tab
+  if (isGecko) {
+    await ext.tabs.update(tabId, { url });
+  } else {
+    await new Promise(resolve => {
+      ext.tabs.update(tabId, { url }, resolve);
+    });
+  }
+
+  await postToControlServer("status", `tab ${tabId} updated`);
+}
+
+async function collectResults() {
+  // now we can set the page timeout timer and wait for pageload test result from content
+  raptorLog("ready to poll for results; turning on page-timeout timer");
+  setTimeoutAlarm("raptor-page-timeout", pageTimeout);
+
+  // wait for pageload test result from content
+  await waitForResults();
+
+  // move on to next cycle (or test complete)
+  await nextCycle();
+}
+
+function checkForTestFinished() {
+  let finished = false;
+
+  switch (testType) {
+    case TEST_BENCHMARK:
+      finished = !isBenchmarkPending;
+      break;
+    case TEST_PAGE_LOAD:
+      if (
+        !isHeroPending &&
+        !isFNBPaintPending &&
+        !isFCPPending &&
+        !isDCFPending &&
+        !isTTFIPending &&
+        !isLoadTimePending
+      ) {
+        finished = true;
+      }
+      break;
+
+    case TEST_SCENARIO:
+      finished = !isScenarioPending;
+      break;
+  }
+
+  return finished;
+}
+
+async function waitForResults() {
+  raptorLog("waiting for results...");
+
+  while (!checkForTestFinished()) {
+    raptorLog("results pending...");
+    await sleep(250);
+  }
+
+  await cancelTimeoutAlarm("raptor-page-timeout");
+
+  await postToControlServer("status", "results received");
+
+  if (geckoProfiling) {
+    await getGeckoProfile();
+  }
+
+  if (screenCapture) {
+    await getScreenCapture();
+  }
+}
+
+async function getScreenCapture() {
+  raptorLog("capturing screenshot");
+
+  try {
+    let screenshotUri;
+
+    if (isGecko) {
+      screenshotUri = await ext.tabs.captureVisibleTab();
+    } else {
+      screenshotUri = await new Promise(resolve =>
+        ext.tabs.captureVisibleTab(resolve)
+      );
+    }
+
+    await postToControlServer("screenshot", [
+      screenshotUri,
+      testName,
+      pageCycle,
+    ]);
+  } catch (e) {
+    raptorLog(`failed to capture screenshot: ${e}`);
+  }
+}
+
+async function startGeckoProfiling() {
+  await postToControlServer(
+    "status",
+    `starting Gecko profiling for threads: ${geckoThreads}`
+  );
+  const features = geckoFeatures
+    ? geckoFeatures.split(",")
+    : ["js", "leaf", "stackwalk", "cpu", "responsiveness"];
+  await ext.geckoProfiler.start({
+    bufferSize: geckoEntries,
+    interval: geckoInterval,
+    features,
+    threads: geckoThreads.split(","),
+  });
+}
+
+async function stopGeckoProfiling() {
+  await postToControlServer("status", "stopping gecko profiling");
+  await ext.geckoProfiler.stop();
+}
+
+async function getGeckoProfile() {
+  // trigger saving the gecko profile, and send the file name to the control server
+  const fileName = `${testName}_pagecycle_${pageCycle}.profile`;
+
+  await postToControlServer("status", `saving gecko profile ${fileName}`);
+  await ext.geckoProfiler.dumpProfileToFile(fileName);
+  await postToControlServer("gecko_profile", fileName);
+
+  // must stop the profiler so it clears the buffer before next cycle
+  await stopGeckoProfiling();
+
+  // resume if we have more pagecycles left
+  if (pageCycle + 1 <= pageCycles) {
+    await startGeckoProfiling();
+  }
+}
+
+async function nextCycle() {
+  pageCycle++;
+  if (isBackgroundTest) {
+    await postToControlServer(
+      "end_background",
+      `bringing app to foreground, pausing for ${
+        foregroundDelay / 1000
+      } seconds`
+    );
+    // wait a bit to be sure the app is in foreground before starting
+    // new test, or finishing test
+    await sleep(foregroundDelay);
+  }
+  if (pageCycle == 1) {
+    const text = `running ${pageCycles} pagecycles of ${testURL}`;
+    await postToControlServer("status", text);
+    // start the profiler if enabled
+    if (geckoProfiling) {
+      await startGeckoProfiling();
+    }
+  }
+  if (pageCycle <= pageCycles) {
+    if (isBackgroundTest) {
+      await postToControlServer(
+        "start_background",
+        `bringing app to background`
+      );
+    }
+
+    await sleep(pageCycleDelay);
+
+    await postToControlServer("status", `begin page cycle ${pageCycle}`);
+
+    switch (testType) {
+      case TEST_BENCHMARK:
+        isBenchmarkPending = true;
+        break;
+
+      case TEST_PAGE_LOAD:
+        if (getHero) {
+          isHeroPending = true;
+          pendingHeroes = Array.from(settings.measure.hero);
+        }
+        if (getFNBPaint) {
+          isFNBPaintPending = true;
+        }
+        if (getFCP) {
+          isFCPPending = true;
+        }
+        if (getDCF) {
+          isDCFPending = true;
+        }
+        if (getTTFI) {
+          isTTFIPending = true;
+        }
+        if (getLoadTime) {
+          isLoadTimePending = true;
+        }
+        break;
+
+      case TEST_SCENARIO:
+        isScenarioPending = true;
+        break;
+    }
+
+    if (newTabPerCycle) {
+      // close previous test tab and open a new one
+      await closeTab(testTabId);
+      testTabId = await openTab();
+    }
+
+    await updateTab(testTabId, testURL);
+
+    if (testType == TEST_SCENARIO) {
+      await startScenarioTimer();
+    }
+
+    // For benchmark or scenario type tests we can proceed directly to
+    // waitForResult. However for page-load tests we must first wait until
+    // we hear back from pageloaderjs that it has been successfully loaded
+    // in the test page and has been invoked; and only then start looking
+    // for measurements.
+    if (testType != TEST_PAGE_LOAD) {
+      await collectResults();
+    }
+
+    await postToControlServer("status", `ended page cycle ${pageCycle}`);
+  } else {
+    await verifyResults();
+  }
+}
+
+async function timeoutAlarmListener() {
+  raptorLog(`raptor-page-timeout on ${testURL}`, "error");
+
+  const pendingMetrics = {
+    hero: isHeroPending,
+    "fnb paint": isFNBPaintPending,
+    fcp: isFCPPending,
+    dcf: isDCFPending,
+    ttfi: isTTFIPending,
+    "load time": isLoadTimePending,
+  };
+
+  let msgData = [testName, testURL, pageCycle];
+  if (testType == TEST_PAGE_LOAD) {
+    msgData.push(pendingMetrics);
+  }
+
+  await postToControlServer("raptor-page-timeout", msgData);
+  await getScreenCapture();
+
+  // call clean-up to shutdown gracefully
+  await cleanUp();
+}
+
+function setTimeoutAlarm(timeoutName, timeoutMS) {
+  // webext alarms require date.now NOT performance.now
+  const now = Date.now(); // eslint-disable-line mozilla/avoid-Date-timing
+  const timeout_when = now + timeoutMS;
+  ext.alarms.create(timeoutName, { when: timeout_when });
+
+  raptorLog(
+    `now is ${now}, set raptor alarm ${timeoutName} to expire ` +
+      `at ${timeout_when}`
+  );
+}
+
+async function cancelTimeoutAlarm(timeoutName) {
+  let cleared = false;
+
+  if (isGecko) {
+    cleared = await ext.alarms.clear(timeoutName);
+  } else {
+    cleared = await new Promise(resolve => {
+      chrome.alarms.clear(timeoutName, resolve);
+    });
+  }
+
+  if (cleared) {
+    raptorLog(`cancelled raptor alarm ${timeoutName}`);
+  } else {
+    raptorLog(`failed to clear raptor alarm ${timeoutName}`, "error");
+  }
+}
+
+function resultListener(request, sender, sendResponse) {
+  raptorLog(`received message from ${sender.tab.url}`);
+
+  // check if this is a message from pageloaderjs indicating it is ready to start
+  if (request.type == "pageloadjs-ready") {
+    raptorLog("received pageloadjs-ready!");
+
+    sendResponse({ text: "pageloadjs-ready-response" });
+    collectResults();
+    return;
+  }
+
+  if (request.type && request.value) {
+    raptorLog(`result: ${request.type} ${request.value}`);
+    sendResponse({ text: `confirmed ${request.type}` });
+
+    if (!(request.type in results.measurements)) {
+      results.measurements[request.type] = [];
+    }
+
+    switch (testType) {
+      case TEST_BENCHMARK:
+        // benchmark results received (all results for that complete benchmark run)
+        raptorLog("received results from benchmark");
+        results.measurements[request.type].push(request.value);
+        isBenchmarkPending = false;
+        break;
+
+      case TEST_PAGE_LOAD:
+        // a single pageload measurement was received
+        if (request.type.indexOf("hero") > -1) {
+          results.measurements[request.type].push(request.value);
+          const _found = request.type.split("hero:")[1];
+          const index = pendingHeroes.indexOf(_found);
+          if (index > -1) {
+            pendingHeroes.splice(index, 1);
+            if (!pendingHeroes.length) {
+              raptorLog("measured all expected hero elements");
+              isHeroPending = false;
+            }
+          }
+        } else if (request.type == "fnbpaint") {
+          results.measurements.fnbpaint.push(request.value);
+          isFNBPaintPending = false;
+        } else if (request.type == "dcf") {
+          results.measurements.dcf.push(request.value);
+          isDCFPending = false;
+        } else if (request.type == "ttfi") {
+          results.measurements.ttfi.push(request.value);
+          isTTFIPending = false;
+        } else if (request.type == "fcp") {
+          results.measurements.fcp.push(request.value);
+          isFCPPending = false;
+        } else if (request.type == "loadtime") {
+          results.measurements.loadtime.push(request.value);
+          isLoadTimePending = false;
+        }
+        break;
+    }
+  } else {
+    raptorLog(`unknown message received from content: ${request}`);
+  }
+}
+
+async function verifyResults() {
+  raptorLog("Verifying results:");
+  raptorLog(results);
+
+  for (var x in results.measurements) {
+    const count = results.measurements[x].length;
+    if (count == pageCycles) {
+      raptorLog(`have ${count} results for ${x}, as expected`);
+    } else {
+      raptorLog(
+        `expected ${pageCycles} results for ${x} but only have ${count}`,
+        "error"
+      );
+    }
+  }
+
+  await postToControlServer("results", results);
+
+  // we're finished, move to cleanup
+  await cleanUp();
+}
+
+async function postToControlServer(msgType, msgData = "") {
+  await new Promise(resolve => {
+    // requires 'control server' running at port 8000 to receive results
+    const xhr = new XMLHttpRequest();
+    xhr.open("POST", `http://${host}:${csPort}/`, true);
+    xhr.setRequestHeader("Content-Type", "application/json");
+
+    xhr.onreadystatechange = () => {
+      if (xhr.readyState == XMLHttpRequest.DONE) {
+        if (xhr.status != 200) {
+          // Failed to send the message. At least add a console error.
+          let msg = msgType;
+          if (msgType != "screenshot") {
+            msg += ` with '${msgData}'`;
+          }
+          raptorLog(`failed to post ${msg} to control server`, "error");
+        }
+
+        resolve();
+      }
+    };
+
+    xhr.send(
+      JSON.stringify({
+        type: `webext_${msgType}`,
+        data: msgData,
+      })
+    );
+  });
+}
+
+async function cleanUp() {
+  // close tab unless raptor debug-mode is enabled
+  if (debugMode == 1) {
+    raptorLog("debug-mode enabled, leaving tab open");
+  } else {
+    await closeTab(testTabId);
+  }
+
+  if (testType == TEST_PAGE_LOAD) {
+    // remove listeners
+    ext.alarms.onAlarm.removeListener(timeoutAlarmListener);
+    ext.runtime.onMessage.removeListener(resultListener);
+  }
+  raptorLog(`${testType} test finished`);
+
+  // if profiling was enabled, stop the profiler - may have already
+  // been stopped but stop again here in cleanup in case of timeout
+  if (geckoProfiling) {
+    await stopGeckoProfiling();
+  }
+
+  // tell the control server we are done and the browser can be shutdown
+  await postToControlServer("shutdownBrowser");
+}
+
+async function raptorRunner() {
+  await postToControlServer("status", "starting raptorRunner");
+
+  if (isBackgroundTest) {
+    await postToControlServer(
+      "status",
+      "raptor test will be backgrounding the app"
+    );
+  }
+
+  await getTestSettings();
+
+  raptorLog(`${testType} test start`);
+
+  ext.alarms.onAlarm.addListener(timeoutAlarmListener);
+  ext.runtime.onMessage.addListener(resultListener);
+
+  // create new empty tab, which starts the test; we want to
+  // wait some time for the browser to settle before beginning
+  const text = `* pausing ${
+    postStartupDelay / 1000
+  } seconds to let browser settle... *`;
+  await postToControlServer("status", text);
+  await sleep(postStartupDelay);
+
+  if (!isGeckoAndroid) {
+    await openTab();
+  }
+
+  testTabId = await getCurrentTabId();
+
+  await nextCycle();
+}
+
+function raptorLog(text, level = "info") {
+  let prefix = "";
+
+  if (level == "error") {
+    prefix = "ERROR: ";
+  }
+
+  console[level](`${prefix}[raptor-runnerjs] ${text}`);
+}
+
+async function init() {
+  const config = getTestConfig();
+  testName = config.test_name;
+  settingsURL = config.test_settings_url;
+  csPort = config.cs_port;
+  benchmarkPort = config.benchmark_port;
+  postStartupDelay = config.post_startup_delay;
+  host = config.host;
+  debugMode = config.debug_mode;
+  browserCycle = config.browser_cycle;
+
+  try {
+    // Chromium based browsers do not support the "browser" namespace and
+    // raise an exception when accessing it.
+    const info = await browser.runtime.getBrowserInfo();
+    results.browser = `${info.name} ${info.version} ${info.buildID}`;
+
+    ext = browser;
+    isGecko = true;
+    isGeckoAndroid = ANDROID_BROWSERS.includes(info.name.toLowerCase);
+  } catch (e) {
+    const regex = /(Chrome)\/([\w\.]+)/;
+    const userAgent = window.navigator.userAgent;
+    results.browser = regex.exec(userAgent).splice(1, 2).join(" ");
+
+    ext = chrome;
+  }
+
+  await postToControlServer("loaded");
+  await postToControlServer("status", `testing on ${results.browser}`);
+  await postToControlServer("status", `test name is: ${testName}`);
+  await postToControlServer("status", `test settings url is: ${settingsURL}`);
+
+  try {
+    if (window.document.readyState != "complete") {
+      await new Promise(resolve => {
+        window.addEventListener("load", resolve);
+        raptorLog("Waiting for load event...");
+      });
+    }
+
+    await raptorRunner();
+  } catch (e) {
+    await postToControlServer("error", [e.message, e.stack]);
+    await postToControlServer("shutdownBrowser");
+  }
+}
+
+init();