593 lines
21 KiB
JavaScript
593 lines
21 KiB
JavaScript
import { Metric } from "./metric.mjs";
|
|
import { params } from "./params.mjs";
|
|
|
|
const performance = globalThis.performance;
|
|
|
|
export class BenchmarkTestStep {
|
|
constructor(testName, testFunction) {
|
|
this.name = testName;
|
|
this.run = testFunction;
|
|
}
|
|
}
|
|
|
|
function getParent(lookupStartNode, path) {
|
|
const parent = path.reduce((root, selector) => {
|
|
const node = root.querySelector(selector);
|
|
return node.shadowRoot ?? node;
|
|
}, lookupStartNode);
|
|
|
|
return parent;
|
|
}
|
|
|
|
class Page {
|
|
constructor(frame) {
|
|
this._frame = frame;
|
|
}
|
|
|
|
layout() {
|
|
const body = this._frame.contentDocument.body.getBoundingClientRect();
|
|
this.layout.e = document.elementFromPoint((body.width / 2) | 0, (body.height / 2) | 0);
|
|
}
|
|
|
|
async waitForElement(selector) {
|
|
return new Promise((resolve) => {
|
|
const resolveIfReady = () => {
|
|
const element = this.querySelector(selector);
|
|
let callback = resolveIfReady;
|
|
if (element)
|
|
callback = () => resolve(element);
|
|
window.requestAnimationFrame(callback);
|
|
};
|
|
resolveIfReady();
|
|
});
|
|
}
|
|
|
|
/**
|
|
* Returns the first element within the document that matches the specified selector, or group of selectors.
|
|
* If no matches are found, null is returned.
|
|
*
|
|
* An optional path param is added to be able to target elements within a shadow DOM or nested shadow DOMs.
|
|
*
|
|
* @example
|
|
* // DOM structure: <todo-app> -> #shadow-root -> <todo-list> -> #shadow-root -> <todo-item>
|
|
* // return PageElement(<todo-item>)
|
|
* querySelector("todo-item", ["todo-app", "todo-list"]);
|
|
*
|
|
* @param {string} selector A string containing one or more selectors to match.
|
|
* @param {string[]} [path] An array containing a path to the parent element.
|
|
* @returns PageElement | null
|
|
*/
|
|
querySelector(selector, path = []) {
|
|
const lookupStartNode = this._frame.contentDocument;
|
|
const element = getParent(lookupStartNode, path).querySelector(selector);
|
|
|
|
if (element === null)
|
|
return null;
|
|
return this._wrapElement(element);
|
|
}
|
|
|
|
/**
|
|
* Returns all elements within the document that matches the specified selector, or group of selectors.
|
|
* If no matches are found, null is returned.
|
|
*
|
|
* An optional path param is added to be able to target elements within a shadow DOM or nested shadow DOMs.
|
|
*
|
|
* @example
|
|
* // DOM structure: <todo-app> -> #shadow-root -> <todo-list> -> #shadow-root -> <todo-item>
|
|
* // return [PageElement(<todo-item>), PageElement(<todo-item>)]
|
|
* querySelectorAll("todo-item", ["todo-app", "todo-list"]);
|
|
*
|
|
* @param {string} selector A string containing one or more selectors to match.
|
|
* @param {string[]} [path] An array containing a path to the parent element.
|
|
* @returns array
|
|
*/
|
|
querySelectorAll(selector, path = []) {
|
|
const lookupStartNode = this._frame.contentDocument;
|
|
const elements = Array.from(getParent(lookupStartNode, path).querySelectorAll(selector));
|
|
for (let i = 0; i < elements.length; i++)
|
|
elements[i] = this._wrapElement(elements[i]);
|
|
return elements;
|
|
}
|
|
|
|
getElementById(id) {
|
|
const element = this._frame.contentDocument.getElementById(id);
|
|
if (element === null)
|
|
return null;
|
|
return this._wrapElement(element);
|
|
}
|
|
|
|
call(functionName) {
|
|
this._frame.contentWindow[functionName]();
|
|
return null;
|
|
}
|
|
|
|
callAsync(functionName) {
|
|
setTimeout(() => {
|
|
this._frame.contentWindow[functionName]();
|
|
}, 0);
|
|
}
|
|
|
|
callToGetElement(functionName) {
|
|
return this._wrapElement(this._frame.contentWindow[functionName]());
|
|
}
|
|
|
|
_wrapElement(element) {
|
|
return new PageElement(element);
|
|
}
|
|
}
|
|
|
|
const NATIVE_OPTIONS = {
|
|
bubbles: true,
|
|
cancellable: true,
|
|
};
|
|
|
|
class PageElement {
|
|
#node;
|
|
|
|
constructor(node) {
|
|
this.#node = node;
|
|
}
|
|
|
|
setValue(value) {
|
|
this.#node.value = value;
|
|
}
|
|
|
|
click() {
|
|
this.#node.click();
|
|
}
|
|
|
|
focus() {
|
|
this.#node.focus();
|
|
}
|
|
|
|
getElementByMethod(name) {
|
|
return new PageElement(this.#node[name]());
|
|
}
|
|
|
|
dispatchEvent(eventName, options = NATIVE_OPTIONS, eventType = Event) {
|
|
if (eventName === "submit")
|
|
// FIXME FireFox doesn't like `new Event('submit')
|
|
this._dispatchSubmitEvent();
|
|
else
|
|
this.#node.dispatchEvent(new eventType(eventName, options));
|
|
}
|
|
|
|
_dispatchSubmitEvent() {
|
|
const submitEvent = document.createEvent("Event");
|
|
submitEvent.initEvent("submit", true, true);
|
|
this.#node.dispatchEvent(submitEvent);
|
|
}
|
|
|
|
enter(type, options = undefined) {
|
|
const ENTER_KEY_CODE = 13;
|
|
return this.dispatchKeyEvent(type, ENTER_KEY_CODE, "Enter", options);
|
|
}
|
|
|
|
dispatchKeyEvent(type, keyCode, key, options) {
|
|
let eventOptions = { bubbles: true, cancelable: true, keyCode, which: keyCode, key };
|
|
if (options !== undefined)
|
|
eventOptions = Object.assign(eventOptions, options);
|
|
const event = new KeyboardEvent(type, eventOptions);
|
|
this.#node.dispatchEvent(event);
|
|
}
|
|
|
|
dispatchMouseEvent(type, offsetX, offsetY, options) {
|
|
const boundingRect = this.#node.getBoundingClientRect();
|
|
const clientX = offsetX + boundingRect.x;
|
|
const clientY = offsetY + boundingRect.y;
|
|
const contentWindow = this.#node.ownerDocument.defaultView;
|
|
const screenX = clientX + contentWindow.screenX;
|
|
const screenY = clientY + contentWindow.screenY;
|
|
let eventOptions = { bubbles: true, cancelable: true, clientX, clientY, screenX, screenY };
|
|
if (options !== undefined)
|
|
eventOptions = Object.assign(eventOptions, options);
|
|
const event = new contentWindow.MouseEvent(type, eventOptions);
|
|
this.#node.dispatchEvent(event);
|
|
}
|
|
|
|
/**
|
|
* Returns the first element found in a node of a PageElement that matches the specified selector, or group of selectors. If a shadow DOM is present in the node, the shadow DOM is used to query.
|
|
* If no matches are found, null is returned.
|
|
*
|
|
* @param {string} selector A string containing one or more selectors to match.
|
|
* @param {string[]} [path] An array containing a path to the parent element.
|
|
* @returns PageElement | null
|
|
*/
|
|
querySelectorInShadowRoot(selector, path = []) {
|
|
const lookupStartNode = this.#node.shadowRoot ?? this.#node;
|
|
const element = getParent(lookupStartNode, path).querySelector(selector);
|
|
|
|
if (element === null)
|
|
return null;
|
|
return new PageElement(element);
|
|
}
|
|
|
|
querySelector(selector) {
|
|
const element = this.#node.querySelector(selector);
|
|
|
|
if (element === null)
|
|
return null;
|
|
return new PageElement(element);
|
|
}
|
|
}
|
|
|
|
function geomeanToScore(geomean) {
|
|
return 1000 / geomean;
|
|
}
|
|
|
|
// The WarmupSuite is used to make sure all runner helper functions and
|
|
// classes are compiled, to avoid unnecessary pauses due to delayed
|
|
// compilation of runner methods in the middle of the measuring cycle.
|
|
const WarmupSuite = {
|
|
name: "Warmup",
|
|
url: "warmup/index.html",
|
|
async prepare(page) {
|
|
await page.waitForElement("#testItem");
|
|
},
|
|
tests: [
|
|
// Make sure to run ever page.method once at least
|
|
new BenchmarkTestStep("WarmingUpPageMethods", (page) => {
|
|
let results = [];
|
|
results.push(page.querySelector(".testItem"));
|
|
results.push(page.querySelectorAll(".item"));
|
|
results.push(page.getElementById("testItem"));
|
|
}),
|
|
new BenchmarkTestStep("WarmingUpPageElementMethods", (page) => {
|
|
const item = page.getElementById("testItem");
|
|
item.setValue("value");
|
|
item.click();
|
|
item.focus();
|
|
item.dispatchEvent("change");
|
|
item.enter("keypress");
|
|
item.dispatchEvent("input");
|
|
item.enter("keyup");
|
|
}),
|
|
new BenchmarkTestStep("WarmingUpPageElementMouseMethods", (page) => {
|
|
const item = page.getElementById("testItem");
|
|
const mouseEventOptions = { clientX: 100, clientY: 100, bubbles: true, cancelable: true };
|
|
const wheelEventOptions = {
|
|
clientX: 200,
|
|
clientY: 200,
|
|
deltaMode: 0,
|
|
delta: -10,
|
|
deltaY: -10,
|
|
bubbles: true,
|
|
cancelable: true,
|
|
};
|
|
item.dispatchEvent("mousedown", mouseEventOptions, MouseEvent);
|
|
item.dispatchEvent("mousemove", mouseEventOptions, MouseEvent);
|
|
item.dispatchEvent("mouseup", mouseEventOptions, MouseEvent);
|
|
item.dispatchEvent("wheel", wheelEventOptions, WheelEvent);
|
|
}),
|
|
],
|
|
};
|
|
|
|
class TestInvoker {
|
|
constructor(syncCallback, asyncCallback, reportCallback) {
|
|
this._syncCallback = syncCallback;
|
|
this._asyncCallback = asyncCallback;
|
|
this._reportCallback = reportCallback;
|
|
}
|
|
}
|
|
|
|
class TimerTestInvoker extends TestInvoker {
|
|
start() {
|
|
return new Promise((resolve) => {
|
|
setTimeout(() => {
|
|
this._syncCallback();
|
|
setTimeout(() => {
|
|
this._asyncCallback();
|
|
requestAnimationFrame(async () => {
|
|
await this._reportCallback();
|
|
resolve();
|
|
});
|
|
}, 0);
|
|
}, params.waitBeforeSync);
|
|
});
|
|
}
|
|
}
|
|
|
|
class RAFTestInvoker extends TestInvoker {
|
|
start() {
|
|
return new Promise((resolve) => {
|
|
if (params.waitBeforeSync)
|
|
setTimeout(() => this._scheduleCallbacks(resolve), params.waitBeforeSync);
|
|
else
|
|
this._scheduleCallbacks(resolve);
|
|
});
|
|
}
|
|
|
|
_scheduleCallbacks(resolve) {
|
|
requestAnimationFrame(() => this._syncCallback());
|
|
requestAnimationFrame(() => {
|
|
setTimeout(() => {
|
|
this._asyncCallback();
|
|
setTimeout(async () => {
|
|
await this._reportCallback();
|
|
resolve();
|
|
}, 0);
|
|
}, 0);
|
|
});
|
|
}
|
|
}
|
|
|
|
// https://stackoverflow.com/a/47593316
|
|
function seededHashRandomNumberGenerator(a) {
|
|
return function () {
|
|
var t = a += 0x6d2b79f5;
|
|
t = Math.imul(t ^ (t >>> 15), t | 1);
|
|
t ^= t + Math.imul(t ^ (t >>> 7), t | 61);
|
|
return (t ^ (t >>> 14)) >>> 0;
|
|
};
|
|
}
|
|
|
|
export class BenchmarkRunner {
|
|
constructor(suites, client) {
|
|
this._suites = suites;
|
|
if (params.useWarmupSuite)
|
|
this._suites = [WarmupSuite, ...suites];
|
|
this._client = client;
|
|
this._page = null;
|
|
this._metrics = null;
|
|
this._iterationCount = params.iterationCount;
|
|
if (params.shuffleSeed !== "off")
|
|
this._suiteOrderRandomNumberGenerator = seededHashRandomNumberGenerator(params.shuffleSeed);
|
|
}
|
|
|
|
async runMultipleIterations(iterationCount) {
|
|
this._iterationCount = iterationCount;
|
|
if (this._client?.willStartFirstIteration)
|
|
await this._client.willStartFirstIteration(iterationCount);
|
|
|
|
const iterationStartLabel = "iteration-start";
|
|
const iterationEndLabel = "iteration-end";
|
|
for (let i = 0; i < iterationCount; i++) {
|
|
performance.mark(iterationStartLabel);
|
|
await this._runAllSuites();
|
|
performance.mark(iterationEndLabel);
|
|
performance.measure(`iteration-${i}`, iterationStartLabel, iterationEndLabel);
|
|
}
|
|
|
|
if (this._client?.didFinishLastIteration)
|
|
await this._client.didFinishLastIteration(this._metrics);
|
|
}
|
|
|
|
_removeFrame() {
|
|
if (this._frame) {
|
|
this._frame.parentNode.removeChild(this._frame);
|
|
this._frame = null;
|
|
}
|
|
}
|
|
|
|
async _appendFrame(src) {
|
|
const frame = document.createElement("iframe");
|
|
const style = frame.style;
|
|
style.width = `${params.viewport.width}px`;
|
|
style.height = `${params.viewport.height}px`;
|
|
style.border = "0px none";
|
|
style.position = "absolute";
|
|
frame.setAttribute("scrolling", "no");
|
|
frame.className = "test-runner";
|
|
style.left = "50%";
|
|
style.top = "50%";
|
|
style.transform = "translate(-50%, -50%)";
|
|
|
|
if (this._client?.willAddTestFrame)
|
|
await this._client.willAddTestFrame(frame);
|
|
|
|
document.body.insertBefore(frame, document.body.firstChild);
|
|
this._frame = frame;
|
|
return frame;
|
|
}
|
|
|
|
async _runAllSuites() {
|
|
this._measuredValues = { tests: {}, total: 0, mean: NaN, geomean: NaN, score: NaN };
|
|
|
|
const prepareStartLabel = "runner-prepare-start";
|
|
const prepareEndLabel = "runner-prepare-end";
|
|
|
|
performance.mark(prepareStartLabel);
|
|
this._removeFrame();
|
|
await this._appendFrame();
|
|
this._page = new Page(this._frame);
|
|
|
|
let suites = [...this._suites];
|
|
if (this._suiteOrderRandomNumberGenerator) {
|
|
// We just do a simple Fisher-Yates shuffle based on the repeated hash of the
|
|
// seed. This is not a high quality RNG, but it's plenty good enough.
|
|
for (let i = 0; i < suites.length - 1; i++) {
|
|
let j = i + (this._suiteOrderRandomNumberGenerator() % (suites.length - i));
|
|
let tmp = suites[i];
|
|
suites[i] = suites[j];
|
|
suites[j] = tmp;
|
|
}
|
|
}
|
|
|
|
performance.mark(prepareEndLabel);
|
|
performance.measure("runner-prepare", prepareStartLabel, prepareEndLabel);
|
|
|
|
for (const suite of suites) {
|
|
if (!suite.disabled)
|
|
await this._runSuite(suite);
|
|
}
|
|
|
|
const finalizeStartLabel = "runner-finalize-start";
|
|
const finalizeEndLabel = "runner-finalize-end";
|
|
|
|
performance.mark(finalizeStartLabel);
|
|
// Remove frame to clear the view for displaying the results.
|
|
this._removeFrame();
|
|
await this._finalize();
|
|
performance.mark(finalizeEndLabel);
|
|
performance.measure("runner-finalize", finalizeStartLabel, finalizeEndLabel);
|
|
}
|
|
|
|
async _runSuite(suite) {
|
|
const suitePrepareStartLabel = `suite-${suite.name}-prepare-start`;
|
|
const suitePrepareEndLabel = `suite-${suite.name}-prepare-end`;
|
|
const suiteStartLabel = `suite-${suite.name}-start`;
|
|
const suiteEndLabel = `suite-${suite.name}-end`;
|
|
|
|
performance.mark(suitePrepareStartLabel);
|
|
await this._prepareSuite(suite);
|
|
performance.mark(suitePrepareEndLabel);
|
|
|
|
performance.mark(suiteStartLabel);
|
|
for (const test of suite.tests)
|
|
await this._runTestAndRecordResults(suite, test);
|
|
performance.mark(suiteEndLabel);
|
|
|
|
performance.measure(`suite-${suite.name}-prepare`, suitePrepareStartLabel, suitePrepareEndLabel);
|
|
performance.measure(`suite-${suite.name}`, suiteStartLabel, suiteEndLabel);
|
|
}
|
|
|
|
async _prepareSuite(suite) {
|
|
return new Promise((resolve) => {
|
|
const frame = this._page._frame;
|
|
frame.onload = async () => {
|
|
await suite.prepare(this._page);
|
|
resolve();
|
|
};
|
|
frame.src = `resources/${suite.url}`;
|
|
});
|
|
}
|
|
|
|
async _runTestAndRecordResults(suite, test) {
|
|
/* eslint-disable-next-line no-async-promise-executor */
|
|
if (this._client?.willRunTest)
|
|
await this._client.willRunTest(suite, test);
|
|
|
|
// Prepare all mark labels outside the measuring loop.
|
|
const startLabel = `${suite.name}.${test.name}-start`;
|
|
const syncEndLabel = `${suite.name}.${test.name}-sync-end`;
|
|
const asyncStartLabel = `${suite.name}.${test.name}-async-start`;
|
|
const asyncEndLabel = `${suite.name}.${test.name}-async-end`;
|
|
|
|
let syncTime;
|
|
let asyncStartTime;
|
|
let asyncTime;
|
|
const runSync = () => {
|
|
if (params.warmupBeforeSync) {
|
|
performance.mark("warmup-start");
|
|
const startTime = performance.now();
|
|
// Infinite loop for the specified ms.
|
|
while (performance.now() - startTime < params.warmupBeforeSync)
|
|
continue;
|
|
performance.mark("warmup-end");
|
|
}
|
|
performance.mark(startLabel);
|
|
const syncStartTime = performance.now();
|
|
test.run(this._page);
|
|
const syncEndTime = performance.now();
|
|
performance.mark(syncEndLabel);
|
|
|
|
syncTime = syncEndTime - syncStartTime;
|
|
|
|
performance.mark(asyncStartLabel);
|
|
asyncStartTime = performance.now();
|
|
};
|
|
const measureAsync = () => {
|
|
// Some browsers don't immediately update the layout for paint.
|
|
// Force the layout here to ensure we're measuring the layout time.
|
|
const height = this._frame.contentDocument.body.getBoundingClientRect().height;
|
|
const asyncEndTime = performance.now();
|
|
asyncTime = asyncEndTime - asyncStartTime;
|
|
this._frame.contentWindow._unusedHeightValue = height; // Prevent dead code elimination.
|
|
performance.mark(asyncEndLabel);
|
|
if (params.warmupBeforeSync)
|
|
performance.measure("warmup", "warmup-start", "warmup-end");
|
|
performance.measure(`${suite.name}.${test.name}-sync`, startLabel, syncEndLabel);
|
|
performance.measure(`${suite.name}.${test.name}-async`, asyncStartLabel, asyncEndLabel);
|
|
};
|
|
const report = () => this._recordTestResults(suite, test, syncTime, asyncTime);
|
|
const invokerClass = params.measurementMethod === "raf" ? RAFTestInvoker : TimerTestInvoker;
|
|
const invoker = new invokerClass(runSync, measureAsync, report);
|
|
|
|
return invoker.start();
|
|
}
|
|
|
|
async _recordTestResults(suite, test, syncTime, asyncTime) {
|
|
// Skip reporting updates for the warmup suite.
|
|
if (suite === WarmupSuite)
|
|
return;
|
|
|
|
const suiteResults = this._measuredValues.tests[suite.name] || { tests: {}, total: 0 };
|
|
const total = syncTime + asyncTime;
|
|
this._measuredValues.tests[suite.name] = suiteResults;
|
|
suiteResults.tests[test.name] = { tests: { Sync: syncTime, Async: asyncTime }, total: total };
|
|
suiteResults.total += total;
|
|
|
|
if (this._client?.didRunTest)
|
|
await this._client.didRunTest(suite, test);
|
|
}
|
|
|
|
async _finalize() {
|
|
this._appendIterationMetrics();
|
|
if (this._client?.didRunSuites) {
|
|
let product = 1;
|
|
const values = [];
|
|
for (const suiteName in this._measuredValues.tests) {
|
|
const suiteTotal = this._measuredValues.tests[suiteName].total;
|
|
product *= suiteTotal;
|
|
values.push(suiteTotal);
|
|
}
|
|
|
|
values.sort((a, b) => a - b); // Avoid the loss of significance for the sum.
|
|
const total = values.reduce((a, b) => a + b);
|
|
const geomean = Math.pow(product, 1 / values.length);
|
|
|
|
this._measuredValues.total = total;
|
|
this._measuredValues.mean = total / values.length;
|
|
this._measuredValues.geomean = geomean;
|
|
this._measuredValues.score = geomeanToScore(geomean);
|
|
await this._client.didRunSuites(this._measuredValues);
|
|
}
|
|
}
|
|
|
|
_appendIterationMetrics() {
|
|
const getMetric = (name, unit = "ms") => this._metrics[name] || (this._metrics[name] = new Metric(name, unit));
|
|
const iterationTotalMetric = (i) => {
|
|
if (i >= params.iterationCount)
|
|
throw new Error(`Requested iteration=${i} does not exist.`);
|
|
return getMetric(`Iteration-${i}-Total`);
|
|
};
|
|
|
|
const collectSubMetrics = (prefix, items, parent) => {
|
|
for (let name in items) {
|
|
const results = items[name];
|
|
const metric = getMetric(prefix + name);
|
|
metric.add(results.total ?? results);
|
|
if (metric.parent !== parent)
|
|
parent.addChild(metric);
|
|
if (results.tests)
|
|
collectSubMetrics(`${metric.name}${Metric.separator}`, results.tests, metric);
|
|
}
|
|
};
|
|
const initializeMetrics = this._metrics === null;
|
|
if (initializeMetrics)
|
|
this._metrics = { __proto__: null };
|
|
|
|
const iterationResults = this._measuredValues.tests;
|
|
collectSubMetrics("", iterationResults);
|
|
|
|
if (initializeMetrics) {
|
|
// Prepare all iteration metrics so they are listed at the end of
|
|
// of the _metrics object, before "Total" and "Score".
|
|
for (let i = 0; i < this._iterationCount; i++)
|
|
iterationTotalMetric(i).description = `Test totals for iteration ${i}`;
|
|
getMetric("Geomean", "ms").description = "Geomean of test totals";
|
|
getMetric("Score", "score").description = "Scaled inverse of the Geomean";
|
|
}
|
|
|
|
const geomean = getMetric("Geomean");
|
|
const iterationTotal = iterationTotalMetric(geomean.length);
|
|
for (const results of Object.values(iterationResults))
|
|
iterationTotal.add(results.total);
|
|
iterationTotal.computeAggregatedMetrics();
|
|
geomean.add(iterationTotal.geomean);
|
|
getMetric("Score").add(geomeanToScore(iterationTotal.geomean));
|
|
|
|
for (const metric of Object.values(this._metrics))
|
|
metric.computeAggregatedMetrics();
|
|
}
|
|
}
|