summaryrefslogtreecommitdiffstats
path: root/layout/tools/reftest
diff options
context:
space:
mode:
authorDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-07 19:33:14 +0000
committerDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-07 19:33:14 +0000
commit36d22d82aa202bb199967e9512281e9a53db42c9 (patch)
tree105e8c98ddea1c1e4784a60a5a6410fa416be2de /layout/tools/reftest
parentInitial commit. (diff)
downloadfirefox-esr-36d22d82aa202bb199967e9512281e9a53db42c9.tar.xz
firefox-esr-36d22d82aa202bb199967e9512281e9a53db42c9.zip
Adding upstream version 115.7.0esr.upstream/115.7.0esr
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'layout/tools/reftest')
-rw-r--r--layout/tools/reftest/README.txt2
-rw-r--r--layout/tools/reftest/ReftestFissionChild.jsm292
-rw-r--r--layout/tools/reftest/ReftestFissionParent.jsm238
-rw-r--r--layout/tools/reftest/api.js163
-rw-r--r--layout/tools/reftest/chrome/userContent-import.css3
-rw-r--r--layout/tools/reftest/chrome/userContent.css23
-rwxr-xr-xlayout/tools/reftest/clean-reftest-output.pl38
-rw-r--r--layout/tools/reftest/fake-global.css1
-rw-r--r--layout/tools/reftest/globals.jsm166
-rw-r--r--layout/tools/reftest/jar.mn72
-rw-r--r--layout/tools/reftest/mach_commands.py297
-rw-r--r--layout/tools/reftest/mach_test_package_commands.py113
-rw-r--r--layout/tools/reftest/manifest.jsm803
-rw-r--r--layout/tools/reftest/manifest.json22
-rw-r--r--layout/tools/reftest/moz.build35
-rw-r--r--layout/tools/reftest/output.py190
-rw-r--r--layout/tools/reftest/reftest-analyzer-structured.xhtml649
-rw-r--r--layout/tools/reftest/reftest-analyzer.xhtml934
-rw-r--r--layout/tools/reftest/reftest-content.js1530
-rwxr-xr-xlayout/tools/reftest/reftest-to-html.pl118
-rw-r--r--layout/tools/reftest/reftest.jsm2020
-rw-r--r--layout/tools/reftest/reftest.xhtml13
-rw-r--r--layout/tools/reftest/reftest/__init__.py164
-rw-r--r--layout/tools/reftest/reftestcommandline.py645
-rw-r--r--layout/tools/reftest/remotereftest.py545
-rw-r--r--layout/tools/reftest/runreftest.py1198
-rw-r--r--layout/tools/reftest/schema.json1
-rw-r--r--layout/tools/reftest/selftest/conftest.py147
-rw-r--r--layout/tools/reftest/selftest/files/assert.html7
-rw-r--r--layout/tools/reftest/selftest/files/crash.html7
-rw-r--r--layout/tools/reftest/selftest/files/defaults.list7
-rw-r--r--layout/tools/reftest/selftest/files/failure-type-interactions.list11
-rw-r--r--layout/tools/reftest/selftest/files/green.html6
-rw-r--r--layout/tools/reftest/selftest/files/invalid-defaults-include.list4
-rw-r--r--layout/tools/reftest/selftest/files/invalid-defaults.list3
-rw-r--r--layout/tools/reftest/selftest/files/invalid-include.list2
-rw-r--r--layout/tools/reftest/selftest/files/leaks.log73
-rw-r--r--layout/tools/reftest/selftest/files/red.html6
-rw-r--r--layout/tools/reftest/selftest/files/reftest-assert.list1
-rw-r--r--layout/tools/reftest/selftest/files/reftest-crash.list1
-rw-r--r--layout/tools/reftest/selftest/files/reftest-fail.list3
-rw-r--r--layout/tools/reftest/selftest/files/reftest-pass.list3
-rw-r--r--layout/tools/reftest/selftest/files/scripttest-pass.html17
-rw-r--r--layout/tools/reftest/selftest/files/types.list5
-rw-r--r--layout/tools/reftest/selftest/python.ini7
-rw-r--r--layout/tools/reftest/selftest/test_python_manifest_parser.py37
-rw-r--r--layout/tools/reftest/selftest/test_reftest_manifest_parser.py72
-rw-r--r--layout/tools/reftest/selftest/test_reftest_output.py162
48 files changed, 10856 insertions, 0 deletions
diff --git a/layout/tools/reftest/README.txt b/layout/tools/reftest/README.txt
new file mode 100644
index 0000000000..ebc77011dd
--- /dev/null
+++ b/layout/tools/reftest/README.txt
@@ -0,0 +1,2 @@
+Reftest documentation has been moved to layout/docs/Reftest.rst and is rendered
+at https://firefox-source-docs.mozilla.org/layout/Reftest.html
diff --git a/layout/tools/reftest/ReftestFissionChild.jsm b/layout/tools/reftest/ReftestFissionChild.jsm
new file mode 100644
index 0000000000..97037d1faf
--- /dev/null
+++ b/layout/tools/reftest/ReftestFissionChild.jsm
@@ -0,0 +1,292 @@
+var EXPORTED_SYMBOLS = ["ReftestFissionChild"];
+
+class ReftestFissionChild extends JSWindowActorChild {
+
+ forwardAfterPaintEventToParent(rects, originalTargetUri, dispatchToSelfAsWell) {
+ if (dispatchToSelfAsWell) {
+ let event = new this.contentWindow.CustomEvent("Reftest:MozAfterPaintFromChild",
+ {bubbles: true, detail: {rects, originalTargetUri}});
+ this.contentWindow.dispatchEvent(event);
+ }
+
+ let parentContext = this.browsingContext.parent;
+ if (parentContext) {
+ try {
+ this.sendAsyncMessage("ForwardAfterPaintEvent",
+ {toBrowsingContext: parentContext, fromBrowsingContext: this.browsingContext,
+ rects, originalTargetUri});
+ } catch (e) {
+ // |this| can be destroyed here and unable to send messages, which is
+ // not a problem, the reftest harness probably torn down the page and
+ // moved on to the next test.
+ Cu.reportError(e);
+ }
+ }
+ }
+
+ handleEvent(evt) {
+ switch (evt.type) {
+ case "MozAfterPaint":
+ // We want to forward any after paint events to our parent document so that
+ // that it reaches the root content document where the main reftest harness
+ // code (reftest-content.js) will process it and update the canvas.
+ var rects = [];
+ for (let r of evt.clientRects) {
+ rects.push({ left: r.left, top: r.top, right: r.right, bottom: r.bottom });
+ }
+ this.forwardAfterPaintEventToParent(rects, this.document.documentURI, /* dispatchToSelfAsWell */ false);
+ break;
+ }
+ }
+
+ transformRect(transform, rect) {
+ let p1 = transform.transformPoint({x: rect.left, y: rect.top});
+ let p2 = transform.transformPoint({x: rect.right, y: rect.top});
+ let p3 = transform.transformPoint({x: rect.left, y: rect.bottom});
+ let p4 = transform.transformPoint({x: rect.right, y: rect.bottom});
+ let quad = new DOMQuad(p1, p2, p3, p4);
+ return quad.getBounds();
+ }
+
+ SetupDisplayportRoot() {
+ let returnStrings = {infoStrings: [], errorStrings: []};
+
+ let contentRootElement = this.contentWindow.document.documentElement;
+ if (!contentRootElement) {
+ return Promise.resolve(returnStrings);
+ }
+
+ // If we don't have the reftest-async-scroll attribute we only look at
+ // the root element for potential display ports to set.
+ if (!contentRootElement.hasAttribute("reftest-async-scroll")) {
+ let winUtils = this.contentWindow.windowUtils;
+ this.setupDisplayportForElement(contentRootElement, winUtils, returnStrings);
+ return Promise.resolve(returnStrings);
+ }
+
+ // Send a msg to the parent side to get the parent side to tell all
+ // process roots to do the displayport setting.
+ let browsingContext = this.browsingContext;
+ let promise = this.sendQuery("TellChildrenToSetupDisplayport", {browsingContext});
+ return promise.then(function(result) {
+ for (let errorString of result.errorStrings) {
+ returnStrings.errorStrings.push(errorString);
+ }
+ for (let infoString of result.infoStrings) {
+ returnStrings.infoStrings.push(infoString);
+ }
+ return returnStrings;
+ },
+ function(reason) {
+ returnStrings.errorStrings.push("SetupDisplayport SendQuery to parent promise rejected: " + reason);
+ return returnStrings;
+ });
+ }
+
+ attrOrDefault(element, attr, def) {
+ return element.hasAttribute(attr) ? Number(element.getAttribute(attr)) : def;
+ }
+
+ setupDisplayportForElement(element, winUtils, returnStrings) {
+ var dpw = this.attrOrDefault(element, "reftest-displayport-w", 0);
+ var dph = this.attrOrDefault(element, "reftest-displayport-h", 0);
+ var dpx = this.attrOrDefault(element, "reftest-displayport-x", 0);
+ var dpy = this.attrOrDefault(element, "reftest-displayport-y", 0);
+ if (dpw !== 0 || dph !== 0 || dpx != 0 || dpy != 0) {
+ returnStrings.infoStrings.push("Setting displayport to <x="+ dpx +", y="+ dpy +", w="+ dpw +", h="+ dph +">");
+ winUtils.setDisplayPortForElement(dpx, dpy, dpw, dph, element, 1);
+ }
+ }
+
+ setupDisplayportForElementSubtree(element, winUtils, returnStrings) {
+ this.setupDisplayportForElement(element, winUtils, returnStrings);
+ for (let c = element.firstElementChild; c; c = c.nextElementSibling) {
+ this.setupDisplayportForElementSubtree(c, winUtils, returnStrings);
+ }
+ if (typeof element.contentDocument !== "undefined" &&
+ element.contentDocument) {
+ returnStrings.infoStrings.push("setupDisplayportForElementSubtree descending into subdocument");
+ this.setupDisplayportForElementSubtree(element.contentDocument.documentElement,
+ element.contentWindow.windowUtils, returnStrings);
+ }
+ }
+
+ setupAsyncScrollOffsetsForElement(element, winUtils, allowFailure, returnStrings) {
+ let sx = this.attrOrDefault(element, "reftest-async-scroll-x", 0);
+ let sy = this.attrOrDefault(element, "reftest-async-scroll-y", 0);
+ if (sx != 0 || sy != 0) {
+ try {
+ // This might fail when called from RecordResult since layers
+ // may not have been constructed yet
+ winUtils.setAsyncScrollOffset(element, sx, sy);
+ return true;
+ } catch (e) {
+ if (allowFailure) {
+ returnStrings.infoStrings.push("setupAsyncScrollOffsetsForElement error calling setAsyncScrollOffset: " + e);
+ } else {
+ returnStrings.errorStrings.push("setupAsyncScrollOffsetsForElement error calling setAsyncScrollOffset: " + e);
+ }
+ }
+ }
+ return false;
+ }
+
+ setupAsyncScrollOffsetsForElementSubtree(element, winUtils, allowFailure, returnStrings) {
+ let updatedAny = this.setupAsyncScrollOffsetsForElement(element, winUtils, returnStrings);
+ for (let c = element.firstElementChild; c; c = c.nextElementSibling) {
+ if (this.setupAsyncScrollOffsetsForElementSubtree(c, winUtils, allowFailure, returnStrings)) {
+ updatedAny = true;
+ }
+ }
+ if (typeof element.contentDocument !== "undefined" &&
+ element.contentDocument) {
+ returnStrings.infoStrings.push("setupAsyncScrollOffsetsForElementSubtree Descending into subdocument");
+ if (this.setupAsyncScrollOffsetsForElementSubtree(element.contentDocument.documentElement,
+ element.contentWindow.windowUtils, allowFailure, returnStrings)) {
+ updatedAny = true;
+ }
+ }
+ return updatedAny;
+ }
+
+ async receiveMessage(msg) {
+ switch (msg.name) {
+ case "ForwardAfterPaintEventToSelfAndParent":
+ {
+ // The embedderElement can be null if the child we got this from was removed.
+ // Not much we can do to transform the rects, but it doesn't matter, the rects
+ // won't reach reftest-content.js.
+ if (msg.data.fromBrowsingContext.embedderElement == null) {
+ this.forwardAfterPaintEventToParent(msg.data.rects, msg.data.originalTargetUri,
+ /* dispatchToSelfAsWell */ true);
+ return;
+ }
+
+ // Transform the rects from fromBrowsingContext to us.
+ // We first translate from the content rect to the border rect of the iframe.
+ let style = this.contentWindow.getComputedStyle(msg.data.fromBrowsingContext.embedderElement);
+ let translate = new DOMMatrixReadOnly().translate(
+ parseFloat(style.paddingLeft) + parseFloat(style.borderLeftWidth),
+ parseFloat(style.paddingTop) + parseFloat(style.borderTopWidth));
+
+ // Then we transform from the iframe to our root frame.
+ // We are guaranteed to be the process with the embedderElement for fromBrowsingContext.
+ let transform = msg.data.fromBrowsingContext.embedderElement.getTransformToViewport();
+ let combined = translate.multiply(transform);
+
+ let newrects = msg.data.rects.map(r => this.transformRect(combined, r))
+
+ this.forwardAfterPaintEventToParent(newrects, msg.data.originalTargetUri, /* dispatchToSelfAsWell */ true);
+ break;
+ }
+
+ case "EmptyMessage":
+ return undefined;
+ case "UpdateLayerTree":
+ {
+ let errorStrings = [];
+ try {
+ if (this.manager.isProcessRoot) {
+ this.contentWindow.windowUtils.updateLayerTree();
+ }
+ } catch (e) {
+ errorStrings.push("updateLayerTree failed: " + e);
+ }
+ return {errorStrings};
+ }
+ case "FlushRendering":
+ {
+ let errorStrings = [];
+ let warningStrings = [];
+ let infoStrings = [];
+
+ try {
+ let {ignoreThrottledAnimations, needsAnimationFrame} = msg.data;
+
+ if (this.manager.isProcessRoot) {
+ var anyPendingPaintsGeneratedInDescendants = false;
+
+ if (needsAnimationFrame) {
+ await new Promise(resolve => this.contentWindow.requestAnimationFrame(resolve));
+ }
+
+ function flushWindow(win) {
+ var utils = win.windowUtils;
+ var afterPaintWasPending = utils.isMozAfterPaintPending;
+
+ var root = win.document.documentElement;
+ if (root && !root.classList.contains("reftest-no-flush")) {
+ try {
+ if (ignoreThrottledAnimations) {
+ utils.flushLayoutWithoutThrottledAnimations();
+ } else {
+ root.getBoundingClientRect();
+ }
+ } catch (e) {
+ warningStrings.push("flushWindow failed: " + e + "\n");
+ }
+ }
+
+ if (!afterPaintWasPending && utils.isMozAfterPaintPending) {
+ infoStrings.push("FlushRendering generated paint for window " + win.location.href);
+ anyPendingPaintsGeneratedInDescendants = true;
+ }
+
+ for (let i = 0; i < win.frames.length; ++i) {
+ try {
+ if (!Cu.isRemoteProxy(win.frames[i])) {
+ flushWindow(win.frames[i]);
+ }
+ } catch (e) {
+ Cu.reportError(e);
+ }
+ }
+ }
+
+ // `contentWindow` will be null if the inner window for this actor
+ // has been navigated away from.
+ if (this.contentWindow) {
+ flushWindow(this.contentWindow);
+ }
+
+ if (anyPendingPaintsGeneratedInDescendants &&
+ !this.contentWindow.windowUtils.isMozAfterPaintPending) {
+ warningStrings.push("Internal error: descendant frame generated a MozAfterPaint event, but the root document doesn't have one!");
+ }
+
+ }
+ } catch (e) {
+ errorStrings.push("flushWindow failed: " + e);
+ }
+ return {errorStrings, warningStrings, infoStrings};
+ }
+
+ case "SetupDisplayport":
+ {
+ let contentRootElement = this.document.documentElement;
+ let winUtils = this.contentWindow.windowUtils;
+ let returnStrings = {infoStrings: [], errorStrings: []};
+ if (contentRootElement) {
+ this.setupDisplayportForElementSubtree(contentRootElement, winUtils, returnStrings);
+ }
+ return returnStrings;
+ }
+
+ case "SetupAsyncScrollOffsets":
+ {
+ let returns = {infoStrings: [], errorStrings: [], updatedAny: false};
+ let contentRootElement = this.document.documentElement;
+
+ if (!contentRootElement) {
+ return returns;
+ }
+
+ let winUtils = this.contentWindow.windowUtils;
+
+ returns.updatedAny = this.setupAsyncScrollOffsetsForElementSubtree(contentRootElement, winUtils, msg.data.allowFailure, returns);
+ return returns;
+ }
+
+ }
+ }
+}
diff --git a/layout/tools/reftest/ReftestFissionParent.jsm b/layout/tools/reftest/ReftestFissionParent.jsm
new file mode 100644
index 0000000000..7cf61bdd55
--- /dev/null
+++ b/layout/tools/reftest/ReftestFissionParent.jsm
@@ -0,0 +1,238 @@
+var EXPORTED_SYMBOLS = ["ReftestFissionParent"];
+
+class ReftestFissionParent extends JSWindowActorParent {
+
+ tellChildrenToFlushRendering(browsingContext, ignoreThrottledAnimations, needsAnimationFrame) {
+ let promises = [];
+ this.tellChildrenToFlushRenderingRecursive(browsingContext, ignoreThrottledAnimations, needsAnimationFrame, promises);
+ return Promise.allSettled(promises);
+ }
+
+ tellChildrenToFlushRenderingRecursive(browsingContext, ignoreThrottledAnimations, needsAnimationFrame, promises) {
+ let cwg = browsingContext.currentWindowGlobal;
+ if (cwg && cwg.isProcessRoot) {
+ let a = cwg.getActor("ReftestFission");
+ if (a) {
+ let responsePromise = a.sendQuery("FlushRendering", {ignoreThrottledAnimations, needsAnimationFrame});
+ promises.push(responsePromise);
+ }
+ }
+
+ for (let context of browsingContext.children) {
+ this.tellChildrenToFlushRenderingRecursive(context, ignoreThrottledAnimations, needsAnimationFrame, promises);
+ }
+ }
+
+ // not including browsingContext
+ getNearestProcessRootProperDescendants(browsingContext) {
+ let result = [];
+ for (let context of browsingContext.children) {
+ this.getNearestProcessRootProperDescendantsRecursive(context, result);
+ }
+ return result;
+ }
+
+ getNearestProcessRootProperDescendantsRecursive(browsingContext, result) {
+ let cwg = browsingContext.currentWindowGlobal;
+ if (cwg && cwg.isProcessRoot) {
+ result.push(browsingContext);
+ return;
+ }
+ for (let context of browsingContext.children) {
+ this.getNearestProcessRootProperDescendantsRecursive(context, result);
+ }
+ }
+
+ // tell children and itself
+ async tellChildrenToUpdateLayerTree(browsingContext) {
+ let errorStrings = [];
+ let infoStrings = [];
+
+ let cwg = browsingContext.currentWindowGlobal;
+ if (!cwg || !cwg.isProcessRoot) {
+ if (cwg) {
+ errorStrings.push("tellChildrenToUpdateLayerTree called on a non process root?");
+ }
+ return {errorStrings, infoStrings};
+ }
+
+ let actor = cwg.getActor("ReftestFission");
+ if (!actor) {
+ return {errorStrings, infoStrings};
+ }
+
+ // When we paint a document we also update the EffectsInfo visible rect in
+ // nsSubDocumentFrame for any remote subdocuments. This visible rect is
+ // used to limit painting for the subdocument in the subdocument's process.
+ // So we want to ensure that the IPC message that updates the visible rect
+ // to the subdocument's process arrives before we paint the subdocument
+ // (otherwise our painting might not be up to date). We do this by sending,
+ // and waiting for reply, an "EmptyMessage" to every direct descendant that
+ // is in another process. Since we send the "EmptyMessage" after the
+ // visible rect update message we know that the visible rect will be
+ // updated by the time we hear back from the "EmptyMessage". Then we can
+ // ask the subdocument process to paint.
+
+ try {
+ let result = await actor.sendQuery("UpdateLayerTree");
+ errorStrings.push(...result.errorStrings);
+ } catch (e) {
+ infoStrings.push("tellChildrenToUpdateLayerTree UpdateLayerTree msg to child rejected: " + e);
+ }
+
+ let descendants = actor.getNearestProcessRootProperDescendants(browsingContext);
+ for (let context of descendants) {
+ let cwg2 = context.currentWindowGlobal;
+ if (cwg2) {
+ if (!cwg2.isProcessRoot) {
+ errorStrings.push("getNearestProcessRootProperDescendants returned a non process root?");
+ }
+ let actor2 = cwg2.getActor("ReftestFission");
+ if (actor2) {
+ try {
+ await actor2.sendQuery("EmptyMessage");
+ } catch(e) {
+ infoStrings.push("tellChildrenToUpdateLayerTree EmptyMessage msg to child rejected: " + e);
+ }
+
+ try {
+ let result2 = await actor2.tellChildrenToUpdateLayerTree(context);
+ errorStrings.push(...result2.errorStrings);
+ infoStrings.push(...result2.infoStrings);
+ } catch (e) {
+ errorStrings.push("tellChildrenToUpdateLayerTree recursive tellChildrenToUpdateLayerTree call rejected: " + e);
+ }
+
+ }
+ }
+ }
+
+ return {errorStrings, infoStrings};
+ }
+
+ tellChildrenToSetupDisplayport(browsingContext, promises) {
+ let cwg = browsingContext.currentWindowGlobal;
+ if (cwg && cwg.isProcessRoot) {
+ let a = cwg.getActor("ReftestFission");
+ if (a) {
+ let responsePromise = a.sendQuery("SetupDisplayport");
+ promises.push(responsePromise);
+ }
+ }
+
+ for (let context of browsingContext.children) {
+ this.tellChildrenToSetupDisplayport(context, promises);
+ }
+ }
+
+ tellChildrenToSetupAsyncScrollOffsets(browsingContext, allowFailure, promises) {
+ let cwg = browsingContext.currentWindowGlobal;
+ if (cwg && cwg.isProcessRoot) {
+ let a = cwg.getActor("ReftestFission");
+ if (a) {
+ let responsePromise = a.sendQuery("SetupAsyncScrollOffsets", {allowFailure});
+ promises.push(responsePromise);
+ }
+ }
+
+ for (let context of browsingContext.children) {
+ this.tellChildrenToSetupAsyncScrollOffsets(context, allowFailure, promises);
+ }
+ }
+
+
+ receiveMessage(msg) {
+ switch (msg.name) {
+ case "ForwardAfterPaintEvent":
+ {
+ let cwg = msg.data.toBrowsingContext.currentWindowGlobal;
+ if (cwg) {
+ let a = cwg.getActor("ReftestFission");
+ if (a) {
+ a.sendAsyncMessage("ForwardAfterPaintEventToSelfAndParent", msg.data);
+ }
+ }
+ break;
+ }
+ case "FlushRendering":
+ {
+ let promise = this.tellChildrenToFlushRendering(msg.data.browsingContext, msg.data.ignoreThrottledAnimations, msg.data.needsAnimationFrame);
+ return promise.then(function (results) {
+ let errorStrings = [];
+ let warningStrings = [];
+ let infoStrings = [];
+ for (let r of results) {
+ if (r.status != "fulfilled") {
+ if (r.status == "pending") {
+ errorStrings.push("FlushRendering sendQuery to child promise still pending?");
+ } else {
+ // We expect actors to go away causing sendQuery's to fail, so
+ // just note it.
+ infoStrings.push("FlushRendering sendQuery to child promise rejected: " + r.reason);
+ }
+ continue;
+ }
+
+ errorStrings.push(...r.value.errorStrings);
+ warningStrings.push(...r.value.warningStrings);
+ infoStrings.push(...r.value.infoStrings);
+ }
+ return {errorStrings, warningStrings, infoStrings};
+ });
+ }
+ case "UpdateLayerTree":
+ {
+ return this.tellChildrenToUpdateLayerTree(msg.data.browsingContext);
+ }
+ case "TellChildrenToSetupDisplayport":
+ {
+ let promises = [];
+ this.tellChildrenToSetupDisplayport(msg.data.browsingContext, promises);
+ return Promise.allSettled(promises).then(function (results) {
+ let errorStrings = [];
+ let infoStrings = [];
+ for (let r of results) {
+ if (r.status != "fulfilled") {
+ // We expect actors to go away causing sendQuery's to fail, so
+ // just note it.
+ infoStrings.push("SetupDisplayport sendQuery to child promise rejected: " + r.reason);
+ continue;
+ }
+
+ errorStrings.push(...r.value.errorStrings);
+ infoStrings.push(...r.value.infoStrings);
+ }
+ return {errorStrings, infoStrings}
+ });
+ }
+
+ case "SetupAsyncScrollOffsets":
+ {
+ let promises = [];
+ this.tellChildrenToSetupAsyncScrollOffsets(this.manager.browsingContext, msg.data.allowFailure, promises);
+ return Promise.allSettled(promises).then(function (results) {
+ let errorStrings = [];
+ let infoStrings = [];
+ let updatedAny = false;
+ for (let r of results) {
+ if (r.status != "fulfilled") {
+ // We expect actors to go away causing sendQuery's to fail, so
+ // just note it.
+ infoStrings.push("SetupAsyncScrollOffsets sendQuery to child promise rejected: " + r.reason);
+ continue;
+ }
+
+ errorStrings.push(...r.value.errorStrings);
+ infoStrings.push(...r.value.infoStrings);
+ if (r.value.updatedAny) {
+ updatedAny = true;
+ }
+ }
+ return {errorStrings, infoStrings, updatedAny};
+ });
+ }
+
+ }
+ }
+
+}
diff --git a/layout/tools/reftest/api.js b/layout/tools/reftest/api.js
new file mode 100644
index 0000000000..a0f20a77ad
--- /dev/null
+++ b/layout/tools/reftest/api.js
@@ -0,0 +1,163 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+const Cm = Components.manager;
+
+var OnRefTestLoad, OnRefTestUnload;
+
+XPCOMUtils.defineLazyServiceGetter(
+ this,
+ "resProto",
+ "@mozilla.org/network/protocol;1?name=resource",
+ "nsISubstitutingProtocolHandler"
+);
+
+XPCOMUtils.defineLazyServiceGetter(
+ this,
+ "aomStartup",
+ "@mozilla.org/addons/addon-manager-startup;1",
+ "amIAddonManagerStartup"
+);
+
+function processTerminated() {
+ return new Promise(resolve => {
+ Services.obs.addObserver(function observe(subject, topic) {
+ if (topic == "ipc:content-shutdown") {
+ Services.obs.removeObserver(observe, topic);
+ resolve();
+ }
+ }, "ipc:content-shutdown");
+ });
+}
+
+function startAndroid(win) {
+ // Add setTimeout here because windows.innerWidth/Height are not set yet.
+ win.setTimeout(function() {
+ OnRefTestLoad(win);
+ }, 0);
+}
+
+function GetMainWindow() {
+ let win = Services.wm.getMostRecentWindow("navigator:browser");
+ if (!win) {
+ // There is no navigator:browser in the geckoview TestRunnerActivity;
+ // try navigator.geckoview instead.
+ win = Services.wm.getMostRecentWindow("navigator:geckoview");
+ }
+ return win;
+}
+
+this.reftest = class extends ExtensionAPI {
+ onStartup() {
+ let uri = Services.io.newURI(
+ "chrome/reftest/res/",
+ null,
+ this.extension.rootURI
+ );
+ resProto.setSubstitutionWithFlags(
+ "reftest",
+ uri,
+ resProto.ALLOW_CONTENT_ACCESS
+ );
+
+ const manifestURI = Services.io.newURI(
+ "manifest.json",
+ null,
+ this.extension.rootURI
+ );
+
+ let manifestDirectives = [
+ [
+ "content",
+ "reftest",
+ "chrome/reftest/content/",
+ "contentaccessible=yes",
+ ],
+ ];
+ if (Services.appinfo.OS == "Android") {
+ manifestDirectives.push([
+ "override",
+ "chrome://global/skin/global.css",
+ "chrome://reftest/content/fake-global.css",
+ ]);
+ }
+ this.chromeHandle = aomStartup.registerChrome(
+ manifestURI,
+ manifestDirectives
+ );
+
+ // Starting tests is handled quite differently on android and desktop.
+ // On Android, OnRefTestLoad() takes over the main browser window so
+ // we just need to call it as soon as the browser window is available.
+ // On desktop, a separate window (dummy) is created and explicitly given
+ // focus (see bug 859339 for details), then tests are launched in a new
+ // top-level window.
+ let win = GetMainWindow();
+ if (Services.appinfo.OS == "Android") {
+ ({ OnRefTestLoad, OnRefTestUnload } = ChromeUtils.import(
+ "resource://reftest/reftest.jsm"
+ ));
+ if (win) {
+ startAndroid(win);
+ } else {
+ // The window type parameter is only available once the window's document
+ // element has been created. The main window has already been created
+ // however and it is in an in-between state which means that you can't
+ // find it by its type nor will domwindowcreated be fired.
+ // So we listen to either initial-document-element-inserted which
+ // indicates when it's okay to search for the main window by type again.
+ Services.obs.addObserver(function observer(aSubject, aTopic, aData) {
+ Services.obs.removeObserver(observer, aTopic);
+ startAndroid(GetMainWindow());
+ }, "initial-document-element-inserted");
+ }
+ return;
+ }
+
+ Services.io.manageOfflineStatus = false;
+ Services.io.offline = false;
+
+ let dummy = Services.ww.openWindow(
+ null,
+ "about:blank",
+ "dummy",
+ "chrome,dialog=no,left=800,height=200,width=200,all",
+ null
+ );
+ dummy.onload = async function() {
+ // Close pre-existing window
+ win.close();
+
+ const { PerTestCoverageUtils } = ChromeUtils.importESModule(
+ "resource://reftest/PerTestCoverageUtils.sys.mjs"
+ );
+ if (PerTestCoverageUtils.enabled) {
+ // In PerTestCoverage mode, wait for the process belonging to the window we just closed
+ // to be terminated, to avoid its shutdown interfering when we reset the counters.
+ await processTerminated();
+ }
+
+ dummy.focus();
+ Services.ww.openWindow(
+ null,
+ "chrome://reftest/content/reftest.xhtml",
+ "_blank",
+ "chrome,dialog=no,all",
+ {}
+ );
+ };
+ }
+
+ onShutdown() {
+ resProto.setSubstitution("reftest", null);
+
+ this.chromeHandle.destruct();
+ this.chromeHandle = null;
+
+ if (Services.appinfo.OS == "Android") {
+ OnRefTestUnload();
+ Cu.unload("resource://reftest/reftest.jsm");
+ }
+ }
+};
diff --git a/layout/tools/reftest/chrome/userContent-import.css b/layout/tools/reftest/chrome/userContent-import.css
new file mode 100644
index 0000000000..e1936a02bc
--- /dev/null
+++ b/layout/tools/reftest/chrome/userContent-import.css
@@ -0,0 +1,3 @@
+.reftest-usercss-import {
+ background-color: lime !important;
+}
diff --git a/layout/tools/reftest/chrome/userContent.css b/layout/tools/reftest/chrome/userContent.css
new file mode 100644
index 0000000000..1106a6ed55
--- /dev/null
+++ b/layout/tools/reftest/chrome/userContent.css
@@ -0,0 +1,23 @@
+@import "invalid.css";
+@import "userContent-import.css";
+
+.reftest-usercss {
+ background: lime !important;
+}
+.RefTest-upperCase {
+ background: lime !important;
+}
+/*
+ * file: URLs have an empty domain.
+ * Android uses a special loopback-to-host address.
+ */
+@-moz-document domain(), domain(10.0.2.2) {
+ .reftest-domain {
+ background: lime !important;
+ }
+}
+@-moz-document domain(example.invalid) {
+ .reftest-xdomain {
+ background: red !important;
+ }
+}
diff --git a/layout/tools/reftest/clean-reftest-output.pl b/layout/tools/reftest/clean-reftest-output.pl
new file mode 100755
index 0000000000..b1959281d5
--- /dev/null
+++ b/layout/tools/reftest/clean-reftest-output.pl
@@ -0,0 +1,38 @@
+#!/usr/bin/perl
+# vim: set shiftwidth=4 tabstop=8 autoindent expandtab:
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# This script is intended to be run over the standard output of a
+# reftest run. It will extract the parts of the output run relevant to
+# reftest and HTML-ize the URLs.
+
+use strict;
+
+print <<EOM
+<html>
+<head>
+<title>reftest output</title>
+</head>
+<body>
+<pre>
+EOM
+;
+
+while (<>) {
+ next unless /REFTEST/;
+ chomp;
+ chop if /\r$/;
+ s,(TEST-)([^\|]*) \| ([^\|]*) \|(.*),\1\2: <a href="\3">\3</a>\4,;
+ s,(IMAGE[^:]*): (data:.*),<a href="\2">\1</a>,;
+ print;
+ print "\n";
+}
+
+print <<EOM
+</pre>
+</body>
+</html>
+EOM
+;
diff --git a/layout/tools/reftest/fake-global.css b/layout/tools/reftest/fake-global.css
new file mode 100644
index 0000000000..66480b04a2
--- /dev/null
+++ b/layout/tools/reftest/fake-global.css
@@ -0,0 +1 @@
+/* This file deliberately left blank. See comment in jar.mn for rationale. */
diff --git a/layout/tools/reftest/globals.jsm b/layout/tools/reftest/globals.jsm
new file mode 100644
index 0000000000..7f1486d5f5
--- /dev/null
+++ b/layout/tools/reftest/globals.jsm
@@ -0,0 +1,166 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+"use strict";
+
+var EXPORTED_SYMBOLS = [];
+
+for (let [key, val] of Object.entries({
+ /* Constants */
+ XHTML_NS: "http://www.w3.org/1999/xhtml",
+ XUL_NS: "http://www.mozilla.org/keymaster/gatekeeper/there.is.only.xul",
+
+ NS_LOCAL_FILE_CONTRACTID: "@mozilla.org/file/local;1",
+ NS_GFXINFO_CONTRACTID: "@mozilla.org/gfx/info;1",
+ IO_SERVICE_CONTRACTID: "@mozilla.org/network/io-service;1",
+ DEBUG_CONTRACTID: "@mozilla.org/xpcom/debug;1",
+ NS_DIRECTORY_SERVICE_CONTRACTID: "@mozilla.org/file/directory_service;1",
+ NS_OBSERVER_SERVICE_CONTRACTID: "@mozilla.org/observer-service;1",
+
+ TYPE_REFTEST_EQUAL: '==',
+ TYPE_REFTEST_NOTEQUAL: '!=',
+ TYPE_LOAD: 'load', // test without a reference (just test that it does
+ // not assert, crash, hang, or leak)
+ TYPE_SCRIPT: 'script', // test contains individual test results
+ TYPE_PRINT: 'print', // test and reference will be printed to PDF's and
+ // compared structurally
+
+ // keep this in sync with reftest-content.js
+ URL_TARGET_TYPE_TEST: 0, // first url
+ URL_TARGET_TYPE_REFERENCE: 1, // second url, if any
+
+ // The order of these constants matters, since when we have a status
+ // listed for a *manifest*, we combine the status with the status for
+ // the test by using the *larger*.
+ // FIXME: In the future, we may also want to use this rule for combining
+ // statuses that are on the same line (rather than making the last one
+ // win).
+ EXPECTED_PASS: 0,
+ EXPECTED_FAIL: 1,
+ EXPECTED_RANDOM: 2,
+ EXPECTED_FUZZY: 3,
+
+ // types of preference value we might want to set for a specific test
+ PREF_BOOLEAN: 0,
+ PREF_STRING: 1,
+ PREF_INTEGER: 2,
+
+ FOCUS_FILTER_ALL_TESTS: "all",
+ FOCUS_FILTER_NEEDS_FOCUS_TESTS: "needs-focus",
+ FOCUS_FILTER_NON_NEEDS_FOCUS_TESTS: "non-needs-focus",
+
+ // "<!--CLEAR-->"
+ BLANK_URL_FOR_CLEARING: "data:text/html;charset=UTF-8,%3C%21%2D%2DCLEAR%2D%2D%3E",
+
+ /* Globals */
+ g: {
+ loadTimeout: 0,
+ timeoutHook: null,
+ remote: false,
+ ignoreWindowSize: false,
+ shuffle: false,
+ repeat: null,
+ runUntilFailure: false,
+ cleanupPendingCrashes: false,
+ totalChunks: 0,
+ thisChunk: 0,
+ containingWindow: null,
+ urlFilterRegex: {},
+ contentGfxInfo: null,
+ focusFilterMode: "all",
+ compareRetainedDisplayLists: false,
+ isCoverageBuild: false,
+
+ browser: undefined,
+ // Are we testing web content loaded in a separate process?
+ browserIsRemote: undefined, // bool
+ // Are we using <iframe mozbrowser>?
+ browserIsIframe: undefined, // bool
+ browserMessageManager: undefined, // bool
+ useDrawSnapshot: undefined, // bool
+ canvas1: undefined,
+ canvas2: undefined,
+ // gCurrentCanvas is non-null between InitCurrentCanvasWithSnapshot and the next
+ // RecordResult.
+ currentCanvas: null,
+ urls: undefined,
+ // Map from URI spec to the number of times it remains to be used
+ uriUseCounts: undefined,
+ // Map from URI spec to the canvas rendered for that URI
+ uriCanvases: undefined,
+ testResults: {
+ // Successful...
+ Pass: 0,
+ LoadOnly: 0,
+ // Unexpected...
+ Exception: 0,
+ FailedLoad: 0,
+ UnexpectedFail: 0,
+ UnexpectedPass: 0,
+ AssertionUnexpected: 0,
+ AssertionUnexpectedFixed: 0,
+ // Known problems...
+ KnownFail : 0,
+ AssertionKnown: 0,
+ Random : 0,
+ Skip: 0,
+ Slow: 0,
+ },
+ totalTests: 0,
+ currentURL: undefined,
+ currentURLTargetType: undefined,
+ testLog: [],
+ logLevel: undefined,
+ logFile: null,
+ logger: undefined,
+ server: undefined,
+ count: 0,
+ assertionCount: 0,
+
+ ioService: undefined,
+ debug: undefined,
+ windowUtils: undefined,
+
+ slowestTestTime: 0,
+ slowestTestURL: undefined,
+ failedUseWidgetLayers: false,
+
+ drawWindowFlags: undefined,
+
+ expectingProcessCrash: false,
+ expectedCrashDumpFiles: [],
+ unexpectedCrashDumpFiles: {},
+ crashDumpDir: undefined,
+ pendingCrashDumpDir: undefined,
+ failedNoPaint: false,
+ failedNoDisplayList: false,
+ failedDisplayList: false,
+ failedOpaqueLayer: false,
+ failedOpaqueLayerMessages: [],
+ failedAssignedLayer: false,
+ failedAssignedLayerMessages: [],
+
+ startAfter: undefined,
+ suiteStarted: false,
+ manageSuite: false,
+
+ prefsToRestore: [],
+ httpServerPort: -1,
+
+ // whether to run slow tests or not
+ runSlowTests: true,
+
+ // whether we should skip caching canvases
+ noCanvasCache: false,
+ recycledCanvases: new Array(),
+ testPrintOutput: null,
+
+ manifestsLoaded: {},
+ // Only dump the sandbox once, because it doesn't depend on the
+ // manifest URL (yet!).
+ dumpedConditionSandbox: false,
+ }
+})) {
+ this[key] = val;
+ EXPORTED_SYMBOLS.push(key);
+}
diff --git a/layout/tools/reftest/jar.mn b/layout/tools/reftest/jar.mn
new file mode 100644
index 0000000000..4b76906575
--- /dev/null
+++ b/layout/tools/reftest/jar.mn
@@ -0,0 +1,72 @@
+reftest.jar:
+# Ref tests
+ content/moz-bool-pref.css (../../../layout/reftests/css-parsing/moz-bool-pref.css)
+ content/editor/reftests/xul (../../../editor/reftests/xul/*)
+ content/bidi (../../reftests/bidi/*)
+ content/box (../../reftests/box/*)
+ content/box-ordinal (../../reftests/box-ordinal/*)
+ content/box-shadow (../../reftests/box-shadow/*)
+ content/bugs (../../reftests/bugs/*)
+ content/css-display (../../reftests/css-display/*)
+ content/fonts (../../reftests/fonts/*)
+ content/forms/input/file (../../reftests/forms/input/file/*)
+ content/forms/input/text (../../reftests/forms/input/text/*)
+ content/forms/placeholder (../../reftests/forms/placeholder/*)
+ content/forms/textbox (../../reftests/forms/textbox/*)
+ content/image-region (../../reftests/image-region/*)
+ content/color-scheme (../../reftests/color-scheme/*)
+ content/invalidation (../../reftests/invalidation/*)
+ content/native-theme (../../reftests/native-theme/*)
+ content/reftest-sanity (../../reftests/reftest-sanity/*)
+ content/text-overflow (../../reftests/text-overflow/*)
+ content/text-shadow (../../reftests/text-shadow/*)
+ content/writing-mode (../../reftests/writing-mode/*)
+ content/xul-document-load (../../reftests/xul-document-load/*)
+ content/xul (../../reftests/xul/*)
+ content/xul/reftest (../../xul/reftest/*)
+ content/toolkit/reftests (../../../toolkit/content/tests/reftests/*)
+ content/osx-theme (../../../toolkit/themes/osx/reftests/*)
+ content/reftest.xhtml (reftest.xhtml)
+
+# Crash tests
+ content/crashtests/dom/svg/crashtests (../../../dom/svg/crashtests/*)
+ content/crashtests/dom/html/crashtests (../../../dom/html/crashtests/*)
+ content/crashtests/dom/base/crashtests (../../../dom/base/crashtests/*)
+ content/crashtests/dom/xul/crashtests (../../../dom/xul/crashtests/*)
+ content/crashtests/dom/xml/crashtests (../../../dom/xml/crashtests/*)
+ content/crashtests/layout/forms/crashtests (../../../layout/forms/crashtests/*)
+ content/crashtests/layout/svg/crashtests (../../../layout/svg/crashtests/*)
+ content/crashtests/layout/tables/crashtests (../../../layout/tables/crashtests/*)
+ content/crashtests/layout/base/crashtests (../../../layout/base/crashtests/*)
+ content/crashtests/layout/xul/tree/crashtests (../../../layout/xul/tree/crashtests/*)
+ content/crashtests/layout/xul/crashtests (../../../layout/xul/crashtests/*)
+ content/crashtests/layout/generic/crashtests (../../../layout/generic/crashtests/*)
+ content/crashtests/layout/style/crashtests (../../../layout/style/crashtests/*)
+ content/crashtests/gfx/tests/crashtests (../../../gfx/tests/crashtests/*)
+ content/crashtests/accessible/tests/crashtests (../../../accessible/tests/crashtests/*)
+ content/crashtests/view/crashtests (../../../view/crashtests/*)
+ content/crashtests/widget/cocoa/crashtests (../../../widget/cocoa/crashtests/*)
+
+ res/globals.jsm (globals.jsm)
+ res/reftest-content.js (reftest-content.js)
+ res/ReftestFissionParent.jsm (ReftestFissionParent.jsm)
+ res/ReftestFissionChild.jsm (ReftestFissionChild.jsm)
+ res/AsyncSpellCheckTestHelper.sys.mjs (../../../editor/AsyncSpellCheckTestHelper.sys.mjs)
+ res/httpd.jsm (../../../netwerk/test/httpserver/httpd.js)
+ res/StructuredLog.sys.mjs (../../../testing/modules/StructuredLog.sys.mjs)
+ res/PerTestCoverageUtils.sys.mjs (../../../tools/code-coverage/PerTestCoverageUtils.sys.mjs)
+ res/input.css (../../../editor/reftests/xul/input.css)
+ res/progress.css (../../../layout/reftests/forms/progress/style.css)
+ res/manifest.jsm (manifest.jsm)
+ res/reftest.jsm (reftest.jsm)
+
+# Android doesn't ship global.css
+# Accessing missing non-test chrome:// files crashes, to avoid relying on
+# non-existing files.
+# A bunch of our older crash- and reftests use it, and it's unclear to what
+# extent they rely on it (ie removing the CSS file may stop the tests testing
+# what they are supposed to test on OSes that do ship global.css).
+# We also would like to move towards shipping less and less XUL/ex-XBL files
+# on android, so adding it everywhere doesn't seem like the right direction.
+# Instead, add an empty file only for reftests:
+ content/fake-global.css (fake-global.css)
diff --git a/layout/tools/reftest/mach_commands.py b/layout/tools/reftest/mach_commands.py
new file mode 100644
index 0000000000..bd99ac4393
--- /dev/null
+++ b/layout/tools/reftest/mach_commands.py
@@ -0,0 +1,297 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import os
+import re
+import sys
+from argparse import Namespace
+
+from mach.decorators import Command
+from mozbuild.base import MachCommandConditions as conditions
+from mozbuild.base import MozbuildObject
+
+parser = None
+
+
+class ReftestRunner(MozbuildObject):
+ """Easily run reftests.
+
+ This currently contains just the basics for running reftests. We may want
+ to hook up result parsing, etc.
+ """
+
+ def __init__(self, *args, **kwargs):
+ MozbuildObject.__init__(self, *args, **kwargs)
+
+ # TODO Bug 794506 remove once mach integrates with virtualenv.
+ build_path = os.path.join(self.topobjdir, "build")
+ if build_path not in sys.path:
+ sys.path.append(build_path)
+
+ self.tests_dir = os.path.join(self.topobjdir, "_tests")
+ self.reftest_dir = os.path.join(self.tests_dir, "reftest")
+
+ def _make_shell_string(self, s):
+ return "'%s'" % re.sub("'", r"'\''", s)
+
+ def _setup_objdir(self, args):
+ # reftest imports will happen from the objdir
+ sys.path.insert(0, self.reftest_dir)
+
+ tests = os.path.join(self.reftest_dir, "tests")
+ if not os.path.isdir(tests) and not os.path.islink(tests):
+ # This symbolic link is used by the desktop tests to
+ # locate the actual test files when running using file:.
+ os.symlink(self.topsrcdir, tests)
+
+ def run_desktop_test(self, **kwargs):
+ """Runs a reftest, in desktop Firefox."""
+ import runreftest
+
+ args = Namespace(**kwargs)
+ if args.suite not in ("reftest", "crashtest", "jstestbrowser"):
+ raise Exception("None or unrecognized reftest suite type.")
+
+ default_manifest = {
+ "reftest": (self.topsrcdir, "layout", "reftests", "reftest.list"),
+ "crashtest": (self.topsrcdir, "testing", "crashtest", "crashtests.list"),
+ "jstestbrowser": (
+ self.topobjdir,
+ "dist",
+ "test-stage",
+ "jsreftest",
+ "tests",
+ "js",
+ "src",
+ "tests",
+ "jstests.list",
+ ),
+ }
+
+ args.extraProfileFiles.append(os.path.join(self.topobjdir, "dist", "plugins"))
+ args.symbolsPath = os.path.join(self.topobjdir, "dist", "crashreporter-symbols")
+ args.sandboxReadWhitelist.extend([self.topsrcdir, self.topobjdir])
+
+ if not args.tests:
+ args.tests = [os.path.join(*default_manifest[args.suite])]
+
+ if args.suite == "jstestbrowser":
+ args.extraProfileFiles.append(
+ os.path.join(
+ self.topobjdir,
+ "dist",
+ "test-stage",
+ "jsreftest",
+ "tests",
+ "js",
+ "src",
+ "tests",
+ "user.js",
+ )
+ )
+
+ self.log_manager.enable_unstructured()
+ try:
+ rv = runreftest.run_test_harness(parser, args)
+ finally:
+ self.log_manager.disable_unstructured()
+
+ return rv
+
+ def run_android_test(self, **kwargs):
+ """Runs a reftest, in an Android application."""
+
+ args = Namespace(**kwargs)
+ if args.suite not in ("reftest", "crashtest", "jstestbrowser"):
+ raise Exception("None or unrecognized reftest suite type.")
+
+ self._setup_objdir(args)
+ import remotereftest
+
+ default_manifest = {
+ "reftest": (self.topsrcdir, "layout", "reftests", "reftest.list"),
+ "crashtest": (self.topsrcdir, "testing", "crashtest", "crashtests.list"),
+ "jstestbrowser": (
+ self.topobjdir,
+ "dist",
+ "test-stage",
+ "jsreftest",
+ "tests",
+ "js",
+ "src",
+ "tests",
+ "jstests.list",
+ ),
+ }
+
+ if not args.tests:
+ args.tests = [os.path.join(*default_manifest[args.suite])]
+
+ args.extraProfileFiles.append(
+ os.path.join(self.topsrcdir, "mobile", "android", "fonts")
+ )
+
+ hyphenation_path = os.path.join(self.topsrcdir, "intl", "locales")
+
+ for (dirpath, dirnames, filenames) in os.walk(hyphenation_path):
+ for filename in filenames:
+ if filename.endswith(".dic"):
+ args.extraProfileFiles.append(os.path.join(dirpath, filename))
+
+ if not args.httpdPath:
+ args.httpdPath = os.path.join(self.tests_dir, "modules")
+ if not args.symbolsPath:
+ args.symbolsPath = os.path.join(self.topobjdir, "crashreporter-symbols")
+ if not args.xrePath:
+ args.xrePath = os.environ.get("MOZ_HOST_BIN")
+ if not args.app:
+ args.app = "org.mozilla.geckoview.test_runner"
+ if not args.utilityPath:
+ args.utilityPath = args.xrePath
+ args.ignoreWindowSize = True
+
+ from mozrunner.devices.android_device import get_adb_path
+
+ if not args.adb_path:
+ args.adb_path = get_adb_path(self)
+
+ if "geckoview" not in args.app:
+ args.e10s = False
+ print("using e10s=False for non-geckoview app")
+
+ # Disable fission until geckoview supports fission by default.
+ # Need fission on Android? Use '--setpref fission.autostart=true'
+ args.disableFission = True
+
+ # A symlink and some path manipulations are required so that test
+ # manifests can be found both locally and remotely (via a url)
+ # using the same relative path.
+ if args.suite == "jstestbrowser":
+ staged_js_dir = os.path.join(
+ self.topobjdir, "dist", "test-stage", "jsreftest"
+ )
+ tests = os.path.join(self.reftest_dir, "jsreftest")
+ if not os.path.isdir(tests) and not os.path.islink(tests):
+ os.symlink(staged_js_dir, tests)
+ args.extraProfileFiles.append(
+ os.path.join(staged_js_dir, "tests", "js", "src", "tests", "user.js")
+ )
+ else:
+ tests = os.path.join(self.reftest_dir, "tests")
+ if not os.path.isdir(tests) and not os.path.islink(tests):
+ os.symlink(self.topsrcdir, tests)
+ for i, path in enumerate(args.tests):
+ # Non-absolute paths are relative to the packaged directory, which
+ # has an extra tests/ at the start
+ if os.path.exists(os.path.abspath(path)):
+ path = os.path.relpath(path, os.path.join(self.topsrcdir))
+ args.tests[i] = os.path.join("tests", path)
+
+ self.log_manager.enable_unstructured()
+ try:
+ rv = remotereftest.run_test_harness(parser, args)
+ finally:
+ self.log_manager.disable_unstructured()
+
+ return rv
+
+
+def process_test_objects(kwargs):
+ """|mach test| works by providing a test_objects argument, from
+ which the test path must be extracted and converted into a normal
+ reftest tests argument."""
+
+ if "test_objects" in kwargs:
+ if kwargs["tests"] is None:
+ kwargs["tests"] = []
+ kwargs["tests"].extend(item["path"] for item in kwargs["test_objects"])
+ del kwargs["test_objects"]
+
+
+def get_parser():
+ import reftestcommandline
+
+ global parser
+ here = os.path.abspath(os.path.dirname(__file__))
+ build_obj = MozbuildObject.from_environment(cwd=here)
+ if conditions.is_android(build_obj):
+ parser = reftestcommandline.RemoteArgumentsParser()
+ else:
+ parser = reftestcommandline.DesktopArgumentsParser()
+ return parser
+
+
+@Command(
+ "reftest",
+ category="testing",
+ description="Run reftests (layout and graphics correctness).",
+ parser=get_parser,
+)
+def run_reftest(command_context, **kwargs):
+ kwargs["suite"] = "reftest"
+ return _run_reftest(command_context, **kwargs)
+
+
+@Command(
+ "jstestbrowser",
+ category="testing",
+ description="Run js/src/tests in the browser.",
+ parser=get_parser,
+)
+def run_jstestbrowser(command_context, **kwargs):
+ if command_context.substs.get("JS_DISABLE_SHELL"):
+ raise Exception(
+ "jstestbrowser requires --enable-js-shell be specified in mozconfig."
+ )
+ command_context._mach_context.commands.dispatch(
+ "build", command_context._mach_context, what=["stage-jstests"]
+ )
+ kwargs["suite"] = "jstestbrowser"
+ return _run_reftest(command_context, **kwargs)
+
+
+@Command(
+ "crashtest",
+ category="testing",
+ description="Run crashtests (Check if crashes on a page).",
+ parser=get_parser,
+)
+def run_crashtest(command_context, **kwargs):
+ kwargs["suite"] = "crashtest"
+ return _run_reftest(command_context, **kwargs)
+
+
+def _run_reftest(command_context, **kwargs):
+ kwargs["topsrcdir"] = command_context.topsrcdir
+ process_test_objects(kwargs)
+ reftest = command_context._spawn(ReftestRunner)
+ # Unstructured logging must be enabled prior to calling
+ # adb which uses an unstructured logger in its constructor.
+ reftest.log_manager.enable_unstructured()
+ if conditions.is_android(command_context):
+ from mozrunner.devices.android_device import (
+ InstallIntent,
+ verify_android_device,
+ )
+
+ install = InstallIntent.NO if kwargs.get("no_install") else InstallIntent.YES
+ verbose = False
+ if (
+ kwargs.get("log_mach_verbose")
+ or kwargs.get("log_tbpl_level") == "debug"
+ or kwargs.get("log_mach_level") == "debug"
+ or kwargs.get("log_raw_level") == "debug"
+ ):
+ verbose = True
+ verify_android_device(
+ command_context,
+ install=install,
+ xre=True,
+ network=True,
+ app=kwargs["app"],
+ device_serial=kwargs["deviceSerial"],
+ verbose=verbose,
+ )
+ return reftest.run_android_test(**kwargs)
+ return reftest.run_desktop_test(**kwargs)
diff --git a/layout/tools/reftest/mach_test_package_commands.py b/layout/tools/reftest/mach_test_package_commands.py
new file mode 100644
index 0000000000..4effd4cfda
--- /dev/null
+++ b/layout/tools/reftest/mach_test_package_commands.py
@@ -0,0 +1,113 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import os
+import sys
+from argparse import Namespace
+from functools import partial
+
+from mach.decorators import Command
+
+here = os.path.abspath(os.path.dirname(__file__))
+logger = None
+
+
+def run_reftest(context, **kwargs):
+ import mozinfo
+ from mozlog.commandline import setup_logging
+
+ if not kwargs.get("log"):
+ kwargs["log"] = setup_logging("reftest", kwargs, {"mach": sys.stdout})
+ global logger
+ logger = kwargs["log"]
+
+ args = Namespace(**kwargs)
+ args.e10s = context.mozharness_config.get("e10s", args.e10s)
+
+ if not args.tests:
+ args.tests = [os.path.join("layout", "reftests", "reftest.list")]
+
+ test_root = os.path.join(context.package_root, "reftest", "tests")
+ normalize = partial(context.normalize_test_path, test_root)
+ args.tests = map(normalize, args.tests)
+
+ if kwargs.get("allow_software_gl_layers"):
+ os.environ["MOZ_LAYERS_ALLOW_SOFTWARE_GL"] = "1"
+
+ if mozinfo.info.get("buildapp") == "mobile/android":
+ return run_reftest_android(context, args)
+ return run_reftest_desktop(context, args)
+
+
+def run_reftest_desktop(context, args):
+ from runreftest import run_test_harness
+
+ args.app = args.app or context.firefox_bin
+ args.extraProfileFiles.append(os.path.join(context.bin_dir, "plugins"))
+ args.utilityPath = context.bin_dir
+ args.sandboxReadWhitelist.append(context.mozharness_workdir)
+ args.extraPrefs.append("layers.acceleration.force-enabled=true")
+
+ logger.info("mach calling runreftest with args: " + str(args))
+
+ return run_test_harness(parser, args)
+
+
+def run_reftest_android(context, args):
+ from remotereftest import run_test_harness
+
+ args.app = args.app or "org.mozilla.geckoview.test_runner"
+ args.utilityPath = context.hostutils
+ args.xrePath = context.hostutils
+ args.httpdPath = context.module_dir
+ args.ignoreWindowSize = True
+
+ config = context.mozharness_config
+ if config:
+ host = os.environ.get("HOST_IP", "10.0.2.2")
+ args.remoteWebServer = config.get("remote_webserver", host)
+ args.httpPort = config.get("http_port", 8854)
+ args.sslPort = config.get("ssl_port", 4454)
+ args.adb_path = config["exes"]["adb"] % {
+ "abs_work_dir": context.mozharness_workdir
+ }
+ args.deviceSerial = os.environ.get("DEVICE_SERIAL", "emulator-5554")
+
+ logger.info("mach calling remotereftest with args: " + str(args))
+
+ return run_test_harness(parser, args)
+
+
+def add_global_arguments(parser):
+ parser.add_argument("--test-suite")
+ parser.add_argument("--reftest-suite")
+ parser.add_argument("--download-symbols")
+ parser.add_argument("--allow-software-gl-layers", action="store_true")
+ parser.add_argument("--no-run-tests", action="store_true")
+
+
+def setup_argument_parser():
+ import mozinfo
+ import reftestcommandline
+
+ global parser
+ mozinfo.find_and_update_from_json(here)
+ if mozinfo.info.get("buildapp") == "mobile/android":
+ parser = reftestcommandline.RemoteArgumentsParser()
+ else:
+ parser = reftestcommandline.DesktopArgumentsParser()
+ add_global_arguments(parser)
+ return parser
+
+
+@Command(
+ "reftest",
+ category="testing",
+ description="Run the reftest harness.",
+ parser=setup_argument_parser,
+)
+def reftest(command_context, **kwargs):
+ command_context._mach_context.activate_mozharness_venv()
+ kwargs["suite"] = "reftest"
+ return run_reftest(command_context._mach_context, **kwargs)
diff --git a/layout/tools/reftest/manifest.jsm b/layout/tools/reftest/manifest.jsm
new file mode 100644
index 0000000000..476b04317b
--- /dev/null
+++ b/layout/tools/reftest/manifest.jsm
@@ -0,0 +1,803 @@
+/* -*- indent-tabs-mode: nil; js-indent-level: 4 -*- /
+/* vim: set shiftwidth=4 tabstop=8 autoindent cindent expandtab: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+"use strict";
+
+var EXPORTED_SYMBOLS = ["ReadTopManifest", "CreateUrls"];
+
+const {
+ NS_GFXINFO_CONTRACTID,
+
+ TYPE_REFTEST_EQUAL,
+ TYPE_REFTEST_NOTEQUAL,
+ TYPE_LOAD,
+ TYPE_SCRIPT,
+ TYPE_PRINT,
+
+ EXPECTED_PASS,
+ EXPECTED_FAIL,
+ EXPECTED_RANDOM,
+ EXPECTED_FUZZY,
+
+ PREF_BOOLEAN,
+ PREF_STRING,
+ PREF_INTEGER,
+
+ FOCUS_FILTER_NEEDS_FOCUS_TESTS,
+ FOCUS_FILTER_NON_NEEDS_FOCUS_TESTS,
+
+ g,
+} = ChromeUtils.import("resource://reftest/globals.jsm");
+const { NetUtil } = ChromeUtils.import("resource://gre/modules/NetUtil.jsm");
+const { AppConstants } = ChromeUtils.importESModule(
+ "resource://gre/modules/AppConstants.sys.mjs"
+);
+
+const NS_SCRIPTSECURITYMANAGER_CONTRACTID = "@mozilla.org/scriptsecuritymanager;1";
+const NS_NETWORK_PROTOCOL_CONTRACTID_PREFIX = "@mozilla.org/network/protocol;1?name=";
+const NS_XREAPPINFO_CONTRACTID = "@mozilla.org/xre/app-info;1";
+
+const RE_PROTOCOL = /^\w+:/;
+const RE_PREF_ITEM = /^(|test-|ref-)pref\((.+?),(.*)\)$/;
+
+
+function ReadTopManifest(aFileURL, aFilter, aManifestID)
+{
+ var url = g.ioService.newURI(aFileURL);
+ if (!url)
+ throw "Expected a file or http URL for the manifest.";
+
+ g.manifestsLoaded = {};
+ ReadManifest(url, aFilter, aManifestID);
+}
+
+// Note: If you materially change the reftest manifest parsing,
+// please keep the parser in layout/tools/reftest/__init__.py in sync.
+function ReadManifest(aURL, aFilter, aManifestID)
+{
+ // Ensure each manifest is only read once. This assumes that manifests that
+ // are included with filters will be read via their include before they are
+ // read directly in the case of a duplicate
+ if (g.manifestsLoaded.hasOwnProperty(aURL.spec)) {
+ if (g.manifestsLoaded[aURL.spec] === null)
+ return;
+ else
+ aFilter = [aFilter[0], aFilter[1], true];
+ }
+ g.manifestsLoaded[aURL.spec] = aFilter[1];
+
+ var secMan = Cc[NS_SCRIPTSECURITYMANAGER_CONTRACTID]
+ .getService(Ci.nsIScriptSecurityManager);
+
+ var listURL = aURL;
+ var channel = NetUtil.newChannel({uri: aURL,
+ loadUsingSystemPrincipal: true});
+ try {
+ var inputStream = channel.open();
+ } catch (e) {
+ g.logger.error("failed to open manifest at : " + aURL.spec);
+ throw e;
+ }
+ if (channel instanceof Ci.nsIHttpChannel
+ && channel.responseStatus != 200) {
+ g.logger.error("HTTP ERROR : " + channel.responseStatus);
+ }
+ var streamBuf = getStreamContent(inputStream);
+ inputStream.close();
+ var lines = streamBuf.split(/\n|\r|\r\n/);
+
+ // The sandbox for fails-if(), etc., condition evaluation. This is not
+ // always required and so is created on demand.
+ var sandbox;
+ function GetOrCreateSandbox() {
+ if (!sandbox) {
+ sandbox = BuildConditionSandbox(aURL);
+ }
+ return sandbox;
+ }
+
+ var lineNo = 0;
+ var urlprefix = "";
+ var defaults = [];
+ var defaultTestPrefSettings = [], defaultRefPrefSettings = [];
+ if (g.compareRetainedDisplayLists) {
+ AddRetainedDisplayListTestPrefs(GetOrCreateSandbox(), defaultTestPrefSettings,
+ defaultRefPrefSettings);
+ }
+ for (var str of lines) {
+ ++lineNo;
+ if (str.charAt(0) == "#")
+ continue; // entire line was a comment
+ var i = str.search(/\s+#/);
+ if (i >= 0)
+ str = str.substring(0, i);
+ // strip leading and trailing whitespace
+ str = str.replace(/^\s*/, '').replace(/\s*$/, '');
+ if (!str || str == "")
+ continue;
+ var items = str.split(/\s+/); // split on whitespace
+
+ if (items[0] == "url-prefix") {
+ if (items.length != 2)
+ throw "url-prefix requires one url in manifest file " + aURL.spec + " line " + lineNo;
+ urlprefix = items[1];
+ continue;
+ }
+
+ if (items[0] == "defaults") {
+ items.shift();
+ defaults = items;
+ continue;
+ }
+
+ var expected_status = EXPECTED_PASS;
+ var allow_silent_fail = false;
+ var minAsserts = 0;
+ var maxAsserts = 0;
+ var needs_focus = false;
+ var slow = false;
+ var skip = false;
+ var testPrefSettings = defaultTestPrefSettings.concat();
+ var refPrefSettings = defaultRefPrefSettings.concat();
+ var fuzzy_delta = { min: 0, max: 2 };
+ var fuzzy_pixels = { min: 0, max: 1 };
+ var chaosMode = false;
+ var wrCapture = { test: false, ref: false };
+ var nonSkipUsed = false;
+ var noAutoFuzz = false;
+
+ var origLength = items.length;
+ items = defaults.concat(items);
+ while (items[0].match(/^(fails|needs-focus|random|skip|asserts|slow|require-or|silentfail|pref|test-pref|ref-pref|fuzzy|chaos-mode|wr-capture|wr-capture-ref|noautofuzz)/)) {
+ var item = items.shift();
+ var stat;
+ var cond;
+ var m = item.match(/^(fails|random|skip|silentfail)-if(\(.*\))$/);
+ if (m) {
+ stat = m[1];
+ // Note: m[2] contains the parentheses, and we want them.
+ cond = Cu.evalInSandbox(m[2], GetOrCreateSandbox());
+ } else if (item.match(/^(fails|random|skip)$/)) {
+ stat = item;
+ cond = true;
+ } else if (item == "needs-focus") {
+ needs_focus = true;
+ cond = false;
+ } else if ((m = item.match(/^asserts\((\d+)(-\d+)?\)$/))) {
+ cond = false;
+ minAsserts = Number(m[1]);
+ maxAsserts = (m[2] == undefined) ? minAsserts
+ : Number(m[2].substring(1));
+ } else if ((m = item.match(/^asserts-if\((.*?),(\d+)(-\d+)?\)$/))) {
+ cond = false;
+ if (Cu.evalInSandbox("(" + m[1] + ")", GetOrCreateSandbox())) {
+ minAsserts = Number(m[2]);
+ maxAsserts =
+ (m[3] == undefined) ? minAsserts
+ : Number(m[3].substring(1));
+ }
+ } else if (item == "slow") {
+ cond = false;
+ slow = true;
+ } else if ((m = item.match(/^require-or\((.*?)\)$/))) {
+ var args = m[1].split(/,/);
+ if (args.length != 2) {
+ throw "Error in manifest file " + aURL.spec + " line " + lineNo + ": wrong number of args to require-or";
+ }
+ var [precondition_str, fallback_action] = args;
+ var preconditions = precondition_str.split(/&&/);
+ cond = false;
+ for (var precondition of preconditions) {
+ if (precondition === "debugMode") {
+ // Currently unimplemented. Requires asynchronous
+ // JSD call + getting an event while no JS is running
+ stat = fallback_action;
+ cond = true;
+ break;
+ } else if (precondition === "true") {
+ // For testing
+ } else {
+ // Unknown precondition. Assume it is unimplemented.
+ stat = fallback_action;
+ cond = true;
+ break;
+ }
+ }
+ } else if ((m = item.match(/^slow-if\((.*?)\)$/))) {
+ cond = false;
+ if (Cu.evalInSandbox("(" + m[1] + ")", GetOrCreateSandbox()))
+ slow = true;
+ } else if (item == "silentfail") {
+ cond = false;
+ allow_silent_fail = true;
+ } else if ((m = item.match(RE_PREF_ITEM))) {
+ cond = false;
+ if (!AddPrefSettings(m[1], m[2], m[3], GetOrCreateSandbox(),
+ testPrefSettings, refPrefSettings)) {
+ throw "Error in pref value in manifest file " + aURL.spec + " line " + lineNo;
+ }
+ } else if ((m = item.match(/^fuzzy\((\d+)-(\d+),(\d+)-(\d+)\)$/))) {
+ cond = false;
+ expected_status = EXPECTED_FUZZY;
+ fuzzy_delta = ExtractRange(m, 1);
+ fuzzy_pixels = ExtractRange(m, 3);
+ } else if ((m = item.match(/^fuzzy-if\((.*?),(\d+)-(\d+),(\d+)-(\d+)\)$/))) {
+ cond = false;
+ if (Cu.evalInSandbox("(" + m[1] + ")", GetOrCreateSandbox())) {
+ expected_status = EXPECTED_FUZZY;
+ fuzzy_delta = ExtractRange(m, 2);
+ fuzzy_pixels = ExtractRange(m, 4);
+ }
+ } else if (item == "chaos-mode") {
+ cond = false;
+ chaosMode = true;
+ } else if (item == "wr-capture") {
+ cond = false;
+ wrCapture.test = true;
+ } else if (item == "wr-capture-ref") {
+ cond = false;
+ wrCapture.ref = true;
+ } else if (item == "noautofuzz") {
+ cond = false;
+ noAutoFuzz = true;
+ } else {
+ throw "Error in manifest file " + aURL.spec + " line " + lineNo + ": unexpected item " + item;
+ }
+
+ if (stat != "skip") {
+ nonSkipUsed = true;
+ }
+
+ if (cond) {
+ if (stat == "fails") {
+ expected_status = EXPECTED_FAIL;
+ } else if (stat == "random") {
+ expected_status = EXPECTED_RANDOM;
+ } else if (stat == "skip") {
+ skip = true;
+ } else if (stat == "silentfail") {
+ allow_silent_fail = true;
+ }
+ }
+ }
+
+ if (items.length > origLength) {
+ // Implies we broke out of the loop before we finished processing
+ // defaults. This means defaults contained an invalid token.
+ throw "Error in manifest file " + aURL.spec + " line " + lineNo + ": invalid defaults token '" + items[0] + "'";
+ }
+
+ if (minAsserts > maxAsserts) {
+ throw "Bad range in manifest file " + aURL.spec + " line " + lineNo;
+ }
+
+ var runHttp = false;
+ var httpDepth;
+ if (items[0] == "HTTP") {
+ runHttp = (aURL.scheme == "file"); // We can't yet run the local HTTP server
+ // for non-local reftests.
+ httpDepth = 0;
+ items.shift();
+ } else if (items[0].match(/HTTP\(\.\.(\/\.\.)*\)/)) {
+ // Accept HTTP(..), HTTP(../..), HTTP(../../..), etc.
+ runHttp = (aURL.scheme == "file"); // We can't yet run the local HTTP server
+ // for non-local reftests.
+ httpDepth = (items[0].length - 5) / 3;
+ items.shift();
+ }
+
+ // do not prefix the url for include commands or urls specifying
+ // a protocol
+ if (urlprefix && items[0] != "include") {
+ if (items.length > 1 && !items[1].match(RE_PROTOCOL)) {
+ items[1] = urlprefix + items[1];
+ }
+ if (items.length > 2 && !items[2].match(RE_PROTOCOL)) {
+ items[2] = urlprefix + items[2];
+ }
+ }
+
+ var principal = secMan.createContentPrincipal(aURL, {});
+
+ if (items[0] == "include") {
+ if (items.length != 2)
+ throw "Error in manifest file " + aURL.spec + " line " + lineNo + ": incorrect number of arguments to include";
+ if (runHttp)
+ throw "Error in manifest file " + aURL.spec + " line " + lineNo + ": use of include with http";
+
+ // If the expected_status is EXPECTED_PASS (the default) then allow
+ // the include. If 'skip' is true, that means there was a skip
+ // or skip-if annotation (with a true condition) on this include
+ // statement, so we should skip the include. Any other expected_status
+ // is disallowed since it's nonintuitive as to what the intended
+ // effect is.
+ if (nonSkipUsed) {
+ throw "Error in manifest file " + aURL.spec + " line " + lineNo + ": include statement with annotation other than 'skip' or 'skip-if'";
+ } else if (skip) {
+ g.logger.info("Skipping included manifest at " + aURL.spec + " line " + lineNo + " due to matching skip condition");
+ } else {
+ // poor man's assertion
+ if (expected_status != EXPECTED_PASS) {
+ throw "Error in manifest file parsing code: we should never get expected_status=" + expected_status + " when nonSkipUsed=false (from " + aURL.spec + " line " + lineNo + ")";
+ }
+
+ var incURI = g.ioService.newURI(items[1], null, listURL);
+ secMan.checkLoadURIWithPrincipal(principal, incURI,
+ Ci.nsIScriptSecurityManager.DISALLOW_SCRIPT);
+
+ // Cannot use nsIFile or similar to manipulate the manifest ID; although it appears
+ // path-like, it does not refer to an actual path in the filesystem.
+ var newManifestID = aManifestID;
+ var included = items[1];
+ // Remove included manifest file name.
+ // eg. dir1/dir2/reftest.list -> dir1/dir2
+ var pos = included.lastIndexOf("/");
+ if (pos <= 0) {
+ included = "";
+ } else {
+ included = included.substring(0, pos);
+ }
+ // Simplify references to parent directories.
+ // eg. dir1/dir2/../dir3 -> dir1/dir3
+ while (included.startsWith("../")) {
+ pos = newManifestID.lastIndexOf("/");
+ if (pos < 0) {
+ pos = 0;
+ }
+ newManifestID = newManifestID.substring(0, pos);
+ included = included.substring(3);
+ }
+ // Use a new manifest ID if the included manifest is in a different directory.
+ if (included.length > 0) {
+ if (newManifestID.length > 0) {
+ newManifestID = newManifestID + "/" + included;
+ } else {
+ // parent directory includes may refer to the topsrcdir
+ newManifestID = included;
+ }
+ }
+ ReadManifest(incURI, aFilter, newManifestID);
+ }
+ } else if (items[0] == TYPE_LOAD || items[0] == TYPE_SCRIPT) {
+ var type = items[0];
+ if (items.length != 2)
+ throw "Error in manifest file " + aURL.spec + " line " + lineNo + ": incorrect number of arguments to " + type;
+ if (type == TYPE_LOAD && expected_status != EXPECTED_PASS)
+ throw "Error in manifest file " + aURL.spec + " line " + lineNo + ": incorrect known failure type for load test";
+ AddTestItem({ type: type,
+ expected: expected_status,
+ manifest: aURL.spec,
+ manifestID: TestIdentifier(aURL.spec, aManifestID),
+ allowSilentFail: allow_silent_fail,
+ minAsserts: minAsserts,
+ maxAsserts: maxAsserts,
+ needsFocus: needs_focus,
+ slow: slow,
+ skip: skip,
+ prefSettings1: testPrefSettings,
+ prefSettings2: refPrefSettings,
+ fuzzyMinDelta: fuzzy_delta.min,
+ fuzzyMaxDelta: fuzzy_delta.max,
+ fuzzyMinPixels: fuzzy_pixels.min,
+ fuzzyMaxPixels: fuzzy_pixels.max,
+ runHttp: runHttp,
+ httpDepth: httpDepth,
+ url1: items[1],
+ url2: null,
+ chaosMode: chaosMode,
+ wrCapture: wrCapture,
+ noAutoFuzz: noAutoFuzz }, aFilter, aManifestID);
+ } else if (items[0] == TYPE_REFTEST_EQUAL || items[0] == TYPE_REFTEST_NOTEQUAL || items[0] == TYPE_PRINT) {
+ if (items.length != 3)
+ throw "Error in manifest file " + aURL.spec + " line " + lineNo + ": incorrect number of arguments to " + items[0];
+
+ if (items[0] == TYPE_REFTEST_NOTEQUAL &&
+ expected_status == EXPECTED_FUZZY &&
+ (fuzzy_delta.min > 0 || fuzzy_pixels.min > 0)) {
+ throw "Error in manifest file " + aURL.spec + " line " + lineNo + ": minimum fuzz must be zero for tests of type " + items[0];
+ }
+
+ var type = items[0];
+ if (g.compareRetainedDisplayLists) {
+ type = TYPE_REFTEST_EQUAL;
+
+ // We expect twice as many assertion failures when comparing
+ // tests because we run each test twice.
+ minAsserts *= 2;
+ maxAsserts *= 2;
+
+ // Skip the test if it is expected to fail in both modes.
+ // It would unexpectedly "pass" in comparison mode mode when
+ // comparing the two failures, which is not a useful result.
+ if (expected_status === EXPECTED_FAIL ||
+ expected_status === EXPECTED_RANDOM) {
+ skip = true;
+ }
+ }
+
+ AddTestItem({ type: type,
+ expected: expected_status,
+ manifest: aURL.spec,
+ manifestID: TestIdentifier(aURL.spec, aManifestID),
+ allowSilentFail: allow_silent_fail,
+ minAsserts: minAsserts,
+ maxAsserts: maxAsserts,
+ needsFocus: needs_focus,
+ slow: slow,
+ skip: skip,
+ prefSettings1: testPrefSettings,
+ prefSettings2: refPrefSettings,
+ fuzzyMinDelta: fuzzy_delta.min,
+ fuzzyMaxDelta: fuzzy_delta.max,
+ fuzzyMinPixels: fuzzy_pixels.min,
+ fuzzyMaxPixels: fuzzy_pixels.max,
+ runHttp: runHttp,
+ httpDepth: httpDepth,
+ url1: items[1],
+ url2: items[2],
+ chaosMode: chaosMode,
+ wrCapture: wrCapture,
+ noAutoFuzz: noAutoFuzz }, aFilter, aManifestID);
+ } else {
+ throw "Error in manifest file " + aURL.spec + " line " + lineNo + ": unknown test type " + items[0];
+ }
+ }
+}
+
+// Read all available data from an input stream and return it
+// as a string.
+function getStreamContent(inputStream)
+{
+ var streamBuf = "";
+ var sis = Cc["@mozilla.org/scriptableinputstream;1"].
+ createInstance(Ci.nsIScriptableInputStream);
+ sis.init(inputStream);
+
+ var available;
+ while ((available = sis.available()) != 0) {
+ streamBuf += sis.read(available);
+ }
+
+ return streamBuf;
+}
+
+// Build the sandbox for fails-if(), etc., condition evaluation.
+function BuildConditionSandbox(aURL) {
+ var sandbox = new Cu.Sandbox(aURL.spec);
+ var xr = Cc[NS_XREAPPINFO_CONTRACTID].getService(Ci.nsIXULRuntime);
+ var appInfo = Cc[NS_XREAPPINFO_CONTRACTID].getService(Ci.nsIXULAppInfo);
+ sandbox.isDebugBuild = g.debug.isDebugBuild;
+ sandbox.isCoverageBuild = g.isCoverageBuild;
+ var prefs = Cc["@mozilla.org/preferences-service;1"].
+ getService(Ci.nsIPrefBranch);
+
+ sandbox.xulRuntime = Cu.cloneInto({widgetToolkit: xr.widgetToolkit, OS: xr.OS, XPCOMABI: xr.XPCOMABI}, sandbox);
+
+ var testRect = g.browser.getBoundingClientRect();
+ sandbox.smallScreen = false;
+ if (g.containingWindow.innerWidth < 800 || g.containingWindow.innerHeight < 1000) {
+ sandbox.smallScreen = true;
+ }
+
+ var gfxInfo = (NS_GFXINFO_CONTRACTID in Cc) && Cc[NS_GFXINFO_CONTRACTID].getService(Ci.nsIGfxInfo);
+ let readGfxInfo = function (obj, key) {
+ if (g.contentGfxInfo && (key in g.contentGfxInfo)) {
+ return g.contentGfxInfo[key];
+ }
+ return obj[key];
+ }
+
+ try {
+ sandbox.d2d = readGfxInfo(gfxInfo, "D2DEnabled");
+ sandbox.dwrite = readGfxInfo(gfxInfo, "DWriteEnabled");
+ sandbox.embeddedInFirefoxReality = readGfxInfo(gfxInfo, "EmbeddedInFirefoxReality");
+ } catch (e) {
+ sandbox.d2d = false;
+ sandbox.dwrite = false;
+ sandbox.embeddedInFirefoxReality = false;
+ }
+
+ var canvasBackend = readGfxInfo(gfxInfo, "AzureCanvasBackend");
+ var contentBackend = readGfxInfo(gfxInfo, "AzureContentBackend");
+
+ sandbox.gpuProcess = gfxInfo.usingGPUProcess;
+ sandbox.azureCairo = canvasBackend == "cairo";
+ sandbox.azureSkia = canvasBackend == "skia";
+ sandbox.skiaContent = contentBackend == "skia";
+ sandbox.azureSkiaGL = false;
+ // true if we are using the same Azure backend for rendering canvas and content
+ sandbox.contentSameGfxBackendAsCanvas = contentBackend == canvasBackend
+ || (contentBackend == "none" && canvasBackend == "cairo");
+
+ sandbox.remoteCanvas = prefs.getBoolPref("gfx.canvas.remote") && sandbox.d2d && sandbox.gpuProcess;
+
+ sandbox.layersGPUAccelerated =
+ g.windowUtils.layerManagerType != "Basic";
+ sandbox.d3d11 =
+ g.windowUtils.layerManagerType == "Direct3D 11";
+ sandbox.d3d9 =
+ g.windowUtils.layerManagerType == "Direct3D 9";
+ sandbox.layersOpenGL =
+ g.windowUtils.layerManagerType == "OpenGL";
+ sandbox.swgl =
+ g.windowUtils.layerManagerType.startsWith("WebRender (Software");
+ sandbox.layersOMTC =
+ g.windowUtils.layerManagerRemote == true;
+
+ // Shortcuts for widget toolkits.
+ sandbox.Android = xr.OS == "Android";
+ sandbox.cocoaWidget = xr.widgetToolkit == "cocoa";
+ sandbox.gtkWidget = xr.widgetToolkit == "gtk";
+ sandbox.qtWidget = xr.widgetToolkit == "qt";
+ sandbox.winWidget = xr.widgetToolkit == "windows";
+
+ sandbox.is64Bit = xr.is64Bit;
+
+ // Use this to annotate reftests that fail in drawSnapshot, but
+ // the reason hasn't been investigated (or fixed) yet.
+ sandbox.useDrawSnapshot = g.useDrawSnapshot;
+ // Use this to annotate reftests that use functionality
+ // that isn't available to drawSnapshot (like any sort of
+ // compositor feature such as async scrolling).
+ sandbox.unsupportedWithDrawSnapshot = g.useDrawSnapshot;
+
+ sandbox.retainedDisplayList =
+ prefs.getBoolPref("layout.display-list.retain") && !sandbox.useDrawSnapshot;
+
+ // Needed to specifically test the new and old behavior. This will eventually be removed.
+ sandbox.retainedDisplayListNew =
+ sandbox.retainedDisplayList && prefs.getBoolPref("layout.display-list.retain.sc");
+
+ // GeckoView is currently uniquely identified by "android + e10s" but
+ // we might want to make this condition more precise in the future.
+ sandbox.geckoview = (sandbox.Android && g.browserIsRemote);
+
+ // Scrollbars that are semi-transparent. See bug 1169666.
+ sandbox.transparentScrollbars = xr.widgetToolkit == "gtk";
+
+ var sysInfo = Cc["@mozilla.org/system-info;1"].getService(Ci.nsIPropertyBag2);
+ if (sandbox.Android) {
+ // This is currently used to distinguish Android 4.0.3 (SDK version 15)
+ // and later from Android 2.x
+ sandbox.AndroidVersion = sysInfo.getPropertyAsInt32("version");
+
+ sandbox.emulator = readGfxInfo(gfxInfo, "adapterDeviceID").includes("Android Emulator");
+ sandbox.device = !sandbox.emulator;
+ }
+
+ sandbox.MinGW = sandbox.winWidget && sysInfo.getPropertyAsBool("isMinGW");
+
+ sandbox.AddressSanitizer = AppConstants.ASAN;
+ sandbox.ThreadSanitizer = AppConstants.TSAN;
+ sandbox.webrtc = AppConstants.MOZ_WEBRTC;
+ sandbox.jxl = AppConstants.MOZ_JXL;
+
+ sandbox.compareRetainedDisplayLists = g.compareRetainedDisplayLists;
+
+ sandbox.release_or_beta = AppConstants.RELEASE_OR_BETA;
+
+ var hh = Cc[NS_NETWORK_PROTOCOL_CONTRACTID_PREFIX + "http"].
+ getService(Ci.nsIHttpProtocolHandler);
+ var httpProps = ["userAgent", "appName", "appVersion", "vendor",
+ "vendorSub", "product", "productSub", "platform",
+ "oscpu", "language", "misc"];
+ sandbox.http = new sandbox.Object();
+ httpProps.forEach((x) => sandbox.http[x] = hh[x]);
+
+ // set to specific Android13 version (Pixel 5 in CI)
+ sandbox.Android13 = sandbox.Android && (sandbox.http["platform"] == "Android 13");
+
+ // Set OSX to be the Mac OS X version, as an integer, or undefined
+ // for other platforms. The integer is formed by 100 times the
+ // major version plus the minor version, so 1006 for 10.6, 1010 for
+ // 10.10, etc.
+ var osxmatch = /Mac OS X (\d+).(\d+)$/.exec(hh.oscpu);
+ sandbox.OSX = osxmatch ? parseInt(osxmatch[1]) * 100 + parseInt(osxmatch[2]) : undefined;
+
+ // config specific prefs
+ sandbox.appleSilicon = prefs.getBoolPref("sandbox.apple_silicon", false);
+
+ // Set a flag on sandbox if the windows default theme is active
+ sandbox.windowsDefaultTheme = g.containingWindow.matchMedia("(-moz-windows-default-theme)").matches;
+ sandbox.gpuProcessForceEnabled = prefs.getBoolPref("layers.gpu-process.force-enabled", false);
+
+ sandbox.prefs = Cu.cloneInto({
+ getBoolPref: function(p) { return prefs.getBoolPref(p); },
+ getIntPref: function(p) { return prefs.getIntPref(p); }
+ }, sandbox, { cloneFunctions: true });
+
+ // Tests shouldn't care about this except for when they need to
+ // crash the content process
+ sandbox.browserIsRemote = g.browserIsRemote;
+ sandbox.browserIsFission = g.browserIsFission;
+
+ try {
+ sandbox.asyncPan = g.containingWindow.docShell.asyncPanZoomEnabled && !sandbox.useDrawSnapshot;
+ } catch (e) {
+ sandbox.asyncPan = false;
+ }
+
+ // Graphics features
+ sandbox.usesRepeatResampling = sandbox.d2d;
+
+ // Running in a test-verify session?
+ sandbox.verify = prefs.getBoolPref("reftest.verify", false);
+
+ // Running with a variant enabled?
+ sandbox.fission = Services.appinfo.fissionAutostart;
+ sandbox.serviceWorkerE10s = true;
+
+ if (!g.dumpedConditionSandbox) {
+ g.logger.info("Dumping representation of sandbox which can be used for expectation annotations");
+ for (let entry of Object.entries(Cu.waiveXrays(sandbox)).sort((a, b) => a[0].localeCompare(b[0]))) {
+ let value = typeof entry[1] === "object" ? JSON.stringify(entry[1]) : entry[1];
+ g.logger.info(` ${entry[0]}: ${value}`);
+ }
+ g.dumpedConditionSandbox = true;
+ }
+
+ return sandbox;
+}
+
+function AddRetainedDisplayListTestPrefs(aSandbox, aTestPrefSettings,
+ aRefPrefSettings) {
+ AddPrefSettings("test-", "layout.display-list.retain", "true", aSandbox,
+ aTestPrefSettings, aRefPrefSettings);
+ AddPrefSettings("ref-", "layout.display-list.retain", "false", aSandbox,
+ aTestPrefSettings, aRefPrefSettings);
+}
+
+function AddPrefSettings(aWhere, aPrefName, aPrefValExpression, aSandbox, aTestPrefSettings, aRefPrefSettings) {
+ var prefVal = Cu.evalInSandbox("(" + aPrefValExpression + ")", aSandbox);
+ var prefType;
+ var valType = typeof(prefVal);
+ if (valType == "boolean") {
+ prefType = PREF_BOOLEAN;
+ } else if (valType == "string") {
+ prefType = PREF_STRING;
+ } else if (valType == "number" && (parseInt(prefVal) == prefVal)) {
+ prefType = PREF_INTEGER;
+ } else {
+ return false;
+ }
+ var setting = { name: aPrefName,
+ type: prefType,
+ value: prefVal };
+
+ if (g.compareRetainedDisplayLists && aPrefName != "layout.display-list.retain") {
+ // ref-pref() is ignored, test-pref() and pref() are added to both
+ if (aWhere != "ref-") {
+ aTestPrefSettings.push(setting);
+ aRefPrefSettings.push(setting);
+ }
+ } else {
+ if (aWhere != "ref-") {
+ aTestPrefSettings.push(setting);
+ }
+ if (aWhere != "test-") {
+ aRefPrefSettings.push(setting);
+ }
+ }
+ return true;
+}
+
+function ExtractRange(matches, startIndex) {
+ return {
+ min: Number(matches[startIndex]),
+ max: Number(matches[startIndex + 1])
+ };
+}
+
+function ServeTestBase(aURL, depth) {
+ var listURL = aURL.QueryInterface(Ci.nsIFileURL);
+ var directory = listURL.file.parent;
+
+ // Allow serving a tree that's an ancestor of the directory containing
+ // the files so that they can use resources in ../ (etc.).
+ var dirPath = "/";
+ while (depth > 0) {
+ dirPath = "/" + directory.leafName + dirPath;
+ directory = directory.parent;
+ --depth;
+ }
+
+ g.count++;
+ var path = "/" + Date.now() + "/" + g.count;
+ g.server.registerDirectory(path + "/", directory);
+ // this one is needed so tests can use example.org urls for cross origin testing
+ g.server.registerDirectory("/", directory);
+
+ return g.ioService.newURI("http://localhost:" + g.httpServerPort + path + dirPath);
+}
+
+function CreateUrls(test) {
+ let secMan = Cc[NS_SCRIPTSECURITYMANAGER_CONTRACTID]
+ .getService(Ci.nsIScriptSecurityManager);
+
+ let manifestURL = g.ioService.newURI(test.manifest);
+
+ let testbase = manifestURL;
+ if (test.runHttp) {
+ testbase = ServeTestBase(manifestURL, test.httpDepth)
+ }
+
+ let testbasePrincipal = secMan.createContentPrincipal(testbase, {});
+ Services.perms.addFromPrincipal(testbasePrincipal, "allowXULXBL", Services.perms.ALLOW_ACTION);
+
+ function FileToURI(file)
+ {
+ if (file === null)
+ return file;
+
+ var testURI = g.ioService.newURI(file, null, testbase);
+ let isChromeOrViewSource = testURI.scheme == "chrome" || testURI.scheme == "view-source";
+ let principal = isChromeOrViewSource ? secMan.getSystemPrincipal() :
+ secMan.createContentPrincipal(manifestURL, {});
+ secMan.checkLoadURIWithPrincipal(principal, testURI,
+ Ci.nsIScriptSecurityManager.DISALLOW_SCRIPT);
+ return testURI;
+ }
+
+ let files = [test.url1, test.url2];
+ [test.url1, test.url2] = files.map(FileToURI);
+
+ return test;
+}
+
+function TestIdentifier(aUrl, aManifestID) {
+ // Construct a platform-independent and location-independent test identifier for
+ // a url; normally the identifier looks like a posix-compliant relative file
+ // path.
+ // Test urls may be simple file names, chrome: urls with full paths, about:blank, etc.
+ if (aUrl.startsWith("about:") || aUrl.startsWith("data:")) {
+ return aUrl;
+ }
+ var pos = aUrl.lastIndexOf("/");
+ var url = (pos < 0) ? aUrl : aUrl.substring(pos + 1);
+ return (aManifestID + "/" + url);
+}
+
+function AddTestItem(aTest, aFilter, aManifestID) {
+ if (!aFilter)
+ aFilter = [null, [], false];
+
+ var identifier = TestIdentifier(aTest.url1, aManifestID);
+ if (aTest.url2 !== null) {
+ identifier = [identifier, aTest.type, TestIdentifier(aTest.url2, aManifestID)];
+ }
+
+ var {url1, url2} = CreateUrls(Object.assign({}, aTest));
+
+ var globalFilter = aFilter[0];
+ var manifestFilter = aFilter[1];
+ var invertManifest = aFilter[2];
+ if (globalFilter && !globalFilter.test(url1.spec)) {
+ if (url2 === null)
+ return;
+ if (globalFilter && !globalFilter.test(url2.spec))
+ return;
+ }
+ if (manifestFilter && !(invertManifest ^ manifestFilter.test(url1.spec))) {
+ if (url2 === null)
+ return;
+ if (manifestFilter && !(invertManifest ^ manifestFilter.test(url2.spec)))
+ return;
+ }
+ if (g.focusFilterMode == FOCUS_FILTER_NEEDS_FOCUS_TESTS &&
+ !aTest.needsFocus)
+ return;
+ if (g.focusFilterMode == FOCUS_FILTER_NON_NEEDS_FOCUS_TESTS &&
+ aTest.needsFocus)
+ return;
+
+ aTest.identifier = identifier;
+ g.urls.push(aTest);
+ // Periodically log progress to avoid no-output timeout on slow platforms.
+ // No-output timeouts during manifest parsing have been a problem for
+ // jsreftests on Android/debug. Any logging resets the no-output timer,
+ // even debug logging which is normally not displayed.
+ if ((g.urls.length % 5000) == 0)
+ g.logger.debug(g.urls.length + " tests found...");
+}
diff --git a/layout/tools/reftest/manifest.json b/layout/tools/reftest/manifest.json
new file mode 100644
index 0000000000..e6f2e0cba2
--- /dev/null
+++ b/layout/tools/reftest/manifest.json
@@ -0,0 +1,22 @@
+{
+ "manifest_version": 2,
+ "name": "Reftest",
+ "version": "1.0",
+
+ "browser_specific_settings": {
+ "gecko": {
+ "id": "reftest@mozilla.org"
+ }
+ },
+
+ "experiment_apis": {
+ "reftest": {
+ "schema": "schema.json",
+ "parent": {
+ "scopes": ["addon_parent"],
+ "script": "api.js",
+ "events": ["startup"]
+ }
+ }
+ }
+}
diff --git a/layout/tools/reftest/moz.build b/layout/tools/reftest/moz.build
new file mode 100644
index 0000000000..da07a9962f
--- /dev/null
+++ b/layout/tools/reftest/moz.build
@@ -0,0 +1,35 @@
+# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*-
+# vim: set filetype=python:
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+with Files("**"):
+ BUG_COMPONENT = ("Testing", "Reftest")
+ SCHEDULES.exclusive = ["reftest", "crashtest"]
+
+XPI_NAME = "reftest"
+USE_EXTENSION_MANIFEST = True
+JAR_MANIFESTS += ["jar.mn"]
+FINAL_TARGET_FILES += [
+ "api.js",
+ "manifest.json",
+ "schema.json",
+]
+
+TEST_HARNESS_FILES.reftest += [
+ "/build/pgo/server-locations.txt",
+ "/testing/mochitest/server.js",
+ "mach_test_package_commands.py",
+ "output.py",
+ "reftestcommandline.py",
+ "remotereftest.py",
+ "runreftest.py",
+]
+
+TEST_HARNESS_FILES.reftest.chrome += [
+ "chrome/userContent-import.css",
+ "chrome/userContent.css",
+]
+
+TEST_HARNESS_FILES.reftest.manifest += ["reftest/__init__.py"]
diff --git a/layout/tools/reftest/output.py b/layout/tools/reftest/output.py
new file mode 100644
index 0000000000..fe435eb545
--- /dev/null
+++ b/layout/tools/reftest/output.py
@@ -0,0 +1,190 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import json
+import threading
+from collections import defaultdict
+
+from mozlog.formatters import TbplFormatter
+from mozrunner.utils import get_stack_fixer_function
+
+
+class ReftestFormatter(TbplFormatter):
+ """
+ Formatter designed to preserve the legacy "tbpl" format in reftest.
+
+ This is needed for both the reftest-analyzer and mozharness log parsing.
+ We can change this format when both reftest-analyzer and mozharness have
+ been changed to read structured logs.
+ """
+
+ def __call__(self, data):
+ if "component" in data and data["component"] == "mozleak":
+ # Output from mozleak requires that no prefix be added
+ # so that mozharness will pick up these failures.
+ return "%s\n" % data["message"]
+
+ formatted = TbplFormatter.__call__(self, data)
+
+ if formatted is None:
+ return
+ if data["action"] == "process_output":
+ return formatted
+ return "REFTEST %s" % formatted
+
+ def log(self, data):
+ prefix = "%s |" % data["level"].upper()
+ return "%s %s\n" % (prefix, data["message"])
+
+ def _format_status(self, data):
+ extra = data.get("extra", {})
+ status = data["status"]
+
+ status_msg = "TEST-"
+ if "expected" in data:
+ status_msg += "UNEXPECTED-%s" % status
+ else:
+ if status not in ("PASS", "SKIP"):
+ status_msg += "KNOWN-"
+ status_msg += status
+ if extra.get("status_msg") == "Random":
+ status_msg += "(EXPECTED RANDOM)"
+ return status_msg
+
+ def test_status(self, data):
+ extra = data.get("extra", {})
+ test = data["test"]
+
+ status_msg = self._format_status(data)
+ output_text = "%s | %s | %s" % (
+ status_msg,
+ test,
+ data.get("subtest", "unknown test"),
+ )
+ if data.get("message"):
+ output_text += " | %s" % data["message"]
+
+ if "reftest_screenshots" in extra:
+ screenshots = extra["reftest_screenshots"]
+ image_1 = screenshots[0]["screenshot"]
+
+ if len(screenshots) == 3:
+ image_2 = screenshots[2]["screenshot"]
+ output_text += (
+ "\nREFTEST IMAGE 1 (TEST): data:image/png;base64,%s\n"
+ "REFTEST IMAGE 2 (REFERENCE): data:image/png;base64,%s"
+ ) % (image_1, image_2)
+ elif len(screenshots) == 1:
+ output_text += "\nREFTEST IMAGE: data:image/png;base64,%s" % image_1
+
+ return output_text + "\n"
+
+ def test_end(self, data):
+ status = data["status"]
+ test = data["test"]
+
+ output_text = ""
+ if status != "OK":
+ status_msg = self._format_status(data)
+ output_text = "%s | %s | %s" % (status_msg, test, data.get("message", ""))
+
+ if output_text:
+ output_text += "\nREFTEST "
+ output_text += "TEST-END | %s" % test
+ return "%s\n" % output_text
+
+ def process_output(self, data):
+ return "%s\n" % data["data"]
+
+ def suite_end(self, data):
+ lines = []
+ summary = data["extra"]["results"]
+ summary["success"] = summary["Pass"] + summary["LoadOnly"]
+ lines.append(
+ "Successful: %(success)s (%(Pass)s pass, %(LoadOnly)s load only)" % summary
+ )
+ summary["unexpected"] = (
+ summary["Exception"]
+ + summary["FailedLoad"]
+ + summary["UnexpectedFail"]
+ + summary["UnexpectedPass"]
+ + summary["AssertionUnexpected"]
+ + summary["AssertionUnexpectedFixed"]
+ )
+ lines.append(
+ (
+ "Unexpected: %(unexpected)s (%(UnexpectedFail)s unexpected fail, "
+ "%(UnexpectedPass)s unexpected pass, "
+ "%(AssertionUnexpected)s unexpected asserts, "
+ "%(FailedLoad)s failed load, "
+ "%(Exception)s exception)"
+ )
+ % summary
+ )
+ summary["known"] = (
+ summary["KnownFail"]
+ + summary["AssertionKnown"]
+ + summary["Random"]
+ + summary["Skip"]
+ + summary["Slow"]
+ )
+ lines.append(
+ (
+ "Known problems: %(known)s ("
+ + "%(KnownFail)s known fail, "
+ + "%(AssertionKnown)s known asserts, "
+ + "%(Random)s random, "
+ + "%(Skip)s skipped, "
+ + "%(Slow)s slow)"
+ )
+ % summary
+ )
+ lines = ["REFTEST INFO | %s" % s for s in lines]
+ lines.append("REFTEST SUITE-END | Shutdown")
+ return "INFO | Result summary:\n{}\n".format("\n".join(lines))
+
+
+class OutputHandler(object):
+ """Process the output of a process during a test run and translate
+ raw data logged from reftest.js to an appropriate structured log action,
+ where applicable.
+ """
+
+ def __init__(self, log, utilityPath, symbolsPath=None):
+ self.stack_fixer_function = get_stack_fixer_function(utilityPath, symbolsPath)
+ self.log = log
+ self.proc_name = None
+ self.results = defaultdict(int)
+
+ def __call__(self, line):
+ # need to return processed messages to appease remoteautomation.py
+ if not line.strip():
+ return []
+ line = line.decode("utf-8", errors="replace")
+
+ try:
+ data = json.loads(line)
+ except ValueError:
+ self.verbatim(line)
+ return [line]
+
+ if isinstance(data, dict) and "action" in data:
+ if data["action"] == "results":
+ for k, v in data["results"].items():
+ self.results[k] += v
+ else:
+ self.log.log_raw(data)
+ else:
+ self.verbatim(json.dumps(data))
+
+ return [data]
+
+ def write(self, data):
+ return self.__call__(data)
+
+ def verbatim(self, line):
+ if self.stack_fixer_function:
+ line = self.stack_fixer_function(line)
+ name = self.proc_name or threading.current_thread().name
+ self.log.process_output(name, line)
diff --git a/layout/tools/reftest/reftest-analyzer-structured.xhtml b/layout/tools/reftest/reftest-analyzer-structured.xhtml
new file mode 100644
index 0000000000..871b1bc08b
--- /dev/null
+++ b/layout/tools/reftest/reftest-analyzer-structured.xhtml
@@ -0,0 +1,649 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!-- -*- Mode: HTML; tab-width: 2; indent-tabs-mode: nil; -*- -->
+<!-- vim: set shiftwidth=2 tabstop=2 autoindent expandtab: -->
+<!-- This Source Code Form is subject to the terms of the Mozilla Public
+ - License, v. 2.0. If a copy of the MPL was not distributed with this
+ - file, You can obtain one at http://mozilla.org/MPL/2.0/. -->
+<!--
+
+Features to add:
+* make the left and right parts of the viewer independently scrollable
+* make the test list filterable
+** default to only showing unexpecteds
+* add other ways to highlight differences other than circling?
+* add zoom/pan to images
+* Add ability to load log via XMLHttpRequest (also triggered via URL param)
+* color the test list based on pass/fail and expected/unexpected/random/skip
+* ability to load multiple logs ?
+** rename them by clicking on the name and editing
+** turn the test list into a collapsing tree view
+** move log loading into popup from viewer UI
+
+-->
+<!DOCTYPE html>
+<html lang="en-US" xml:lang="en-US" xmlns="http://www.w3.org/1999/xhtml">
+<head>
+ <title>Reftest analyzer</title>
+ <style type="text/css"><![CDATA[
+
+ html, body { margin: 0; }
+ html { padding: 0; }
+ body { padding: 4px; }
+
+ #pixelarea, #itemlist, #images { position: absolute; }
+ #itemlist, #images { overflow: auto; }
+ #pixelarea { top: 0; left: 0; width: 320px; height: 84px; overflow: visible }
+ #itemlist { top: 84px; left: 0; width: 320px; bottom: 0; }
+ #images { top: 0; bottom: 0; left: 320px; right: 0; }
+
+ #leftpane { width: 320px; }
+ #images { position: fixed; top: 10px; left: 340px; }
+
+ form#imgcontrols { margin: 0; display: block; }
+
+ #itemlist > table { border-collapse: collapse; }
+ #itemlist > table > tbody > tr > td { border: 1px solid; padding: 1px; }
+ #itemlist td.activeitem { background-color: yellow; }
+
+ /*
+ #itemlist > table > tbody > tr.pass > td.url { background: lime; }
+ #itemlist > table > tbody > tr.fail > td.url { background: red; }
+ */
+
+ #magnification > svg { display: block; width: 84px; height: 84px; }
+
+ #pixelinfo { font: small sans-serif; position: absolute; width: 200px; left: 84px; }
+ #pixelinfo table { border-collapse: collapse; }
+ #pixelinfo table th { white-space: nowrap; text-align: left; padding: 0; }
+ #pixelinfo table td { font-family: monospace; padding: 0 0 0 0.25em; }
+
+ #pixelhint { display: inline; color: #88f; cursor: help; }
+ #pixelhint > * { display: none; position: absolute; margin: 8px 0 0 8px; padding: 4px; width: 400px; background: #ffa; color: black; box-shadow: 3px 3px 2px #888; z-index: 1; }
+ #pixelhint:hover { color: #000; }
+ #pixelhint:hover > * { display: block; }
+ #pixelhint p { margin: 0; }
+ #pixelhint p + p { margin-top: 1em; }
+
+ ]]></style>
+ <script type="text/javascript"><![CDATA[
+
+var XLINK_NS = "http://www.w3.org/1999/xlink";
+var SVG_NS = "http://www.w3.org/2000/svg";
+var IMAGE_NOT_AVAILABLE = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAKAAAAASCAYAAADczdVTAAAACXBIWXMAAAsTAAALEwEAmpwYAAAHy0lEQVRoge2aX2hb5xnGf2dYabROgqQkpMuKnWUJLmxHMFaa/SscteQiF5EvUgqLctEVrDJKK1+MolzkQr4IctgW+SLIheJc1BpFpswJw92FbaZsTCGTL0465AtntUekJdJ8lByVHbnnwLsLKbKdSJbiZBVjeuAYn+/P+z3fc97vfd9zbEVEhB566BK+1m0CPfx/o+eAPXQVbR3QqVapOl8FlR46h0O1Wu02iacCZfsasMKEz8vbx1JYE6fY/dXx6mEbFObPcvDVDBlznpc9G+2r8xNcvLqK2w39r4UI+fs7tFjmytgFFu718865EIebPGincI3zFz7Bcrtx97/GL0P+p+IPbSOgRwXtW3vpewqL/a/g5rgf39hit2m0hGUAHOHrrq3trmef4/lDB7Ay57n01zuPZXPX7jUunv+Yf9ktR7D/0CHca7/n3KXPsHbAuynkCWCZptgiImKLaVqP9NuW1bT9ceybpr3j+WJbYrVa3rbEatGZi2uixvWdrysilmWKae2M+5PqlktoosayLfubcrN10dAk24aynUsIxMVsadwUs+EX7dEyAlaXLqMoCj6fj5HkUqO9MD+Govjx+xXcXi+uoRAhvwuv182Z8Ws4AJUlxoZ8uNxuvF43ii/EtdXNNUuV68lR/IqC4gsxPj7KkE/BF5qmClRXrzFSt+/1ulDOjLNU6eQ4OcyPDqH4hhg5O4LicuN2K4xcvk6jjHUKJM8O1fvcKMoZkouFOq1VPp1OcuXGAvrvfsv0lWmSySTzN0sdH+jyYhK/ouB2e/G6XfjPJikBVG8SUhT8fl99nwVGfQp+vx+f4iO5VO1AtwJjfgXF58M/kqSVJP9ef0xuAI6NlwWmL41xxqeg+PyMXr72yBqW3cI4JaZHh1DcXrxeLy5liORiB7q1PiZFyeV0mQqz9TRZeUmFVUGLSjqdkgCIFp2RTCosEJOiiIihSyKWkDl9WYrFnCQCCNF0w0QmHhBQJTEzJ+nZSQmAoEYks2KIGBkJgASiM5I3LbGMnCSCCEQl38GJMvMZiag1e+nlFcmmIgKaZEwREaPGhWGZ1VfEMFZkNj4sgCSyhoihSzwSlqCGoAUlEo1IJByW+Oxyh+dZJJ+eklhiRnIrRcnrM6KCxLOmiNiipyICSGR2pTY2O1m7T2XEsNrrJmJLfjkn6amwoMbFaMEhG28eAVtzExErW3sOBCWVzkpmNiEqCOEZ2RyLTT3eJAKaMhVEUMOSXjHEtg3JTIUFkNTK9rGwbQrWm2xGb6QoWxIqEtdtEWO28aDtoi6JSFCAjUtL1AUzJA4SSW/IZ2VjjU0V0zEBJBiJSzwWk1g8IZEAAmrdidrBkoSKxB4IW08tGVNEzIxoIJM5a8v4SQ1RY5lGSy6x8xScz6QkHFBre1Zre49nH+y1KDEQLV7TcyU1LBCtHVppp9smxk2dYAMtHXA7blZWNJDZ4sZ4MxPbdHjrbc3WNuvOq4YlkYhLLBaXeKx2sLcrBUS2ScFtUbUBh3WgajvgOYgGuKjw4Rsqb1uvkssbWLbJXFQFqL/I9IEKa2WzYcqy16E2BNteB1R+cuwoRwcHGRx4nlfenWMuPclRDx3goSraqd+7Gj/Y5d76SrXLu3VKLYW1rMZbo/QpB4+9zt6fT1I0Law/LRMBaLzC7ePNuSgL7/2GpcotLr7+AZG5t9gH0Fa3zuFq1tiWG4DKs5tebV1NDDW1XYd26iWO9A8wODjAUfUN5ubm+Ch4ZFuuLRzQoVwqUCqXyN9fg3tFSuUShVIZhyr5O2vo94o42DwD/PP23fq8Bf5urLO+BoHBwxzc20c++wcmz+lAkWLFATwcf3+YDwIDhMYmuDw+wt5j5+C5ZwDYP/gSoLP6xX5+fOIkJ47/lIP8g49/Nc3tDj59OZUiRR3uFYsAVO/eZoE1yvkyeA6gAaff+zU3SxUcp8LilQucnoFTP3hhix19/garlQqFW9eZOBti9Mqt9mubXwBw+NALeDC4cfVDzgP3i3keUN/nf4uo+hEver/DRaK84/9mY/72uoFTKVMolVn5/HPgPvlSmVKhRL2bSrlEqVyidH8N/d7t2u/lakfcKneLgM4rvxhncbXA6tI8kTffB+0NjnrAqZYplcrk83ceXdtzgB+psHD7S/pfPs7JkydQB1x8dnWS2SVje9GaxkVLl+DmNNC4NJn/S6JxH5nJyNRwrW7Qi7oMgxBMyd9molvmRKO1cExgshG6l9NTEhkOynAkLlOJoKBuhPV8ZlK0h9aNTqVbv3ltEK/VIiAQEN0yZVLbuM+aImLoEgts3VdsJrfFil1M1/ZSv9RAROaWO8n/hkyF1Q3bgeFGygvPrDRG5Wcf1IJbq9rlNrrNbra96aqlUVMSWrNnNiw5uw23T/4o4Xq7FtA29h2My3K9WtETgRZr13UxdIk+pGswkpCcsX0N2OZD9BOgWqFsgWePp20KWb0ywkDgEIa8y55Gq0O5XKHP7cGz++l/haxWylgOuD17aG7eoVpxwL27RX8b27jZ42n1qdahXKrg2bfnUW0eQ7edoD232l+/LPp2pHvNfh8eT2f8/3sO2AZLyRAvns6gqToLOgxP6Uz87HvdoNJDF9E1B6ysLrLw5yW+3PUNvv3dH/L9wX3doNFDl9E1B+yhB+j9O1YPXcZ/AAl9BWJNvZE7AAAAAElFTkSuQmCC";
+
+var gPhases = null;
+
+var gIDCache = {};
+
+var gMagPixPaths = []; // 2D array of array-of-two <path> objects used in the pixel magnifier
+var gMagWidth = 5; // number of zoomed in pixels to show horizontally
+var gMagHeight = 5; // number of zoomed in pixels to show vertically
+var gMagZoom = 16; // size of the zoomed in pixels
+var gImage1Data; // ImageData object for the reference image
+var gImage2Data; // ImageData object for the test output image
+var gFlashingPixels = []; // array of <path> objects that should be flashed due to pixel color mismatch
+var gParams;
+
+function ID(id) {
+ if (!(id in gIDCache))
+ gIDCache[id] = document.getElementById(id);
+ return gIDCache[id];
+}
+
+function hash_parameters() {
+ var result = { };
+ var params = window.location.hash.substr(1).split(/[&;]/);
+ for (var i = 0; i < params.length; i++) {
+ var parts = params[i].split("=");
+ result[parts[0]] = unescape(unescape(parts[1]));
+ }
+ return result;
+}
+
+function load() {
+ gPhases = [ ID("entry"), ID("loading"), ID("viewer") ];
+ build_mag();
+ gParams = hash_parameters();
+ if (gParams.log) {
+ show_phase("loading");
+ process_log(gParams.log);
+ } else if (gParams.logurl) {
+ show_phase("loading");
+ var req = new XMLHttpRequest();
+ req.onreadystatechange = function() {
+ if (req.readyState === 4) {
+ process_log(req.responseText);
+ }
+ };
+ req.open('GET', gParams.logurl, true);
+ req.send();
+ }
+ window.addEventListener('keypress', handle_keyboard_shortcut);
+ ID("image1").addEventListener('error', image_load_error);
+ ID("image2").addEventListener('error', image_load_error);
+}
+
+function image_load_error(e) {
+ e.target.setAttributeNS(XLINK_NS, "xlink:href", IMAGE_NOT_AVAILABLE);
+}
+
+function build_mag() {
+ var mag = ID("mag");
+
+ var r = document.createElementNS(SVG_NS, "rect");
+ r.setAttribute("x", gMagZoom * -gMagWidth / 2);
+ r.setAttribute("y", gMagZoom * -gMagHeight / 2);
+ r.setAttribute("width", gMagZoom * gMagWidth);
+ r.setAttribute("height", gMagZoom * gMagHeight);
+ mag.appendChild(r);
+
+ mag.setAttribute("transform", "translate(" + (gMagZoom * (gMagWidth / 2) + 1) + "," + (gMagZoom * (gMagHeight / 2) + 1) + ")");
+
+ for (var x = 0; x < gMagWidth; x++) {
+ gMagPixPaths[x] = [];
+ for (var y = 0; y < gMagHeight; y++) {
+ var p1 = document.createElementNS(SVG_NS, "path");
+ p1.setAttribute("d", "M" + ((x - gMagWidth / 2) + 1) * gMagZoom + "," + (y - gMagHeight / 2) * gMagZoom + "h" + -gMagZoom + "v" + gMagZoom);
+ p1.setAttribute("stroke", "black");
+ p1.setAttribute("stroke-width", "1px");
+ p1.setAttribute("fill", "#aaa");
+
+ var p2 = document.createElementNS(SVG_NS, "path");
+ p2.setAttribute("d", "M" + ((x - gMagWidth / 2) + 1) * gMagZoom + "," + (y - gMagHeight / 2) * gMagZoom + "v" + gMagZoom + "h" + -gMagZoom);
+ p2.setAttribute("stroke", "black");
+ p2.setAttribute("stroke-width", "1px");
+ p2.setAttribute("fill", "#888");
+
+ mag.appendChild(p1);
+ mag.appendChild(p2);
+ gMagPixPaths[x][y] = [p1, p2];
+ }
+ }
+
+ var flashedOn = false;
+ setInterval(function() {
+ flashedOn = !flashedOn;
+ flash_pixels(flashedOn);
+ }, 500);
+}
+
+function show_phase(phaseid) {
+ for (var i in gPhases) {
+ var phase = gPhases[i];
+ phase.style.display = (phase.id == phaseid) ? "" : "none";
+ }
+
+ if (phase == "viewer")
+ ID("images").style.display = "none";
+}
+
+function fileentry_changed() {
+ show_phase("loading");
+ var input = ID("fileentry");
+ var files = input.files;
+ if (files.length > 0) {
+ // Only handle the first file; don't handle multiple selection.
+ // The parts of the log we care about are ASCII-only. Since we
+ // can ignore lines we don't care about, best to read in as
+ // iso-8859-1, which guarantees we don't get decoding errors.
+ var fileReader = new FileReader();
+ fileReader.onload = function(e) {
+ var log = null;
+
+ log = e.target.result;
+
+ if (log)
+ process_log(log);
+ else
+ show_phase("entry");
+ }
+ fileReader.readAsText(files[0], "iso-8859-1");
+ }
+ // So the user can process the same filename again (after
+ // overwriting the log), clear the value on the form input so we
+ // will always get an onchange event.
+ input.value = "";
+}
+
+function log_pasted() {
+ show_phase("loading");
+ var entry = ID("logentry");
+ var log = entry.value;
+ entry.value = "";
+ process_log(log);
+}
+
+var gTestItems;
+
+function process_log(contents) {
+ var lines = contents.split(/[\r\n]+/);
+ gTestItems = [];
+ for (var j in lines) {
+ var line = lines[j];
+ try {
+ var data = JSON.parse(line);
+ } catch(e) {
+ continue;
+ }
+ // Ignore duplicated output in logcat.
+ if (!data.action == "test_end" && data.status != "FAIL")
+ continue;
+
+ if (!data.hasOwnProperty("extra") ||
+ !data.extra.hasOwnProperty("reftest_screenshots")) {
+ continue;
+ }
+
+ var url = data.test;
+ var screenshots = data.extra.reftest_screenshots;
+ gTestItems.push(
+ {
+ pass: data.status === "PASS",
+ // only one of the following three should ever be true
+ unexpected: data.hasOwnProperty("expected"),
+ random: false,
+ skip: data.status == "SKIP",
+ url: url,
+ images: [],
+ imageLabels: []
+ });
+
+ var item = gTestItems[gTestItems.length - 1];
+ item.images.push("data:image/png;base64," + screenshots[0].screenshot);
+ item.imageLabels.push(screenshots[0].url);
+ if (screenshots.length > 1) {
+ item.images.push("data:image/png;base64," + screenshots[2].screenshot);
+ item.imageLabels.push(screenshots[2].url);
+ }
+ }
+ build_viewer();
+}
+
+function build_viewer() {
+ if (gTestItems.length == 0) {
+ show_phase("entry");
+ return;
+ }
+
+ var cell = ID("itemlist");
+ while (cell.childNodes.length > 0)
+ cell.removeChild(cell.childNodes[cell.childNodes.length - 1]);
+
+ var table = document.createElement("table");
+ var tbody = document.createElement("tbody");
+ table.appendChild(tbody);
+
+ for (var i in gTestItems) {
+ var item = gTestItems[i];
+
+ // optional url filter for only showing unexpected results
+ if (parseInt(gParams.only_show_unexpected) && !item.unexpected)
+ continue;
+
+ // XXX regardless skip expected pass items until we have filtering UI
+ if (item.pass && !item.unexpected)
+ continue;
+
+ var tr = document.createElement("tr");
+ var rowclass = item.pass ? "pass" : "fail";
+ var td;
+ var text;
+
+ td = document.createElement("td");
+ text = "";
+ if (item.unexpected) { text += "!"; rowclass += " unexpected"; }
+ if (item.random) { text += "R"; rowclass += " random"; }
+ if (item.skip) { text += "S"; rowclass += " skip"; }
+ td.appendChild(document.createTextNode(text));
+ tr.appendChild(td);
+
+ td = document.createElement("td");
+ td.id = "item" + i;
+ td.className = "url";
+ // Only display part of URL after "/mozilla/".
+ var match = item.url.match(/\/mozilla\/(.*)/);
+ text = document.createTextNode(match ? match[1] : item.url);
+ if (item.images.length > 0) {
+ var a = document.createElement("a");
+ a.href = "javascript:show_images(" + i + ")";
+ a.appendChild(text);
+ td.appendChild(a);
+ } else {
+ td.appendChild(text);
+ }
+ tr.appendChild(td);
+
+ tbody.appendChild(tr);
+ }
+
+ cell.appendChild(table);
+
+ show_phase("viewer");
+}
+
+function get_image_data(src, whenReady) {
+ var img = new Image();
+ img.onload = function() {
+ var canvas = document.createElement("canvas");
+ canvas.width = img.naturalWidth;
+ canvas.height = img.naturalHeight;
+
+ var ctx = canvas.getContext("2d");
+ ctx.drawImage(img, 0, 0);
+
+ whenReady(ctx.getImageData(0, 0, img.naturalWidth, img.naturalHeight));
+ };
+ img.src = src;
+}
+
+function sync_svg_size(imageData) {
+ // We need the size of the 'svg' and its 'image' elements to match the size
+ // of the ImageData objects that we're going to read pixels from or else our
+ // magnify() function will be very broken.
+ ID("svg").setAttribute("width", imageData.width);
+ ID("svg").setAttribute("height", imageData.height);
+}
+
+function show_images(i) {
+ var item = gTestItems[i];
+ var cell = ID("images");
+
+ // Remove activeitem class from any existing elements
+ var activeItems = document.querySelectorAll(".activeitem");
+ for (var activeItemIdx = activeItems.length; activeItemIdx-- != 0;) {
+ activeItems[activeItemIdx].classList.remove("activeitem");
+ }
+
+ ID("item" + i).classList.add("activeitem");
+ ID("image1").style.display = "";
+ ID("image2").style.display = "none";
+ ID("diffrect").style.display = "none";
+ ID("imgcontrols").reset();
+
+ ID("image1").setAttributeNS(XLINK_NS, "xlink:href", item.images[0]);
+ // Making the href be #image1 doesn't seem to work
+ ID("feimage1").setAttributeNS(XLINK_NS, "xlink:href", item.images[0]);
+ if (item.images.length == 1) {
+ ID("imgcontrols").style.display = "none";
+ } else {
+ ID("imgcontrols").style.display = "";
+
+ ID("image2").setAttributeNS(XLINK_NS, "xlink:href", item.images[1]);
+ // Making the href be #image2 doesn't seem to work
+ ID("feimage2").setAttributeNS(XLINK_NS, "xlink:href", item.images[1]);
+
+ ID("label1").textContent = 'Image ' + item.imageLabels[0];
+ ID("label2").textContent = 'Image ' + item.imageLabels[1];
+ }
+
+ cell.style.display = "";
+
+ get_image_data(item.images[0], function(data) { gImage1Data = data; sync_svg_size(gImage1Data); });
+ get_image_data(item.images[1], function(data) { gImage2Data = data });
+}
+
+function show_image(i) {
+ if (i == 1) {
+ ID("image1").style.display = "";
+ ID("image2").style.display = "none";
+ } else {
+ ID("image1").style.display = "none";
+ ID("image2").style.display = "";
+ }
+}
+
+function handle_keyboard_shortcut(event) {
+ switch (event.charCode) {
+ case 49: // "1" key
+ document.getElementById("radio1").checked = true;
+ show_image(1);
+ break;
+ case 50: // "2" key
+ document.getElementById("radio2").checked = true;
+ show_image(2);
+ break;
+ case 100: // "d" key
+ document.getElementById("differences").click();
+ break;
+ case 112: // "p" key
+ shift_images(-1);
+ break;
+ case 110: // "n" key
+ shift_images(1);
+ break;
+ }
+}
+
+function shift_images(dir) {
+ var activeItem = document.querySelector(".activeitem");
+ if (!activeItem) {
+ return;
+ }
+ for (var elm = activeItem; elm; elm = elm.parentElement) {
+ if (elm.tagName != "tr") {
+ continue;
+ }
+ elm = dir > 0 ? elm.nextElementSibling : elm.previousElementSibling;
+ if (elm) {
+ elm.getElementsByTagName("a")[0].click();
+ }
+ return;
+ }
+}
+
+function show_differences(cb) {
+ ID("diffrect").style.display = cb.checked ? "" : "none";
+}
+
+function flash_pixels(on) {
+ var stroke = on ? "red" : "black";
+ var strokeWidth = on ? "2px" : "1px";
+ for (var i = 0; i < gFlashingPixels.length; i++) {
+ gFlashingPixels[i].setAttribute("stroke", stroke);
+ gFlashingPixels[i].setAttribute("stroke-width", strokeWidth);
+ }
+}
+
+function cursor_point(evt) {
+ var m = evt.target.getScreenCTM().inverse();
+ var p = ID("svg").createSVGPoint();
+ p.x = evt.clientX;
+ p.y = evt.clientY;
+ p = p.matrixTransform(m);
+ return { x: Math.floor(p.x), y: Math.floor(p.y) };
+}
+
+function hex2(i) {
+ return (i < 16 ? "0" : "") + i.toString(16);
+}
+
+function canvas_pixel_as_hex(data, x, y) {
+ var offset = (y * data.width + x) * 4;
+ var r = data.data[offset];
+ var g = data.data[offset + 1];
+ var b = data.data[offset + 2];
+ return "#" + hex2(r) + hex2(g) + hex2(b);
+}
+
+function hex_as_rgb(hex) {
+ return "rgb(" + [parseInt(hex.substring(1, 3), 16), parseInt(hex.substring(3, 5), 16), parseInt(hex.substring(5, 7), 16)] + ")";
+}
+
+function magnify(evt) {
+ var { x: x, y: y } = cursor_point(evt);
+ var centerPixelColor1, centerPixelColor2;
+
+ var dx_lo = -Math.floor(gMagWidth / 2);
+ var dx_hi = Math.floor(gMagWidth / 2);
+ var dy_lo = -Math.floor(gMagHeight / 2);
+ var dy_hi = Math.floor(gMagHeight / 2);
+
+ flash_pixels(false);
+ gFlashingPixels = [];
+ for (var j = dy_lo; j <= dy_hi; j++) {
+ for (var i = dx_lo; i <= dx_hi; i++) {
+ var px = x + i;
+ var py = y + j;
+ var p1 = gMagPixPaths[i + dx_hi][j + dy_hi][0];
+ var p2 = gMagPixPaths[i + dx_hi][j + dy_hi][1];
+ // Here we just use the dimensions of gImage1Data since we expect test
+ // and reference to have the same dimensions.
+ if (px < 0 || py < 0 || px >= gImage1Data.width || py >= gImage1Data.height) {
+ p1.setAttribute("fill", "#aaa");
+ p2.setAttribute("fill", "#888");
+ } else {
+ var color1 = canvas_pixel_as_hex(gImage1Data, x + i, y + j);
+ var color2 = canvas_pixel_as_hex(gImage2Data, x + i, y + j);
+ p1.setAttribute("fill", color1);
+ p2.setAttribute("fill", color2);
+ if (color1 != color2) {
+ gFlashingPixels.push(p1, p2);
+ p1.parentNode.appendChild(p1);
+ p2.parentNode.appendChild(p2);
+ }
+ if (i == 0 && j == 0) {
+ centerPixelColor1 = color1;
+ centerPixelColor2 = color2;
+ }
+ }
+ }
+ }
+ flash_pixels(true);
+ show_pixelinfo(x, y, centerPixelColor1, hex_as_rgb(centerPixelColor1), centerPixelColor2, hex_as_rgb(centerPixelColor2));
+}
+
+function show_pixelinfo(x, y, pix1rgb, pix1hex, pix2rgb, pix2hex) {
+ var pixelinfo = ID("pixelinfo");
+ ID("coords").textContent = [x, y];
+ ID("pix1hex").textContent = pix1hex;
+ ID("pix1rgb").textContent = pix1rgb;
+ ID("pix2hex").textContent = pix2hex;
+ ID("pix2rgb").textContent = pix2rgb;
+}
+
+ ]]></script>
+
+</head>
+<body onload="load()">
+
+<div id="entry">
+
+<h1>Reftest analyzer: load raw structured log</h1>
+
+<p>Either paste your log into this textarea:<br />
+<textarea cols="80" rows="10" id="logentry"/><br/>
+<input type="button" value="Process pasted log" onclick="log_pasted()" /></p>
+
+<p>... or load it from a file:<br/>
+<input type="file" id="fileentry" onchange="fileentry_changed()" />
+</p>
+</div>
+
+<div id="loading" style="display:none">Loading log...</div>
+
+<div id="viewer" style="display:none">
+ <div id="pixelarea">
+ <div id="pixelinfo">
+ <table>
+ <tbody>
+ <tr><th>Pixel at:</th><td colspan="2" id="coords"/></tr>
+ <tr><th>Image 1:</th><td id="pix1rgb"></td><td id="pix1hex"></td></tr>
+ <tr><th>Image 2:</th><td id="pix2rgb"></td><td id="pix2hex"></td></tr>
+ </tbody>
+ </table>
+ <div>
+ <div id="pixelhint">★
+ <div>
+ <p>Move the mouse over the reftest image on the right to show
+ magnified pixels on the left. The color information above is for
+ the pixel centered in the magnified view.</p>
+ <p>Image 1 is shown in the upper triangle of each pixel and Image 2
+ is shown in the lower triangle.</p>
+ </div>
+ </div>
+ </div>
+ </div>
+ <div id="magnification">
+ <svg xmlns="http://www.w3.org/2000/svg" width="84" height="84" shape-rendering="optimizeSpeed">
+ <g id="mag"/>
+ </svg>
+ </div>
+ </div>
+ <div id="itemlist"></div>
+ <div id="images" style="display:none">
+ <form id="imgcontrols">
+ <input id="radio1" type="radio" name="which" value="0" onchange="show_image(1)" checked="checked" /><label id="label1" title="1" for="radio1">Image 1</label>
+ <input id="radio2" type="radio" name="which" value="1" onchange="show_image(2)" /><label id="label2" title="2" for="radio2">Image 2</label>
+ <label><input id="differences" type="checkbox" onchange="show_differences(this)" />Circle differences</label>
+ </form>
+ <svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" version="1.1" width="800" height="1000" id="svg">
+ <defs>
+ <!-- use sRGB to avoid loss of data -->
+ <filter id="showDifferences" x="0%" y="0%" width="100%" height="100%"
+ style="color-interpolation-filters: sRGB">
+ <feImage id="feimage1" result="img1" xlink:href="#image1" />
+ <feImage id="feimage2" result="img2" xlink:href="#image2" />
+ <!-- inv1 and inv2 are the images with RGB inverted -->
+ <feComponentTransfer result="inv1" in="img1">
+ <feFuncR type="linear" slope="-1" intercept="1" />
+ <feFuncG type="linear" slope="-1" intercept="1" />
+ <feFuncB type="linear" slope="-1" intercept="1" />
+ </feComponentTransfer>
+ <feComponentTransfer result="inv2" in="img2">
+ <feFuncR type="linear" slope="-1" intercept="1" />
+ <feFuncG type="linear" slope="-1" intercept="1" />
+ <feFuncB type="linear" slope="-1" intercept="1" />
+ </feComponentTransfer>
+ <!-- w1 will have non-white pixels anywhere that img2
+ is brighter than img1, and w2 for the reverse.
+ It would be nice not to have to go through these
+ intermediate states, but feComposite
+ type="arithmetic" can't transform the RGB channels
+ and leave the alpha channel untouched. -->
+ <feComposite result="w1" in="img1" in2="inv2" operator="arithmetic" k2="1" k3="1" />
+ <feComposite result="w2" in="img2" in2="inv1" operator="arithmetic" k2="1" k3="1" />
+ <!-- c1 will have non-black pixels anywhere that img2
+ is brighter than img1, and c2 for the reverse -->
+ <feComponentTransfer result="c1" in="w1">
+ <feFuncR type="linear" slope="-1" intercept="1" />
+ <feFuncG type="linear" slope="-1" intercept="1" />
+ <feFuncB type="linear" slope="-1" intercept="1" />
+ </feComponentTransfer>
+ <feComponentTransfer result="c2" in="w2">
+ <feFuncR type="linear" slope="-1" intercept="1" />
+ <feFuncG type="linear" slope="-1" intercept="1" />
+ <feFuncB type="linear" slope="-1" intercept="1" />
+ </feComponentTransfer>
+ <!-- c will be nonblack (and fully on) for every pixel+component where there are differences -->
+ <feComposite result="c" in="c1" in2="c2" operator="arithmetic" k2="255" k3="255" />
+ <!-- a will be opaque for every pixel with differences and transparent for all others -->
+ <feColorMatrix result="a" type="matrix" values="0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 0 0" />
+
+ <!-- a, dilated by 1 pixel -->
+ <feMorphology result="dila1" in="a" operator="dilate" radius="1" />
+ <!-- a, dilated by 2 pixels -->
+ <feMorphology result="dila2" in="dila1" operator="dilate" radius="1" />
+
+ <!-- all the pixels in the 2-pixel dilation of a but not in the 1-pixel dilation, to highlight the diffs -->
+ <feComposite result="highlight" in="dila2" in2="dila1" operator="out" />
+
+ <feFlood result="red" flood-color="red" />
+ <feComposite result="redhighlight" in="red" in2="highlight" operator="in" />
+ <feFlood result="black" flood-color="black" flood-opacity="0.5" />
+ <feMerge>
+ <feMergeNode in="black" />
+ <feMergeNode in="redhighlight" />
+ </feMerge>
+ </filter>
+ </defs>
+ <g onmousemove="magnify(evt)">
+ <image x="0" y="0" width="100%" height="100%" id="image1" />
+ <image x="0" y="0" width="100%" height="100%" id="image2" />
+ </g>
+ <rect id="diffrect" filter="url(#showDifferences)" pointer-events="none" x="0" y="0" width="100%" height="100%" />
+ </svg>
+ </div>
+</div>
+
+</body>
+</html>
diff --git a/layout/tools/reftest/reftest-analyzer.xhtml b/layout/tools/reftest/reftest-analyzer.xhtml
new file mode 100644
index 0000000000..3977f489d3
--- /dev/null
+++ b/layout/tools/reftest/reftest-analyzer.xhtml
@@ -0,0 +1,934 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!-- -*- Mode: HTML; tab-width: 2; indent-tabs-mode: nil; -*- -->
+<!-- vim: set shiftwidth=2 tabstop=2 autoindent expandtab: -->
+<!-- This Source Code Form is subject to the terms of the Mozilla Public
+ - License, v. 2.0. If a copy of the MPL was not distributed with this
+ - file, You can obtain one at http://mozilla.org/MPL/2.0/. -->
+<!--
+
+Features to add:
+* make the left and right parts of the viewer independently scrollable
+* make the test list filterable
+** default to only showing unexpecteds
+* add other ways to highlight differences other than circling?
+* add zoom/pan to images
+* Add ability to load log via XMLHttpRequest (also triggered via URL param)
+* color the test list based on pass/fail and expected/unexpected/random/skip
+* ability to load multiple logs ?
+** rename them by clicking on the name and editing
+** turn the test list into a collapsing tree view
+** move log loading into popup from viewer UI
+
+-->
+<!DOCTYPE html>
+<html lang="en-US" xml:lang="en-US" xmlns="http://www.w3.org/1999/xhtml">
+<head>
+ <title>Reftest analyzer</title>
+ <style type="text/css"><![CDATA[
+
+ html, body { margin: 0; }
+ html { padding: 0; }
+ body { padding: 4px; }
+
+ #pixelarea, #itemlist, #images { position: absolute; }
+ #itemlist, #images { overflow: auto; }
+ #pixelarea { top: 0; left: 0; width: 320px; height: 84px; overflow: visible }
+ #itemlist { top: 84px; left: 0; width: 320px; bottom: 0; }
+ #images { top: 0; bottom: 0; left: 320px; right: 0; }
+
+ #leftpane { width: 320px; }
+ #images { position: fixed; top: 10px; left: 340px; }
+
+ form#imgcontrols { margin: 0; display: block; }
+
+ #itemlist > table { border-collapse: collapse; }
+ #itemlist > table > tbody > tr > td { border: 1px solid; padding: 1px; }
+ #itemlist td.activeitem { background-color: yellow; }
+
+ /*
+ #itemlist > table > tbody > tr.pass > td.url { background: lime; }
+ #itemlist > table > tbody > tr.fail > td.url { background: red; }
+ */
+
+ #magnification > svg { display: block; width: 84px; height: 84px; }
+
+ #pixelinfo { font: small sans-serif; position: absolute; width: 200px; left: 84px; }
+ #pixelinfo table { border-collapse: collapse; }
+ #pixelinfo table th { white-space: nowrap; text-align: left; padding: 0; }
+ #pixelinfo table td { font-family: monospace; padding: 0 0 0 0.25em; }
+
+ #pixelhint { display: inline; color: #88f; cursor: help; }
+ #pixelhint > * { display: none; position: absolute; margin: 8px 0 0 8px; padding: 4px; width: 400px; background: #ffa; color: black; box-shadow: 3px 3px 2px #888; z-index: 1; }
+ #pixelhint:hover { color: #000; }
+ #pixelhint:hover > * { display: block; }
+ #pixelhint p { margin: 0; }
+ #pixelhint p + p { margin-top: 1em; }
+
+ ]]></style>
+ <script type="text/javascript"><![CDATA[
+
+var XLINK_NS = "http://www.w3.org/1999/xlink";
+var SVG_NS = "http://www.w3.org/2000/svg";
+var IMAGE_NOT_AVAILABLE = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAKAAAAASCAYAAADczdVTAAAACXBIWXMAAAsTAAALEwEAmpwYAAAHy0lEQVRoge2aX2hb5xnGf2dYabROgqQkpMuKnWUJLmxHMFaa/SscteQiF5EvUgqLctEVrDJKK1+MolzkQr4IctgW+SLIheJc1BpFpswJw92FbaZsTCGTL0465AtntUekJdJ8lByVHbnnwLsLKbKdSJbiZBVjeuAYn+/P+z3fc97vfd9zbEVEhB566BK+1m0CPfx/o+eAPXQVbR3QqVapOl8FlR46h0O1Wu02iacCZfsasMKEz8vbx1JYE6fY/dXx6mEbFObPcvDVDBlznpc9G+2r8xNcvLqK2w39r4UI+fs7tFjmytgFFu718865EIebPGincI3zFz7Bcrtx97/GL0P+p+IPbSOgRwXtW3vpewqL/a/g5rgf39hit2m0hGUAHOHrrq3trmef4/lDB7Ay57n01zuPZXPX7jUunv+Yf9ktR7D/0CHca7/n3KXPsHbAuynkCWCZptgiImKLaVqP9NuW1bT9ceybpr3j+WJbYrVa3rbEatGZi2uixvWdrysilmWKae2M+5PqlktoosayLfubcrN10dAk24aynUsIxMVsadwUs+EX7dEyAlaXLqMoCj6fj5HkUqO9MD+Govjx+xXcXi+uoRAhvwuv182Z8Ws4AJUlxoZ8uNxuvF43ii/EtdXNNUuV68lR/IqC4gsxPj7KkE/BF5qmClRXrzFSt+/1ulDOjLNU6eQ4OcyPDqH4hhg5O4LicuN2K4xcvk6jjHUKJM8O1fvcKMoZkouFOq1VPp1OcuXGAvrvfsv0lWmSySTzN0sdH+jyYhK/ouB2e/G6XfjPJikBVG8SUhT8fl99nwVGfQp+vx+f4iO5VO1AtwJjfgXF58M/kqSVJP9ef0xuAI6NlwWmL41xxqeg+PyMXr72yBqW3cI4JaZHh1DcXrxeLy5liORiB7q1PiZFyeV0mQqz9TRZeUmFVUGLSjqdkgCIFp2RTCosEJOiiIihSyKWkDl9WYrFnCQCCNF0w0QmHhBQJTEzJ+nZSQmAoEYks2KIGBkJgASiM5I3LbGMnCSCCEQl38GJMvMZiag1e+nlFcmmIgKaZEwREaPGhWGZ1VfEMFZkNj4sgCSyhoihSzwSlqCGoAUlEo1IJByW+Oxyh+dZJJ+eklhiRnIrRcnrM6KCxLOmiNiipyICSGR2pTY2O1m7T2XEsNrrJmJLfjkn6amwoMbFaMEhG28eAVtzExErW3sOBCWVzkpmNiEqCOEZ2RyLTT3eJAKaMhVEUMOSXjHEtg3JTIUFkNTK9rGwbQrWm2xGb6QoWxIqEtdtEWO28aDtoi6JSFCAjUtL1AUzJA4SSW/IZ2VjjU0V0zEBJBiJSzwWk1g8IZEAAmrdidrBkoSKxB4IW08tGVNEzIxoIJM5a8v4SQ1RY5lGSy6x8xScz6QkHFBre1Zre49nH+y1KDEQLV7TcyU1LBCtHVppp9smxk2dYAMtHXA7blZWNJDZ4sZ4MxPbdHjrbc3WNuvOq4YlkYhLLBaXeKx2sLcrBUS2ScFtUbUBh3WgajvgOYgGuKjw4Rsqb1uvkssbWLbJXFQFqL/I9IEKa2WzYcqy16E2BNteB1R+cuwoRwcHGRx4nlfenWMuPclRDx3goSraqd+7Gj/Y5d76SrXLu3VKLYW1rMZbo/QpB4+9zt6fT1I0Law/LRMBaLzC7ePNuSgL7/2GpcotLr7+AZG5t9gH0Fa3zuFq1tiWG4DKs5tebV1NDDW1XYd26iWO9A8wODjAUfUN5ubm+Ch4ZFuuLRzQoVwqUCqXyN9fg3tFSuUShVIZhyr5O2vo94o42DwD/PP23fq8Bf5urLO+BoHBwxzc20c++wcmz+lAkWLFATwcf3+YDwIDhMYmuDw+wt5j5+C5ZwDYP/gSoLP6xX5+fOIkJ47/lIP8g49/Nc3tDj59OZUiRR3uFYsAVO/eZoE1yvkyeA6gAaff+zU3SxUcp8LilQucnoFTP3hhix19/garlQqFW9eZOBti9Mqt9mubXwBw+NALeDC4cfVDzgP3i3keUN/nf4uo+hEver/DRaK84/9mY/72uoFTKVMolVn5/HPgPvlSmVKhRL2bSrlEqVyidH8N/d7t2u/lakfcKneLgM4rvxhncbXA6tI8kTffB+0NjnrAqZYplcrk83ceXdtzgB+psHD7S/pfPs7JkydQB1x8dnWS2SVje9GaxkVLl+DmNNC4NJn/S6JxH5nJyNRwrW7Qi7oMgxBMyd9molvmRKO1cExgshG6l9NTEhkOynAkLlOJoKBuhPV8ZlK0h9aNTqVbv3ltEK/VIiAQEN0yZVLbuM+aImLoEgts3VdsJrfFil1M1/ZSv9RAROaWO8n/hkyF1Q3bgeFGygvPrDRG5Wcf1IJbq9rlNrrNbra96aqlUVMSWrNnNiw5uw23T/4o4Xq7FtA29h2My3K9WtETgRZr13UxdIk+pGswkpCcsX0N2OZD9BOgWqFsgWePp20KWb0ywkDgEIa8y55Gq0O5XKHP7cGz++l/haxWylgOuD17aG7eoVpxwL27RX8b27jZ42n1qdahXKrg2bfnUW0eQ7edoD232l+/LPp2pHvNfh8eT2f8/3sO2AZLyRAvns6gqToLOgxP6Uz87HvdoNJDF9E1B6ysLrLw5yW+3PUNvv3dH/L9wX3doNFDl9E1B+yhB+j9O1YPXcZ/AAl9BWJNvZE7AAAAAElFTkSuQmCC";
+
+var gPhases = null;
+
+var gIDCache = {};
+
+var gMagPixPaths = []; // 2D array of array-of-two <path> objects used in the pixel magnifier
+var gMagWidth = 5; // number of zoomed in pixels to show horizontally
+var gMagHeight = 5; // number of zoomed in pixels to show vertically
+var gMagZoom = 16; // size of the zoomed in pixels
+var gImage1Data; // ImageData object for the reference image
+var gImage2Data; // ImageData object for the test output image
+var gFlashingPixels = []; // array of <path> objects that should be flashed due to pixel color mismatch
+var gParams;
+
+function ID(id) {
+ if (!(id in gIDCache))
+ gIDCache[id] = document.getElementById(id);
+ return gIDCache[id];
+}
+
+function hash_parameters() {
+ var result = { };
+ var params = window.location.hash.substr(1).split(/[&;]/);
+ for (var i = 0; i < params.length; i++) {
+ var parts = params[i].split("=");
+ result[parts[0]] = unescape(unescape(parts[1]));
+ }
+ return result;
+}
+
+function load() {
+ gPhases = [ ID("entry"), ID("loading"), ID("viewer") ];
+ build_mag();
+ gParams = hash_parameters();
+ if (gParams.log) {
+ show_phase("loading");
+ process_log(gParams.log);
+ } else if (gParams.logurl) {
+ show_phase("loading");
+ var req = new XMLHttpRequest();
+ req.onreadystatechange = function() {
+ if (req.readyState === 4) {
+ process_log(req.responseText);
+ }
+ };
+ req.open('GET', gParams.logurl, true);
+ req.send();
+ }
+ window.addEventListener('keypress', handle_keyboard_shortcut);
+ window.addEventListener('keydown', handle_keydown);
+ ID("image1").addEventListener('error', image_load_error);
+ ID("image2").addEventListener('error', image_load_error);
+}
+
+function image_load_error(e) {
+ e.target.setAttributeNS(XLINK_NS, "xlink:href", IMAGE_NOT_AVAILABLE);
+}
+
+function build_mag() {
+ var mag = ID("mag");
+
+ var r = document.createElementNS(SVG_NS, "rect");
+ r.setAttribute("x", gMagZoom * -gMagWidth / 2);
+ r.setAttribute("y", gMagZoom * -gMagHeight / 2);
+ r.setAttribute("width", gMagZoom * gMagWidth);
+ r.setAttribute("height", gMagZoom * gMagHeight);
+ mag.appendChild(r);
+
+ mag.setAttribute("transform", "translate(" + (gMagZoom * (gMagWidth / 2) + 1) + "," + (gMagZoom * (gMagHeight / 2) + 1) + ")");
+
+ for (var x = 0; x < gMagWidth; x++) {
+ gMagPixPaths[x] = [];
+ for (var y = 0; y < gMagHeight; y++) {
+ var p1 = document.createElementNS(SVG_NS, "path");
+ p1.setAttribute("d", "M" + ((x - gMagWidth / 2) + 1) * gMagZoom + "," + (y - gMagHeight / 2) * gMagZoom + "h" + -gMagZoom + "v" + gMagZoom);
+ p1.setAttribute("stroke", "black");
+ p1.setAttribute("stroke-width", "1px");
+ p1.setAttribute("fill", "#aaa");
+
+ var p2 = document.createElementNS(SVG_NS, "path");
+ p2.setAttribute("d", "M" + ((x - gMagWidth / 2) + 1) * gMagZoom + "," + (y - gMagHeight / 2) * gMagZoom + "v" + gMagZoom + "h" + -gMagZoom);
+ p2.setAttribute("stroke", "black");
+ p2.setAttribute("stroke-width", "1px");
+ p2.setAttribute("fill", "#888");
+
+ mag.appendChild(p1);
+ mag.appendChild(p2);
+ gMagPixPaths[x][y] = [p1, p2];
+ }
+ }
+
+ var flashedOn = false;
+ setInterval(function() {
+ flashedOn = !flashedOn;
+ flash_pixels(flashedOn);
+ }, 500);
+}
+
+function show_phase(phaseid) {
+ for (var i in gPhases) {
+ var phase = gPhases[i];
+ phase.style.display = (phase.id == phaseid) ? "" : "none";
+ }
+
+ if (phase == "viewer")
+ ID("images").style.display = "none";
+}
+
+function fileentry_changed() {
+ show_phase("loading");
+ var input = ID("fileentry");
+ var files = input.files;
+ if (files.length > 0) {
+ // Only handle the first file; don't handle multiple selection.
+ // The parts of the log we care about are ASCII-only. Since we
+ // can ignore lines we don't care about, best to read in as
+ // iso-8859-1, which guarantees we don't get decoding errors.
+ var fileReader = new FileReader();
+ fileReader.onload = function(e) {
+ var log = null;
+
+ log = e.target.result;
+
+ if (log)
+ process_log(log);
+ else
+ show_phase("entry");
+ }
+ fileReader.readAsText(files[0], "iso-8859-1");
+ }
+ // So the user can process the same filename again (after
+ // overwriting the log), clear the value on the form input so we
+ // will always get an onchange event.
+ input.value = "";
+}
+
+function log_pasted() {
+ show_phase("loading");
+ var entry = ID("logentry");
+ var log = entry.value;
+ entry.value = "";
+ process_log(log);
+}
+
+var gTestItems;
+
+// This function is not used in production code, but can be invoked manually
+// from the devtools console in order to test changes to the parsing regexes
+// in process_log.
+function test_parsing() {
+ // Note that the logs in these testcases have been manually edited to strip
+ // out stuff for brevity.
+ var testcases = [
+ { "name": "empty log",
+ "log": "",
+ "expected": { "pass": 0, "unexpected": 0, "random": 0, "skip": 0 },
+ "expected_images": 0,
+ },
+ { "name": "android log",
+ "log": `[task 2018-12-28T10:36:45.718Z] 10:36:45 INFO - REFTEST TEST-START | a == b
+[task 2018-12-28T10:36:45.719Z] 10:36:45 INFO - REFTEST TEST-LOAD | a | 78 / 275 (28%)
+[task 2018-12-28T10:36:56.138Z] 10:36:56 INFO - REFTEST TEST-LOAD | b | 78 / 275 (28%)
+[task 2018-12-28T10:37:06.559Z] 10:37:06 INFO - REFTEST TEST-UNEXPECTED-FAIL | a == b | image comparison, max difference: 255, number of differing pixels: 5950
+[task 2018-12-28T10:37:06.568Z] 10:37:06 INFO - REFTEST IMAGE 1 (TEST): data:image/png;base64,
+[task 2018-12-28T10:37:06.577Z] 10:37:06 INFO - REFTEST IMAGE 2 (REFERENCE): data:image/png;base64,
+[task 2018-12-28T10:37:06.577Z] 10:37:06 INFO - REFTEST INFO | Saved log: stuff trimmed here
+[task 2018-12-28T10:37:06.582Z] 10:37:06 INFO - REFTEST TEST-END | a == b
+[task 2018-12-28T10:37:06.583Z] 10:37:06 INFO - REFTEST TEST-START | a2 == b2
+[task 2018-12-28T10:37:06.583Z] 10:37:06 INFO - REFTEST TEST-LOAD | a2 | 79 / 275 (28%)
+[task 2018-12-28T10:37:06.584Z] 10:37:06 INFO - REFTEST TEST-LOAD | b2 | 79 / 275 (28%)
+[task 2018-12-28T10:37:16.982Z] 10:37:16 INFO - REFTEST TEST-PASS | a2 == b2 | image comparison, max difference: 0, number of differing pixels: 0
+[task 2018-12-28T10:37:16.982Z] 10:37:16 INFO - REFTEST TEST-END | a2 == b2`,
+ "expected": { "pass": 1, "unexpected": 1, "random": 0, "skip": 0 },
+ "expected_images": 2,
+ },
+ { "name": "local reftest run (Linux)",
+ "log": `REFTEST TEST-START | file:///a == file:///b
+REFTEST TEST-LOAD | file:///a | 73 / 86 (84%)
+REFTEST TEST-LOAD | file:///b | 73 / 86 (84%)
+REFTEST TEST-PASS | file:///a == file:///b | image comparison, max difference: 0, number of differing pixels: 0
+REFTEST TEST-END | file:///a == file:///b`,
+ "expected": { "pass": 1, "unexpected": 0, "random": 0, "skip": 0 },
+ "expected_images": 0,
+ },
+ { "name": "wpt reftests (Linux automation)",
+ "log": `16:50:43 INFO - TEST-START | /a
+16:50:43 INFO - PID 4276 | 1548694243694 Marionette INFO Testing http://web-platform.test:8000/a == http://web-platform.test:8000/b
+16:50:43 INFO - PID 4276 | 1548694243963 Marionette INFO No differences allowed
+16:50:44 INFO - TEST-PASS | /a | took 370ms
+16:50:44 INFO - TEST-START | /a2
+16:50:44 INFO - PID 4276 | 1548694244066 Marionette INFO Testing http://web-platform.test:8000/a2 == http://web-platform.test:8000/b2
+16:50:44 INFO - PID 4276 | 1548694244792 Marionette INFO No differences allowed
+16:50:44 INFO - PID 4276 | 1548694244792 Marionette INFO Found 28 pixels different, maximum difference per channel 14
+16:50:44 INFO - TEST-UNEXPECTED-FAIL | /a2 | Testing http://web-platform.test:8000/a2 == http://web-platform.test:8000/b2
+16:50:44 INFO - REFTEST IMAGE 1 (TEST): data:image/png;base64,
+16:50:44 INFO - REFTEST IMAGE 2 (REFERENCE): data:image/png;base64,
+16:50:44 INFO - TEST-INFO took 840ms`,
+ "expected": { "pass": 1, "unexpected": 1, "random": 0, "skip": 0 },
+ "expected_images": 2,
+ },
+ { "name": "windows log",
+ "log": `12:17:14 INFO - REFTEST TEST-START | a == b
+12:17:14 INFO - REFTEST TEST-LOAD | a | 1603 / 2053 (78%)
+12:17:14 INFO - REFTEST TEST-LOAD | b | 1603 / 2053 (78%)
+12:17:14 INFO - REFTEST TEST-PASS(EXPECTED RANDOM) | a == b | image comparison, max difference: 0, number of differing pixels: 0
+12:17:14 INFO - REFTEST TEST-END | a == b
+12:17:14 INFO - REFTEST TEST-START | a2 == b2
+12:17:14 INFO - REFTEST TEST-LOAD | a2 | 1604 / 2053 (78%)
+12:17:14 INFO - REFTEST TEST-LOAD | b2 | 1604 / 2053 (78%)
+12:17:14 INFO - REFTEST TEST-UNEXPECTED-FAIL | a2 == b2 | image comparison, max difference: 255, number of differing pixels: 9976
+12:17:14 INFO - REFTEST IMAGE 1 (TEST): data:image/png;base64,
+12:17:14 INFO - REFTEST IMAGE 2 (REFERENCE): data:image/png;base64,
+12:17:14 INFO - REFTEST INFO | Saved log: stuff trimmed here
+12:17:14 INFO - REFTEST TEST-END | a2 == b2
+12:01:09 INFO - REFTEST TEST-START | a3 == b3
+12:01:09 INFO - REFTEST TEST-LOAD | a3 | 66 / 189 (34%)
+12:01:09 INFO - REFTEST TEST-LOAD | b3 | 66 / 189 (34%)
+12:01:09 INFO - REFTEST TEST-KNOWN-FAIL | a3 == b3 | image comparison, max difference: 255, number of differing pixels: 9654
+12:01:09 INFO - REFTEST TEST-END | a3 == b3`,
+ "expected": { "pass": 1, "unexpected": 1, "random": 1, "skip": 0 },
+ "expected_images": 2,
+ },
+ { "name": "webrender wrench log (windows)",
+ "log": `[task 2018-12-29T04:29:48.800Z] REFTEST a == b
+[task 2018-12-29T04:29:48.984Z] REFTEST a2 == b2
+[task 2018-12-29T04:29:49.053Z] REFTEST TEST-UNEXPECTED-FAIL | a2 == b2 | image comparison, max difference: 255, number of differing pixels: 3128
+[task 2018-12-29T04:29:49.053Z] REFTEST IMAGE 1 (TEST): data:image/png;
+[task 2018-12-29T04:29:49.053Z] REFTEST IMAGE 2 (REFERENCE): data:image/png;
+[task 2018-12-29T04:29:49.053Z] REFTEST TEST-END | a2 == b2`,
+ "expected": { "pass": 0, "unexpected": 1, "random": 0, "skip": 0 },
+ "expected_images": 2,
+ },
+ { "name": "wpt reftests (Linux local; Bug 1530008)",
+ "log": `SUITE-START | Running 1 tests
+TEST-START | /css/css-backgrounds/border-image-6.html
+TEST-UNEXPECTED-FAIL | /css/css-backgrounds/border-image-6.html | Testing http://web-platform.test:8000/css/css-backgrounds/border-image-6.html == http://web-platform.test:8000/css/css-backgrounds/border-image-6-ref.html
+REFTEST IMAGE 1 (TEST): data:image/png;base64,
+REFTEST IMAGE 2 (REFERENCE): data:image/png;base64,
+TEST-INFO took 425ms
+SUITE-END | took 2s`,
+ "expected": { "pass": 0, "unexpected": 1, "random": 0, "skip": 0 },
+ "expected_images": 2,
+ },
+ { "name": "wpt reftests (taskcluster log from macOS CI)",
+ "log": `[task 2020-06-26T01:35:29.065Z] 01:35:29 INFO - TEST-START | /html/rendering/non-replaced-elements/the-page/iframe-scrolling-attribute-values.html
+[task 2020-06-26T01:35:29.065Z] 01:35:29 INFO - PID 1353 | 1593135329040 Marionette INFO Testing http://web-platform.test:8000/html/rendering/non-replaced-elements/the-page/iframe-scrolling-attribute-values.html == http://web-platform.test:8000/html/rendering/non-replaced-elements/the-page/iframe-scrolling-attribute-values-ref.html
+[task 2020-06-26T01:35:29.673Z] 01:35:29 INFO - PID 1353 | 1593135329633 Marionette INFO No differences allowed
+[task 2020-06-26T01:35:29.726Z] 01:35:29 INFO - TEST-KNOWN-INTERMITTENT-FAIL | /html/rendering/non-replaced-elements/the-page/iframe-scrolling-attribute-values.html | took 649ms
+[task 2020-06-26T01:35:29.726Z] 01:35:29 INFO - REFTEST IMAGE 1 (TEST): data:image/png;
+[task 2020-06-26T01:35:29.726Z] 01:35:29 INFO - REFTEST IMAGE 2 (REFERENCE): data:image/png;`,
+ "expected": { "pass": 0, "unexpected": 0, "random": 1, "skip": 0 },
+ "expected_images": 2,
+ },
+ { "name": "wpt reftests (taskcluster log from Windows CI)",
+ "log": `[task 2020-06-26T01:41:19.205Z] 01:41:19 INFO - TEST-START | /html/rendering/non-replaced-elements/the-page/iframe-scrolling-attribute-values.html
+[task 2020-06-26T01:41:19.214Z] 01:41:19 INFO - PID 5920 | 1593135679202 Marionette WARN [24] http://web-platform.test:8000/css/WOFF2/metadatadisplay-schema-license-022-ref.xht overflows viewport (width: 783, height: 731)
+[task 2020-06-26T01:41:19.214Z] 01:41:19 INFO - PID 9692 | 1593135679208 Marionette INFO Testing http://web-platform.test:8000/html/rendering/non-replaced-elements/the-page/iframe-scrolling-attribute-values.html == http://web-platform.test:8000/html/rendering/non-replaced-elements/the-page/iframe-scrolling-attribute-values-ref.html
+[task 2020-06-26T01:41:19.638Z] 01:41:19 INFO - PID 9692 | 1593135679627 Marionette INFO No differences allowed
+[task 2020-06-26T01:41:19.688Z] 01:41:19 INFO - TEST-KNOWN-INTERMITTENT-PASS | /html/rendering/non-replaced-elements/the-page/iframe-scrolling-attribute-values.html | took 474ms
+[task 2020-06-26T01:41:19.688Z] 01:41:19 INFO - REFTEST IMAGE 1 (TEST): data:image/png;
+[task 2020-06-26T01:41:19.689Z] 01:41:19 INFO - REFTEST IMAGE 2 (REFERENCE): data:image/png;`,
+ "expected": { "pass": 1, "unexpected": 0, "random": 1, "skip": 0 },
+ "expected_images": 2,
+ },
+ { "name": "local reftest run with timestamps (Linux; Bug 1167712)",
+ "log": ` 0:05.21 REFTEST TEST-START | a
+ 0:05.21 REFTEST REFTEST TEST-LOAD | a | 0 / 1 (0%)
+ 0:05.27 REFTEST REFTEST TEST-LOAD | b | 0 / 1 (0%)
+ 0:05.66 REFTEST TEST-UNEXPECTED-FAIL | a | image comparison (==), max difference: 106, number of differing pixels: 800
+ 0:05.67 REFTEST REFTEST IMAGE 1 (TEST): data:image/png;base64,
+ 0:05.67 REFTEST REFTEST IMAGE 2 (REFERENCE): data:image/png;base64,
+ 0:05.73 REFTEST REFTEST TEST-END | a`,
+ "expected": { "pass": 0, "unexpected": 1, "random": 0, "skip": 0 },
+ "expected_images": 2,
+ },
+ { "name": "reftest run with whitespace compressed (Treeherder; Bug 1084322)",
+ "log": ` REFTEST TEST-START | a
+REFTEST TEST-LOAD | a | 0 / 1 (0%)
+REFTEST TEST-LOAD | b | 0 / 1 (0%)
+REFTEST TEST-UNEXPECTED-FAIL | a | image comparison (==), max difference: 106, number of differing pixels: 800
+REFTEST REFTEST IMAGE 1 (TEST): data:image/png;base64,
+REFTEST REFTEST IMAGE 2 (REFERENCE): data:image/png;base64,
+REFTEST REFTEST TEST-END | a`,
+ "expected": { "pass": 0, "unexpected": 1, "random": 0, "skip": 0 },
+ "expected_images": 2,
+ },
+ ];
+
+ var current_test = 0;
+
+ // Override the build_viewer function invoked at the end of process_log to
+ // actually just check the results of parsing.
+ build_viewer = function() {
+ var expected = testcases[current_test].expected;
+ var expected_images = testcases[current_test].expected_images;
+ for (var result of gTestItems) {
+ for (let type in expected) { // type is "pass", "unexpected" etc.
+ if (result[type]) {
+ expected[type]--;
+ }
+ }
+ }
+ var failed = false;
+ for (let type in expected) {
+ if (expected[type] != 0) {
+ console.log(`Failure: for testcase ${testcases[current_test].name} got ${expected[type]} fewer ${type} results than expected!`);
+ failed = true;
+ }
+ }
+
+ let total_images = 0;
+ for (var result of gTestItems) {
+ total_images += result.images.length;
+ }
+ if (total_images !== expected_images) {
+ console.log(`Failure: for testcase ${testcases[current_test].name} got ${total_images} images, expected ${expected_images}`);
+ failed = true;
+ }
+
+ if (!failed) {
+ console.log(`Success for testcase ${testcases[current_test].name}`);
+ }
+ };
+
+ while (current_test < testcases.length) {
+ process_log(testcases[current_test].log);
+ current_test++;
+ }
+}
+
+function process_log(contents) {
+ var lines = contents.split(/[\r\n]+/);
+ gTestItems = [];
+ for (var j in lines) {
+
+ // !!!!!!
+ // When making any changes to this code, please add a test to the
+ // test_parsing function above, and ensure all existing tests pass.
+ // !!!!!!
+
+ var line = lines[j];
+ // Ignore duplicated output in logcat.
+ if (line.match(/I\/Gecko.*?REFTEST/))
+ continue;
+ var match = line.match(/^.*?(?:REFTEST\s+)+(.*)$/);
+ if (!match) {
+ // WPT reftests don't always have the "REFTEST" prefix but do have
+ // mozharness prefixing. Trying to match both prefixes optionally with a
+ // single regex either makes an unreadable mess or matches everything so
+ // we do them separately.
+ match = line.match(/^(?:.*? (?:INFO|ERROR) -\s+)(.*)$/);
+ }
+ if (match)
+ line = match[1];
+ match = line.match(/^(TEST-PASS|TEST-UNEXPECTED-PASS|TEST-FAIL|TEST-KNOWN-FAIL|TEST-UNEXPECTED-FAIL|TEST-DEBUG-INFO|TEST-KNOWN-INTERMITTENT-FAIL|TEST-KNOWN-INTERMITTENT-PASS)(\(EXPECTED RANDOM\)|) \| ([^\|]+)(?: \|(.*)|$)/);
+ if (match) {
+ var state = match[1];
+ var random = match[2];
+ var url = match[3];
+ var extra = match[4];
+ gTestItems.push(
+ {
+ pass: !state.match(/DEBUG-INFO$|FAIL$/),
+ // only one of the following three should ever be true
+ unexpected: !!state.match(/^TEST-UNEXPECTED/),
+ random: (random == "(EXPECTED RANDOM)" || state == "TEST-KNOWN-INTERMITTENT-FAIL" || state == "TEST-KNOWN-INTERMITTENT-PASS"),
+ skip: (extra == " (SKIP)"),
+ url: url,
+ images: [],
+ imageLabels: []
+ });
+ continue;
+ }
+ match = line.match(/^IMAGE([^:]*): (data:.*)$/);
+ if (match) {
+ var item = gTestItems[gTestItems.length - 1];
+ item.images.push(match[2]);
+ item.imageLabels.push(match[1]);
+ }
+ }
+
+ build_viewer();
+}
+
+function build_viewer() {
+ if (gTestItems.length == 0) {
+ show_phase("entry");
+ return;
+ }
+
+ var cell = ID("itemlist");
+ while (cell.childNodes.length > 0)
+ cell.removeChild(cell.childNodes[cell.childNodes.length - 1]);
+
+ var table = document.createElement("table");
+ var tbody = document.createElement("tbody");
+ table.appendChild(tbody);
+
+ for (var i in gTestItems) {
+ var item = gTestItems[i];
+
+ // optional url filter for only showing unexpected results
+ if (parseInt(gParams.only_show_unexpected) && !item.unexpected)
+ continue;
+
+ // XXX regardless skip expected pass items until we have filtering UI
+ if (item.pass && !item.unexpected)
+ continue;
+
+ var tr = document.createElement("tr");
+ var rowclass = item.pass ? "pass" : "fail";
+ var td;
+ var text;
+
+ td = document.createElement("td");
+ text = "";
+ if (item.unexpected) { text += "!"; rowclass += " unexpected"; }
+ if (item.random) { text += "R"; rowclass += " random"; }
+ if (item.skip) { text += "S"; rowclass += " skip"; }
+ td.appendChild(document.createTextNode(text));
+ tr.appendChild(td);
+
+ td = document.createElement("td");
+ td.id = "item" + i;
+ td.className = "url";
+ // Only display part of URL after "/mozilla/".
+ var match = item.url.match(/\/mozilla\/(.*)/);
+ text = document.createTextNode(match ? match[1] : item.url);
+ if (item.images.length > 0) {
+ var a = document.createElement("a");
+ a.href = "javascript:show_images(" + i + ")";
+ a.appendChild(text);
+ td.appendChild(a);
+ } else {
+ td.appendChild(text);
+ }
+ tr.appendChild(td);
+
+ tbody.appendChild(tr);
+ }
+
+ cell.appendChild(table);
+
+ show_phase("viewer");
+}
+
+function get_image_data(src, whenReady) {
+ var img = new Image();
+ img.onload = function() {
+ var canvas = document.createElement("canvas");
+ canvas.width = img.naturalWidth;
+ canvas.height = img.naturalHeight;
+
+ var ctx = canvas.getContext("2d");
+ ctx.drawImage(img, 0, 0);
+
+ whenReady(ctx.getImageData(0, 0, img.naturalWidth, img.naturalHeight));
+ };
+ img.src = src;
+}
+
+function sync_svg_size(imageData) {
+ // We need the size of the 'svg' and its 'image' elements to match the size
+ // of the ImageData objects that we're going to read pixels from or else our
+ // magnify() function will be very broken.
+ ID("svg").setAttribute("width", imageData.width);
+ ID("svg").setAttribute("height", imageData.height);
+}
+
+function show_images(i) {
+ var item = gTestItems[i];
+ var cell = ID("images");
+
+ // Remove activeitem class from any existing elements
+ var activeItems = document.querySelectorAll(".activeitem");
+ for (var activeItemIdx = activeItems.length; activeItemIdx-- != 0;) {
+ activeItems[activeItemIdx].classList.remove("activeitem");
+ }
+
+ ID("item" + i).classList.add("activeitem");
+ ID("image1").style.display = "";
+ ID("image2").style.display = "none";
+ ID("diffrect").style.display = "none";
+ ID("imgcontrols").reset();
+ ID("pixel-differences").textContent = "";
+
+ ID("image1").setAttributeNS(XLINK_NS, "xlink:href", item.images[0]);
+ // Making the href be #image1 doesn't seem to work
+ ID("feimage1").setAttributeNS(XLINK_NS, "xlink:href", item.images[0]);
+ if (item.images.length == 1) {
+ ID("imgcontrols").style.display = "none";
+ } else {
+ ID("imgcontrols").style.display = "";
+
+ ID("image2").setAttributeNS(XLINK_NS, "xlink:href", item.images[1]);
+ // Making the href be #image2 doesn't seem to work
+ ID("feimage2").setAttributeNS(XLINK_NS, "xlink:href", item.images[1]);
+
+ ID("label1").textContent = 'Image ' + item.imageLabels[0];
+ ID("label2").textContent = 'Image ' + item.imageLabels[1];
+ }
+
+ cell.style.display = "";
+
+ let loaded = [false, false];
+
+ function images_loaded(id) {
+ loaded[id] = true;
+ if (loaded.every(x => x)) {
+ update_pixel_difference_text()
+ }
+ }
+
+ get_image_data(item.images[0], function(data) { gImage1Data = data; sync_svg_size(gImage1Data); images_loaded(0)});
+ get_image_data(item.images[1], function(data) { gImage2Data = data; images_loaded(1)});
+
+}
+
+function update_pixel_difference_text() {
+ let differenceText;
+ if (gImage1Data.height !== gImage2Data.height ||
+ gImage1Data.width !== gImage2Data.width) {
+ differenceText = "Images are different sizes"
+ } else {
+ let [numPixels, maxPerChannel] = get_pixel_differences();
+ if (!numPixels) {
+ differenceText = "Images are identical";
+ } else {
+ differenceText = `Maximum difference per channel ${maxPerChannel}, ${numPixels} pixels differ`;
+ }
+ }
+ // Disable this for now, because per bug 1633504, the numbers may be
+ // inaccurate and dependent on the browser's configuration.
+ ID("pixel-differences").textContent = differenceText;
+}
+
+function get_pixel_differences() {
+ let numPixels = 0;
+ let maxPerChannel = 0;
+ for (var i=0; i<gImage1Data.data.length; i+=4) {
+ let r1 = gImage1Data.data[i];
+ let r2 = gImage2Data.data[i];
+ let g1 = gImage1Data.data[i+1];
+ let g2 = gImage2Data.data[i+1];
+ let b1 = gImage1Data.data[i+2];
+ let b2 = gImage2Data.data[i+2];
+ // Ignore alpha.
+ if (r1 == r2 && g1 == g2 && b1 == b2) {
+ continue;
+ }
+ numPixels += 1;
+ let maxDiff = Math.max(Math.abs(r1-r2),
+ Math.abs(g1-g2),
+ Math.abs(b1-b2));
+ if (maxDiff > maxPerChannel) {
+ maxPerChannel = maxDiff
+ }
+ }
+ return [numPixels, maxPerChannel];
+}
+
+function show_image(i) {
+ if (i == 1) {
+ ID("image1").style.display = "";
+ ID("image2").style.display = "none";
+ } else {
+ ID("image1").style.display = "none";
+ ID("image2").style.display = "";
+ }
+}
+
+function handle_keyboard_shortcut(event) {
+ switch (event.charCode) {
+ case 49: // "1" key
+ document.getElementById("radio1").checked = true;
+ show_image(1);
+ break;
+ case 50: // "2" key
+ document.getElementById("radio2").checked = true;
+ show_image(2);
+ break;
+ case 100: // "d" key
+ document.getElementById("differences").click();
+ break;
+ case 112: // "p" key
+ shift_images(-1);
+ break;
+ case 110: // "n" key
+ shift_images(1);
+ break;
+ }
+}
+
+function handle_keydown(event) {
+ switch (event.keyCode) {
+ case 37: // left arrow
+ move_pixel(-1, 0);
+ break;
+ case 38: // up arrow
+ move_pixel(0,-1);
+ break;
+ case 39: // right arrow
+ move_pixel(1, 0);
+ break;
+ case 40: // down arrow
+ move_pixel(0, 1);
+ break;
+ }
+}
+
+function shift_images(dir) {
+ var activeItem = document.querySelector(".activeitem");
+ if (!activeItem) {
+ return;
+ }
+ for (var elm = activeItem; elm; elm = elm.parentElement) {
+ if (elm.tagName != "tr") {
+ continue;
+ }
+ elm = dir > 0 ? elm.nextElementSibling : elm.previousElementSibling;
+ if (elm) {
+ elm.getElementsByTagName("a")[0].click();
+ }
+ return;
+ }
+}
+
+function show_differences(cb) {
+ ID("diffrect").style.display = cb.checked ? "" : "none";
+}
+
+function flash_pixels(on) {
+ var stroke = on ? "red" : "black";
+ var strokeWidth = on ? "2px" : "1px";
+ for (var i = 0; i < gFlashingPixels.length; i++) {
+ gFlashingPixels[i].setAttribute("stroke", stroke);
+ gFlashingPixels[i].setAttribute("stroke-width", strokeWidth);
+ }
+}
+
+function cursor_point(evt) {
+ var m = evt.target.getScreenCTM().inverse();
+ var p = ID("svg").createSVGPoint();
+ p.x = evt.clientX;
+ p.y = evt.clientY;
+ p = p.matrixTransform(m);
+ return { x: Math.floor(p.x), y: Math.floor(p.y) };
+}
+
+function hex2(i) {
+ return (i < 16 ? "0" : "") + i.toString(16);
+}
+
+function canvas_pixel_as_hex(data, x, y) {
+ var offset = (y * data.width + x) * 4;
+ var r = data.data[offset];
+ var g = data.data[offset + 1];
+ var b = data.data[offset + 2];
+ return "#" + hex2(r) + hex2(g) + hex2(b);
+}
+
+function hex_as_rgb(hex) {
+ return "rgb(" + [parseInt(hex.substring(1, 3), 16), parseInt(hex.substring(3, 5), 16), parseInt(hex.substring(5, 7), 16)] + ")";
+}
+
+function magnify(evt) {
+ var { x: x, y: y } = cursor_point(evt);
+ do_magnify(x, y);
+}
+
+function do_magnify(x, y) {
+ var centerPixelColor1, centerPixelColor2;
+
+ var dx_lo = -Math.floor(gMagWidth / 2);
+ var dx_hi = Math.floor(gMagWidth / 2);
+ var dy_lo = -Math.floor(gMagHeight / 2);
+ var dy_hi = Math.floor(gMagHeight / 2);
+
+ flash_pixels(false);
+ gFlashingPixels = [];
+ for (var j = dy_lo; j <= dy_hi; j++) {
+ for (var i = dx_lo; i <= dx_hi; i++) {
+ var px = x + i;
+ var py = y + j;
+ var p1 = gMagPixPaths[i + dx_hi][j + dy_hi][0];
+ var p2 = gMagPixPaths[i + dx_hi][j + dy_hi][1];
+ // Here we just use the dimensions of gImage1Data since we expect test
+ // and reference to have the same dimensions.
+ if (px < 0 || py < 0 || px >= gImage1Data.width || py >= gImage1Data.height) {
+ p1.setAttribute("fill", "#aaa");
+ p2.setAttribute("fill", "#888");
+ } else {
+ var color1 = canvas_pixel_as_hex(gImage1Data, x + i, y + j);
+ var color2 = canvas_pixel_as_hex(gImage2Data, x + i, y + j);
+ p1.setAttribute("fill", color1);
+ p2.setAttribute("fill", color2);
+ if (color1 != color2) {
+ gFlashingPixels.push(p1, p2);
+ p1.parentNode.appendChild(p1);
+ p2.parentNode.appendChild(p2);
+ }
+ if (i == 0 && j == 0) {
+ centerPixelColor1 = color1;
+ centerPixelColor2 = color2;
+ }
+ }
+ }
+ }
+ flash_pixels(true);
+ show_pixelinfo(x, y, centerPixelColor1, hex_as_rgb(centerPixelColor1), centerPixelColor2, hex_as_rgb(centerPixelColor2));
+}
+
+function show_pixelinfo(x, y, pix1rgb, pix1hex, pix2rgb, pix2hex) {
+ var pixelinfo = ID("pixelinfo");
+ ID("coords").textContent = [x, y];
+ ID("pix1hex").textContent = pix1hex;
+ ID("pix1rgb").textContent = pix1rgb;
+ ID("pix2hex").textContent = pix2hex;
+ ID("pix2rgb").textContent = pix2rgb;
+}
+
+function move_pixel(deltax, deltay) {
+ coords = ID("coords").textContent.split(',');
+ x = parseInt(coords[0]);
+ y = parseInt(coords[1]);
+ if (isNaN(x) || isNaN(y)) {
+ return;
+ }
+ x = x + deltax;
+ y = y + deltay;
+ if (x >= 0 && y >= 0 && x < gImage1Data.width && y < gImage1Data.height) {
+ do_magnify(x, y);
+ }
+}
+
+ ]]></script>
+
+</head>
+<body onload="load()">
+
+<div id="entry">
+
+<h1>Reftest analyzer: load reftest log</h1>
+
+<p>Either paste your log into this textarea:<br />
+<textarea cols="80" rows="10" id="logentry"/><br/>
+<input type="button" value="Process pasted log" onclick="log_pasted()" /></p>
+
+<p>... or load it from a file:<br/>
+<input type="file" id="fileentry" onchange="fileentry_changed()" />
+</p>
+</div>
+
+<div id="loading" style="display:none">Loading log...</div>
+
+<div id="viewer" style="display:none">
+ <div id="pixelarea">
+ <div id="pixelinfo">
+ <table>
+ <tbody>
+ <tr><th>Pixel at:</th><td colspan="2" id="coords"/></tr>
+ <tr><th>Image 1:</th><td id="pix1rgb"></td><td id="pix1hex"></td></tr>
+ <tr><th>Image 2:</th><td id="pix2rgb"></td><td id="pix2hex"></td></tr>
+ </tbody>
+ </table>
+ <div>
+ <div id="pixelhint">★
+ <div>
+ <p>Move the mouse over the reftest image on the right to show
+ magnified pixels on the left. The color information above is for
+ the pixel centered in the magnified view.</p>
+ <p>Image 1 is shown in the upper triangle of each pixel and Image 2
+ is shown in the lower triangle.</p>
+ </div>
+ </div>
+ </div>
+ </div>
+ <div id="magnification">
+ <svg xmlns="http://www.w3.org/2000/svg" width="84" height="84" shape-rendering="optimizeSpeed">
+ <g id="mag"/>
+ </svg>
+ </div>
+ </div>
+ <div id="itemlist"></div>
+ <div id="images" style="display:none">
+ <form id="imgcontrols">
+ <input id="radio1" type="radio" name="which" value="0" onchange="show_image(1)" checked="checked" /><label id="label1" title="1" for="radio1">Image 1</label>
+ <input id="radio2" type="radio" name="which" value="1" onchange="show_image(2)" /><label id="label2" title="2" for="radio2">Image 2</label>
+ <label><input id="differences" type="checkbox" onchange="show_differences(this)" />Circle differences</label>
+ </form>
+ <svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" version="1.1" width="800" height="1000" id="svg">
+ <defs>
+ <!-- use sRGB to avoid loss of data -->
+ <filter id="showDifferences" x="0%" y="0%" width="100%" height="100%"
+ style="color-interpolation-filters: sRGB">
+ <feImage id="feimage1" result="img1" xlink:href="#image1" />
+ <feImage id="feimage2" result="img2" xlink:href="#image2" />
+ <!-- inv1 and inv2 are the images with RGB inverted -->
+ <feComponentTransfer result="inv1" in="img1">
+ <feFuncR type="linear" slope="-1" intercept="1" />
+ <feFuncG type="linear" slope="-1" intercept="1" />
+ <feFuncB type="linear" slope="-1" intercept="1" />
+ </feComponentTransfer>
+ <feComponentTransfer result="inv2" in="img2">
+ <feFuncR type="linear" slope="-1" intercept="1" />
+ <feFuncG type="linear" slope="-1" intercept="1" />
+ <feFuncB type="linear" slope="-1" intercept="1" />
+ </feComponentTransfer>
+ <!-- w1 will have non-white pixels anywhere that img2
+ is brighter than img1, and w2 for the reverse.
+ It would be nice not to have to go through these
+ intermediate states, but feComposite
+ type="arithmetic" can't transform the RGB channels
+ and leave the alpha channel untouched. -->
+ <feComposite result="w1" in="img1" in2="inv2" operator="arithmetic" k2="1" k3="1" />
+ <feComposite result="w2" in="img2" in2="inv1" operator="arithmetic" k2="1" k3="1" />
+ <!-- c1 will have non-black pixels anywhere that img2
+ is brighter than img1, and c2 for the reverse -->
+ <feComponentTransfer result="c1" in="w1">
+ <feFuncR type="linear" slope="-1" intercept="1" />
+ <feFuncG type="linear" slope="-1" intercept="1" />
+ <feFuncB type="linear" slope="-1" intercept="1" />
+ </feComponentTransfer>
+ <feComponentTransfer result="c2" in="w2">
+ <feFuncR type="linear" slope="-1" intercept="1" />
+ <feFuncG type="linear" slope="-1" intercept="1" />
+ <feFuncB type="linear" slope="-1" intercept="1" />
+ </feComponentTransfer>
+ <!-- c will be nonblack (and fully on) for every pixel+component where there are differences -->
+ <feComposite result="c" in="c1" in2="c2" operator="arithmetic" k2="255" k3="255" />
+ <!-- a will be opaque for every pixel with differences and transparent for all others -->
+ <feColorMatrix result="a" type="matrix" values="0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 0 0" />
+
+ <!-- a, dilated by 1 pixel -->
+ <feMorphology result="dila1" in="a" operator="dilate" radius="1" />
+ <!-- a, dilated by 2 pixels -->
+ <feMorphology result="dila2" in="dila1" operator="dilate" radius="1" />
+
+ <!-- all the pixels in the 2-pixel dilation of a but not in the 1-pixel dilation, to highlight the diffs -->
+ <feComposite result="highlight" in="dila2" in2="dila1" operator="out" />
+
+ <feFlood result="red" flood-color="red" />
+ <feComposite result="redhighlight" in="red" in2="highlight" operator="in" />
+ <feFlood result="black" flood-color="black" flood-opacity="0.5" />
+ <feMerge>
+ <feMergeNode in="black" />
+ <feMergeNode in="redhighlight" />
+ </feMerge>
+ </filter>
+ </defs>
+ <g onmousemove="magnify(evt)">
+ <image x="0" y="0" width="100%" height="100%" id="image1" />
+ <image x="0" y="0" width="100%" height="100%" id="image2" />
+ </g>
+ <rect id="diffrect" filter="url(#showDifferences)" pointer-events="none" x="0" y="0" width="100%" height="100%" />
+ </svg>
+ <div id="pixel-differences"></div>
+ </div>
+</div>
+
+</body>
+</html>
diff --git a/layout/tools/reftest/reftest-content.js b/layout/tools/reftest/reftest-content.js
new file mode 100644
index 0000000000..2703cbc850
--- /dev/null
+++ b/layout/tools/reftest/reftest-content.js
@@ -0,0 +1,1530 @@
+/* -*- indent-tabs-mode: nil; js-indent-level: 4 -*- /
+/* vim: set shiftwidth=4 tabstop=8 autoindent cindent expandtab: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+const XHTML_NS = "http://www.w3.org/1999/xhtml";
+
+const DEBUG_CONTRACTID = "@mozilla.org/xpcom/debug;1";
+const PRINTSETTINGS_CONTRACTID = "@mozilla.org/gfx/printsettings-service;1";
+const NS_OBSERVER_SERVICE_CONTRACTID = "@mozilla.org/observer-service;1";
+const NS_GFXINFO_CONTRACTID = "@mozilla.org/gfx/info;1";
+const IO_SERVICE_CONTRACTID = "@mozilla.org/network/io-service;1"
+
+// "<!--CLEAR-->"
+const BLANK_URL_FOR_CLEARING = "data:text/html;charset=UTF-8,%3C%21%2D%2DCLEAR%2D%2D%3E";
+
+const { setTimeout, clearTimeout } = ChromeUtils.importESModule(
+ "resource://gre/modules/Timer.sys.mjs"
+);
+const { onSpellCheck } = ChromeUtils.importESModule(
+ "resource://reftest/AsyncSpellCheckTestHelper.sys.mjs"
+);
+
+// This will load chrome Custom Elements inside chrome documents:
+ChromeUtils.importESModule("resource://gre/modules/CustomElementsListener.sys.mjs");
+
+var gBrowserIsRemote;
+var gHaveCanvasSnapshot = false;
+var gCurrentURL;
+var gCurrentURLRecordResults;
+var gCurrentURLTargetType;
+var gCurrentTestType;
+var gTimeoutHook = null;
+var gFailureTimeout = null;
+var gFailureReason;
+var gAssertionCount = 0;
+var gUpdateCanvasPromiseResolver = null;
+
+var gDebug;
+var gVerbose = false;
+
+var gCurrentTestStartTime;
+var gClearingForAssertionCheck = false;
+
+const TYPE_LOAD = 'load'; // test without a reference (just test that it does
+ // not assert, crash, hang, or leak)
+const TYPE_SCRIPT = 'script'; // test contains individual test results
+const TYPE_PRINT = 'print'; // test and reference will be printed to PDF's and
+ // compared structurally
+
+// keep this in sync with globals.jsm
+const URL_TARGET_TYPE_TEST = 0; // first url
+const URL_TARGET_TYPE_REFERENCE = 1; // second url, if any
+
+function webNavigation() {
+ return docShell.QueryInterface(Ci.nsIWebNavigation);
+}
+
+function webProgress() {
+ return docShell.QueryInterface(Ci.nsIInterfaceRequestor).getInterface(Ci.nsIWebProgress);
+}
+
+function windowUtilsForWindow(w) {
+ return w.windowUtils;
+}
+
+function windowUtils() {
+ return windowUtilsForWindow(content);
+}
+
+function IDForEventTarget(event)
+{
+ try {
+ return "'" + event.target.getAttribute('id') + "'";
+ } catch (ex) {
+ return "<unknown>";
+ }
+}
+
+var progressListener = {
+ onStateChange(webprogress, request, flags, status) {
+ let uri;
+ try {
+ request.QueryInterface(Ci.nsIChannel);
+ uri = request.originalURI.spec;
+ } catch (ex) {
+ return;
+ }
+ const WPL = Ci.nsIWebProgressListener;
+ const endFlags = WPL.STATE_STOP | WPL.STATE_IS_WINDOW | WPL.STATE_IS_NETWORK;
+ if ((flags & endFlags) == endFlags) {
+ OnDocumentLoad(uri);
+ }
+ },
+ QueryInterface: ChromeUtils.generateQI([
+ "nsIWebProgressListener",
+ "nsISupportsWeakReference",
+ ]),
+};
+
+function OnInitialLoad()
+{
+ removeEventListener("load", OnInitialLoad, true);
+
+ gDebug = Cc[DEBUG_CONTRACTID].getService(Ci.nsIDebug2);
+ if (gDebug.isDebugBuild) {
+ gAssertionCount = gDebug.assertionCount;
+ }
+ gVerbose = !!Services.env.get("MOZ_REFTEST_VERBOSE");
+
+ RegisterMessageListeners();
+
+ var initInfo = SendContentReady();
+ gBrowserIsRemote = initInfo.remote;
+
+ webProgress().addProgressListener(progressListener, Ci.nsIWebProgress.NOTIFY_STATE_WINDOW);
+
+ LogInfo("Using browser remote="+ gBrowserIsRemote +"\n");
+}
+
+function SetFailureTimeout(cb, timeout, uri)
+{
+ var targetTime = Date.now() + timeout;
+
+ var wrapper = function() {
+ // Timeouts can fire prematurely in some cases (e.g. in chaos mode). If this
+ // happens, set another timeout for the remaining time.
+ let remainingMs = targetTime - Date.now();
+ if (remainingMs > 0) {
+ SetFailureTimeout(cb, remainingMs);
+ } else {
+ cb();
+ }
+ }
+
+ // Once OnDocumentLoad is called to handle the 'load' event it will update
+ // this error message to reflect what stage of the processing it has reached
+ // as it advances to each stage in turn.
+ gFailureReason = "timed out after " + timeout +
+ " ms waiting for 'load' event for " + uri;
+ gFailureTimeout = setTimeout(wrapper, timeout);
+}
+
+function StartTestURI(type, uri, uriTargetType, timeout)
+{
+ // The GC is only able to clean up compartments after the CC runs. Since
+ // the JS ref tests disable the normal browser chrome and do not otherwise
+ // create substatial DOM garbage, the CC tends not to run enough normally.
+ windowUtils().runNextCollectorTimer();
+
+ gCurrentTestType = type;
+ gCurrentURL = uri;
+ gCurrentURLTargetType = uriTargetType;
+ gCurrentURLRecordResults = 0;
+
+ gCurrentTestStartTime = Date.now();
+ if (gFailureTimeout != null) {
+ SendException("program error managing timeouts\n");
+ }
+ SetFailureTimeout(LoadFailed, timeout, uri);
+
+ LoadURI(gCurrentURL);
+}
+
+function setupTextZoom(contentRootElement) {
+ if (!contentRootElement || !contentRootElement.hasAttribute('reftest-text-zoom'))
+ return;
+ docShell.browsingContext.textZoom =
+ contentRootElement.getAttribute('reftest-text-zoom');
+}
+
+function setupFullZoom(contentRootElement) {
+ if (!contentRootElement || !contentRootElement.hasAttribute('reftest-zoom'))
+ return;
+ docShell.browsingContext.fullZoom =
+ contentRootElement.getAttribute('reftest-zoom');
+}
+
+function resetZoomAndTextZoom() {
+ docShell.browsingContext.fullZoom = 1.0;
+ docShell.browsingContext.textZoom = 1.0;
+}
+
+function doPrintMode(contentRootElement) {
+ // use getAttribute because className works differently in HTML and SVG
+ if (contentRootElement &&
+ contentRootElement.hasAttribute('class')) {
+ var classList = contentRootElement.getAttribute('class').split(/\s+/);
+ if (classList.includes("reftest-print")) {
+ SendException("reftest-print is obsolete, use reftest-paged instead");
+ return;
+ }
+ return classList.includes("reftest-paged");
+ }
+}
+
+function setupPrintMode(contentRootElement) {
+ var PSSVC =
+ Cc[PRINTSETTINGS_CONTRACTID].getService(Ci.nsIPrintSettingsService);
+ var ps = PSSVC.createNewPrintSettings();
+ ps.paperWidth = 5;
+ ps.paperHeight = 3;
+
+ // Override any os-specific unwriteable margins
+ ps.unwriteableMarginTop = 0;
+ ps.unwriteableMarginLeft = 0;
+ ps.unwriteableMarginBottom = 0;
+ ps.unwriteableMarginRight = 0;
+
+ ps.headerStrLeft = "";
+ ps.headerStrCenter = "";
+ ps.headerStrRight = "";
+ ps.footerStrLeft = "";
+ ps.footerStrCenter = "";
+ ps.footerStrRight = "";
+
+ const printBackgrounds = (() => {
+ const attr = contentRootElement.getAttribute("reftest-paged-backgrounds");
+ return !attr || attr != "false";
+ })();
+ ps.printBGColors = printBackgrounds;
+ ps.printBGImages = printBackgrounds;
+
+ docShell.contentViewer.setPageModeForTesting(/* aPageMode */ true, ps);
+}
+
+// Message the parent process to ask it to print the current page to a PDF file.
+function printToPdf() {
+ let currentDoc = content.document;
+ let isPrintSelection = false;
+ let printRange = '';
+
+ if (currentDoc) {
+ let contentRootElement = currentDoc.documentElement;
+ printRange = contentRootElement.getAttribute("reftest-print-range") || '';
+ }
+
+ if (printRange) {
+ if (printRange === 'selection') {
+ isPrintSelection = true;
+ } else if (!printRange.split(',').every(range => /^[1-9]\d*-[1-9]\d*$/.test(range))) {
+ SendException("invalid value for reftest-print-range");
+ return;
+ }
+ }
+
+ SendStartPrint(isPrintSelection, printRange);
+}
+
+function attrOrDefault(element, attr, def) {
+ return element.hasAttribute(attr) ? Number(element.getAttribute(attr)) : def;
+}
+
+function setupViewport(contentRootElement) {
+ if (!contentRootElement) {
+ return;
+ }
+
+ var sw = attrOrDefault(contentRootElement, "reftest-scrollport-w", 0);
+ var sh = attrOrDefault(contentRootElement, "reftest-scrollport-h", 0);
+ if (sw !== 0 || sh !== 0) {
+ LogInfo("Setting viewport to <w=" + sw + ", h=" + sh + ">");
+ windowUtils().setVisualViewportSize(sw, sh);
+ }
+
+ var res = attrOrDefault(contentRootElement, "reftest-resolution", 1);
+ if (res !== 1) {
+ LogInfo("Setting resolution to " + res);
+ windowUtils().setResolutionAndScaleTo(res);
+ }
+
+ // XXX support viewconfig when needed
+}
+
+
+function setupDisplayport(contentRootElement) {
+ let promise = content.windowGlobalChild.getActor("ReftestFission").SetupDisplayportRoot();
+ return promise.then(function(result) {
+ for (let errorString of result.errorStrings) {
+ LogError(errorString);
+ }
+ for (let infoString of result.infoStrings) {
+ LogInfo(infoString);
+ }
+ },
+ function(reason) {
+ LogError("SetupDisplayportRoot returned promise rejected: " + reason);
+ });
+}
+
+// Returns whether any offsets were updated
+function setupAsyncScrollOffsets(options) {
+ let currentDoc = content.document;
+ let contentRootElement = currentDoc ? currentDoc.documentElement : null;
+
+ if (!contentRootElement || !contentRootElement.hasAttribute("reftest-async-scroll")) {
+ return Promise.resolve(false);
+ }
+
+ let allowFailure = options.allowFailure;
+ let promise = content.windowGlobalChild.getActor("ReftestFission").sendQuery("SetupAsyncScrollOffsets", {allowFailure});
+ return promise.then(function(result) {
+ for (let errorString of result.errorStrings) {
+ LogError(errorString);
+ }
+ for (let infoString of result.infoStrings) {
+ LogInfo(infoString);
+ }
+ return result.updatedAny;
+ },
+ function(reason) {
+ LogError("SetupAsyncScrollOffsets SendQuery to parent promise rejected: " + reason);
+ return false;
+ });
+}
+
+function setupAsyncZoom(options) {
+ var currentDoc = content.document;
+ var contentRootElement = currentDoc ? currentDoc.documentElement : null;
+
+ if (!contentRootElement || !contentRootElement.hasAttribute('reftest-async-zoom'))
+ return false;
+
+ var zoom = attrOrDefault(contentRootElement, "reftest-async-zoom", 1);
+ if (zoom != 1) {
+ try {
+ windowUtils().setAsyncZoom(contentRootElement, zoom);
+ return true;
+ } catch (e) {
+ if (!options.allowFailure) {
+ throw e;
+ }
+ }
+ }
+ return false;
+}
+
+
+function resetDisplayportAndViewport() {
+ // XXX currently the displayport configuration lives on the
+ // presshell and so is "reset" on nav when we get a new presshell.
+}
+
+function shouldWaitForPendingPaints() {
+ // if gHaveCanvasSnapshot is false, we're not taking snapshots so
+ // there is no need to wait for pending paints to be flushed.
+ return gHaveCanvasSnapshot && windowUtils().isMozAfterPaintPending;
+}
+
+function shouldWaitForReftestWaitRemoval(contentRootElement) {
+ // use getAttribute because className works differently in HTML and SVG
+ return contentRootElement &&
+ contentRootElement.hasAttribute('class') &&
+ contentRootElement.getAttribute('class').split(/\s+/)
+ .includes("reftest-wait");
+}
+
+function shouldSnapshotWholePage(contentRootElement) {
+ // use getAttribute because className works differently in HTML and SVG
+ return contentRootElement &&
+ contentRootElement.hasAttribute('class') &&
+ contentRootElement.getAttribute('class').split(/\s+/)
+ .includes("reftest-snapshot-all");
+}
+
+function shouldNotFlush(contentRootElement) {
+ // use getAttribute because className works differently in HTML and SVG
+ return contentRootElement &&
+ contentRootElement.hasAttribute('class') &&
+ contentRootElement.getAttribute('class').split(/\s+/)
+ .includes("reftest-no-flush");
+}
+
+function getNoPaintElements(contentRootElement) {
+ return contentRootElement.getElementsByClassName('reftest-no-paint');
+}
+function getNoDisplayListElements(contentRootElement) {
+ return contentRootElement.getElementsByClassName('reftest-no-display-list');
+}
+function getDisplayListElements(contentRootElement) {
+ return contentRootElement.getElementsByClassName('reftest-display-list');
+}
+
+function getOpaqueLayerElements(contentRootElement) {
+ return contentRootElement.getElementsByClassName('reftest-opaque-layer');
+}
+
+function getAssignedLayerMap(contentRootElement) {
+ var layerNameToElementsMap = {};
+ var elements = contentRootElement.querySelectorAll('[reftest-assigned-layer]');
+ for (var i = 0; i < elements.length; ++i) {
+ var element = elements[i];
+ var layerName = element.getAttribute('reftest-assigned-layer');
+ if (!(layerName in layerNameToElementsMap)) {
+ layerNameToElementsMap[layerName] = [];
+ }
+ layerNameToElementsMap[layerName].push(element);
+ }
+ return layerNameToElementsMap;
+}
+
+const FlushMode = {
+ ALL: 0,
+ IGNORE_THROTTLED_ANIMATIONS: 1
+};
+
+// Initial state. When the document has loaded and all MozAfterPaint events and
+// all explicit paint waits are flushed, we can fire the MozReftestInvalidate
+// event and move to the next state.
+const STATE_WAITING_TO_FIRE_INVALIDATE_EVENT = 0;
+// When reftest-wait has been removed from the root element, we can move to the
+// next state.
+const STATE_WAITING_FOR_REFTEST_WAIT_REMOVAL = 1;
+// When spell checking is done on all spell-checked elements, we can move to the
+// next state.
+const STATE_WAITING_FOR_SPELL_CHECKS = 2;
+// When any pending compositor-side repaint requests have been flushed, we can
+// move to the next state.
+const STATE_WAITING_FOR_APZ_FLUSH = 3;
+// When all MozAfterPaint events and all explicit paint waits are flushed, we're
+// done and can move to the COMPLETED state.
+const STATE_WAITING_TO_FINISH = 4;
+const STATE_COMPLETED = 5;
+
+async function FlushRendering(aFlushMode) {
+ let browsingContext = content.docShell.browsingContext;
+ let ignoreThrottledAnimations = (aFlushMode === FlushMode.IGNORE_THROTTLED_ANIMATIONS);
+ // Ensure the refresh driver ticks at least once, this ensures some
+ // preference changes take effect.
+ let needsAnimationFrame = IsSnapshottableTestType();
+ try {
+ let result = await content.windowGlobalChild.getActor("ReftestFission").sendQuery("FlushRendering", {browsingContext, ignoreThrottledAnimations, needsAnimationFrame});
+ for (let errorString of result.errorStrings) {
+ LogError(errorString);
+ }
+ for (let warningString of result.warningStrings) {
+ LogWarning(warningString);
+ }
+ for (let infoString of result.infoStrings) {
+ LogInfo(infoString);
+ }
+ } catch (reason) {
+ // We expect actors to go away causing sendQuery's to fail, so
+ // just note it.
+ LogInfo("FlushRendering sendQuery to parent rejected: " + reason);
+ }
+}
+
+function WaitForTestEnd(contentRootElement, inPrintMode, spellCheckedElements, forURL) {
+ // WaitForTestEnd works via the MakeProgress function below. It is responsible for
+ // moving through the states listed above and calling FlushRendering. We also listen
+ // for a number of events, the most important of which is the AfterPaintListener,
+ // which is responsible for updating the canvas after paints. In a fission world
+ // FlushRendering and updating the canvas must necessarily be async operations.
+ // During these async operations we want to wait for them to finish and we don't
+ // want to try to do anything else (what would we even want to do while only some of
+ // the processes involved have flushed layout or updated their layer trees?). So
+ // we call OperationInProgress whenever we are about to go back to the event loop
+ // during one of these calls, and OperationCompleted when it finishes. This prevents
+ // anything else from running while we wait and getting us into a confused state. We
+ // then record anything that happens while we are waiting to make sure that the
+ // right actions are triggered. The possible actions are basically calling
+ // MakeProgress from a setTimeout, and updating the canvas for an after paint event.
+ // The after paint listener just stashes the rects and we update them after a
+ // completed MakeProgress call. This is handled by
+ // HandlePendingTasksAfterMakeProgress, which also waits for any pending after paint
+ // events. The general sequence of events is:
+ // - MakeProgress
+ // - HandlePendingTasksAfterMakeProgress
+ // - wait for after paint event if one is pending
+ // - update canvas for after paint events we have received
+ // - MakeProgress
+ // etc
+
+ function CheckForLivenessOfContentRootElement() {
+ if (contentRootElement && Cu.isDeadWrapper(contentRootElement)) {
+ contentRootElement = null;
+ }
+ }
+
+ var setTimeoutCallMakeProgressWhenComplete = false;
+
+ var operationInProgress = false;
+ function OperationInProgress() {
+ if (operationInProgress != false) {
+ LogWarning("Nesting atomic operations?");
+ }
+ operationInProgress = true;
+ }
+ function OperationCompleted() {
+ if (operationInProgress != true) {
+ LogWarning("Mismatched OperationInProgress/OperationCompleted calls?");
+ }
+ operationInProgress = false;
+ if (setTimeoutCallMakeProgressWhenComplete) {
+ setTimeoutCallMakeProgressWhenComplete = false;
+ setTimeout(CallMakeProgress, 0);
+ }
+ }
+ function AssertNoOperationInProgress() {
+ if (operationInProgress) {
+ LogWarning("AssertNoOperationInProgress but operationInProgress");
+ }
+ }
+
+ var updateCanvasPending = false;
+ var updateCanvasRects = [];
+
+ var stopAfterPaintReceived = false;
+ var currentDoc = content.document;
+ var state = STATE_WAITING_TO_FIRE_INVALIDATE_EVENT;
+
+ var setTimeoutMakeProgressPending = false;
+
+ function CallSetTimeoutMakeProgress() {
+ if (setTimeoutMakeProgressPending) {
+ return;
+ }
+ setTimeoutMakeProgressPending = true;
+ setTimeout(CallMakeProgress, 0);
+ }
+
+ // This should only ever be called from a timeout.
+ function CallMakeProgress() {
+ if (operationInProgress) {
+ setTimeoutCallMakeProgressWhenComplete = true;
+ return;
+ }
+ setTimeoutMakeProgressPending = false;
+ MakeProgress();
+ }
+
+ var waitingForAnAfterPaint = false;
+
+ // Updates the canvas if there are pending updates for it. Checks if we
+ // need to call MakeProgress.
+ function HandlePendingTasksAfterMakeProgress() {
+ AssertNoOperationInProgress();
+
+ if ((state == STATE_WAITING_TO_FIRE_INVALIDATE_EVENT || state == STATE_WAITING_TO_FINISH) &&
+ shouldWaitForPendingPaints()) {
+ LogInfo("HandlePendingTasksAfterMakeProgress waiting for a MozAfterPaint");
+ // We are in a state where we wait for MozAfterPaint to clear and a
+ // MozAfterPaint event is pending, give it a chance to fire, but don't
+ // let anything else run.
+ waitingForAnAfterPaint = true;
+ OperationInProgress();
+ return;
+ }
+
+ if (updateCanvasPending) {
+ LogInfo("HandlePendingTasksAfterMakeProgress updating canvas");
+ updateCanvasPending = false;
+ let rects = updateCanvasRects;
+ updateCanvasRects = [];
+ OperationInProgress();
+ CheckForLivenessOfContentRootElement();
+ let promise = SendUpdateCanvasForEvent(forURL, rects, contentRootElement);
+ promise.then(function () {
+ OperationCompleted();
+ // After paint events are fired immediately after a paint (one
+ // of the things that can call us). Don't confuse ourselves by
+ // firing synchronously if we triggered the paint ourselves.
+ CallSetTimeoutMakeProgress();
+ });
+ }
+ }
+
+ // true if rectA contains rectB
+ function Contains(rectA, rectB) {
+ return (rectA.left <= rectB.left && rectB.right <= rectA.right && rectA.top <= rectB.top && rectB.bottom <= rectA.bottom);
+ }
+ // true if some rect in rectList contains rect
+ function ContainedIn(rectList, rect) {
+ for (let i = 0; i < rectList.length; ++i) {
+ if (Contains(rectList[i], rect)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ function AfterPaintListener(event) {
+ LogInfo("AfterPaintListener in " + event.target.document.location.href);
+ if (event.target.document != currentDoc) {
+ // ignore paint events for subframes or old documents in the window.
+ // Invalidation in subframes will cause invalidation in the toplevel document anyway.
+ return;
+ }
+
+ updateCanvasPending = true;
+ for (let r of event.clientRects) {
+ if (ContainedIn(updateCanvasRects, r)) {
+ continue;
+ }
+
+ // Copy the rect; it's content and we are chrome, which means if the
+ // document goes away (and it can in some crashtests) our reference
+ // to it will be turned into a dead wrapper that we can't acccess.
+ updateCanvasRects.push({ left: r.left, top: r.top, right: r.right, bottom: r.bottom });
+ }
+
+ if (waitingForAnAfterPaint) {
+ waitingForAnAfterPaint = false;
+ OperationCompleted();
+ }
+
+ if (!operationInProgress) {
+ HandlePendingTasksAfterMakeProgress();
+ }
+ // Otherwise we know that eventually after the operation finishes we
+ // will get a MakeProgress and/or HandlePendingTasksAfterMakeProgress
+ // call, so we don't need to do anything.
+ }
+
+ function FromChildAfterPaintListener(event) {
+ LogInfo("FromChildAfterPaintListener from " + event.detail.originalTargetUri);
+
+ updateCanvasPending = true;
+ for (let r of event.detail.rects) {
+ if (ContainedIn(updateCanvasRects, r)) {
+ continue;
+ }
+
+ // Copy the rect; it's content and we are chrome, which means if the
+ // document goes away (and it can in some crashtests) our reference
+ // to it will be turned into a dead wrapper that we can't acccess.
+ updateCanvasRects.push({ left: r.left, top: r.top, right: r.right, bottom: r.bottom });
+ }
+
+ if (!operationInProgress) {
+ HandlePendingTasksAfterMakeProgress();
+ }
+ // Otherwise we know that eventually after the operation finishes we
+ // will get a MakeProgress and/or HandlePendingTasksAfterMakeProgress
+ // call, so we don't need to do anything.
+ }
+
+ function AttrModifiedListener() {
+ LogInfo("AttrModifiedListener fired");
+ // Wait for the next return-to-event-loop before continuing --- for
+ // example, the attribute may have been modified in an subdocument's
+ // load event handler, in which case we need load event processing
+ // to complete and unsuppress painting before we check isMozAfterPaintPending.
+ CallSetTimeoutMakeProgress();
+ }
+
+ function RemoveListeners() {
+ // OK, we can end the test now.
+ removeEventListener("MozAfterPaint", AfterPaintListener, false);
+ removeEventListener("Reftest:MozAfterPaintFromChild", FromChildAfterPaintListener, false);
+ CheckForLivenessOfContentRootElement();
+ if (contentRootElement) {
+ contentRootElement.removeEventListener("DOMAttrModified", AttrModifiedListener);
+ }
+ gTimeoutHook = null;
+ // Make sure we're in the COMPLETED state just in case
+ // (this may be called via the test-timeout hook)
+ state = STATE_COMPLETED;
+ }
+
+ // Everything that could cause shouldWaitForXXX() to
+ // change from returning true to returning false is monitored via some kind
+ // of event listener which eventually calls this function.
+ function MakeProgress() {
+ if (state >= STATE_COMPLETED) {
+ LogInfo("MakeProgress: STATE_COMPLETED");
+ return;
+ }
+
+ LogInfo("MakeProgress");
+
+ // We don't need to flush styles any more when we are in the state
+ // after reftest-wait has removed.
+ OperationInProgress();
+ let promise = Promise.resolve(undefined);
+ if (state != STATE_WAITING_TO_FINISH) {
+ // If we are waiting for the MozReftestInvalidate event we don't want
+ // to flush throttled animations. Flushing throttled animations can
+ // continue to cause new MozAfterPaint events even when all the
+ // rendering we're concerned about should have ceased. Since
+ // MozReftestInvalidate won't be sent until we finish waiting for all
+ // MozAfterPaint events, we should avoid flushing throttled animations
+ // here or else we'll never leave this state.
+ flushMode = (state === STATE_WAITING_TO_FIRE_INVALIDATE_EVENT)
+ ? FlushMode.IGNORE_THROTTLED_ANIMATIONS
+ : FlushMode.ALL;
+ promise = FlushRendering(flushMode);
+ }
+ promise.then(function () {
+ OperationCompleted();
+ MakeProgress2();
+ // If there is an operation in progress then we know there will be
+ // a MakeProgress call is will happen after it finishes.
+ if (!operationInProgress) {
+ HandlePendingTasksAfterMakeProgress();
+ }
+ });
+ }
+
+ function MakeProgress2() {
+ switch (state) {
+ case STATE_WAITING_TO_FIRE_INVALIDATE_EVENT: {
+ LogInfo("MakeProgress: STATE_WAITING_TO_FIRE_INVALIDATE_EVENT");
+ if (shouldWaitForPendingPaints() || updateCanvasPending) {
+ gFailureReason = "timed out waiting for pending paint count to reach zero";
+ if (shouldWaitForPendingPaints()) {
+ gFailureReason += " (waiting for MozAfterPaint)";
+ LogInfo("MakeProgress: waiting for MozAfterPaint");
+ }
+ if (updateCanvasPending) {
+ gFailureReason += " (waiting for updateCanvasPending)";
+ LogInfo("MakeProgress: waiting for updateCanvasPending");
+ }
+ return;
+ }
+
+ state = STATE_WAITING_FOR_REFTEST_WAIT_REMOVAL;
+ CheckForLivenessOfContentRootElement();
+ var hasReftestWait = shouldWaitForReftestWaitRemoval(contentRootElement);
+ // Notify the test document that now is a good time to test some invalidation
+ LogInfo("MakeProgress: dispatching MozReftestInvalidate");
+ if (contentRootElement) {
+ var elements = getNoPaintElements(contentRootElement);
+ for (var i = 0; i < elements.length; ++i) {
+ windowUtils().checkAndClearPaintedState(elements[i]);
+ }
+ elements = getNoDisplayListElements(contentRootElement);
+ for (var i = 0; i < elements.length; ++i) {
+ windowUtils().checkAndClearDisplayListState(elements[i]);
+ }
+ elements = getDisplayListElements(contentRootElement);
+ for (var i = 0; i < elements.length; ++i) {
+ windowUtils().checkAndClearDisplayListState(elements[i]);
+ }
+ var notification = content.document.createEvent("Events");
+ notification.initEvent("MozReftestInvalidate", true, false);
+ contentRootElement.dispatchEvent(notification);
+ }
+
+ if (!inPrintMode && doPrintMode(contentRootElement)) {
+ LogInfo("MakeProgress: setting up print mode");
+ setupPrintMode(contentRootElement);
+ }
+
+ if (hasReftestWait && !shouldWaitForReftestWaitRemoval(contentRootElement)) {
+ // MozReftestInvalidate handler removed reftest-wait.
+ // We expect something to have been invalidated...
+ OperationInProgress();
+ let promise = FlushRendering(FlushMode.ALL);
+ promise.then(function () {
+ OperationCompleted();
+ if (!updateCanvasPending && !shouldWaitForPendingPaints()) {
+ LogWarning("MozInvalidateEvent didn't invalidate");
+ }
+ MakeProgress();
+ });
+ return;
+ }
+ // Try next state
+ MakeProgress();
+ return;
+ }
+
+ case STATE_WAITING_FOR_REFTEST_WAIT_REMOVAL:
+ LogInfo("MakeProgress: STATE_WAITING_FOR_REFTEST_WAIT_REMOVAL");
+ CheckForLivenessOfContentRootElement();
+ if (shouldWaitForReftestWaitRemoval(contentRootElement)) {
+ gFailureReason = "timed out waiting for reftest-wait to be removed";
+ LogInfo("MakeProgress: waiting for reftest-wait to be removed");
+ return;
+ }
+
+ if (shouldNotFlush(contentRootElement)) {
+ // If reftest-no-flush is specified, we need to set
+ // updateCanvasPending explicitly to take the latest snapshot
+ // since animation changes on the compositor thread don't invoke
+ // any MozAfterPaint events at all.
+ // NOTE: We don't add any rects to updateCanvasRects here since
+ // SendUpdateCanvasForEvent() will handle this case properly
+ // without any rects.
+ updateCanvasPending = true;
+ }
+ // Try next state
+ state = STATE_WAITING_FOR_SPELL_CHECKS;
+ MakeProgress();
+ return;
+
+ case STATE_WAITING_FOR_SPELL_CHECKS:
+ LogInfo("MakeProgress: STATE_WAITING_FOR_SPELL_CHECKS");
+ if (numPendingSpellChecks) {
+ gFailureReason = "timed out waiting for spell checks to end";
+ LogInfo("MakeProgress: waiting for spell checks to end");
+ return;
+ }
+
+ state = STATE_WAITING_FOR_APZ_FLUSH;
+ LogInfo("MakeProgress: STATE_WAITING_FOR_APZ_FLUSH");
+ gFailureReason = "timed out waiting for APZ flush to complete";
+
+ var os = Cc[NS_OBSERVER_SERVICE_CONTRACTID].getService(Ci.nsIObserverService);
+ var flushWaiter = function(aSubject, aTopic, aData) {
+ if (aTopic) LogInfo("MakeProgress: apz-repaints-flushed fired");
+ os.removeObserver(flushWaiter, "apz-repaints-flushed");
+ state = STATE_WAITING_TO_FINISH;
+ if (operationInProgress) {
+ CallSetTimeoutMakeProgress();
+ } else {
+ MakeProgress();
+ }
+ };
+ os.addObserver(flushWaiter, "apz-repaints-flushed");
+
+ var willSnapshot = IsSnapshottableTestType();
+ CheckForLivenessOfContentRootElement();
+ var noFlush = !shouldNotFlush(contentRootElement);
+ if (noFlush && willSnapshot && windowUtils().flushApzRepaints()) {
+ LogInfo("MakeProgress: done requesting APZ flush");
+ } else {
+ LogInfo("MakeProgress: APZ flush not required");
+ flushWaiter(null, null, null);
+ }
+ return;
+
+ case STATE_WAITING_FOR_APZ_FLUSH:
+ LogInfo("MakeProgress: STATE_WAITING_FOR_APZ_FLUSH");
+ // Nothing to do here; once we get the apz-repaints-flushed event
+ // we will go to STATE_WAITING_TO_FINISH
+ return;
+
+ case STATE_WAITING_TO_FINISH:
+ LogInfo("MakeProgress: STATE_WAITING_TO_FINISH");
+ if (shouldWaitForPendingPaints() || updateCanvasPending) {
+ gFailureReason = "timed out waiting for pending paint count to " +
+ "reach zero (after reftest-wait removed and switch to print mode)";
+ if (shouldWaitForPendingPaints()) {
+ gFailureReason += " (waiting for MozAfterPaint)";
+ LogInfo("MakeProgress: waiting for MozAfterPaint");
+ }
+ if (updateCanvasPending) {
+ gFailureReason += " (waiting for updateCanvasPending)";
+ LogInfo("MakeProgress: waiting for updateCanvasPending");
+ }
+ return;
+ }
+ CheckForLivenessOfContentRootElement();
+ if (contentRootElement) {
+ var elements = getNoPaintElements(contentRootElement);
+ for (var i = 0; i < elements.length; ++i) {
+ if (windowUtils().checkAndClearPaintedState(elements[i])) {
+ SendFailedNoPaint();
+ }
+ }
+ // We only support retained display lists in the content process
+ // right now, so don't fail reftest-no-display-list tests when
+ // we don't have e10s.
+ if (gBrowserIsRemote) {
+ elements = getNoDisplayListElements(contentRootElement);
+ for (var i = 0; i < elements.length; ++i) {
+ if (windowUtils().checkAndClearDisplayListState(elements[i])) {
+ SendFailedNoDisplayList();
+ }
+ }
+ elements = getDisplayListElements(contentRootElement);
+ for (var i = 0; i < elements.length; ++i) {
+ if (!windowUtils().checkAndClearDisplayListState(elements[i])) {
+ SendFailedDisplayList();
+ }
+ }
+ }
+ }
+
+ if (!IsSnapshottableTestType()) {
+ // If we're not snapshotting the test, at least do a sync round-trip
+ // to the compositor to ensure that all the rendering messages
+ // related to this test get processed. Otherwise problems triggered
+ // by this test may only manifest as failures in a later test.
+ LogInfo("MakeProgress: Doing sync flush to compositor");
+ gFailureReason = "timed out while waiting for sync compositor flush"
+ windowUtils().syncFlushCompositor();
+ }
+
+ LogInfo("MakeProgress: Completed");
+ state = STATE_COMPLETED;
+ gFailureReason = "timed out while taking snapshot (bug in harness?)";
+ RemoveListeners();
+ CheckForLivenessOfContentRootElement();
+ CheckForProcessCrashExpectation(contentRootElement);
+ setTimeout(RecordResult, 0, forURL);
+ return;
+ }
+ }
+
+ LogInfo("WaitForTestEnd: Adding listeners");
+ addEventListener("MozAfterPaint", AfterPaintListener, false);
+ addEventListener("Reftest:MozAfterPaintFromChild", FromChildAfterPaintListener, false);
+
+ // If contentRootElement is null then shouldWaitForReftestWaitRemoval will
+ // always return false so we don't need a listener anyway
+ CheckForLivenessOfContentRootElement();
+ if (contentRootElement) {
+ contentRootElement.addEventListener("DOMAttrModified", AttrModifiedListener);
+ }
+ gTimeoutHook = RemoveListeners;
+
+ // Listen for spell checks on spell-checked elements.
+ var numPendingSpellChecks = spellCheckedElements.length;
+ function decNumPendingSpellChecks() {
+ --numPendingSpellChecks;
+ if (operationInProgress) {
+ CallSetTimeoutMakeProgress();
+ } else {
+ MakeProgress();
+ }
+ }
+ for (let editable of spellCheckedElements) {
+ try {
+ onSpellCheck(editable, decNumPendingSpellChecks);
+ } catch (err) {
+ // The element may not have an editor, so ignore it.
+ setTimeout(decNumPendingSpellChecks, 0);
+ }
+ }
+
+ // Take a full snapshot now that all our listeners are set up. This
+ // ensures it's impossible for us to miss updates between taking the snapshot
+ // and adding our listeners.
+ OperationInProgress();
+ let promise = SendInitCanvasWithSnapshot(forURL);
+ promise.then(function () {
+ OperationCompleted();
+ MakeProgress();
+ });
+}
+
+async function OnDocumentLoad(uri)
+{
+ if (gClearingForAssertionCheck) {
+ if (uri == BLANK_URL_FOR_CLEARING) {
+ DoAssertionCheck();
+ return;
+ }
+
+ // It's likely the previous test document reloads itself and causes the
+ // attempt of loading blank page fails. In this case we should retry
+ // loading the blank page.
+ LogInfo("Retry loading a blank page");
+ setTimeout(LoadURI, 0, BLANK_URL_FOR_CLEARING);
+ return;
+ }
+
+ if (uri != gCurrentURL) {
+ LogInfo("OnDocumentLoad fired for previous document");
+ // Ignore load events for previous documents.
+ return;
+ }
+
+ var currentDoc = content && content.document;
+
+ // Collect all editable, spell-checked elements. It may be the case that
+ // not all the elements that match this selector will be spell checked: for
+ // example, a textarea without a spellcheck attribute may have a parent with
+ // spellcheck=false, or script may set spellcheck=false on an element whose
+ // markup sets it to true. But that's OK since onSpellCheck detects the
+ // absence of spell checking, too.
+ var querySelector =
+ '*[class~="spell-checked"],' +
+ 'textarea:not([spellcheck="false"]),' +
+ 'input[spellcheck]:-moz-any([spellcheck=""],[spellcheck="true"]),' +
+ '*[contenteditable]:-moz-any([contenteditable=""],[contenteditable="true"])';
+ var spellCheckedElements = currentDoc ? currentDoc.querySelectorAll(querySelector) : [];
+
+ var contentRootElement = currentDoc ? currentDoc.documentElement : null;
+ currentDoc = null;
+ setupFullZoom(contentRootElement);
+ setupTextZoom(contentRootElement);
+ setupViewport(contentRootElement);
+ await setupDisplayport(contentRootElement);
+ var inPrintMode = false;
+
+ async function AfterOnLoadScripts() {
+ // Regrab the root element, because the document may have changed.
+ var contentRootElement =
+ content.document ? content.document.documentElement : null;
+
+ // Flush the document in case it got modified in a load event handler.
+ await FlushRendering(FlushMode.ALL);
+
+ // Take a snapshot now.
+ let painted = await SendInitCanvasWithSnapshot(uri);
+
+ if (contentRootElement && Cu.isDeadWrapper(contentRootElement)) {
+ contentRootElement = null;
+ }
+
+ if (!inPrintMode && doPrintMode(contentRootElement) ||
+ // If we didn't force a paint above, in
+ // InitCurrentCanvasWithSnapshot, so we should wait for a
+ // paint before we consider them done.
+ !painted) {
+ LogInfo("AfterOnLoadScripts belatedly entering WaitForTestEnd");
+ // Go into reftest-wait mode belatedly.
+ WaitForTestEnd(contentRootElement, inPrintMode, [], uri);
+ } else {
+ CheckForProcessCrashExpectation(contentRootElement);
+ RecordResult(uri);
+ }
+ }
+
+ if (shouldWaitForReftestWaitRemoval(contentRootElement) ||
+ spellCheckedElements.length) {
+ // Go into reftest-wait mode immediately after painting has been
+ // unsuppressed, after the onload event has finished dispatching.
+ gFailureReason = "timed out waiting for test to complete (trying to get into WaitForTestEnd)";
+ LogInfo("OnDocumentLoad triggering WaitForTestEnd");
+ setTimeout(function () { WaitForTestEnd(contentRootElement, inPrintMode, spellCheckedElements, uri); }, 0);
+ } else {
+ if (doPrintMode(contentRootElement)) {
+ LogInfo("OnDocumentLoad setting up print mode");
+ setupPrintMode(contentRootElement);
+ inPrintMode = true;
+ }
+
+ // Since we can't use a bubbling-phase load listener from chrome,
+ // this is a capturing phase listener. So do setTimeout twice, the
+ // first to get us after the onload has fired in the content, and
+ // the second to get us after any setTimeout(foo, 0) in the content.
+ gFailureReason = "timed out waiting for test to complete (waiting for onload scripts to complete)";
+ LogInfo("OnDocumentLoad triggering AfterOnLoadScripts");
+ setTimeout(function () { setTimeout(AfterOnLoadScripts, 0); }, 0);
+ }
+}
+
+function CheckForProcessCrashExpectation(contentRootElement)
+{
+ if (contentRootElement &&
+ contentRootElement.hasAttribute('class') &&
+ contentRootElement.getAttribute('class').split(/\s+/)
+ .includes("reftest-expect-process-crash")) {
+ SendExpectProcessCrash();
+ }
+}
+
+async function RecordResult(forURL)
+{
+ if (forURL != gCurrentURL) {
+ LogInfo("RecordResult fired for previous document");
+ return;
+ }
+
+ if (gCurrentURLRecordResults > 0) {
+ LogInfo("RecordResult fired extra times");
+ FinishTestItem();
+ return;
+ }
+ gCurrentURLRecordResults++;
+
+ LogInfo("RecordResult fired");
+
+ var currentTestRunTime = Date.now() - gCurrentTestStartTime;
+
+ clearTimeout(gFailureTimeout);
+ gFailureReason = null;
+ gFailureTimeout = null;
+ gCurrentURL = null;
+ gCurrentURLTargetType = undefined;
+
+ if (gCurrentTestType == TYPE_PRINT) {
+ printToPdf();
+ return;
+ }
+ if (gCurrentTestType == TYPE_SCRIPT) {
+ var error = '';
+ var testwindow = content;
+
+ if (testwindow.wrappedJSObject)
+ testwindow = testwindow.wrappedJSObject;
+
+ var testcases;
+ if (!testwindow.getTestCases || typeof testwindow.getTestCases != "function") {
+ // Force an unexpected failure to alert the test author to fix the test.
+ error = "test must provide a function getTestCases(). (SCRIPT)\n";
+ }
+ else if (!(testcases = testwindow.getTestCases())) {
+ // Force an unexpected failure to alert the test author to fix the test.
+ error = "test's getTestCases() must return an Array-like Object. (SCRIPT)\n";
+ }
+ else if (testcases.length == 0) {
+ // This failure may be due to a JavaScript Engine bug causing
+ // early termination of the test. If we do not allow silent
+ // failure, the driver will report an error.
+ }
+
+ var results = [ ];
+ if (!error) {
+ // FIXME/bug 618176: temporary workaround
+ for (var i = 0; i < testcases.length; ++i) {
+ var test = testcases[i];
+ results.push({ passed: test.testPassed(),
+ description: test.testDescription() });
+ }
+ //results = testcases.map(function(test) {
+ // return { passed: test.testPassed(),
+ // description: test.testDescription() };
+ }
+
+ SendScriptResults(currentTestRunTime, error, results);
+ FinishTestItem();
+ return;
+ }
+
+ // Setup async scroll offsets now in case SynchronizeForSnapshot is not
+ // called (due to reftest-no-sync-layers being supplied, or in the single
+ // process case).
+ let changedAsyncScrollZoom = await setupAsyncScrollOffsets({allowFailure:true});
+ if (setupAsyncZoom({allowFailure:true})) {
+ changedAsyncScrollZoom = true;
+ }
+ if (changedAsyncScrollZoom && !gBrowserIsRemote) {
+ sendAsyncMessage("reftest:UpdateWholeCanvasForInvalidation");
+ }
+
+ SendTestDone(currentTestRunTime);
+ FinishTestItem();
+}
+
+function LoadFailed()
+{
+ if (gTimeoutHook) {
+ gTimeoutHook();
+ }
+ gFailureTimeout = null;
+ SendFailedLoad(gFailureReason);
+}
+
+function FinishTestItem()
+{
+ gHaveCanvasSnapshot = false;
+}
+
+function DoAssertionCheck()
+{
+ gClearingForAssertionCheck = false;
+
+ var numAsserts = 0;
+ if (gDebug.isDebugBuild) {
+ var newAssertionCount = gDebug.assertionCount;
+ numAsserts = newAssertionCount - gAssertionCount;
+ gAssertionCount = newAssertionCount;
+ }
+ SendAssertionCount(numAsserts);
+}
+
+function LoadURI(uri)
+{
+ let loadURIOptions = {
+ triggeringPrincipal: Services.scriptSecurityManager.getSystemPrincipal(),
+ };
+ webNavigation().loadURI(Services.io.newURI(uri), loadURIOptions);
+}
+
+function LogError(str)
+{
+ if (gVerbose) {
+ sendSyncMessage("reftest:Log", { type: "error", msg: str });
+ } else {
+ sendAsyncMessage("reftest:Log", { type: "error", msg: str });
+ }
+}
+
+function LogWarning(str)
+{
+ if (gVerbose) {
+ sendSyncMessage("reftest:Log", { type: "warning", msg: str });
+ } else {
+ sendAsyncMessage("reftest:Log", { type: "warning", msg: str });
+ }
+}
+
+function LogInfo(str)
+{
+ if (gVerbose) {
+ sendSyncMessage("reftest:Log", { type: "info", msg: str });
+ } else {
+ sendAsyncMessage("reftest:Log", { type: "info", msg: str });
+ }
+}
+
+function IsSnapshottableTestType()
+{
+ // Script, load-only, and PDF-print tests do not need any snapshotting.
+ return !(gCurrentTestType == TYPE_SCRIPT ||
+ gCurrentTestType == TYPE_LOAD ||
+ gCurrentTestType == TYPE_PRINT);
+}
+
+const SYNC_DEFAULT = 0x0;
+const SYNC_ALLOW_DISABLE = 0x1;
+// Returns a promise that resolve when the snapshot is done.
+function SynchronizeForSnapshot(flags)
+{
+ if (!IsSnapshottableTestType()) {
+ return Promise.resolve(undefined);
+ }
+
+ if (flags & SYNC_ALLOW_DISABLE) {
+ var docElt = content.document.documentElement;
+ if (docElt &&
+ (docElt.hasAttribute("reftest-no-sync-layers") ||
+ shouldNotFlush(docElt))) {
+ LogInfo("Test file chose to skip SynchronizeForSnapshot");
+ return Promise.resolve(undefined);
+ }
+ }
+
+ let browsingContext = content.docShell.browsingContext;
+ let promise = content.windowGlobalChild.getActor("ReftestFission").sendQuery("UpdateLayerTree", {browsingContext});
+ return promise.then(function (result) {
+ for (let errorString of result.errorStrings) {
+ LogError(errorString);
+ }
+ for (let infoString of result.infoStrings) {
+ LogInfo(infoString);
+ }
+
+ // Setup async scroll offsets now, because any scrollable layers should
+ // have had their AsyncPanZoomControllers created.
+ return setupAsyncScrollOffsets({allowFailure:false}).then(function(result) {
+ setupAsyncZoom({allowFailure:false});
+ });
+ }, function(reason) {
+ // We expect actors to go away causing sendQuery's to fail, so
+ // just note it.
+ LogInfo("UpdateLayerTree sendQuery to parent rejected: " + reason);
+
+ // Setup async scroll offsets now, because any scrollable layers should
+ // have had their AsyncPanZoomControllers created.
+ return setupAsyncScrollOffsets({allowFailure:false}).then(function(result) {
+ setupAsyncZoom({allowFailure:false});
+ });
+ });
+}
+
+function RegisterMessageListeners()
+{
+ addMessageListener(
+ "reftest:Clear",
+ function (m) { RecvClear() }
+ );
+ addMessageListener(
+ "reftest:LoadScriptTest",
+ function (m) { RecvLoadScriptTest(m.json.uri, m.json.timeout); }
+ );
+ addMessageListener(
+ "reftest:LoadPrintTest",
+ function (m) { RecvLoadPrintTest(m.json.uri, m.json.timeout); }
+ );
+ addMessageListener(
+ "reftest:LoadTest",
+ function (m) { RecvLoadTest(m.json.type, m.json.uri,
+ m.json.uriTargetType,
+ m.json.timeout); }
+ );
+ addMessageListener(
+ "reftest:ResetRenderingState",
+ function (m) { RecvResetRenderingState(); }
+ );
+ addMessageListener(
+ "reftest:PrintDone",
+ function (m) { RecvPrintDone(m.json.status, m.json.fileName); }
+ );
+ addMessageListener(
+ "reftest:UpdateCanvasWithSnapshotDone",
+ function (m) { RecvUpdateCanvasWithSnapshotDone(m.json.painted); }
+ );
+}
+
+function RecvClear()
+{
+ gClearingForAssertionCheck = true;
+ LoadURI(BLANK_URL_FOR_CLEARING);
+}
+
+function RecvLoadTest(type, uri, uriTargetType, timeout)
+{
+ StartTestURI(type, uri, uriTargetType, timeout);
+}
+
+function RecvLoadScriptTest(uri, timeout)
+{
+ StartTestURI(TYPE_SCRIPT, uri, URL_TARGET_TYPE_TEST, timeout);
+}
+
+function RecvLoadPrintTest(uri, timeout)
+{
+ StartTestURI(TYPE_PRINT, uri, URL_TARGET_TYPE_TEST, timeout);
+}
+
+function RecvResetRenderingState()
+{
+ resetZoomAndTextZoom();
+ resetDisplayportAndViewport();
+}
+
+function RecvPrintDone(status, fileName)
+{
+ const currentTestRunTime = Date.now() - gCurrentTestStartTime;
+ SendPrintResult(currentTestRunTime, status, fileName);
+ FinishTestItem();
+}
+
+function RecvUpdateCanvasWithSnapshotDone(painted)
+{
+ gUpdateCanvasPromiseResolver(painted);
+}
+
+function SendAssertionCount(numAssertions)
+{
+ sendAsyncMessage("reftest:AssertionCount", { count: numAssertions });
+}
+
+function SendContentReady()
+{
+ let gfxInfo = (NS_GFXINFO_CONTRACTID in Cc) && Cc[NS_GFXINFO_CONTRACTID].getService(Ci.nsIGfxInfo);
+
+ let info = {};
+
+ try {
+ info.D2DEnabled = gfxInfo.D2DEnabled;
+ info.DWriteEnabled = gfxInfo.DWriteEnabled;
+ info.EmbeddedInFirefoxReality = gfxInfo.EmbeddedInFirefoxReality;
+ } catch (e) {
+ info.D2DEnabled = false;
+ info.DWriteEnabled = false;
+ info.EmbeddedInFirefoxReality = false;
+ }
+
+ info.AzureCanvasBackend = gfxInfo.AzureCanvasBackend;
+ info.AzureContentBackend = gfxInfo.AzureContentBackend;
+
+ return sendSyncMessage("reftest:ContentReady", { 'gfx': info })[0];
+}
+
+function SendException(what)
+{
+ sendAsyncMessage("reftest:Exception", { what: what });
+}
+
+function SendFailedLoad(why)
+{
+ sendAsyncMessage("reftest:FailedLoad", { why: why });
+}
+
+function SendFailedNoPaint()
+{
+ sendAsyncMessage("reftest:FailedNoPaint");
+}
+
+function SendFailedNoDisplayList()
+{
+ sendAsyncMessage("reftest:FailedNoDisplayList");
+}
+
+function SendFailedDisplayList()
+{
+ sendAsyncMessage("reftest:FailedDisplayList");
+}
+
+function SendFailedOpaqueLayer(why)
+{
+ sendAsyncMessage("reftest:FailedOpaqueLayer", { why: why });
+}
+
+function SendFailedAssignedLayer(why)
+{
+ sendAsyncMessage("reftest:FailedAssignedLayer", { why: why });
+}
+
+// Returns a promise that resolves to a bool that indicates if a snapshot was taken.
+async function SendInitCanvasWithSnapshot(forURL)
+{
+ if (forURL != gCurrentURL) {
+ LogInfo("SendInitCanvasWithSnapshot called for previous document");
+ // Lie and say we painted because it doesn't matter, this is a test we
+ // are already done with that is clearing out. Then AfterOnLoadScripts
+ // should finish quicker if that is who is calling us.
+ return Promise.resolve(true);
+ }
+
+ // If we're in the same process as the top-level XUL window, then
+ // drawing that window will also update our layers, so no
+ // synchronization is needed.
+ //
+ // NB: this is a test-harness optimization only, it must not
+ // affect the validity of the tests.
+ if (gBrowserIsRemote) {
+ await SynchronizeForSnapshot(SYNC_DEFAULT);
+ let promise = new Promise(resolve => { gUpdateCanvasPromiseResolver = resolve; });
+ sendAsyncMessage("reftest:InitCanvasWithSnapshot");
+
+ gHaveCanvasSnapshot = await promise;
+ return gHaveCanvasSnapshot;
+ }
+
+ // For in-process browser, we have to make a synchronous request
+ // here to make the above optimization valid, so that MozWaitPaint
+ // events dispatched (synchronously) during painting are received
+ // before we check the paint-wait counter. For out-of-process
+ // browser though, it doesn't wrt correctness whether this request
+ // is sync or async.
+ let promise = new Promise(resolve => { gUpdateCanvasPromiseResolver = resolve; });
+ sendAsyncMessage("reftest:InitCanvasWithSnapshot");
+
+ gHaveCanvasSnapshot = await promise;
+ return Promise.resolve(gHaveCanvasSnapshot);
+}
+
+function SendScriptResults(runtimeMs, error, results)
+{
+ sendAsyncMessage("reftest:ScriptResults",
+ { runtimeMs: runtimeMs, error: error, results: results });
+}
+
+function SendStartPrint(isPrintSelection, printRange)
+{
+ sendAsyncMessage("reftest:StartPrint", { isPrintSelection, printRange });
+}
+
+function SendPrintResult(runtimeMs, status, fileName)
+{
+ sendAsyncMessage("reftest:PrintResult",
+ { runtimeMs: runtimeMs, status: status, fileName: fileName });
+}
+
+function SendExpectProcessCrash(runtimeMs)
+{
+ sendAsyncMessage("reftest:ExpectProcessCrash");
+}
+
+function SendTestDone(runtimeMs)
+{
+ sendAsyncMessage("reftest:TestDone", { runtimeMs: runtimeMs });
+}
+
+function roundTo(x, fraction)
+{
+ return Math.round(x/fraction)*fraction;
+}
+
+function elementDescription(element)
+{
+ return '<' + element.localName +
+ [].slice.call(element.attributes).map((attr) =>
+ ` ${attr.nodeName}="${attr.value}"`).join('') +
+ '>';
+}
+
+async function SendUpdateCanvasForEvent(forURL, rectList, contentRootElement)
+{
+ if (forURL != gCurrentURL) {
+ LogInfo("SendUpdateCanvasForEvent called for previous document");
+ // This is a test we are already done with that is clearing out.
+ // Don't do anything.
+ return;
+ }
+
+ var win = content;
+ var scale = docShell.browsingContext.fullZoom;
+
+ var rects = [ ];
+ if (shouldSnapshotWholePage(contentRootElement)) {
+ // See comments in SendInitCanvasWithSnapshot() re: the split
+ // logic here.
+ if (!gBrowserIsRemote) {
+ sendSyncMessage("reftest:UpdateWholeCanvasForInvalidation");
+ } else {
+ await SynchronizeForSnapshot(SYNC_ALLOW_DISABLE);
+ let promise = new Promise(resolve => { gUpdateCanvasPromiseResolver = resolve; });
+ sendAsyncMessage("reftest:UpdateWholeCanvasForInvalidation");
+ await promise;
+ }
+ return;
+ }
+
+ var message;
+
+ if (!windowUtils().isMozAfterPaintPending) {
+ // Webrender doesn't have invalidation, and animations on the compositor
+ // don't invoke any MozAfterEvent which means we have no invalidated
+ // rect so we just invalidate the whole screen once we don't have
+ // anymore paints pending. This will force the snapshot.
+
+ LogInfo("Sending update whole canvas for invalidation");
+ message = "reftest:UpdateWholeCanvasForInvalidation";
+ } else {
+ LogInfo("SendUpdateCanvasForEvent with " + rectList.length + " rects");
+ for (var i = 0; i < rectList.length; ++i) {
+ var r = rectList[i];
+ // Set left/top/right/bottom to "device pixel" boundaries
+ var left = Math.floor(roundTo(r.left * scale, 0.001));
+ var top = Math.floor(roundTo(r.top * scale, 0.001));
+ var right = Math.ceil(roundTo(r.right * scale, 0.001));
+ var bottom = Math.ceil(roundTo(r.bottom * scale, 0.001));
+ LogInfo("Rect: " + left + " " + top + " " + right + " " + bottom);
+
+ rects.push({ left: left, top: top, right: right, bottom: bottom });
+ }
+
+ message = "reftest:UpdateCanvasForInvalidation";
+ }
+
+ // See comments in SendInitCanvasWithSnapshot() re: the split
+ // logic here.
+ if (!gBrowserIsRemote) {
+ sendSyncMessage(message, { rects: rects });
+ } else {
+ await SynchronizeForSnapshot(SYNC_ALLOW_DISABLE);
+ let promise = new Promise(resolve => { gUpdateCanvasPromiseResolver = resolve; });
+ sendAsyncMessage(message, { rects: rects });
+ await promise;
+ }
+}
+
+if (content.document.readyState == "complete") {
+ // load event has already fired for content, get started
+ OnInitialLoad();
+} else {
+ addEventListener("load", OnInitialLoad, true);
+}
diff --git a/layout/tools/reftest/reftest-to-html.pl b/layout/tools/reftest/reftest-to-html.pl
new file mode 100755
index 0000000000..3fc2380e9e
--- /dev/null
+++ b/layout/tools/reftest/reftest-to-html.pl
@@ -0,0 +1,118 @@
+#!/usr/bin/perl
+
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+print <<EOD
+<html>
+<head>
+<title>reftest output</title>
+<style type="text/css">
+/* must be in this order */
+.PASS { background-color: green; }
+.FAIL { background-color: red; }
+.XFAIL { background-color: #999300; }
+.WEIRDPASS { background-color: #00FFED; }
+.PASSRANDOM { background-color: #598930; }
+.FAILRANDOM, td.XFAILRANDOM { background-color: #99402A; }
+
+.FAILIMAGES { }
+img { margin: 5px; width: 80px; height: 100px; }
+img.testresult { border: 2px solid red; }
+img.testref { border: 2px solid green; }
+a { color: inherit; }
+.always { display: inline ! important; }
+</style>
+</head>
+<body>
+<p>
+<span class="PASS always"><input type="checkbox" checked="true" onclick="var s = document.styleSheets[0].cssRules[0].style; if (s.display == 'none') s.display = null; else s.display = 'none';">PASS</span>&nbsp;
+<span class="FAIL always"><input type="checkbox" checked="true" onclick="var s = document.styleSheets[0].cssRules[1].style; if (s.display == 'none') s.display = null; else s.display = 'none';">UNEXPECTED FAIL</span>&nbsp;
+<span class="XFAIL always"><input type="checkbox" checked="true" onclick="var s = document.styleSheets[0].cssRules[2].style; if (s.display == 'none') s.display = null; else s.display = 'none';">KNOWN FAIL</span>&nbsp;
+<span class="WEIRDPASS always"><input type="checkbox" checked="true" onclick="var s = document.styleSheets[0].cssRules[3].style; if (s.display == 'none') s.display = null; else s.display = 'none';">UNEXPECTED PASS</span>&nbsp;
+<span class="PASSRANDOM always"><input type="checkbox" checked="true" onclick="var s = document.styleSheets[0].cssRules[4].style; if (s.display == 'none') s.display = null; else s.display = 'none';">PASS (Random)</span>&nbsp;
+<span class="FAILRANDOM always"><input type="checkbox" checked="true" onclick="var s = document.styleSheets[0].cssRules[5].style; if (s.display == 'none') s.display = null; else s.display = 'none';">FAIL (Random)</span>&nbsp;
+</p>
+<table>
+EOD
+;
+
+sub readcleanline {
+ my $l = <>;
+ chomp $l;
+ chop $l if ($l =~ /\r$/);
+ return $l;
+}
+
+sub do_html {
+ my ($l) = @_;
+
+ $l =~ s,(file:[^ ]*),<a href="\1">\1</a>,g;
+ $l =~ s,(data:[^ ]*),<a href="\1">\1</a>,g;
+
+ return $l;
+}
+
+$l = 0;
+
+while (<>) {
+ $l++;
+ next unless /^REFTEST/;
+
+ chomp;
+ chop if /\r$/;
+
+ s/^REFTEST *//;
+
+ my $randomresult = 0;
+ if (/EXPECTED RANDOM/) {
+ s/\(EXPECTED RANDOM\)//;
+ $randomresult = 1;
+ }
+
+ if (/^TEST-PASS \| (.*)$/) {
+ my $class = $randomresult ? "PASSRANDOM" : "PASS";
+ print '<tr><td class="' . $class . '">' . do_html($1) . "</td></tr>\n";
+ } elsif (/^TEST-UNEXPECTED-(....) \| (.*)$/) {
+ if ($randomresult) {
+ die "Error on line $l: UNEXPECTED with test marked random?!";
+ }
+ my $class = ($1 eq "PASS") ? "WEIRDPASS" : "FAIL";
+ print '<tr><td class="' . $class . '">' . do_html($2) . "</td></tr>\n";
+
+ # UNEXPECTED results can be followed by one or two images
+ $testline = &readcleanline;
+
+ print '<tr><td class="FAILIMAGES">';
+
+ if ($testline =~ /REFTEST IMAGE: (data:.*)$/) {
+ print '<a href="' . $1 . '"><img class="testresult" src="' . $1 . '"></a>';
+ } elsif ($testline =~ /REFTEST IMAGE 1 \(TEST\): (data:.*)$/) {
+ $refline = &readcleanline;
+ print '<a href="' . $1 . '"><img class="testresult" src="' . $1 . '"></a>';
+ {
+ die "Error on line $l" unless $refline =~ /REFTEST IMAGE 2 \(REFERENCE\): (data:.*)$/;
+ print '<a href="' . $1 . '"><img class="testref" src="' . $1 . '"></a>';
+ }
+
+ } else {
+ die "Error on line $l";
+ }
+
+ print "</td></tr>\n";
+ } elsif (/^TEST-KNOWN-FAIL \| (.*$)/) {
+ my $class = $randomresult ? "XFAILRANDOM" : "XFAIL";
+ print '<tr><td class="' . $class . '">' . do_html($1) . "</td></tr>\n";
+ } else {
+ print STDERR "Unknown Line: " . $_ . "\n";
+ print "<tr><td><pre>" . $_ . "</pre></td></tr>\n";
+ }
+}
+
+print <<EOD
+</table>
+</body>
+</html>
+EOD
+;
diff --git a/layout/tools/reftest/reftest.jsm b/layout/tools/reftest/reftest.jsm
new file mode 100644
index 0000000000..c304caeb49
--- /dev/null
+++ b/layout/tools/reftest/reftest.jsm
@@ -0,0 +1,2020 @@
+/* -*- indent-tabs-mode: nil; js-indent-level: 4 -*- /
+/* vim: set shiftwidth=4 tabstop=8 autoindent cindent expandtab: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+"use strict";
+
+var EXPORTED_SYMBOLS = [
+ "OnRefTestLoad",
+ "OnRefTestUnload",
+];
+
+const { FileUtils } = ChromeUtils.importESModule(
+ "resource://gre/modules/FileUtils.sys.mjs"
+);
+const {
+ XHTML_NS,
+ XUL_NS,
+
+ IO_SERVICE_CONTRACTID,
+ DEBUG_CONTRACTID,
+ NS_DIRECTORY_SERVICE_CONTRACTID,
+ NS_OBSERVER_SERVICE_CONTRACTID,
+
+ TYPE_REFTEST_EQUAL,
+ TYPE_REFTEST_NOTEQUAL,
+ TYPE_LOAD,
+ TYPE_SCRIPT,
+ TYPE_PRINT,
+
+ URL_TARGET_TYPE_TEST,
+ URL_TARGET_TYPE_REFERENCE,
+
+ EXPECTED_PASS,
+ EXPECTED_FAIL,
+ EXPECTED_RANDOM,
+ EXPECTED_FUZZY,
+
+ PREF_BOOLEAN,
+ PREF_STRING,
+ PREF_INTEGER,
+
+ FOCUS_FILTER_NON_NEEDS_FOCUS_TESTS,
+
+ g,
+} = ChromeUtils.import("resource://reftest/globals.jsm");
+const { HttpServer } = ChromeUtils.import("resource://reftest/httpd.jsm");
+const { ReadTopManifest, CreateUrls } = ChromeUtils.import(
+ "resource://reftest/manifest.jsm"
+);
+const { StructuredLogger } = ChromeUtils.importESModule(
+ "resource://reftest/StructuredLog.sys.mjs"
+);
+const { PerTestCoverageUtils } = ChromeUtils.importESModule(
+ "resource://reftest/PerTestCoverageUtils.sys.mjs"
+);
+const { XPCOMUtils } = ChromeUtils.importESModule(
+ "resource://gre/modules/XPCOMUtils.sys.mjs"
+);
+
+const { E10SUtils } = ChromeUtils.importESModule(
+ "resource://gre/modules/E10SUtils.sys.mjs"
+);
+
+const lazy = {};
+
+XPCOMUtils.defineLazyServiceGetters(lazy, {
+ proxyService: [
+ "@mozilla.org/network/protocol-proxy-service;1",
+ "nsIProtocolProxyService",
+ ],
+});
+
+function HasUnexpectedResult()
+{
+ return g.testResults.Exception > 0 ||
+ g.testResults.FailedLoad > 0 ||
+ g.testResults.UnexpectedFail > 0 ||
+ g.testResults.UnexpectedPass > 0 ||
+ g.testResults.AssertionUnexpected > 0 ||
+ g.testResults.AssertionUnexpectedFixed > 0;
+}
+
+// By default we just log to stdout
+var gDumpFn = function(line) {
+ dump(line);
+ if (g.logFile) {
+ g.logFile.writeString(line);
+ }
+}
+var gDumpRawLog = function(record) {
+ // Dump JSON representation of data on a single line
+ var line = "\n" + JSON.stringify(record) + "\n";
+ dump(line);
+
+ if (g.logFile) {
+ g.logFile.writeString(line);
+ }
+}
+g.logger = new StructuredLogger('reftest', gDumpRawLog);
+var logger = g.logger;
+
+function TestBuffer(str)
+{
+ logger.debug(str);
+ g.testLog.push(str);
+}
+
+function isAndroidDevice() {
+ var xr = Cc["@mozilla.org/xre/app-info;1"].getService(Ci.nsIXULRuntime);
+ // This is the best we can do for now; maybe in the future we'll have
+ // more correct detection of this case.
+ return xr.OS == "Android" &&
+ g.browserIsRemote;
+}
+
+function FlushTestBuffer()
+{
+ // In debug mode, we've dumped all these messages already.
+ if (g.logLevel !== 'debug') {
+ for (var i = 0; i < g.testLog.length; ++i) {
+ logger.info("Saved log: " + g.testLog[i]);
+ }
+ }
+ g.testLog = [];
+}
+
+function LogWidgetLayersFailure()
+{
+ logger.error(
+ "Screen resolution is too low - USE_WIDGET_LAYERS was disabled. " +
+ (g.browserIsRemote ?
+ "Since E10s is enabled, there is no fallback rendering path!" :
+ "The fallback rendering path is not reliably consistent with on-screen rendering."));
+
+ logger.error(
+ "If you cannot increase your screen resolution you can try reducing " +
+ "gecko's pixel scaling by adding something like '--setpref " +
+ "layout.css.devPixelsPerPx=1.0' to your './mach reftest' command " +
+ "(possibly as an alias in ~/.mozbuild/machrc). Note that this is " +
+ "inconsistent with CI testing, and may interfere with HighDPI/" +
+ "reftest-zoom tests.");
+}
+
+function AllocateCanvas()
+{
+ if (g.recycledCanvases.length > 0) {
+ return g.recycledCanvases.shift();
+ }
+
+ var canvas = g.containingWindow.document.createElementNS(XHTML_NS, "canvas");
+ var r = g.browser.getBoundingClientRect();
+ canvas.setAttribute("width", Math.ceil(r.width));
+ canvas.setAttribute("height", Math.ceil(r.height));
+
+ return canvas;
+}
+
+function ReleaseCanvas(canvas)
+{
+ // store a maximum of 2 canvases, if we're not caching
+ if (!g.noCanvasCache || g.recycledCanvases.length < 2) {
+ g.recycledCanvases.push(canvas);
+ }
+}
+
+function IDForEventTarget(event)
+{
+ try {
+ return "'" + event.target.getAttribute('id') + "'";
+ } catch (ex) {
+ return "<unknown>";
+ }
+}
+
+function OnRefTestLoad(win)
+{
+ g.crashDumpDir = Cc[NS_DIRECTORY_SERVICE_CONTRACTID]
+ .getService(Ci.nsIProperties)
+ .get("ProfD", Ci.nsIFile);
+ g.crashDumpDir.append("minidumps");
+
+ g.pendingCrashDumpDir = Cc[NS_DIRECTORY_SERVICE_CONTRACTID]
+ .getService(Ci.nsIProperties)
+ .get("UAppData", Ci.nsIFile);
+ g.pendingCrashDumpDir.append("Crash Reports");
+ g.pendingCrashDumpDir.append("pending");
+
+ g.browserIsRemote = Services.appinfo.browserTabsRemoteAutostart;
+ g.browserIsFission = Services.appinfo.fissionAutostart;
+
+ var prefs = Cc["@mozilla.org/preferences-service;1"].
+ getService(Ci.nsIPrefBranch);
+ g.browserIsIframe = prefs.getBoolPref("reftest.browser.iframe.enabled", false);
+ g.useDrawSnapshot = prefs.getBoolPref("reftest.use-draw-snapshot", false);
+
+ g.logLevel = prefs.getStringPref("reftest.logLevel", "info");
+
+ if (win === undefined || win == null) {
+ win = window;
+ }
+ if (g.containingWindow == null && win != null) {
+ g.containingWindow = win;
+ }
+
+ if (g.browserIsIframe) {
+ g.browser = g.containingWindow.document.createElementNS(XHTML_NS, "iframe");
+ g.browser.setAttribute("mozbrowser", "");
+ } else {
+ g.browser = g.containingWindow.document.createElementNS(XUL_NS, "xul:browser");
+ }
+ g.browser.setAttribute("id", "browser");
+ g.browser.setAttribute("type", "content");
+ g.browser.setAttribute("primary", "true");
+ g.browser.setAttribute("remote", g.browserIsRemote ? "true" : "false");
+ // Make sure the browser element is exactly 800x1000, no matter
+ // what size our window is
+ g.browser.setAttribute("style", "padding: 0px; margin: 0px; border:none; min-width: 800px; min-height: 1000px; max-width: 800px; max-height: 1000px; color-scheme: env(-moz-content-preferred-color-scheme)");
+
+ if (Services.appinfo.OS == "Android") {
+ let doc = g.containingWindow.document.getElementById('main-window');
+ while (doc.hasChildNodes()) {
+ doc.firstChild.remove();
+ }
+ doc.appendChild(g.browser);
+ // TODO Bug 1156817: reftests don't have most of GeckoView infra so we
+ // can't register this actor
+ ChromeUtils.unregisterWindowActor("LoadURIDelegate");
+ } else {
+ document.getElementById("reftest-window").appendChild(g.browser);
+ }
+
+ g.browserMessageManager = g.browser.frameLoader.messageManager;
+ // The content script waits for the initial onload, then notifies
+ // us.
+ RegisterMessageListenersAndLoadContentScript(false);
+}
+
+function InitAndStartRefTests()
+{
+ /* These prefs are optional, so we don't need to spit an error to the log */
+ try {
+ var prefs = Cc["@mozilla.org/preferences-service;1"].
+ getService(Ci.nsIPrefBranch);
+ } catch(e) {
+ logger.error("EXCEPTION: " + e);
+ }
+
+ try {
+ prefs.setBoolPref("android.widget_paints_background", false);
+ } catch (e) {}
+
+ // If fission is enabled, then also put data: URIs in the default web process,
+ // since most reftests run in the file process, and this will make data:
+ // <iframe>s OOP.
+ if (g.browserIsFission) {
+ prefs.setBoolPref("browser.tabs.remote.dataUriInDefaultWebProcess", true);
+ }
+
+ /* set the g.loadTimeout */
+ try {
+ g.loadTimeout = prefs.getIntPref("reftest.timeout");
+ } catch(e) {
+ g.loadTimeout = 5 * 60 * 1000; //5 minutes as per bug 479518
+ }
+
+ /* Get the logfile for android tests */
+ try {
+ var logFile = prefs.getStringPref("reftest.logFile");
+ if (logFile) {
+ var f = FileUtils.File(logFile);
+ var out = FileUtils.openFileOutputStream(f, FileUtils.MODE_WRONLY | FileUtils.MODE_CREATE);
+ g.logFile = Cc["@mozilla.org/intl/converter-output-stream;1"]
+ .createInstance(Ci.nsIConverterOutputStream);
+ g.logFile.init(out, null);
+ }
+ } catch(e) {}
+
+ g.remote = prefs.getBoolPref("reftest.remote", false);
+
+ g.ignoreWindowSize = prefs.getBoolPref("reftest.ignoreWindowSize", false);
+
+ /* Support for running a chunk (subset) of tests. In separate try as this is optional */
+ try {
+ g.totalChunks = prefs.getIntPref("reftest.totalChunks");
+ g.thisChunk = prefs.getIntPref("reftest.thisChunk");
+ }
+ catch(e) {
+ g.totalChunks = 0;
+ g.thisChunk = 0;
+ }
+
+ try {
+ g.focusFilterMode = prefs.getStringPref("reftest.focusFilterMode");
+ } catch(e) {}
+
+ try {
+ g.isCoverageBuild = prefs.getBoolPref("reftest.isCoverageBuild");
+ } catch(e) {}
+
+ try {
+ g.compareRetainedDisplayLists = prefs.getBoolPref("reftest.compareRetainedDisplayLists");
+ } catch (e) {}
+
+ try {
+ // We have to set print.always_print_silent or a print dialog would
+ // appear for each print operation, which would interrupt the test run.
+ prefs.setBoolPref("print.always_print_silent", true);
+ } catch (e) {
+ /* uh oh, print reftests may not work... */
+ logger.warning("Failed to set silent printing pref, EXCEPTION: " + e);
+ }
+
+ g.windowUtils = g.containingWindow.windowUtils;
+ if (!g.windowUtils || !g.windowUtils.compareCanvases)
+ throw "nsIDOMWindowUtils inteface missing";
+
+ g.ioService = Cc[IO_SERVICE_CONTRACTID].getService(Ci.nsIIOService);
+ g.debug = Cc[DEBUG_CONTRACTID].getService(Ci.nsIDebug2);
+
+ RegisterProcessCrashObservers();
+
+ if (g.remote) {
+ g.server = null;
+ } else {
+ g.server = new HttpServer();
+ }
+ try {
+ if (g.server)
+ StartHTTPServer();
+ } catch (ex) {
+ //g.browser.loadURI('data:text/plain,' + ex);
+ ++g.testResults.Exception;
+ logger.error("EXCEPTION: " + ex);
+ DoneTests();
+ }
+
+ // Focus the content browser.
+ if (g.focusFilterMode != FOCUS_FILTER_NON_NEEDS_FOCUS_TESTS) {
+ var fm = Cc["@mozilla.org/focus-manager;1"].getService(Ci.nsIFocusManager);
+ if (fm.activeWindow != g.containingWindow) {
+ Focus();
+ }
+ g.browser.addEventListener("focus", ReadTests, true);
+ g.browser.focus();
+ } else {
+ ReadTests();
+ }
+}
+
+function StartHTTPServer()
+{
+ g.server.registerContentType("sjs", "sjs");
+ g.server.start(-1);
+
+ g.server.identity.add("http", "example.org", "80");
+ g.server.identity.add("https", "example.org", "443");
+
+ const proxyFilter = {
+ proxyInfo: lazy.proxyService.newProxyInfo(
+ "http", // type of proxy
+ "localhost", //proxy host
+ g.server.identity.primaryPort, // proxy host port
+ "", // auth header
+ "", // isolation key
+ 0, // flags
+ 4096, // timeout
+ null // failover proxy
+ ),
+
+ applyFilter(channel, defaultProxyInfo, callback) {
+ if (channel.URI.host == "example.org") {
+ callback.onProxyFilterResult(this.proxyInfo);
+ } else {
+ callback.onProxyFilterResult(defaultProxyInfo);
+ }
+ },
+ };
+
+ lazy.proxyService.registerChannelFilter(proxyFilter, 0);
+
+ g.httpServerPort = g.server.identity.primaryPort;
+}
+
+// Perform a Fisher-Yates shuffle of the array.
+function Shuffle(array)
+{
+ for (var i = array.length - 1; i > 0; i--) {
+ var j = Math.floor(Math.random() * (i + 1));
+ var temp = array[i];
+ array[i] = array[j];
+ array[j] = temp;
+ }
+}
+
+function ReadTests() {
+ try {
+ if (g.focusFilterMode != FOCUS_FILTER_NON_NEEDS_FOCUS_TESTS) {
+ g.browser.removeEventListener("focus", ReadTests, true);
+ }
+
+ g.urls = [];
+ var prefs = Cc["@mozilla.org/preferences-service;1"].
+ getService(Ci.nsIPrefBranch);
+
+ /* There are three modes implemented here:
+ * 1) reftest.manifests
+ * 2) reftest.manifests and reftest.manifests.dumpTests
+ * 3) reftest.tests
+ *
+ * The first will parse the specified manifests, then immediately
+ * run the tests. The second will parse the manifests, save the test
+ * objects to a file and exit. The third will load a file of test
+ * objects and run them.
+ *
+ * The latter two modes are used to pass test data back and forth
+ * with python harness.
+ */
+ let manifests = prefs.getStringPref("reftest.manifests", null);
+ let dumpTests = prefs.getStringPref("reftest.manifests.dumpTests", null);
+ let testList = prefs.getStringPref("reftest.tests", null);
+
+ if ((testList && manifests) || !(testList || manifests)) {
+ logger.error("Exactly one of reftest.manifests or reftest.tests must be specified.");
+ logger.debug("reftest.manifests is: " + manifests);
+ logger.error("reftest.tests is: " + testList);
+ DoneTests();
+ }
+
+ if (testList) {
+ logger.debug("Reading test objects from: " + testList);
+ let promise = IOUtils.readJSON(testList).then(function onSuccess(json) {
+ g.urls = json.map(CreateUrls);
+ StartTests();
+ }).catch(function onFailure(e) {
+ logger.error("Failed to load test objects: " + e);
+ DoneTests();
+ });
+ } else if (manifests) {
+ // Parse reftest manifests
+ logger.debug("Reading " + manifests.length + " manifests");
+ manifests = JSON.parse(manifests);
+ g.urlsFilterRegex = manifests[null];
+
+ var globalFilter = null;
+ if (manifests.hasOwnProperty("")) {
+ let filterAndId = manifests[""];
+ if (!Array.isArray(filterAndId)) {
+ logger.error(`manifest[""] should be an array`);
+ DoneTests();
+ }
+ if (filterAndId.length === 0) {
+ logger.error(`manifest[""] should contain a filter pattern in the 1st item`);
+ DoneTests();
+ }
+ let filter = filterAndId[0];
+ if (typeof filter !== "string") {
+ logger.error(`The first item of manifest[""] should be a string`);
+ DoneTests();
+ }
+ globalFilter = new RegExp(filter);
+ delete manifests[""];
+ }
+
+ var manifestURLs = Object.keys(manifests);
+
+ // Ensure we read manifests from higher up the directory tree first so that we
+ // process includes before reading the included manifest again
+ manifestURLs.sort(function(a,b) {return a.length - b.length})
+ manifestURLs.forEach(function(manifestURL) {
+ logger.info("Reading manifest " + manifestURL);
+ var manifestInfo = manifests[manifestURL];
+ var filter = manifestInfo[0] ? new RegExp(manifestInfo[0]) : null;
+ var manifestID = manifestInfo[1];
+ ReadTopManifest(manifestURL, [globalFilter, filter, false], manifestID);
+ });
+
+ if (dumpTests) {
+ logger.debug("Dumping test objects to file: " + dumpTests);
+ IOUtils.writeJSON(dumpTests, g.urls, { flush: true }).then(
+ function onSuccess() {
+ DoneTests();
+ },
+ function onFailure(reason) {
+ logger.error("failed to write test data: " + reason);
+ DoneTests();
+ }
+ )
+ } else {
+ logger.debug("Running " + g.urls.length + " test objects");
+ g.manageSuite = true;
+ g.urls = g.urls.map(CreateUrls);
+ StartTests();
+ }
+ }
+ } catch(e) {
+ ++g.testResults.Exception;
+ logger.error("EXCEPTION: " + e);
+ DoneTests();
+ }
+}
+
+function StartTests()
+{
+ /* These prefs are optional, so we don't need to spit an error to the log */
+ try {
+ var prefs = Cc["@mozilla.org/preferences-service;1"].
+ getService(Ci.nsIPrefBranch);
+ } catch(e) {
+ logger.error("EXCEPTION: " + e);
+ }
+
+ g.noCanvasCache = prefs.getIntPref("reftest.nocache", false);
+
+ g.shuffle = prefs.getBoolPref("reftest.shuffle", false);
+
+ g.runUntilFailure = prefs.getBoolPref("reftest.runUntilFailure", false);
+
+ g.verify = prefs.getBoolPref("reftest.verify", false);
+
+ g.cleanupPendingCrashes = prefs.getBoolPref("reftest.cleanupPendingCrashes", false);
+
+ // Check if there are any crash dump files from the startup procedure, before
+ // we start running the first test. Otherwise the first test might get
+ // blamed for producing a crash dump file when that was not the case.
+ CleanUpCrashDumpFiles();
+
+ // When we repeat this function is called again, so really only want to set
+ // g.repeat once.
+ if (g.repeat == null) {
+ g.repeat = prefs.getIntPref("reftest.repeat", 0);
+ }
+
+ g.runSlowTests = prefs.getIntPref("reftest.skipslowtests", false);
+
+ if (g.shuffle) {
+ g.noCanvasCache = true;
+ }
+
+ try {
+ BuildUseCounts();
+
+ // Filter tests which will be skipped to get a more even distribution when chunking
+ // tURLs is a temporary array containing all active tests
+ var tURLs = new Array();
+ for (var i = 0; i < g.urls.length; ++i) {
+ if (g.urls[i].skip)
+ continue;
+
+ if (g.urls[i].needsFocus && !Focus())
+ continue;
+
+ if (g.urls[i].slow && !g.runSlowTests)
+ continue;
+
+ tURLs.push(g.urls[i]);
+ }
+
+ var numActiveTests = tURLs.length;
+
+ if (g.totalChunks > 0 && g.thisChunk > 0) {
+ // Calculate start and end indices of this chunk if tURLs array were
+ // divided evenly
+ var testsPerChunk = tURLs.length / g.totalChunks;
+ var start = Math.round((g.thisChunk-1) * testsPerChunk);
+ var end = Math.round(g.thisChunk * testsPerChunk);
+ numActiveTests = end - start;
+
+ // Map these indices onto the g.urls array. This avoids modifying the
+ // g.urls array which prevents skipped tests from showing up in the log
+ start = g.thisChunk == 1 ? 0 : g.urls.indexOf(tURLs[start]);
+ end = g.thisChunk == g.totalChunks ? g.urls.length : g.urls.indexOf(tURLs[end + 1]) - 1;
+
+ logger.info("Running chunk " + g.thisChunk + " out of " + g.totalChunks + " chunks. " +
+ "tests " + (start+1) + "-" + end + "/" + g.urls.length);
+
+ g.urls = g.urls.slice(start, end);
+ }
+
+ if (g.manageSuite && !g.suiteStarted) {
+ var ids = {};
+ g.urls.forEach(function(test) {
+ if (!(test.manifestID in ids)) {
+ ids[test.manifestID] = [];
+ }
+ ids[test.manifestID].push(test.identifier);
+ });
+ var suite = prefs.getStringPref('reftest.suite', 'reftest');
+ logger.suiteStart(ids, suite, {"skipped": g.urls.length - numActiveTests});
+ g.suiteStarted = true
+ }
+
+ if (g.shuffle) {
+ Shuffle(g.urls);
+ }
+
+ g.totalTests = g.urls.length;
+ if (!g.totalTests && !g.verify && !g.repeat)
+ throw "No tests to run";
+
+ g.uriCanvases = {};
+
+ PerTestCoverageUtils.beforeTest()
+ .then(StartCurrentTest)
+ .catch(e => {
+ logger.error("EXCEPTION: " + e);
+ DoneTests();
+ });
+ } catch (ex) {
+ //g.browser.loadURI('data:text/plain,' + ex);
+ ++g.testResults.Exception;
+ logger.error("EXCEPTION: " + ex);
+ DoneTests();
+ }
+}
+
+function OnRefTestUnload()
+{
+}
+
+function AddURIUseCount(uri)
+{
+ if (uri == null)
+ return;
+
+ var spec = uri.spec;
+ if (spec in g.uriUseCounts) {
+ g.uriUseCounts[spec]++;
+ } else {
+ g.uriUseCounts[spec] = 1;
+ }
+}
+
+function BuildUseCounts()
+{
+ if (g.noCanvasCache) {
+ return;
+ }
+
+ g.uriUseCounts = {};
+ for (var i = 0; i < g.urls.length; ++i) {
+ var url = g.urls[i];
+ if (!url.skip &&
+ (url.type == TYPE_REFTEST_EQUAL ||
+ url.type == TYPE_REFTEST_NOTEQUAL)) {
+ if (url.prefSettings1.length == 0) {
+ AddURIUseCount(g.urls[i].url1);
+ }
+ if (url.prefSettings2.length == 0) {
+ AddURIUseCount(g.urls[i].url2);
+ }
+ }
+ }
+}
+
+// Return true iff this window is focused when this function returns.
+function Focus()
+{
+ var fm = Cc["@mozilla.org/focus-manager;1"].getService(Ci.nsIFocusManager);
+ fm.focusedWindow = g.containingWindow;
+
+ try {
+ var dock = Cc["@mozilla.org/widget/macdocksupport;1"].getService(Ci.nsIMacDockSupport);
+ dock.activateApplication(true);
+ } catch(ex) {
+ }
+
+ return true;
+}
+
+function Blur()
+{
+ // On non-remote reftests, this will transfer focus to the dummy window
+ // we created to hold focus for non-needs-focus tests. Buggy tests
+ // (ones which require focus but don't request needs-focus) will then
+ // fail.
+ g.containingWindow.blur();
+}
+
+async function StartCurrentTest()
+{
+ g.testLog = [];
+
+ // make sure we don't run tests that are expected to kill the browser
+ while (g.urls.length > 0) {
+ var test = g.urls[0];
+ logger.testStart(test.identifier);
+ if (test.skip) {
+ ++g.testResults.Skip;
+ logger.testEnd(test.identifier, "SKIP");
+ g.urls.shift();
+ } else if (test.needsFocus && !Focus()) {
+ // FIXME: Marking this as a known fail is dangerous! What
+ // if it starts failing all the time?
+ ++g.testResults.Skip;
+ logger.testEnd(test.identifier, "SKIP", null, "(COULDN'T GET FOCUS)");
+ g.urls.shift();
+ } else if (test.slow && !g.runSlowTests) {
+ ++g.testResults.Slow;
+ logger.testEnd(test.identifier, "SKIP", null, "(SLOW)");
+ g.urls.shift();
+ } else {
+ break;
+ }
+ }
+
+ if ((g.urls.length == 0 && g.repeat == 0) ||
+ (g.runUntilFailure && HasUnexpectedResult())) {
+ await RestoreChangedPreferences();
+ DoneTests();
+ } else if (g.urls.length == 0 && g.repeat > 0) {
+ // Repeat
+ g.repeat--;
+ ReadTests();
+ } else {
+ if (g.urls[0].chaosMode) {
+ g.windowUtils.enterChaosMode();
+ }
+ if (!g.urls[0].needsFocus) {
+ Blur();
+ }
+ var currentTest = g.totalTests - g.urls.length;
+ g.containingWindow.document.title = "reftest: " + currentTest + " / " + g.totalTests +
+ " (" + Math.floor(100 * (currentTest / g.totalTests)) + "%)";
+ StartCurrentURI(URL_TARGET_TYPE_TEST);
+ }
+}
+
+// A simplified version of the function with the same name in tabbrowser.js.
+function updateBrowserRemotenessByURL(aBrowser, aURL) {
+ var oa = E10SUtils.predictOriginAttributes({ browser: aBrowser });
+ let remoteType = E10SUtils.getRemoteTypeForURI(
+ aURL,
+ aBrowser.ownerGlobal.docShell.nsILoadContext.useRemoteTabs,
+ aBrowser.ownerGlobal.docShell.nsILoadContext.useRemoteSubframes,
+ aBrowser.remoteType,
+ aBrowser.currentURI,
+ oa
+ );
+ // Things get confused if we switch to not-remote
+ // for chrome:// URIs, so lets not for now.
+ if (remoteType == E10SUtils.NOT_REMOTE &&
+ g.browserIsRemote) {
+ remoteType = aBrowser.remoteType;
+ }
+ if (aBrowser.remoteType != remoteType) {
+ if (remoteType == E10SUtils.NOT_REMOTE) {
+ aBrowser.removeAttribute("remote");
+ aBrowser.removeAttribute("remoteType");
+ } else {
+ aBrowser.setAttribute("remote", "true");
+ aBrowser.setAttribute("remoteType", remoteType);
+ }
+ aBrowser.changeRemoteness({ remoteType });
+ aBrowser.construct();
+
+ g.browserMessageManager = aBrowser.frameLoader.messageManager;
+ RegisterMessageListenersAndLoadContentScript(true);
+ return new Promise(resolve => { g.resolveContentReady = resolve; });
+ }
+
+ return Promise.resolve();
+}
+
+// This logic should match SpecialPowersParent._applyPrefs.
+function PrefRequiresRefresh(name) {
+ return name == "layout.css.prefers-color-scheme.content-override" ||
+ name.startsWith("ui.") ||
+ name.startsWith("browser.display.") ||
+ name.startsWith("font.");
+}
+
+async function StartCurrentURI(aURLTargetType)
+{
+ const isStartingRef = (aURLTargetType == URL_TARGET_TYPE_REFERENCE);
+
+ g.currentURL = g.urls[0][isStartingRef ? "url2" : "url1"].spec;
+ g.currentURLTargetType = aURLTargetType;
+
+ await RestoreChangedPreferences();
+
+ var prefs = Cc["@mozilla.org/preferences-service;1"].
+ getService(Ci.nsIPrefBranch);
+
+ const prefSettings =
+ g.urls[0][isStartingRef ? "prefSettings2" : "prefSettings1"];
+
+ var prefsRequireRefresh = false;
+
+ if (prefSettings.length > 0) {
+ var badPref = undefined;
+ try {
+ prefSettings.forEach(function(ps) {
+ let prefExists = false;
+ try {
+ let prefType = prefs.getPrefType(ps.name);
+ prefExists = (prefType != prefs.PREF_INVALID);
+ } catch (e) {
+ }
+ if (!prefExists) {
+ logger.info("Pref " + ps.name + " not found, will be added");
+ }
+
+ let oldVal = undefined;
+ if (prefExists) {
+ if (ps.type == PREF_BOOLEAN) {
+ try {
+ oldVal = prefs.getBoolPref(ps.name);
+ } catch (e) {
+ badPref = "boolean preference '" + ps.name + "'";
+ throw "bad pref";
+ }
+ } else if (ps.type == PREF_STRING) {
+ try {
+ oldVal = prefs.getStringPref(ps.name);
+ } catch (e) {
+ badPref = "string preference '" + ps.name + "'";
+ throw "bad pref";
+ }
+ } else if (ps.type == PREF_INTEGER) {
+ try {
+ oldVal = prefs.getIntPref(ps.name);
+ } catch (e) {
+ badPref = "integer preference '" + ps.name + "'";
+ throw "bad pref";
+ }
+ } else {
+ throw "internal error - unknown preference type";
+ }
+ }
+ if (!prefExists || oldVal != ps.value) {
+ var requiresRefresh = PrefRequiresRefresh(ps.name);
+ prefsRequireRefresh = prefsRequireRefresh || requiresRefresh;
+ g.prefsToRestore.push( { name: ps.name,
+ type: ps.type,
+ value: oldVal,
+ requiresRefresh,
+ prefExisted: prefExists } );
+ var value = ps.value;
+ if (ps.type == PREF_BOOLEAN) {
+ prefs.setBoolPref(ps.name, value);
+ } else if (ps.type == PREF_STRING) {
+ prefs.setStringPref(ps.name, value);
+ value = '"' + value + '"';
+ } else if (ps.type == PREF_INTEGER) {
+ prefs.setIntPref(ps.name, value);
+ }
+ logger.info("SET PREFERENCE pref(" + ps.name + "," + value + ")");
+ }
+ });
+ } catch (e) {
+ if (e == "bad pref") {
+ var test = g.urls[0];
+ if (test.expected == EXPECTED_FAIL) {
+ logger.testEnd(test.identifier, "FAIL", "FAIL",
+ "(SKIPPED; " + badPref + " not known or wrong type)");
+ ++g.testResults.Skip;
+ } else {
+ logger.testEnd(test.identifier, "FAIL", "PASS",
+ badPref + " not known or wrong type");
+ ++g.testResults.UnexpectedFail;
+ }
+
+ // skip the test that had a bad preference
+ g.urls.shift();
+ await StartCurrentTest();
+ return;
+ } else {
+ throw e;
+ }
+ }
+ }
+
+ if (prefSettings.length == 0 &&
+ g.uriCanvases[g.currentURL] &&
+ (g.urls[0].type == TYPE_REFTEST_EQUAL ||
+ g.urls[0].type == TYPE_REFTEST_NOTEQUAL) &&
+ g.urls[0].maxAsserts == 0) {
+ // Pretend the document loaded --- RecordResult will notice
+ // there's already a canvas for this URL
+ g.containingWindow.setTimeout(RecordResult, 0);
+ } else {
+ var currentTest = g.totalTests - g.urls.length;
+ // Log this to preserve the same overall log format,
+ // should be removed if the format is updated
+ gDumpFn("REFTEST TEST-LOAD | " + g.currentURL + " | " + currentTest + " / " + g.totalTests +
+ " (" + Math.floor(100 * (currentTest / g.totalTests)) + "%)\n");
+ TestBuffer("START " + g.currentURL);
+ await updateBrowserRemotenessByURL(g.browser, g.currentURL);
+
+ if (prefsRequireRefresh) {
+ await new Promise(resolve => g.containingWindow.requestAnimationFrame(resolve));
+ }
+
+ var type = g.urls[0].type
+ if (TYPE_SCRIPT == type) {
+ SendLoadScriptTest(g.currentURL, g.loadTimeout);
+ } else if (TYPE_PRINT == type) {
+ SendLoadPrintTest(g.currentURL, g.loadTimeout);
+ } else {
+ SendLoadTest(type, g.currentURL, g.currentURLTargetType, g.loadTimeout);
+ }
+ }
+}
+
+function DoneTests()
+{
+ PerTestCoverageUtils.afterTest()
+ .catch(e => logger.error("EXCEPTION: " + e))
+ .then(() => {
+ if (g.manageSuite) {
+ g.suiteStarted = false
+ logger.suiteEnd({'results': g.testResults});
+ } else {
+ logger.logData('results', {results: g.testResults});
+ }
+ logger.info("Slowest test took " + g.slowestTestTime + "ms (" + g.slowestTestURL + ")");
+ logger.info("Total canvas count = " + g.recycledCanvases.length);
+ if (g.failedUseWidgetLayers) {
+ LogWidgetLayersFailure();
+ }
+
+ function onStopped() {
+ if (g.logFile) {
+ g.logFile.close();
+ g.logFile = null;
+ }
+ let appStartup = Cc["@mozilla.org/toolkit/app-startup;1"].getService(Ci.nsIAppStartup);
+ appStartup.quit(Ci.nsIAppStartup.eForceQuit);
+ }
+ if (g.server) {
+ g.server.stop(onStopped);
+ }
+ else {
+ onStopped();
+ }
+ });
+}
+
+function UpdateCanvasCache(url, canvas)
+{
+ var spec = url.spec;
+
+ --g.uriUseCounts[spec];
+
+ if (g.uriUseCounts[spec] == 0) {
+ ReleaseCanvas(canvas);
+ delete g.uriCanvases[spec];
+ } else if (g.uriUseCounts[spec] > 0) {
+ g.uriCanvases[spec] = canvas;
+ } else {
+ throw "Use counts were computed incorrectly";
+ }
+}
+
+// Recompute drawWindow flags for every drawWindow operation.
+// We have to do this every time since our window can be
+// asynchronously resized (e.g. by the window manager, to make
+// it fit on screen) at unpredictable times.
+// Fortunately this is pretty cheap.
+async function DoDrawWindow(ctx, x, y, w, h)
+{
+ if (g.useDrawSnapshot) {
+ try {
+ let image = await g.browser.drawSnapshot(x, y, w, h, 1.0, "#fff");
+ ctx.drawImage(image, x, y);
+ } catch (ex) {
+ logger.error(g.currentURL + " | drawSnapshot failed: " + ex);
+ ++g.testResults.Exception;
+ }
+ return;
+ }
+
+ var flags = ctx.DRAWWINDOW_DRAW_CARET | ctx.DRAWWINDOW_DRAW_VIEW;
+ var testRect = g.browser.getBoundingClientRect();
+ if (g.ignoreWindowSize ||
+ (0 <= testRect.left &&
+ 0 <= testRect.top &&
+ g.containingWindow.innerWidth >= testRect.right &&
+ g.containingWindow.innerHeight >= testRect.bottom)) {
+ // We can use the window's retained layer manager
+ // because the window is big enough to display the entire
+ // browser element
+ flags |= ctx.DRAWWINDOW_USE_WIDGET_LAYERS;
+ } else if (g.browserIsRemote) {
+ logger.error(g.currentURL + " | can't drawWindow remote content");
+ ++g.testResults.Exception;
+ }
+
+ if (g.drawWindowFlags != flags) {
+ // Every time the flags change, dump the new state.
+ g.drawWindowFlags = flags;
+ var flagsStr = "DRAWWINDOW_DRAW_CARET | DRAWWINDOW_DRAW_VIEW";
+ if (flags & ctx.DRAWWINDOW_USE_WIDGET_LAYERS) {
+ flagsStr += " | DRAWWINDOW_USE_WIDGET_LAYERS";
+ } else {
+ // Output a special warning because we need to be able to detect
+ // this whenever it happens.
+ LogWidgetLayersFailure();
+ g.failedUseWidgetLayers = true;
+ }
+ logger.info("drawWindow flags = " + flagsStr +
+ "; window size = " + g.containingWindow.innerWidth + "," + g.containingWindow.innerHeight +
+ "; test browser size = " + testRect.width + "," + testRect.height);
+ }
+
+ TestBuffer("DoDrawWindow " + x + "," + y + "," + w + "," + h);
+ ctx.save();
+ ctx.translate(x, y);
+ ctx.drawWindow(g.containingWindow, x, y, w, h, "rgb(255,255,255)",
+ g.drawWindowFlags);
+ ctx.restore();
+}
+
+async function InitCurrentCanvasWithSnapshot()
+{
+ TestBuffer("Initializing canvas snapshot");
+
+ if (g.urls[0].type == TYPE_LOAD || g.urls[0].type == TYPE_SCRIPT || g.urls[0].type == TYPE_PRINT) {
+ // We don't want to snapshot this kind of test
+ return false;
+ }
+
+ if (!g.currentCanvas) {
+ g.currentCanvas = AllocateCanvas();
+ }
+
+ var ctx = g.currentCanvas.getContext("2d");
+ await DoDrawWindow(ctx, 0, 0, g.currentCanvas.width, g.currentCanvas.height);
+ return true;
+}
+
+async function UpdateCurrentCanvasForInvalidation(rects)
+{
+ TestBuffer("Updating canvas for invalidation");
+
+ if (!g.currentCanvas) {
+ return;
+ }
+
+ var ctx = g.currentCanvas.getContext("2d");
+ for (var i = 0; i < rects.length; ++i) {
+ var r = rects[i];
+ // Set left/top/right/bottom to pixel boundaries
+ var left = Math.floor(r.left);
+ var top = Math.floor(r.top);
+ var right = Math.ceil(r.right);
+ var bottom = Math.ceil(r.bottom);
+
+ // Clamp the values to the canvas size
+ left = Math.max(0, Math.min(left, g.currentCanvas.width));
+ top = Math.max(0, Math.min(top, g.currentCanvas.height));
+ right = Math.max(0, Math.min(right, g.currentCanvas.width));
+ bottom = Math.max(0, Math.min(bottom, g.currentCanvas.height));
+
+ await DoDrawWindow(ctx, left, top, right - left, bottom - top);
+ }
+}
+
+async function UpdateWholeCurrentCanvasForInvalidation()
+{
+ TestBuffer("Updating entire canvas for invalidation");
+
+ if (!g.currentCanvas) {
+ return;
+ }
+
+ var ctx = g.currentCanvas.getContext("2d");
+ await DoDrawWindow(ctx, 0, 0, g.currentCanvas.width, g.currentCanvas.height);
+}
+
+function RecordResult(testRunTime, errorMsg, typeSpecificResults)
+{
+ TestBuffer("RecordResult fired");
+
+ // Keep track of which test was slowest, and how long it took.
+ if (testRunTime > g.slowestTestTime) {
+ g.slowestTestTime = testRunTime;
+ g.slowestTestURL = g.currentURL;
+ }
+
+ // Not 'const ...' because of 'EXPECTED_*' value dependency.
+ var outputs = {};
+ outputs[EXPECTED_PASS] = {
+ true: {s: ["PASS", "PASS"], n: "Pass"},
+ false: {s: ["FAIL", "PASS"], n: "UnexpectedFail"}
+ };
+ outputs[EXPECTED_FAIL] = {
+ true: {s: ["PASS", "FAIL"], n: "UnexpectedPass"},
+ false: {s: ["FAIL", "FAIL"], n: "KnownFail"}
+ };
+ outputs[EXPECTED_RANDOM] = {
+ true: {s: ["PASS", "PASS"], n: "Random"},
+ false: {s: ["FAIL", "FAIL"], n: "Random"}
+ };
+ // for EXPECTED_FUZZY we need special handling because we can have
+ // Pass, UnexpectedPass, or UnexpectedFail
+
+ if ((g.currentURLTargetType == URL_TARGET_TYPE_TEST && g.urls[0].wrCapture.test) ||
+ (g.currentURLTargetType == URL_TARGET_TYPE_REFERENCE && g.urls[0].wrCapture.ref)) {
+ logger.info("Running webrender capture");
+ g.windowUtils.wrCapture();
+ }
+
+ var output;
+ var extra;
+
+ if (g.urls[0].type == TYPE_LOAD) {
+ ++g.testResults.LoadOnly;
+ logger.testStatus(g.urls[0].identifier, "(LOAD ONLY)", "PASS", "PASS");
+ g.currentCanvas = null;
+ FinishTestItem();
+ return;
+ }
+ if (g.urls[0].type == TYPE_PRINT) {
+ switch (g.currentURLTargetType) {
+ case URL_TARGET_TYPE_TEST:
+ // First document has been loaded.
+ g.testPrintOutput = typeSpecificResults;
+ // Proceed to load the second document.
+ CleanUpCrashDumpFiles();
+ StartCurrentURI(URL_TARGET_TYPE_REFERENCE);
+ break;
+ case URL_TARGET_TYPE_REFERENCE:
+ let pathToTestPdf = g.testPrintOutput;
+ let pathToRefPdf = typeSpecificResults;
+ comparePdfs(pathToTestPdf, pathToRefPdf, function(error, results) {
+ let expected = g.urls[0].expected;
+ // TODO: We should complain here if results is empty!
+ // (If it's empty, we'll spuriously succeed, regardless of
+ // our expectations)
+ if (error) {
+ output = outputs[expected][false];
+ extra = { status_msg: output.n };
+ ++g.testResults[output.n];
+ logger.testEnd(g.urls[0].identifier, output.s[0], output.s[1],
+ error.message, null, extra);
+ } else {
+ let outputPair = outputs[expected];
+ if (expected === EXPECTED_FAIL) {
+ let failureResults = results.filter(function (result) { return !result.passed });
+ if (failureResults.length > 0) {
+ // We got an expected failure. Let's get rid of the
+ // passes from the results so we don't trigger
+ // TEST_UNEXPECTED_PASS logging for those.
+ results = failureResults;
+ }
+ // (else, we expected a failure but got none!
+ // Leave results untouched so we can log them.)
+ }
+ results.forEach(function(result) {
+ output = outputPair[result.passed];
+ let extra = { status_msg: output.n };
+ ++g.testResults[output.n];
+ logger.testEnd(g.urls[0].identifier, output.s[0], output.s[1],
+ result.description, null, extra);
+ });
+ }
+ FinishTestItem();
+ });
+ break;
+ default:
+ throw "Unexpected state.";
+ }
+ return;
+ }
+ if (g.urls[0].type == TYPE_SCRIPT) {
+ var expected = g.urls[0].expected;
+
+ if (errorMsg) {
+ // Force an unexpected failure to alert the test author to fix the test.
+ expected = EXPECTED_PASS;
+ } else if (typeSpecificResults.length == 0) {
+ // This failure may be due to a JavaScript Engine bug causing
+ // early termination of the test. If we do not allow silent
+ // failure, report an error.
+ if (!g.urls[0].allowSilentFail)
+ errorMsg = "No test results reported. (SCRIPT)\n";
+ else
+ logger.info("An expected silent failure occurred");
+ }
+
+ if (errorMsg) {
+ output = outputs[expected][false];
+ extra = { status_msg: output.n };
+ ++g.testResults[output.n];
+ logger.testStatus(g.urls[0].identifier, errorMsg, output.s[0], output.s[1], null, null, extra);
+ FinishTestItem();
+ return;
+ }
+
+ var anyFailed = typeSpecificResults.some(function(result) { return !result.passed; });
+ var outputPair;
+ if (anyFailed && expected == EXPECTED_FAIL) {
+ // If we're marked as expected to fail, and some (but not all) tests
+ // passed, treat those tests as though they were marked random
+ // (since we can't tell whether they were really intended to be
+ // marked failing or not).
+ outputPair = { true: outputs[EXPECTED_RANDOM][true],
+ false: outputs[expected][false] };
+ } else {
+ outputPair = outputs[expected];
+ }
+ var index = 0;
+ typeSpecificResults.forEach(function(result) {
+ var output = outputPair[result.passed];
+ var extra = { status_msg: output.n };
+
+ ++g.testResults[output.n];
+ logger.testStatus(g.urls[0].identifier, result.description + " item " + (++index),
+ output.s[0], output.s[1], null, null, extra);
+ });
+
+ if (anyFailed && expected == EXPECTED_PASS) {
+ FlushTestBuffer();
+ }
+
+ FinishTestItem();
+ return;
+ }
+
+ const isRecordingRef =
+ (g.currentURLTargetType == URL_TARGET_TYPE_REFERENCE);
+ const prefSettings =
+ g.urls[0][isRecordingRef ? "prefSettings2" : "prefSettings1"];
+
+ if (prefSettings.length == 0 && g.uriCanvases[g.currentURL]) {
+ g.currentCanvas = g.uriCanvases[g.currentURL];
+ }
+ if (g.currentCanvas == null) {
+ logger.error(g.currentURL, "program error managing snapshots");
+ ++g.testResults.Exception;
+ }
+ g[isRecordingRef ? "canvas2" : "canvas1"] = g.currentCanvas;
+ g.currentCanvas = null;
+
+ ResetRenderingState();
+
+ switch (g.currentURLTargetType) {
+ case URL_TARGET_TYPE_TEST:
+ // First document has been loaded.
+ // Proceed to load the second document.
+
+ CleanUpCrashDumpFiles();
+ StartCurrentURI(URL_TARGET_TYPE_REFERENCE);
+ break;
+ case URL_TARGET_TYPE_REFERENCE:
+ // Both documents have been loaded. Compare the renderings and see
+ // if the comparison result matches the expected result specified
+ // in the manifest.
+
+ // number of different pixels
+ var differences;
+ // whether the two renderings match:
+ var equal;
+ var maxDifference = {};
+ // whether the allowed fuzziness from the annotations is exceeded
+ // by the actual comparison results
+ var fuzz_exceeded = false;
+
+ // what is expected on this platform (PASS, FAIL, RANDOM, or FUZZY)
+ var expected = g.urls[0].expected;
+
+ differences = g.windowUtils.compareCanvases(g.canvas1, g.canvas2, maxDifference);
+
+ if (g.urls[0].noAutoFuzz) {
+ // Autofuzzing is disabled
+ } else if (isAndroidDevice() && maxDifference.value <= 2 && differences > 0) {
+ // Autofuzz for WR on Android physical devices: Reduce any
+ // maxDifference of 2 to 0, because we get a lot of off-by-ones
+ // and off-by-twos that are very random and hard to annotate.
+ // In cases where the difference on any pixel component is more
+ // than 2 we require manual annotation. Note that this applies
+ // to both == tests and != tests, so != tests don't
+ // inadvertently pass due to a random off-by-one pixel
+ // difference.
+ logger.info(`REFTEST wr-on-android dropping fuzz of (${maxDifference.value}, ${differences}) to (0, 0)`);
+ maxDifference.value = 0;
+ differences = 0;
+ }
+
+ equal = (differences == 0);
+
+ if (maxDifference.value > 0 && equal) {
+ throw "Inconsistent result from compareCanvases.";
+ }
+
+ if (expected == EXPECTED_FUZZY) {
+ logger.info(`REFTEST fuzzy test ` +
+ `(${g.urls[0].fuzzyMinDelta}, ${g.urls[0].fuzzyMinPixels}) <= ` +
+ `(${maxDifference.value}, ${differences}) <= ` +
+ `(${g.urls[0].fuzzyMaxDelta}, ${g.urls[0].fuzzyMaxPixels})`);
+ fuzz_exceeded = maxDifference.value > g.urls[0].fuzzyMaxDelta ||
+ differences > g.urls[0].fuzzyMaxPixels;
+ equal = !fuzz_exceeded &&
+ maxDifference.value >= g.urls[0].fuzzyMinDelta &&
+ differences >= g.urls[0].fuzzyMinPixels;
+ }
+
+ var failedExtraCheck = g.failedNoPaint || g.failedNoDisplayList || g.failedDisplayList || g.failedOpaqueLayer || g.failedAssignedLayer;
+
+ // whether the comparison result matches what is in the manifest
+ var test_passed = (equal == (g.urls[0].type == TYPE_REFTEST_EQUAL)) && !failedExtraCheck;
+
+ if (expected != EXPECTED_FUZZY) {
+ output = outputs[expected][test_passed];
+ } else if (test_passed) {
+ output = {s: ["PASS", "PASS"], n: "Pass"};
+ } else if (g.urls[0].type == TYPE_REFTEST_EQUAL &&
+ !failedExtraCheck &&
+ !fuzz_exceeded) {
+ // If we get here, that means we had an '==' type test where
+ // at least one of the actual difference values was below the
+ // allowed range, but nothing else was wrong. So let's produce
+ // UNEXPECTED-PASS in this scenario. Also, if we enter this
+ // branch, 'equal' must be false so let's assert that to guard
+ // against logic errors.
+ if (equal) {
+ throw "Logic error in reftest.jsm fuzzy test handling!";
+ }
+ output = {s: ["PASS", "FAIL"], n: "UnexpectedPass"};
+ } else {
+ // In all other cases we fail the test
+ output = {s: ["FAIL", "PASS"], n: "UnexpectedFail"};
+ }
+ extra = { status_msg: output.n };
+
+ ++g.testResults[output.n];
+
+ // It's possible that we failed both an "extra check" and the normal comparison, but we don't
+ // have a way to annotate these separately, so just print an error for the extra check failures.
+ if (failedExtraCheck) {
+ var failures = [];
+ if (g.failedNoPaint) {
+ failures.push("failed reftest-no-paint");
+ }
+ if (g.failedNoDisplayList) {
+ failures.push("failed reftest-no-display-list");
+ }
+ if (g.failedDisplayList) {
+ failures.push("failed reftest-display-list");
+ }
+ // The g.failed*Messages arrays will contain messages from both the test and the reference.
+ if (g.failedOpaqueLayer) {
+ failures.push("failed reftest-opaque-layer: " + g.failedOpaqueLayerMessages.join(", "));
+ }
+ if (g.failedAssignedLayer) {
+ failures.push("failed reftest-assigned-layer: " + g.failedAssignedLayerMessages.join(", "));
+ }
+ var failureString = failures.join(", ");
+ logger.testStatus(g.urls[0].identifier, failureString, output.s[0], output.s[1], null, null, extra);
+ } else {
+ var message = "image comparison, max difference: " + maxDifference.value +
+ ", number of differing pixels: " + differences;
+ if (!test_passed && expected == EXPECTED_PASS ||
+ !test_passed && expected == EXPECTED_FUZZY ||
+ test_passed && expected == EXPECTED_FAIL) {
+ if (!equal) {
+ extra.max_difference = maxDifference.value;
+ extra.differences = differences;
+ var image1 = g.canvas1.toDataURL();
+ var image2 = g.canvas2.toDataURL();
+ extra.reftest_screenshots = [
+ {url:g.urls[0].identifier[0],
+ screenshot: image1.slice(image1.indexOf(",") + 1)},
+ g.urls[0].identifier[1],
+ {url:g.urls[0].identifier[2],
+ screenshot: image2.slice(image2.indexOf(",") + 1)}
+ ];
+ extra.image1 = image1;
+ extra.image2 = image2;
+ } else {
+ var image1 = g.canvas1.toDataURL();
+ extra.reftest_screenshots = [
+ {url:g.urls[0].identifier[0],
+ screenshot: image1.slice(image1.indexOf(",") + 1)}
+ ];
+ extra.image1 = image1;
+ }
+ }
+ logger.testStatus(g.urls[0].identifier, message, output.s[0], output.s[1], null, null, extra);
+
+ if (g.noCanvasCache) {
+ ReleaseCanvas(g.canvas1);
+ ReleaseCanvas(g.canvas2);
+ } else {
+ if (g.urls[0].prefSettings1.length == 0) {
+ UpdateCanvasCache(g.urls[0].url1, g.canvas1);
+ }
+ if (g.urls[0].prefSettings2.length == 0) {
+ UpdateCanvasCache(g.urls[0].url2, g.canvas2);
+ }
+ }
+ }
+
+ if ((!test_passed && expected == EXPECTED_PASS) || (test_passed && expected == EXPECTED_FAIL)) {
+ FlushTestBuffer();
+ }
+
+ CleanUpCrashDumpFiles();
+ FinishTestItem();
+ break;
+ default:
+ throw "Unexpected state.";
+ }
+}
+
+function LoadFailed(why)
+{
+ ++g.testResults.FailedLoad;
+ if (!why) {
+ // reftest-content.js sets an initial reason before it sets the
+ // timeout that will call us with the currently set reason, so we
+ // should never get here. If we do then there's a logic error
+ // somewhere. Perhaps tests are somehow running overlapped and the
+ // timeout for one test is not being cleared before the timeout for
+ // another is set? Maybe there's some sort of race?
+ logger.error("load failed with unknown reason (we should always have a reason!)");
+ }
+ logger.testStatus(g.urls[0].identifier, "load failed: " + why, "FAIL", "PASS");
+ FlushTestBuffer();
+ FinishTestItem();
+}
+
+function RemoveExpectedCrashDumpFiles()
+{
+ if (g.expectingProcessCrash) {
+ for (let crashFilename of g.expectedCrashDumpFiles) {
+ let file = g.crashDumpDir.clone();
+ file.append(crashFilename);
+ if (file.exists()) {
+ file.remove(false);
+ }
+ }
+ }
+ g.expectedCrashDumpFiles.length = 0;
+}
+
+function FindUnexpectedCrashDumpFiles()
+{
+ if (!g.crashDumpDir.exists()) {
+ return;
+ }
+
+ let entries = g.crashDumpDir.directoryEntries;
+ if (!entries) {
+ return;
+ }
+
+ let foundCrashDumpFile = false;
+ while (entries.hasMoreElements()) {
+ let file = entries.nextFile;
+ let path = String(file.path);
+ if (path.match(/\.(dmp|extra)$/) && !g.unexpectedCrashDumpFiles[path]) {
+ if (!foundCrashDumpFile) {
+ ++g.testResults.UnexpectedFail;
+ foundCrashDumpFile = true;
+ if (g.currentURL) {
+ logger.testStatus(g.urls[0].identifier, "crash-check", "FAIL", "PASS", "This test left crash dumps behind, but we weren't expecting it to!");
+ } else {
+ logger.error("Harness startup left crash dumps behind, but we weren't expecting it to!");
+ }
+ }
+ logger.info("Found unexpected crash dump file " + path);
+ g.unexpectedCrashDumpFiles[path] = true;
+ }
+ }
+}
+
+function RemovePendingCrashDumpFiles()
+{
+ if (!g.pendingCrashDumpDir.exists()) {
+ return;
+ }
+
+ let entries = g.pendingCrashDumpDir.directoryEntries;
+ while (entries.hasMoreElements()) {
+ let file = entries.nextFile;
+ if (file.isFile()) {
+ file.remove(false);
+ logger.info("This test left pending crash dumps; deleted "+file.path);
+ }
+ }
+}
+
+function CleanUpCrashDumpFiles()
+{
+ RemoveExpectedCrashDumpFiles();
+ FindUnexpectedCrashDumpFiles();
+ if (g.cleanupPendingCrashes) {
+ RemovePendingCrashDumpFiles();
+ }
+ g.expectingProcessCrash = false;
+}
+
+function FinishTestItem()
+{
+ logger.testEnd(g.urls[0].identifier, "OK");
+
+ // Replace document with BLANK_URL_FOR_CLEARING in case there are
+ // assertions when unloading.
+ logger.debug("Loading a blank page");
+ // After clearing, content will notify us of the assertion count
+ // and tests will continue.
+ SendClear();
+ g.failedNoPaint = false;
+ g.failedNoDisplayList = false;
+ g.failedDisplayList = false;
+ g.failedOpaqueLayer = false;
+ g.failedOpaqueLayerMessages = [];
+ g.failedAssignedLayer = false;
+ g.failedAssignedLayerMessages = [];
+}
+
+async function DoAssertionCheck(numAsserts)
+{
+ if (g.debug.isDebugBuild) {
+ if (g.browserIsRemote) {
+ // Count chrome-process asserts too when content is out of
+ // process.
+ var newAssertionCount = g.debug.assertionCount;
+ var numLocalAsserts = newAssertionCount - g.assertionCount;
+ g.assertionCount = newAssertionCount;
+
+ numAsserts += numLocalAsserts;
+ }
+
+ var minAsserts = g.urls[0].minAsserts;
+ var maxAsserts = g.urls[0].maxAsserts;
+
+ if (numAsserts < minAsserts) {
+ ++g.testResults.AssertionUnexpectedFixed;
+ } else if (numAsserts > maxAsserts) {
+ ++g.testResults.AssertionUnexpected;
+ } else if (numAsserts != 0) {
+ ++g.testResults.AssertionKnown;
+ }
+ logger.assertionCount(g.urls[0].identifier, numAsserts, minAsserts, maxAsserts);
+ }
+
+ if (g.urls[0].chaosMode) {
+ g.windowUtils.leaveChaosMode();
+ }
+
+ // And start the next test.
+ g.urls.shift();
+ await StartCurrentTest();
+}
+
+function ResetRenderingState()
+{
+ SendResetRenderingState();
+ // We would want to clear any viewconfig here, if we add support for it
+}
+
+async function RestoreChangedPreferences()
+{
+ if (!g.prefsToRestore.length) {
+ return;
+ }
+ var prefs = Cc["@mozilla.org/preferences-service;1"].
+ getService(Ci.nsIPrefBranch);
+ var requiresRefresh = false;
+ g.prefsToRestore.reverse();
+ g.prefsToRestore.forEach(function(ps) {
+ requiresRefresh = requiresRefresh || ps.requiresRefresh;
+ if (ps.prefExisted) {
+ var value = ps.value;
+ if (ps.type == PREF_BOOLEAN) {
+ prefs.setBoolPref(ps.name, value);
+ } else if (ps.type == PREF_STRING) {
+ prefs.setStringPref(ps.name, value);
+ value = '"' + value + '"';
+ } else if (ps.type == PREF_INTEGER) {
+ prefs.setIntPref(ps.name, value);
+ }
+ logger.info("RESTORE PREFERENCE pref(" + ps.name + "," + value + ")");
+ } else {
+ prefs.clearUserPref(ps.name);
+ logger.info("RESTORE PREFERENCE pref(" + ps.name + ", <no value set>) (clearing user pref)");
+ }
+ });
+
+ g.prefsToRestore = [];
+
+ if (requiresRefresh) {
+ await new Promise(resolve => g.containingWindow.requestAnimationFrame(resolve));
+ }
+}
+
+function RegisterMessageListenersAndLoadContentScript(aReload)
+{
+ g.browserMessageManager.addMessageListener(
+ "reftest:AssertionCount",
+ function (m) { RecvAssertionCount(m.json.count); }
+ );
+ g.browserMessageManager.addMessageListener(
+ "reftest:ContentReady",
+ function (m) { return RecvContentReady(m.data); }
+ );
+ g.browserMessageManager.addMessageListener(
+ "reftest:Exception",
+ function (m) { RecvException(m.json.what) }
+ );
+ g.browserMessageManager.addMessageListener(
+ "reftest:FailedLoad",
+ function (m) { RecvFailedLoad(m.json.why); }
+ );
+ g.browserMessageManager.addMessageListener(
+ "reftest:FailedNoPaint",
+ function (m) { RecvFailedNoPaint(); }
+ );
+ g.browserMessageManager.addMessageListener(
+ "reftest:FailedNoDisplayList",
+ function (m) { RecvFailedNoDisplayList(); }
+ );
+ g.browserMessageManager.addMessageListener(
+ "reftest:FailedDisplayList",
+ function (m) { RecvFailedDisplayList(); }
+ );
+ g.browserMessageManager.addMessageListener(
+ "reftest:FailedOpaqueLayer",
+ function (m) { RecvFailedOpaqueLayer(m.json.why); }
+ );
+ g.browserMessageManager.addMessageListener(
+ "reftest:FailedAssignedLayer",
+ function (m) { RecvFailedAssignedLayer(m.json.why); }
+ );
+ g.browserMessageManager.addMessageListener(
+ "reftest:InitCanvasWithSnapshot",
+ function (m) { RecvInitCanvasWithSnapshot(); }
+ );
+ g.browserMessageManager.addMessageListener(
+ "reftest:Log",
+ function (m) { RecvLog(m.json.type, m.json.msg); }
+ );
+ g.browserMessageManager.addMessageListener(
+ "reftest:ScriptResults",
+ function (m) { RecvScriptResults(m.json.runtimeMs, m.json.error, m.json.results); }
+ );
+ g.browserMessageManager.addMessageListener(
+ "reftest:StartPrint",
+ function (m) { RecvStartPrint(m.json.isPrintSelection, m.json.printRange); }
+ );
+ g.browserMessageManager.addMessageListener(
+ "reftest:PrintResult",
+ function (m) { RecvPrintResult(m.json.runtimeMs, m.json.status, m.json.fileName); }
+ );
+ g.browserMessageManager.addMessageListener(
+ "reftest:TestDone",
+ function (m) { RecvTestDone(m.json.runtimeMs); }
+ );
+ g.browserMessageManager.addMessageListener(
+ "reftest:UpdateCanvasForInvalidation",
+ function (m) { RecvUpdateCanvasForInvalidation(m.json.rects); }
+ );
+ g.browserMessageManager.addMessageListener(
+ "reftest:UpdateWholeCanvasForInvalidation",
+ function (m) { RecvUpdateWholeCanvasForInvalidation(); }
+ );
+ g.browserMessageManager.addMessageListener(
+ "reftest:ExpectProcessCrash",
+ function (m) { RecvExpectProcessCrash(); }
+ );
+
+ g.browserMessageManager.loadFrameScript("resource://reftest/reftest-content.js", true, true);
+
+ if (aReload) {
+ return;
+ }
+
+ ChromeUtils.registerWindowActor("ReftestFission", {
+ parent: {
+ moduleURI: "resource://reftest/ReftestFissionParent.jsm",
+ },
+ child: {
+ moduleURI: "resource://reftest/ReftestFissionChild.jsm",
+ events: {
+ MozAfterPaint: {},
+ },
+ },
+ allFrames: true,
+ includeChrome: true,
+ });
+}
+
+async function RecvAssertionCount(count)
+{
+ await DoAssertionCheck(count);
+}
+
+function RecvContentReady(info)
+{
+ if (g.resolveContentReady) {
+ g.resolveContentReady();
+ g.resolveContentReady = null;
+ } else {
+ g.contentGfxInfo = info.gfx;
+ InitAndStartRefTests();
+ }
+ return { remote: g.browserIsRemote };
+}
+
+function RecvException(what)
+{
+ logger.error(g.currentURL + " | " + what);
+ ++g.testResults.Exception;
+}
+
+function RecvFailedLoad(why)
+{
+ LoadFailed(why);
+}
+
+function RecvFailedNoPaint()
+{
+ g.failedNoPaint = true;
+}
+
+function RecvFailedNoDisplayList()
+{
+ g.failedNoDisplayList = true;
+}
+
+function RecvFailedDisplayList()
+{
+ g.failedDisplayList = true;
+}
+
+function RecvFailedOpaqueLayer(why) {
+ g.failedOpaqueLayer = true;
+ g.failedOpaqueLayerMessages.push(why);
+}
+
+function RecvFailedAssignedLayer(why) {
+ g.failedAssignedLayer = true;
+ g.failedAssignedLayerMessages.push(why);
+}
+
+async function RecvInitCanvasWithSnapshot()
+{
+ var painted = await InitCurrentCanvasWithSnapshot();
+ SendUpdateCurrentCanvasWithSnapshotDone(painted);
+}
+
+function RecvLog(type, msg)
+{
+ msg = "[CONTENT] " + msg;
+ if (type == "info") {
+ TestBuffer(msg);
+ } else if (type == "warning") {
+ logger.warning(msg);
+ } else if (type == "error") {
+ logger.error("REFTEST TEST-UNEXPECTED-FAIL | " + g.currentURL + " | " + msg + "\n");
+ ++g.testResults.Exception;
+ } else {
+ logger.error("REFTEST TEST-UNEXPECTED-FAIL | " + g.currentURL + " | unknown log type " + type + "\n");
+ ++g.testResults.Exception;
+ }
+}
+
+function RecvScriptResults(runtimeMs, error, results)
+{
+ RecordResult(runtimeMs, error, results);
+}
+
+function RecvStartPrint(isPrintSelection, printRange)
+{
+ let fileName =`reftest-print-${Date.now()}-`;
+ crypto.getRandomValues(new Uint8Array(4)).forEach(x => fileName += x.toString(16));
+ fileName += ".pdf"
+ let file = Services.dirsvc.get("TmpD", Ci.nsIFile);
+ file.append(fileName);
+
+ let PSSVC = Cc["@mozilla.org/gfx/printsettings-service;1"].getService(Ci.nsIPrintSettingsService);
+ let ps = PSSVC.createNewPrintSettings();
+ ps.printSilent = true;
+ ps.printBGImages = true;
+ ps.printBGColors = true;
+ ps.unwriteableMarginTop = 0;
+ ps.unwriteableMarginRight = 0;
+ ps.unwriteableMarginLeft = 0;
+ ps.unwriteableMarginBottom = 0;
+ ps.outputDestination = Ci.nsIPrintSettings.kOutputDestinationFile;
+ ps.toFileName = file.path;
+ ps.outputFormat = Ci.nsIPrintSettings.kOutputFormatPDF;
+ ps.printSelectionOnly = isPrintSelection;
+ if (printRange) {
+ ps.pageRanges = printRange.split(',').map(function(r) {
+ let range = r.split('-');
+ return [+range[0] || 1, +range[1] || 1]
+ }).flat();
+ }
+
+ var prefs = Cc["@mozilla.org/preferences-service;1"].
+ getService(Ci.nsIPrefBranch);
+ ps.printInColor = prefs.getBoolPref("print.print_in_color", true);
+
+ g.browser.browsingContext.print(ps)
+ .then(() => SendPrintDone(Cr.NS_OK, file.path))
+ .catch(exception => SendPrintDone(exception.code, file.path));
+}
+
+function RecvPrintResult(runtimeMs, status, fileName)
+{
+ if (!Components.isSuccessCode(status)) {
+ logger.error("REFTEST TEST-UNEXPECTED-FAIL | " + g.currentURL + " | error during printing\n");
+ ++g.testResults.Exception;
+ }
+ RecordResult(runtimeMs, '', fileName);
+}
+
+function RecvTestDone(runtimeMs)
+{
+ RecordResult(runtimeMs, '', [ ]);
+}
+
+async function RecvUpdateCanvasForInvalidation(rects)
+{
+ await UpdateCurrentCanvasForInvalidation(rects);
+ SendUpdateCurrentCanvasWithSnapshotDone(true);
+}
+
+async function RecvUpdateWholeCanvasForInvalidation()
+{
+ await UpdateWholeCurrentCanvasForInvalidation();
+ SendUpdateCurrentCanvasWithSnapshotDone(true);
+}
+
+function OnProcessCrashed(subject, topic, data)
+{
+ let id;
+ let additionalDumps;
+ let propbag = subject.QueryInterface(Ci.nsIPropertyBag2);
+
+ if (topic == "ipc:content-shutdown") {
+ id = propbag.get("dumpID");
+ }
+
+ if (id) {
+ g.expectedCrashDumpFiles.push(id + ".dmp");
+ g.expectedCrashDumpFiles.push(id + ".extra");
+ }
+
+ if (additionalDumps && additionalDumps.length != 0) {
+ for (const name of additionalDumps.split(',')) {
+ g.expectedCrashDumpFiles.push(id + "-" + name + ".dmp");
+ }
+ }
+}
+
+function RegisterProcessCrashObservers()
+{
+ var os = Cc[NS_OBSERVER_SERVICE_CONTRACTID]
+ .getService(Ci.nsIObserverService);
+ os.addObserver(OnProcessCrashed, "ipc:content-shutdown");
+}
+
+function RecvExpectProcessCrash()
+{
+ g.expectingProcessCrash = true;
+}
+
+function SendClear()
+{
+ g.browserMessageManager.sendAsyncMessage("reftest:Clear");
+}
+
+function SendLoadScriptTest(uri, timeout)
+{
+ g.browserMessageManager.sendAsyncMessage("reftest:LoadScriptTest",
+ { uri: uri, timeout: timeout });
+}
+
+function SendLoadPrintTest(uri, timeout)
+{
+ g.browserMessageManager.sendAsyncMessage("reftest:LoadPrintTest",
+ { uri: uri, timeout: timeout });
+}
+
+function SendLoadTest(type, uri, uriTargetType, timeout)
+{
+ g.browserMessageManager.sendAsyncMessage("reftest:LoadTest",
+ { type: type, uri: uri,
+ uriTargetType: uriTargetType,
+ timeout: timeout }
+ );
+}
+
+function SendResetRenderingState()
+{
+ g.browserMessageManager.sendAsyncMessage("reftest:ResetRenderingState");
+}
+
+function SendPrintDone(status, fileName)
+{
+ g.browserMessageManager.sendAsyncMessage("reftest:PrintDone", { status, fileName });
+}
+
+function SendUpdateCurrentCanvasWithSnapshotDone(painted)
+{
+ g.browserMessageManager.sendAsyncMessage("reftest:UpdateCanvasWithSnapshotDone", { painted });
+}
+
+var pdfjsHasLoaded;
+
+function pdfjsHasLoadedPromise() {
+ if (pdfjsHasLoaded === undefined) {
+ pdfjsHasLoaded = new Promise((resolve, reject) => {
+ let doc = g.containingWindow.document;
+ const script = doc.createElement("script");
+ script.src = "resource://pdf.js/build/pdf.js";
+ script.onload = resolve;
+ script.onerror = () => reject(new Error("PDF.js script load failed."));
+ doc.documentElement.appendChild(script);
+ });
+ }
+
+ return pdfjsHasLoaded;
+}
+
+function readPdf(path, callback) {
+ IOUtils.read(path).then(function (data) {
+ pdfjsLib.GlobalWorkerOptions.workerSrc = "resource://pdf.js/build/pdf.worker.js";
+ pdfjsLib.getDocument({
+ data: data
+ }).promise.then(function (pdf) {
+ callback(null, pdf);
+ }, function (e) {
+ callback(new Error(`Couldn't parse ${path}, exception: ${e}`));
+ });
+ return;
+ }, function (e) {
+ callback(new Error(`Couldn't read PDF ${path}, exception: ${e}`));
+ });
+}
+
+function comparePdfs(pathToTestPdf, pathToRefPdf, callback) {
+ pdfjsHasLoadedPromise().then(() =>
+ Promise.all([pathToTestPdf, pathToRefPdf].map(function(path) {
+ return new Promise(function(resolve, reject) {
+ readPdf(path, function(error, pdf) {
+ // Resolve or reject outer promise. reject and resolve are
+ // passed to the callback function given as first arguments
+ // to the Promise constructor.
+ if (error) {
+ reject(error);
+ } else {
+ resolve(pdf);
+ }
+ });
+ });
+ }))).then(function(pdfs) {
+ let numberOfPages = pdfs[1].numPages;
+ let sameNumberOfPages = numberOfPages === pdfs[0].numPages;
+
+ let resultPromises = [Promise.resolve({
+ passed: sameNumberOfPages,
+ description: "Expected number of pages: " + numberOfPages +
+ ", got " + pdfs[0].numPages
+ })];
+
+ if (sameNumberOfPages) {
+ for (let i = 0; i < numberOfPages; i++) {
+ let pageNum = i + 1;
+ let testPagePromise = pdfs[0].getPage(pageNum);
+ let refPagePromise = pdfs[1].getPage(pageNum);
+ resultPromises.push(new Promise(function(resolve, reject) {
+ Promise.all([testPagePromise, refPagePromise]).then(function(pages) {
+ let testTextPromise = pages[0].getTextContent();
+ let refTextPromise = pages[1].getTextContent();
+ Promise.all([testTextPromise, refTextPromise]).then(function(texts) {
+ let testTextItems = texts[0].items;
+ let refTextItems = texts[1].items;
+ let testText;
+ let refText;
+ let passed = refTextItems.every(function(o, i) {
+ refText = o.str;
+ if (!testTextItems[i]) {
+ return false;
+ }
+ testText = testTextItems[i].str;
+ return testText === refText;
+ });
+ let description;
+ if (passed) {
+ if (testTextItems.length > refTextItems.length) {
+ passed = false;
+ description = "Page " + pages[0].pageNumber +
+ " contains unexpected text like '" +
+ testTextItems[refTextItems.length].str + "'";
+ } else {
+ description = "Page " + pages[0].pageNumber +
+ " contains same text"
+ }
+ } else {
+ description = "Expected page " + pages[0].pageNumber +
+ " to contain text '" + refText;
+ if (testText) {
+ description += "' but found '" + testText +
+ "' instead";
+ }
+ }
+ resolve({
+ passed: passed,
+ description: description
+ });
+ }, reject);
+ }, reject);
+ }));
+ }
+ }
+
+ Promise.all(resultPromises).then(function (results) {
+ callback(null, results);
+ });
+ }, function(error) {
+ callback(error);
+ });
+}
diff --git a/layout/tools/reftest/reftest.xhtml b/layout/tools/reftest/reftest.xhtml
new file mode 100644
index 0000000000..924d9e73d2
--- /dev/null
+++ b/layout/tools/reftest/reftest.xhtml
@@ -0,0 +1,13 @@
+<!-- vim: set shiftwidth=4 tabstop=8 autoindent expandtab: -->
+<!-- This Source Code Form is subject to the terms of the Mozilla Public
+ - License, v. 2.0. If a copy of the MPL was not distributed with this
+ - file, You can obtain one at http://mozilla.org/MPL/2.0/. -->
+<window xmlns="http://www.mozilla.org/keymaster/gatekeeper/there.is.only.xul"
+ id="reftest-window"
+ hidechrome="true"
+ onload="OnRefTestLoad();"
+ onunload="OnRefTestUnload();"
+ style="background:white; overflow:hidden">
+ <script type="application/ecmascript" src="resource://reftest/reftest.jsm" />
+ <!-- The reftest browser element is dynamically created, here -->
+</window>
diff --git a/layout/tools/reftest/reftest/__init__.py b/layout/tools/reftest/reftest/__init__.py
new file mode 100644
index 0000000000..f82a07ec44
--- /dev/null
+++ b/layout/tools/reftest/reftest/__init__.py
@@ -0,0 +1,164 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import io
+import os
+import re
+
+import six
+
+RE_COMMENT = re.compile(r"\s+#")
+RE_HTTP = re.compile(r"HTTP\((\.\.(\/\.\.)*)\)")
+RE_PROTOCOL = re.compile(r"^\w+:")
+FAILURE_TYPES = (
+ "fails",
+ "fails-if",
+ "needs-focus",
+ "random",
+ "random-if",
+ "silentfail",
+ "silentfail-if",
+ "skip",
+ "skip-if",
+ "slow",
+ "slow-if",
+ "fuzzy",
+ "fuzzy-if",
+ "require-or",
+ "asserts",
+ "asserts-if",
+)
+PREF_ITEMS = (
+ "pref",
+ "test-pref",
+ "ref-pref",
+)
+RE_ANNOTATION = re.compile(r"(.*)\((.*)\)")
+
+
+class ReftestManifest(object):
+ """Represents a parsed reftest manifest."""
+
+ def __init__(self, finder=None):
+ self.path = None
+ self.dirs = set()
+ self.files = set()
+ self.manifests = set()
+ self.tests = []
+ self.finder = finder
+
+ def load(self, path):
+ """Parse a reftest manifest file."""
+
+ def add_test(file, annotations, referenced_test=None):
+ # We can't package about:, data:, or chrome: URIs.
+ # Discarding data isn't correct for a parser. But retaining
+ # all data isn't currently a requirement.
+ if RE_PROTOCOL.match(file):
+ return
+ test = os.path.normpath(os.path.join(mdir, urlprefix + file))
+ if test in self.files:
+ # if test path has already been added, make no changes, to
+ # avoid duplicate paths in self.tests
+ return
+ self.files.add(test)
+ self.dirs.add(os.path.dirname(test))
+ test_dict = {
+ "path": test,
+ "here": os.path.dirname(test),
+ "manifest": normalized_path,
+ "name": os.path.basename(test),
+ "head": "",
+ "support-files": "",
+ "subsuite": "",
+ }
+ if referenced_test:
+ test_dict["referenced-test"] = referenced_test
+ for annotation in annotations:
+ m = RE_ANNOTATION.match(annotation)
+ if m:
+ if m.group(1) not in test_dict:
+ test_dict[m.group(1)] = m.group(2)
+ else:
+ test_dict[m.group(1)] += ";" + m.group(2)
+ else:
+ test_dict[annotation] = None
+ self.tests.append(test_dict)
+
+ normalized_path = os.path.normpath(os.path.abspath(path))
+ self.manifests.add(normalized_path)
+ if not self.path:
+ self.path = normalized_path
+
+ mdir = os.path.dirname(normalized_path)
+ self.dirs.add(mdir)
+
+ if self.finder:
+ lines = self.finder.get(path).read().splitlines()
+ else:
+ with io.open(path, "r", encoding="utf-8") as fh:
+ lines = fh.read().splitlines()
+
+ urlprefix = ""
+ defaults = []
+ for i, line in enumerate(lines):
+ lineno = i + 1
+ line = six.ensure_text(line)
+
+ # Entire line is a comment.
+ if line.startswith("#"):
+ continue
+
+ # Comments can begin mid line. Strip them.
+ m = RE_COMMENT.search(line)
+ if m:
+ line = line[: m.start()]
+ line = line.strip()
+ if not line:
+ continue
+
+ items = line.split()
+ if items[0] == "defaults":
+ defaults = items[1:]
+ continue
+
+ items = defaults + items
+ annotations = []
+ for i in range(len(items)):
+ item = items[i]
+
+ if item.startswith(FAILURE_TYPES) or item.startswith(PREF_ITEMS):
+ annotations += [item]
+ continue
+ if item == "HTTP":
+ continue
+
+ m = RE_HTTP.match(item)
+ if m:
+ # Need to package the referenced directory.
+ self.dirs.add(os.path.normpath(os.path.join(mdir, m.group(1))))
+ continue
+
+ if i < len(defaults):
+ raise ValueError(
+ "Error parsing manifest {}, line {}: "
+ "Invalid defaults token '{}'".format(path, lineno, item)
+ )
+
+ if item == "url-prefix":
+ urlprefix = items[i + 1]
+ break
+
+ if item == "include":
+ self.load(os.path.join(mdir, items[i + 1]))
+ break
+
+ if item == "load" or item == "script":
+ add_test(items[i + 1], annotations)
+ break
+
+ if item == "==" or item == "!=" or item == "print":
+ add_test(items[i + 1], annotations)
+ add_test(items[i + 2], annotations, items[i + 1])
+ break
diff --git a/layout/tools/reftest/reftestcommandline.py b/layout/tools/reftest/reftestcommandline.py
new file mode 100644
index 0000000000..e2f0baff8c
--- /dev/null
+++ b/layout/tools/reftest/reftestcommandline.py
@@ -0,0 +1,645 @@
+import argparse
+import os
+import sys
+from collections import OrderedDict
+
+import mozinfo
+import mozlog
+from six.moves.urllib.parse import urlparse
+
+here = os.path.abspath(os.path.dirname(__file__))
+
+
+class ReftestArgumentsParser(argparse.ArgumentParser):
+ def __init__(self, **kwargs):
+ super(ReftestArgumentsParser, self).__init__(**kwargs)
+
+ # Try to import a MozbuildObject. Success indicates that we are
+ # running from a source tree. This allows some defaults to be set
+ # from the source tree.
+ try:
+ from mozbuild.base import MozbuildObject
+
+ self.build_obj = MozbuildObject.from_environment(cwd=here)
+ except ImportError:
+ self.build_obj = None
+
+ self.add_argument(
+ "--xre-path",
+ action="store",
+ type=str,
+ dest="xrePath",
+ # individual scripts will set a sane default
+ default=None,
+ help="absolute path to directory containing XRE (probably xulrunner)",
+ )
+
+ self.add_argument(
+ "--symbols-path",
+ action="store",
+ type=str,
+ dest="symbolsPath",
+ default=None,
+ help="absolute path to directory containing breakpad symbols, "
+ "or the URL of a zip file containing symbols",
+ )
+
+ self.add_argument(
+ "--debugger",
+ action="store",
+ dest="debugger",
+ help="use the given debugger to launch the application",
+ )
+
+ self.add_argument(
+ "--debugger-args",
+ action="store",
+ dest="debuggerArgs",
+ help="pass the given args to the debugger _before_ "
+ "the application on the command line",
+ )
+
+ self.add_argument(
+ "--debugger-interactive",
+ action="store_true",
+ dest="debuggerInteractive",
+ help="prevents the test harness from redirecting "
+ "stdout and stderr for interactive debuggers",
+ )
+
+ self.add_argument(
+ "--appname",
+ action="store",
+ type=str,
+ dest="app",
+ default=None,
+ help="absolute path to application, overriding default",
+ )
+
+ self.add_argument(
+ "--extra-profile-file",
+ action="append",
+ dest="extraProfileFiles",
+ default=[],
+ help="copy specified files/dirs to testing profile",
+ )
+
+ self.add_argument(
+ "--timeout",
+ action="store",
+ dest="timeout",
+ type=int,
+ default=300, # 5 minutes per bug 479518
+ help="reftest will timeout in specified number of seconds. "
+ "[default %(default)s].",
+ )
+
+ self.add_argument(
+ "--leak-threshold",
+ action="store",
+ type=int,
+ dest="defaultLeakThreshold",
+ default=0,
+ help="fail if the number of bytes leaked in default "
+ "processes through refcounted objects (or bytes "
+ "in classes with MOZ_COUNT_CTOR and MOZ_COUNT_DTOR) "
+ "is greater than the given number",
+ )
+
+ self.add_argument(
+ "--utility-path",
+ action="store",
+ type=str,
+ dest="utilityPath",
+ default=self.build_obj.bindir if self.build_obj else None,
+ help="absolute path to directory containing utility "
+ "programs (xpcshell, ssltunnel, certutil)",
+ )
+
+ self.add_argument(
+ "--total-chunks",
+ type=int,
+ dest="totalChunks",
+ help="how many chunks to split the tests up into",
+ )
+
+ self.add_argument(
+ "--this-chunk",
+ type=int,
+ dest="thisChunk",
+ help="which chunk to run between 1 and --total-chunks",
+ )
+
+ self.add_argument(
+ "--log-file",
+ action="store",
+ type=str,
+ dest="logFile",
+ default=None,
+ help="file to log output to in addition to stdout",
+ )
+
+ self.add_argument(
+ "--skip-slow-tests",
+ dest="skipSlowTests",
+ action="store_true",
+ default=False,
+ help="skip tests marked as slow when running",
+ )
+
+ self.add_argument(
+ "--ignore-window-size",
+ dest="ignoreWindowSize",
+ action="store_true",
+ default=False,
+ help="ignore the window size, which may cause spurious "
+ "failures and passes",
+ )
+
+ self.add_argument(
+ "--install-extension",
+ action="append",
+ dest="extensionsToInstall",
+ default=[],
+ help="install the specified extension in the testing profile. "
+ "The extension file's name should be <id>.xpi where <id> is "
+ "the extension's id as indicated in its install.rdf. "
+ "An optional path can be specified too.",
+ )
+
+ self.add_argument(
+ "--marionette",
+ default=None,
+ help="host:port to use when connecting to Marionette",
+ )
+
+ self.add_argument(
+ "--marionette-socket-timeout", default=None, help=argparse.SUPPRESS
+ )
+
+ self.add_argument(
+ "--marionette-startup-timeout", default=None, help=argparse.SUPPRESS
+ )
+
+ self.add_argument(
+ "--setenv",
+ action="append",
+ type=str,
+ default=[],
+ dest="environment",
+ metavar="NAME=VALUE",
+ help="sets the given variable in the application's " "environment",
+ )
+
+ self.add_argument(
+ "--filter",
+ action="store",
+ type=str,
+ dest="filter",
+ help="specifies a regular expression (as could be passed to the JS "
+ "RegExp constructor) to test against URLs in the reftest manifest; "
+ "only test items that have a matching test URL will be run.",
+ )
+
+ self.add_argument(
+ "--shuffle",
+ action="store_true",
+ default=False,
+ dest="shuffle",
+ help="run reftests in random order",
+ )
+
+ self.add_argument(
+ "--run-until-failure",
+ action="store_true",
+ default=False,
+ dest="runUntilFailure",
+ help="stop running on the first failure. Useful for RR recordings.",
+ )
+
+ self.add_argument(
+ "--repeat",
+ action="store",
+ type=int,
+ default=0,
+ dest="repeat",
+ help="number of times the select test(s) will be executed. Useful for "
+ "finding intermittent failures.",
+ )
+
+ self.add_argument(
+ "--focus-filter-mode",
+ action="store",
+ type=str,
+ dest="focusFilterMode",
+ default="all",
+ help="filters tests to run by whether they require focus. "
+ "Valid values are `all', `needs-focus', or `non-needs-focus'. "
+ "Defaults to `all'.",
+ )
+
+ self.add_argument(
+ "--disable-e10s",
+ action="store_false",
+ default=True,
+ dest="e10s",
+ help="disables content processes",
+ )
+
+ self.add_argument(
+ "--disable-fission",
+ action="store_true",
+ default=False,
+ dest="disableFission",
+ help="Run tests with fission (site isolation) disabled.",
+ )
+
+ self.add_argument(
+ "--setpref",
+ action="append",
+ type=str,
+ default=[],
+ dest="extraPrefs",
+ metavar="PREF=VALUE",
+ help="defines an extra user preference",
+ )
+
+ self.add_argument(
+ "--reftest-extension-path",
+ action="store",
+ dest="reftestExtensionPath",
+ help="Path to the reftest extension",
+ )
+
+ self.add_argument(
+ "--special-powers-extension-path",
+ action="store",
+ dest="specialPowersExtensionPath",
+ help="Path to the special powers extension",
+ )
+
+ self.add_argument(
+ "--suite",
+ choices=["reftest", "crashtest", "jstestbrowser"],
+ default=None,
+ help=argparse.SUPPRESS,
+ )
+
+ self.add_argument(
+ "--cleanup-crashes",
+ action="store_true",
+ dest="cleanupCrashes",
+ default=False,
+ help="Delete pending crash reports before running tests.",
+ )
+
+ self.add_argument(
+ "--max-retries",
+ type=int,
+ dest="maxRetries",
+ default=4,
+ help="The maximum number of attempts to try and recover from a "
+ "crash before aborting the test run [default 4].",
+ )
+
+ self.add_argument(
+ "tests",
+ metavar="TEST_PATH",
+ nargs="*",
+ help="Path to test file, manifest file, or directory containing "
+ "tests. For jstestbrowser, the relative path can be either from "
+ "topsrcdir or the staged area "
+ "($OBJDIR/dist/test-stage/jsreftest/tests)",
+ )
+
+ self.add_argument(
+ "--sandbox-read-whitelist",
+ action="append",
+ dest="sandboxReadWhitelist",
+ default=[],
+ help="Path to add to the sandbox whitelist.",
+ )
+
+ self.add_argument(
+ "--verify",
+ action="store_true",
+ default=False,
+ help="Run tests in verification mode: Run many times in different "
+ "ways, to see if there are intermittent failures.",
+ )
+
+ self.add_argument(
+ "--verify-max-time",
+ type=int,
+ default=3600,
+ help="Maximum time, in seconds, to run in --verify mode..",
+ )
+
+ self.add_argument(
+ "--enable-webrender",
+ action="store_true",
+ dest="enable_webrender",
+ default=False,
+ help="Enable the WebRender compositor in Gecko.",
+ )
+
+ self.add_argument(
+ "--headless",
+ action="store_true",
+ dest="headless",
+ default=False,
+ help="Run tests in headless mode.",
+ )
+
+ self.add_argument(
+ "--topsrcdir",
+ action="store",
+ type=str,
+ dest="topsrcdir",
+ default=None,
+ help="Path to source directory",
+ )
+
+ mozlog.commandline.add_logging_group(self)
+
+ def get_ip(self):
+ import moznetwork
+
+ if os.name != "nt":
+ return moznetwork.get_ip()
+ else:
+ self.error("ERROR: you must specify a --remote-webserver=<ip address>\n")
+
+ def set_default_suite(self, options):
+ manifests = OrderedDict(
+ [
+ ("reftest.list", "reftest"),
+ ("crashtests.list", "crashtest"),
+ ("jstests.list", "jstestbrowser"),
+ ]
+ )
+
+ for test_path in options.tests:
+ file_name = os.path.basename(test_path)
+ if file_name in manifests:
+ options.suite = manifests[file_name]
+ return
+
+ for test_path in options.tests:
+ for manifest_file, suite in manifests.iteritems():
+ if os.path.exists(os.path.join(test_path, manifest_file)):
+ options.suite = suite
+ return
+
+ self.error(
+ "Failed to determine test suite; supply --suite to set this explicitly"
+ )
+
+ def validate(self, options, reftest):
+ if not options.tests:
+ # Can't just set this in the argument parser because mach will set a default
+ self.error(
+ "Must supply at least one path to a manifest file, "
+ "test directory, or test file to run."
+ )
+
+ if options.suite is None:
+ self.set_default_suite(options)
+
+ if options.totalChunks is not None and options.thisChunk is None:
+ self.error("thisChunk must be specified when totalChunks is specified")
+
+ if options.totalChunks:
+ if not 1 <= options.thisChunk <= options.totalChunks:
+ self.error("thisChunk must be between 1 and totalChunks")
+
+ if not options.disableFission and not options.e10s:
+ self.error("Fission is not supported without e10s.")
+
+ if options.logFile:
+ options.logFile = reftest.getFullPath(options.logFile)
+
+ if options.xrePath is not None:
+ if not os.access(options.xrePath, os.F_OK):
+ self.error("--xre-path '%s' not found" % options.xrePath)
+ if not os.path.isdir(options.xrePath):
+ self.error("--xre-path '%s' is not a directory" % options.xrePath)
+ options.xrePath = reftest.getFullPath(options.xrePath)
+
+ if options.reftestExtensionPath is None:
+ if self.build_obj is not None:
+ reftestExtensionPath = os.path.join(
+ self.build_obj.distdir, "xpi-stage", "reftest"
+ )
+ else:
+ reftestExtensionPath = os.path.join(here, "reftest")
+ options.reftestExtensionPath = os.path.normpath(reftestExtensionPath)
+
+ if options.specialPowersExtensionPath is None:
+ if self.build_obj is not None:
+ specialPowersExtensionPath = os.path.join(
+ self.build_obj.distdir, "xpi-stage", "specialpowers"
+ )
+ else:
+ specialPowersExtensionPath = os.path.join(here, "specialpowers")
+ options.specialPowersExtensionPath = os.path.normpath(
+ specialPowersExtensionPath
+ )
+
+ options.leakThresholds = {
+ "default": options.defaultLeakThreshold,
+ "tab": options.defaultLeakThreshold,
+ }
+
+ if mozinfo.isWin:
+ if mozinfo.info["bits"] == 32:
+ # See bug 1408554.
+ options.leakThresholds["tab"] = 3000
+ else:
+ # See bug 1404482.
+ options.leakThresholds["tab"] = 100
+
+ if options.topsrcdir is None:
+ if self.build_obj:
+ options.topsrcdir = self.build_obj.topsrcdir
+ else:
+ options.topsrcdir = os.getcwd()
+
+
+class DesktopArgumentsParser(ReftestArgumentsParser):
+ def __init__(self, **kwargs):
+ super(DesktopArgumentsParser, self).__init__(**kwargs)
+
+ self.add_argument(
+ "--run-tests-in-parallel",
+ action="store_true",
+ default=False,
+ dest="runTestsInParallel",
+ help="run tests in parallel if possible",
+ )
+
+ def _prefs_gpu(self):
+ if mozinfo.os != "win":
+ return ["layers.acceleration.force-enabled=true"]
+ return []
+
+ def validate(self, options, reftest):
+ super(DesktopArgumentsParser, self).validate(options, reftest)
+
+ if options.runTestsInParallel:
+ if options.logFile is not None:
+ self.error("cannot specify logfile with parallel tests")
+ if options.totalChunks is not None or options.thisChunk is not None:
+ self.error(
+ "cannot specify thisChunk or totalChunks with parallel tests"
+ )
+ if options.focusFilterMode != "all":
+ self.error("cannot specify focusFilterMode with parallel tests")
+ if options.debugger is not None:
+ self.error("cannot specify a debugger with parallel tests")
+
+ if options.debugger:
+ # valgrind and some debuggers may cause Gecko to start slowly. Make sure
+ # marionette waits long enough to connect.
+ options.marionette_startup_timeout = 900
+ options.marionette_socket_timeout = 540
+
+ if not options.tests:
+ self.error("No test files specified.")
+
+ if options.app is None:
+ if (
+ self.build_obj
+ and self.build_obj.substs["MOZ_BUILD_APP"] != "mobile/android"
+ ):
+ from mozbuild.base import BinaryNotFoundException
+
+ try:
+ bin_dir = self.build_obj.get_binary_path()
+ except BinaryNotFoundException as e:
+ print("{}\n\n{}\n".format(e, e.help()), file=sys.stderr)
+ sys.exit(1)
+ else:
+ bin_dir = None
+
+ if bin_dir:
+ options.app = bin_dir
+
+ if options.symbolsPath and len(urlparse(options.symbolsPath).scheme) < 2:
+ options.symbolsPath = reftest.getFullPath(options.symbolsPath)
+
+ options.utilityPath = reftest.getFullPath(options.utilityPath)
+
+
+class RemoteArgumentsParser(ReftestArgumentsParser):
+ def __init__(self, **kwargs):
+ super(RemoteArgumentsParser, self).__init__()
+
+ # app, xrePath and utilityPath variables are set in main function
+ self.set_defaults(
+ logFile="reftest.log", app="", xrePath="", utilityPath="", localLogName=None
+ )
+
+ self.add_argument(
+ "--adbpath",
+ action="store",
+ type=str,
+ dest="adb_path",
+ default=None,
+ help="Path to adb binary.",
+ )
+
+ self.add_argument(
+ "--deviceSerial",
+ action="store",
+ type=str,
+ dest="deviceSerial",
+ help="adb serial number of remote device. This is required "
+ "when more than one device is connected to the host. "
+ "Use 'adb devices' to see connected devices.",
+ )
+
+ self.add_argument(
+ "--remote-webserver",
+ action="store",
+ type=str,
+ dest="remoteWebServer",
+ help="IP address of the remote web server.",
+ )
+
+ self.add_argument(
+ "--http-port",
+ action="store",
+ type=str,
+ dest="httpPort",
+ help="http port of the remote web server.",
+ )
+
+ self.add_argument(
+ "--ssl-port",
+ action="store",
+ type=str,
+ dest="sslPort",
+ help="ssl port of the remote web server.",
+ )
+
+ self.add_argument(
+ "--remoteTestRoot",
+ action="store",
+ type=str,
+ dest="remoteTestRoot",
+ help="Remote directory to use as test root "
+ "(eg. /data/local/tmp/test_root).",
+ )
+
+ self.add_argument(
+ "--httpd-path",
+ action="store",
+ type=str,
+ dest="httpdPath",
+ help="Path to the httpd.js file.",
+ )
+
+ self.add_argument(
+ "--no-install",
+ action="store_true",
+ default=False,
+ help="Skip the installation of the APK.",
+ )
+
+ def validate_remote(self, options):
+ DEFAULT_HTTP_PORT = 8888
+ DEFAULT_SSL_PORT = 4443
+
+ if options.remoteWebServer is None:
+ options.remoteWebServer = self.get_ip()
+
+ if options.remoteWebServer == "127.0.0.1":
+ self.error(
+ "ERROR: Either you specified the loopback for the remote webserver or ",
+ "your local IP cannot be detected. "
+ "Please provide the local ip in --remote-webserver",
+ )
+
+ if not options.httpPort:
+ options.httpPort = DEFAULT_HTTP_PORT
+
+ if not options.sslPort:
+ options.sslPort = DEFAULT_SSL_PORT
+
+ if options.xrePath is None:
+ self.error(
+ "ERROR: You must specify the path to the controller xre directory"
+ )
+ else:
+ # Ensure xrepath is a full path
+ options.xrePath = os.path.abspath(options.xrePath)
+
+ # httpd-path is specified by standard makefile targets and may be specified
+ # on the command line to select a particular version of httpd.js. If not
+ # specified, try to select the one from hostutils.zip, as required in
+ # bug 882932.
+ if not options.httpdPath:
+ options.httpdPath = os.path.join(options.utilityPath, "components")
+
+ return options
diff --git a/layout/tools/reftest/remotereftest.py b/layout/tools/reftest/remotereftest.py
new file mode 100644
index 0000000000..8d5e3fb44f
--- /dev/null
+++ b/layout/tools/reftest/remotereftest.py
@@ -0,0 +1,545 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import datetime
+import os
+import posixpath
+import shutil
+import signal
+import subprocess
+import sys
+import tempfile
+import time
+import traceback
+from contextlib import closing
+
+import mozcrash
+import reftestcommandline
+from mozdevice import ADBDeviceFactory, RemoteProcessMonitor
+from output import OutputHandler
+from runreftest import RefTest, ReftestResolver, build_obj
+from six.moves.urllib_request import urlopen
+
+# We need to know our current directory so that we can serve our test files from it.
+SCRIPT_DIRECTORY = os.path.abspath(os.path.realpath(os.path.dirname(__file__)))
+
+
+class RemoteReftestResolver(ReftestResolver):
+ def absManifestPath(self, path):
+ script_abs_path = os.path.join(SCRIPT_DIRECTORY, path)
+ if os.path.exists(script_abs_path):
+ rv = script_abs_path
+ elif os.path.exists(os.path.abspath(path)):
+ rv = os.path.abspath(path)
+ else:
+ print("Could not find manifest %s" % script_abs_path, file=sys.stderr)
+ sys.exit(1)
+ return os.path.normpath(rv)
+
+ def manifestURL(self, options, path):
+ # Dynamically build the reftest URL if possible, beware that
+ # args[0] should exist 'inside' webroot. It's possible for
+ # this url to have a leading "..", but reftest.js will fix
+ # that. Use the httpdPath to determine if we are running in
+ # production or locally. If we are running the jsreftests
+ # locally, strip text up to jsreftest. We want the docroot of
+ # the server to include a link jsreftest that points to the
+ # test-stage location of the test files. The desktop oriented
+ # setup has already created a link for tests which points
+ # directly into the source tree. For the remote tests we need
+ # a separate symbolic link to point to the staged test files.
+ if "jsreftest" not in path or os.environ.get("MOZ_AUTOMATION"):
+ relPath = os.path.relpath(path, SCRIPT_DIRECTORY)
+ else:
+ relPath = "jsreftest/" + path.split("jsreftest/")[-1]
+ return "http://%s:%s/%s" % (options.remoteWebServer, options.httpPort, relPath)
+
+
+class ReftestServer:
+ """Web server used to serve Reftests, for closer fidelity to the real web.
+ It is virtually identical to the server used in mochitest and will only
+ be used for running reftests remotely.
+ Bug 581257 has been filed to refactor this wrapper around httpd.js into
+ it's own class and use it in both remote and non-remote testing."""
+
+ def __init__(self, options, scriptDir, log):
+ self.log = log
+ self.utilityPath = options.utilityPath
+ self.xrePath = options.xrePath
+ self.profileDir = options.serverProfilePath
+ self.webServer = options.remoteWebServer
+ self.httpPort = options.httpPort
+ self.scriptDir = scriptDir
+ self.httpdPath = os.path.abspath(options.httpdPath)
+ if options.remoteWebServer == "10.0.2.2":
+ # probably running an Android emulator and 10.0.2.2 will
+ # not be visible from host
+ shutdownServer = "127.0.0.1"
+ else:
+ shutdownServer = self.webServer
+ self.shutdownURL = "http://%(server)s:%(port)s/server/shutdown" % {
+ "server": shutdownServer,
+ "port": self.httpPort,
+ }
+
+ def start(self):
+ "Run the Refest server, returning the process ID of the server."
+
+ env = dict(os.environ)
+ env["XPCOM_DEBUG_BREAK"] = "warn"
+ bin_suffix = ""
+ if sys.platform in ("win32", "msys", "cygwin"):
+ env["PATH"] = env["PATH"] + ";" + self.xrePath
+ bin_suffix = ".exe"
+ else:
+ if "LD_LIBRARY_PATH" not in env or env["LD_LIBRARY_PATH"] is None:
+ env["LD_LIBRARY_PATH"] = self.xrePath
+ else:
+ env["LD_LIBRARY_PATH"] = ":".join(
+ [self.xrePath, env["LD_LIBRARY_PATH"]]
+ )
+
+ args = [
+ "-g",
+ self.xrePath,
+ "-f",
+ os.path.join(self.httpdPath, "httpd.js"),
+ "-e",
+ "const _PROFILE_PATH = '%(profile)s';const _SERVER_PORT = "
+ "'%(port)s'; const _SERVER_ADDR ='%(server)s';"
+ % {
+ "profile": self.profileDir.replace("\\", "\\\\"),
+ "port": self.httpPort,
+ "server": self.webServer,
+ },
+ "-f",
+ os.path.join(self.scriptDir, "server.js"),
+ ]
+
+ xpcshell = os.path.join(self.utilityPath, "xpcshell" + bin_suffix)
+
+ if not os.access(xpcshell, os.F_OK):
+ raise Exception("xpcshell not found at %s" % xpcshell)
+ if RemoteProcessMonitor.elf_arm(xpcshell):
+ raise Exception(
+ "xpcshell at %s is an ARM binary; please use "
+ "the --utility-path argument to specify the path "
+ "to a desktop version." % xpcshell
+ )
+
+ self._process = subprocess.Popen([xpcshell] + args, env=env)
+ pid = self._process.pid
+ if pid < 0:
+ self.log.error(
+ "TEST-UNEXPECTED-FAIL | remotereftests.py | Error starting server."
+ )
+ return 2
+ self.log.info("INFO | remotereftests.py | Server pid: %d" % pid)
+
+ def ensureReady(self, timeout):
+ assert timeout >= 0
+
+ aliveFile = os.path.join(self.profileDir, "server_alive.txt")
+ i = 0
+ while i < timeout:
+ if os.path.exists(aliveFile):
+ break
+ time.sleep(1)
+ i += 1
+ else:
+ self.log.error(
+ "TEST-UNEXPECTED-FAIL | remotereftests.py | "
+ "Timed out while waiting for server startup."
+ )
+ self.stop()
+ return 1
+
+ def stop(self):
+ if hasattr(self, "_process"):
+ try:
+ with closing(urlopen(self.shutdownURL)) as c:
+ c.read()
+
+ rtncode = self._process.poll()
+ if rtncode is None:
+ self._process.terminate()
+ except Exception:
+ self.log.info("Failed to shutdown server at %s" % self.shutdownURL)
+ traceback.print_exc()
+ self._process.kill()
+
+
+class RemoteReftest(RefTest):
+ use_marionette = False
+ resolver_cls = RemoteReftestResolver
+
+ def __init__(self, options, scriptDir):
+ RefTest.__init__(self, options.suite)
+ self.run_by_manifest = False
+ self.scriptDir = scriptDir
+ self.localLogName = options.localLogName
+
+ verbose = False
+ if (
+ options.log_mach_verbose
+ or options.log_tbpl_level == "debug"
+ or options.log_mach_level == "debug"
+ or options.log_raw_level == "debug"
+ ):
+ verbose = True
+ print("set verbose!")
+ expected = options.app.split("/")[-1]
+ self.device = ADBDeviceFactory(
+ adb=options.adb_path or "adb",
+ device=options.deviceSerial,
+ test_root=options.remoteTestRoot,
+ verbose=verbose,
+ run_as_package=expected,
+ )
+ if options.remoteTestRoot is None:
+ options.remoteTestRoot = posixpath.join(self.device.test_root, "reftest")
+ options.remoteProfile = posixpath.join(options.remoteTestRoot, "profile")
+ options.remoteLogFile = posixpath.join(options.remoteTestRoot, "reftest.log")
+ options.logFile = options.remoteLogFile
+ self.remoteProfile = options.remoteProfile
+ self.remoteTestRoot = options.remoteTestRoot
+
+ if not options.ignoreWindowSize:
+ parts = self.device.get_info("screen")["screen"][0].split()
+ width = int(parts[0].split(":")[1])
+ height = int(parts[1].split(":")[1])
+ if width < 1366 or height < 1050:
+ self.error(
+ "ERROR: Invalid screen resolution %sx%s, "
+ "please adjust to 1366x1050 or higher" % (width, height)
+ )
+
+ self._populate_logger(options)
+ self.outputHandler = OutputHandler(
+ self.log, options.utilityPath, options.symbolsPath
+ )
+
+ self.SERVER_STARTUP_TIMEOUT = 90
+
+ self.remoteCache = os.path.join(options.remoteTestRoot, "cache/")
+
+ # Check that Firefox is installed
+ expected = options.app.split("/")[-1]
+ if not self.device.is_app_installed(expected):
+ raise Exception("%s is not installed on this device" % expected)
+ self.device.run_as_package = expected
+ self.device.clear_logcat()
+
+ self.device.rm(self.remoteCache, force=True, recursive=True)
+
+ procName = options.app.split("/")[-1]
+ self.device.stop_application(procName)
+ if self.device.process_exist(procName):
+ self.log.error("unable to kill %s before starting tests!" % procName)
+
+ def findPath(self, paths, filename=None):
+ for path in paths:
+ p = path
+ if filename:
+ p = os.path.join(p, filename)
+ if os.path.exists(self.getFullPath(p)):
+ return path
+ return None
+
+ def startWebServer(self, options):
+ """Create the webserver on the host and start it up"""
+ remoteXrePath = options.xrePath
+ remoteUtilityPath = options.utilityPath
+
+ paths = [options.xrePath]
+ if build_obj:
+ paths.append(os.path.join(build_obj.topobjdir, "dist", "bin"))
+ options.xrePath = self.findPath(paths)
+ if options.xrePath is None:
+ print(
+ "ERROR: unable to find xulrunner path for %s, "
+ "please specify with --xre-path" % (os.name)
+ )
+ return 1
+ paths.append("bin")
+ paths.append(os.path.join("..", "bin"))
+
+ xpcshell = "xpcshell"
+ if os.name == "nt":
+ xpcshell += ".exe"
+
+ if options.utilityPath:
+ paths.insert(0, options.utilityPath)
+ options.utilityPath = self.findPath(paths, xpcshell)
+ if options.utilityPath is None:
+ print(
+ "ERROR: unable to find utility path for %s, "
+ "please specify with --utility-path" % (os.name)
+ )
+ return 1
+
+ options.serverProfilePath = tempfile.mkdtemp()
+ self.server = ReftestServer(options, self.scriptDir, self.log)
+ retVal = self.server.start()
+ if retVal:
+ return retVal
+ retVal = self.server.ensureReady(self.SERVER_STARTUP_TIMEOUT)
+ if retVal:
+ return retVal
+
+ options.xrePath = remoteXrePath
+ options.utilityPath = remoteUtilityPath
+ return 0
+
+ def stopWebServer(self, options):
+ self.server.stop()
+
+ def killNamedProc(self, pname, orphans=True):
+ """Kill processes matching the given command name"""
+ try:
+ import psutil
+ except ImportError as e:
+ self.log.warning("Unable to import psutil: %s" % str(e))
+ self.log.warning("Unable to verify that %s is not already running." % pname)
+ return
+
+ self.log.info("Checking for %s processes..." % pname)
+
+ for proc in psutil.process_iter():
+ try:
+ if proc.name() == pname:
+ procd = proc.as_dict(attrs=["pid", "ppid", "name", "username"])
+ if proc.ppid() == 1 or not orphans:
+ self.log.info("killing %s" % procd)
+ try:
+ os.kill(
+ proc.pid, getattr(signal, "SIGKILL", signal.SIGTERM)
+ )
+ except Exception as e:
+ self.log.info(
+ "Failed to kill process %d: %s" % (proc.pid, str(e))
+ )
+ else:
+ self.log.info("NOT killing %s (not an orphan?)" % procd)
+ except Exception:
+ # may not be able to access process info for all processes
+ continue
+
+ def createReftestProfile(self, options, **kwargs):
+ profile = RefTest.createReftestProfile(
+ self,
+ options,
+ server=options.remoteWebServer,
+ port=options.httpPort,
+ **kwargs
+ )
+ profileDir = profile.profile
+ prefs = {}
+ prefs["app.update.url.android"] = ""
+ prefs["reftest.remote"] = True
+ prefs["datareporting.policy.dataSubmissionPolicyBypassAcceptance"] = True
+ # move necko cache to a location that can be cleaned up
+ prefs["browser.cache.disk.parent_directory"] = self.remoteCache
+
+ prefs["layout.css.devPixelsPerPx"] = "1.0"
+ # Because Fennec is a little wacky (see bug 1156817) we need to load the
+ # reftest pages at 1.0 zoom, rather than zooming to fit the CSS viewport.
+ prefs["apz.allow_zooming"] = False
+
+ # Set the extra prefs.
+ profile.set_preferences(prefs)
+
+ try:
+ self.device.push(profileDir, options.remoteProfile)
+ # make sure the parent directories of the profile which
+ # may have been created by the push, also have their
+ # permissions set to allow access.
+ self.device.chmod(options.remoteTestRoot, recursive=True)
+ except Exception:
+ print("Automation Error: Failed to copy profiledir to device")
+ raise
+
+ return profile
+
+ def environment(self, env=None, crashreporter=True, **kwargs):
+ # Since running remote, do not mimic the local env: do not copy os.environ
+ if env is None:
+ env = {}
+
+ if crashreporter:
+ env["MOZ_CRASHREPORTER_NO_REPORT"] = "1"
+ env["MOZ_CRASHREPORTER"] = "1"
+ env["MOZ_CRASHREPORTER_SHUTDOWN"] = "1"
+ else:
+ env["MOZ_CRASHREPORTER_DISABLE"] = "1"
+
+ # Crash on non-local network connections by default.
+ # MOZ_DISABLE_NONLOCAL_CONNECTIONS can be set to "0" to temporarily
+ # enable non-local connections for the purposes of local testing.
+ # Don't override the user's choice here. See bug 1049688.
+ env.setdefault("MOZ_DISABLE_NONLOCAL_CONNECTIONS", "1")
+
+ # Send an env var noting that we are in automation. Passing any
+ # value except the empty string will declare the value to exist.
+ #
+ # This may be used to disabled network connections during testing, e.g.
+ # Switchboard & telemetry uploads.
+ env.setdefault("MOZ_IN_AUTOMATION", "1")
+
+ # Set WebRTC logging in case it is not set yet.
+ env.setdefault("R_LOG_LEVEL", "6")
+ env.setdefault("R_LOG_DESTINATION", "stderr")
+ env.setdefault("R_LOG_VERBOSE", "1")
+
+ return env
+
+ def buildBrowserEnv(self, options, profileDir):
+ browserEnv = RefTest.buildBrowserEnv(self, options, profileDir)
+ # remove desktop environment not used on device
+ if "XPCOM_MEM_BLOAT_LOG" in browserEnv:
+ del browserEnv["XPCOM_MEM_BLOAT_LOG"]
+ return browserEnv
+
+ def runApp(
+ self,
+ options,
+ cmdargs=None,
+ timeout=None,
+ debuggerInfo=None,
+ symbolsPath=None,
+ valgrindPath=None,
+ valgrindArgs=None,
+ valgrindSuppFiles=None,
+ **profileArgs
+ ):
+ if cmdargs is None:
+ cmdargs = []
+
+ if self.use_marionette:
+ cmdargs.append("-marionette")
+
+ binary = options.app
+ profile = self.createReftestProfile(options, **profileArgs)
+
+ # browser environment
+ env = self.buildBrowserEnv(options, profile.profile)
+
+ rpm = RemoteProcessMonitor(
+ binary,
+ self.device,
+ self.log,
+ self.outputHandler,
+ options.remoteLogFile,
+ self.remoteProfile,
+ )
+ startTime = datetime.datetime.now()
+ status = 0
+ profileDirectory = self.remoteProfile + "/"
+ cmdargs.extend(("-no-remote", "-profile", profileDirectory))
+
+ pid = rpm.launch(
+ binary,
+ debuggerInfo,
+ None,
+ cmdargs,
+ env=env,
+ e10s=options.e10s,
+ )
+ self.log.info("remotereftest.py | Application pid: %d" % pid)
+ if not rpm.wait(timeout):
+ status = 1
+ self.log.info(
+ "remotereftest.py | Application ran for: %s"
+ % str(datetime.datetime.now() - startTime)
+ )
+ crashed = self.check_for_crashes(symbolsPath, rpm.last_test_seen)
+ if crashed:
+ status = 1
+
+ self.cleanup(profile.profile)
+ return status
+
+ def check_for_crashes(self, symbols_path, last_test_seen):
+ """
+ Pull any minidumps from remote profile and log any associated crashes.
+ """
+ try:
+ dump_dir = tempfile.mkdtemp()
+ remote_crash_dir = posixpath.join(self.remoteProfile, "minidumps")
+ if not self.device.is_dir(remote_crash_dir):
+ return False
+ self.device.pull(remote_crash_dir, dump_dir)
+ crashed = mozcrash.log_crashes(
+ self.log, dump_dir, symbols_path, test=last_test_seen
+ )
+ finally:
+ try:
+ shutil.rmtree(dump_dir)
+ except Exception as e:
+ self.log.warning(
+ "unable to remove directory %s: %s" % (dump_dir, str(e))
+ )
+ return crashed
+
+ def cleanup(self, profileDir):
+ self.device.rm(self.remoteTestRoot, force=True, recursive=True)
+ self.device.rm(self.remoteProfile, force=True, recursive=True)
+ self.device.rm(self.remoteCache, force=True, recursive=True)
+ RefTest.cleanup(self, profileDir)
+
+
+def run_test_harness(parser, options):
+ reftest = RemoteReftest(options, SCRIPT_DIRECTORY)
+ parser.validate_remote(options)
+ parser.validate(options, reftest)
+
+ # Hack in a symbolic link for jsreftest in the SCRIPT_DIRECTORY
+ # which is the document root for the reftest web server. This
+ # allows a separate redirection for the jsreftests which must
+ # run through the web server using the staged tests files and
+ # the desktop which will use the tests symbolic link to find
+ # the JavaScript tests.
+ jsreftest_target = str(os.path.join(SCRIPT_DIRECTORY, "jsreftest"))
+ if os.environ.get("MOZ_AUTOMATION"):
+ os.system("ln -s ../jsreftest " + jsreftest_target)
+ else:
+ jsreftest_source = os.path.join(
+ build_obj.topobjdir, "dist", "test-stage", "jsreftest"
+ )
+ if not os.path.islink(jsreftest_target):
+ os.symlink(jsreftest_source, jsreftest_target)
+
+ # Despite our efforts to clean up servers started by this script, in practice
+ # we still see infrequent cases where a process is orphaned and interferes
+ # with future tests, typically because the old server is keeping the port in use.
+ # Try to avoid those failures by checking for and killing servers before
+ # trying to start new ones.
+ reftest.killNamedProc("ssltunnel")
+ reftest.killNamedProc("xpcshell")
+
+ # Start the webserver
+ retVal = reftest.startWebServer(options)
+ if retVal:
+ return retVal
+
+ retVal = 0
+ try:
+ if options.verify:
+ retVal = reftest.verifyTests(options.tests, options)
+ else:
+ retVal = reftest.runTests(options.tests, options)
+ except Exception:
+ print("Automation Error: Exception caught while running tests")
+ traceback.print_exc()
+ retVal = 1
+
+ reftest.stopWebServer(options)
+
+ return retVal
+
+
+if __name__ == "__main__":
+ parser = reftestcommandline.RemoteArgumentsParser()
+ options = parser.parse_args()
+ sys.exit(run_test_harness(parser, options))
diff --git a/layout/tools/reftest/runreftest.py b/layout/tools/reftest/runreftest.py
new file mode 100644
index 0000000000..763acc105d
--- /dev/null
+++ b/layout/tools/reftest/runreftest.py
@@ -0,0 +1,1198 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+"""
+Runs the reftest test harness.
+"""
+import json
+import multiprocessing
+import os
+import platform
+import posixpath
+import re
+import shutil
+import signal
+import subprocess
+import sys
+import tempfile
+import threading
+from collections import defaultdict
+from datetime import datetime, timedelta
+
+SCRIPT_DIRECTORY = os.path.abspath(os.path.realpath(os.path.dirname(__file__)))
+if SCRIPT_DIRECTORY not in sys.path:
+ sys.path.insert(0, SCRIPT_DIRECTORY)
+
+import mozcrash
+import mozdebug
+import mozfile
+import mozinfo
+import mozleak
+import mozlog
+import mozprocess
+import mozprofile
+import mozrunner
+from manifestparser import TestManifest
+from manifestparser import filters as mpf
+from mozrunner.utils import get_stack_fixer_function, test_environment
+from mozscreenshot import dump_screen, printstatus
+from six import reraise, string_types
+from six.moves import range
+
+try:
+ from marionette_driver.addons import Addons
+ from marionette_driver.marionette import Marionette
+except ImportError as e: # noqa
+ # Defer ImportError until attempt to use Marionette.
+ # Python 3 deletes the exception once the except block
+ # is exited. Save a version to raise later.
+ e_save = ImportError(str(e))
+
+ def reraise_(*args, **kwargs):
+ raise (e_save) # noqa
+
+ Marionette = reraise_
+
+import reftestcommandline
+from output import OutputHandler, ReftestFormatter
+
+here = os.path.abspath(os.path.dirname(__file__))
+
+try:
+ from mozbuild.base import MozbuildObject
+
+ build_obj = MozbuildObject.from_environment(cwd=here)
+except ImportError:
+ build_obj = None
+
+
+def categoriesToRegex(categoryList):
+ return "\\(" + ", ".join(["(?P<%s>\\d+) %s" % c for c in categoryList]) + "\\)"
+
+
+summaryLines = [
+ ("Successful", [("pass", "pass"), ("loadOnly", "load only")]),
+ (
+ "Unexpected",
+ [
+ ("fail", "unexpected fail"),
+ ("pass", "unexpected pass"),
+ ("asserts", "unexpected asserts"),
+ ("fixedAsserts", "unexpected fixed asserts"),
+ ("failedLoad", "failed load"),
+ ("exception", "exception"),
+ ],
+ ),
+ (
+ "Known problems",
+ [
+ ("knownFail", "known fail"),
+ ("knownAsserts", "known asserts"),
+ ("random", "random"),
+ ("skipped", "skipped"),
+ ("slow", "slow"),
+ ],
+ ),
+]
+
+
+if sys.version_info[0] == 3:
+
+ def reraise_(tp_, value_, tb_=None):
+ if value_ is None:
+ value_ = tp_()
+ if value_.__traceback__ is not tb_:
+ raise value_.with_traceback(tb_)
+ raise value_
+
+
+else:
+ exec("def reraise_(tp_, value_, tb_=None):\n raise tp_, value_, tb_\n")
+
+
+def update_mozinfo():
+ """walk up directories to find mozinfo.json update the info"""
+ # TODO: This should go in a more generic place, e.g. mozinfo
+
+ path = SCRIPT_DIRECTORY
+ dirs = set()
+ while path != os.path.expanduser("~"):
+ if path in dirs:
+ break
+ dirs.add(path)
+ path = os.path.split(path)[0]
+ mozinfo.find_and_update_from_json(*dirs)
+
+
+# Python's print is not threadsafe.
+printLock = threading.Lock()
+
+
+class ReftestThread(threading.Thread):
+ def __init__(self, cmdargs):
+ threading.Thread.__init__(self)
+ self.cmdargs = cmdargs
+ self.summaryMatches = {}
+ self.retcode = -1
+ for text, _ in summaryLines:
+ self.summaryMatches[text] = None
+
+ def run(self):
+ with printLock:
+ print("Starting thread with", self.cmdargs)
+ sys.stdout.flush()
+ process = subprocess.Popen(self.cmdargs, stdout=subprocess.PIPE)
+ for chunk in self.chunkForMergedOutput(process.stdout):
+ with printLock:
+ print(chunk, end=" ")
+ sys.stdout.flush()
+ self.retcode = process.wait()
+
+ def chunkForMergedOutput(self, logsource):
+ """Gather lines together that should be printed as one atomic unit.
+ Individual test results--anything between 'REFTEST TEST-START' and
+ 'REFTEST TEST-END' lines--are an atomic unit. Lines with data from
+ summaries are parsed and the data stored for later aggregation.
+ Other lines are considered their own atomic units and are permitted
+ to intermix freely."""
+ testStartRegex = re.compile("^REFTEST TEST-START")
+ testEndRegex = re.compile("^REFTEST TEST-END")
+ summaryHeadRegex = re.compile("^REFTEST INFO \\| Result summary:")
+ summaryRegexFormatString = (
+ "^REFTEST INFO \\| (?P<message>{text}): (?P<total>\\d+) {regex}"
+ )
+ summaryRegexStrings = [
+ summaryRegexFormatString.format(
+ text=text, regex=categoriesToRegex(categories)
+ )
+ for (text, categories) in summaryLines
+ ]
+ summaryRegexes = [re.compile(regex) for regex in summaryRegexStrings]
+
+ for line in logsource:
+ if testStartRegex.search(line) is not None:
+ chunkedLines = [line]
+ for lineToBeChunked in logsource:
+ chunkedLines.append(lineToBeChunked)
+ if testEndRegex.search(lineToBeChunked) is not None:
+ break
+ yield "".join(chunkedLines)
+ continue
+
+ haveSuppressedSummaryLine = False
+ for regex in summaryRegexes:
+ match = regex.search(line)
+ if match is not None:
+ self.summaryMatches[match.group("message")] = match
+ haveSuppressedSummaryLine = True
+ break
+ if haveSuppressedSummaryLine:
+ continue
+
+ if summaryHeadRegex.search(line) is None:
+ yield line
+
+
+class ReftestResolver(object):
+ def defaultManifest(self, suite):
+ return {
+ "reftest": "reftest.list",
+ "crashtest": "crashtests.list",
+ "jstestbrowser": "jstests.list",
+ }[suite]
+
+ def directoryManifest(self, suite, path):
+ return os.path.join(path, self.defaultManifest(suite))
+
+ def findManifest(self, suite, test_file, subdirs=True):
+ """Return a tuple of (manifest-path, filter-string) for running test_file.
+
+ test_file is a path to a test or a manifest file
+ """
+ rv = []
+ default_manifest = self.defaultManifest(suite)
+ relative_path = None
+ if not os.path.isabs(test_file):
+ relative_path = test_file
+ test_file = self.absManifestPath(test_file)
+
+ if os.path.isdir(test_file):
+ for dirpath, dirnames, filenames in os.walk(test_file):
+ if default_manifest in filenames:
+ rv.append((os.path.join(dirpath, default_manifest), None))
+ # We keep recursing into subdirectories which means that in the case
+ # of include directives we get the same manifest multiple times.
+ # However reftest.js will only read each manifest once
+
+ if (
+ len(rv) == 0
+ and relative_path
+ and suite == "jstestbrowser"
+ and build_obj
+ ):
+ # The relative path can be from staging area.
+ staged_js_dir = os.path.join(
+ build_obj.topobjdir, "dist", "test-stage", "jsreftest"
+ )
+ staged_file = os.path.join(staged_js_dir, "tests", relative_path)
+ return self.findManifest(suite, staged_file, subdirs)
+ elif test_file.endswith(".list"):
+ if os.path.exists(test_file):
+ rv = [(test_file, None)]
+ else:
+ dirname, pathname = os.path.split(test_file)
+ found = True
+ while not os.path.exists(os.path.join(dirname, default_manifest)):
+ dirname, suffix = os.path.split(dirname)
+ pathname = posixpath.join(suffix, pathname)
+ if os.path.dirname(dirname) == dirname:
+ found = False
+ break
+ if found:
+ rv = [
+ (
+ os.path.join(dirname, default_manifest),
+ r".*%s(?:[#?].*)?$" % pathname.replace("?", "\?"),
+ )
+ ]
+
+ return rv
+
+ def absManifestPath(self, path):
+ return os.path.normpath(os.path.abspath(path))
+
+ def manifestURL(self, options, path):
+ return "file://%s" % path
+
+ def resolveManifests(self, options, tests):
+ suite = options.suite
+ manifests = {}
+ for testPath in tests:
+ for manifest, filter_str in self.findManifest(suite, testPath):
+ if manifest not in manifests:
+ manifests[manifest] = set()
+ manifests[manifest].add(filter_str)
+ manifests_by_url = {}
+ for key in manifests.keys():
+ id = os.path.relpath(
+ os.path.abspath(os.path.dirname(key)), options.topsrcdir
+ )
+ id = id.replace(os.sep, posixpath.sep)
+ if None in manifests[key]:
+ manifests[key] = (None, id)
+ else:
+ manifests[key] = ("|".join(list(manifests[key])), id)
+ url = self.manifestURL(options, key)
+ manifests_by_url[url] = manifests[key]
+ return manifests_by_url
+
+
+class RefTest(object):
+ oldcwd = os.getcwd()
+ resolver_cls = ReftestResolver
+ use_marionette = True
+
+ def __init__(self, suite):
+ update_mozinfo()
+ self.lastTestSeen = None
+ self.lastTest = None
+ self.haveDumpedScreen = False
+ self.resolver = self.resolver_cls()
+ self.log = None
+ self.outputHandler = None
+ self.testDumpFile = os.path.join(tempfile.gettempdir(), "reftests.json")
+ self.currentManifest = "No test started"
+
+ self.run_by_manifest = True
+ if suite in ("crashtest", "jstestbrowser"):
+ self.run_by_manifest = False
+
+ def _populate_logger(self, options):
+ if self.log:
+ return
+
+ self.log = getattr(options, "log", None)
+ if self.log:
+ return
+
+ mozlog.commandline.log_formatters["tbpl"] = (
+ ReftestFormatter,
+ "Reftest specific formatter for the"
+ "benefit of legacy log parsers and"
+ "tools such as the reftest analyzer",
+ )
+ fmt_options = {}
+ if not options.log_tbpl_level and os.environ.get("MOZ_REFTEST_VERBOSE"):
+ options.log_tbpl_level = fmt_options["level"] = "debug"
+ self.log = mozlog.commandline.setup_logging(
+ "reftest harness", options, {"tbpl": sys.stdout}, fmt_options
+ )
+
+ def getFullPath(self, path):
+ "Get an absolute path relative to self.oldcwd."
+ return os.path.normpath(os.path.join(self.oldcwd, os.path.expanduser(path)))
+
+ def createReftestProfile(
+ self,
+ options,
+ tests=None,
+ manifests=None,
+ server="localhost",
+ port=0,
+ profile_to_clone=None,
+ prefs=None,
+ ):
+ """Sets up a profile for reftest.
+
+ :param options: Object containing command line options
+ :param tests: List of test objects to run
+ :param manifests: List of manifest files to parse (only takes effect
+ if tests were not passed in)
+ :param server: Server name to use for http tests
+ :param profile_to_clone: Path to a profile to use as the basis for the
+ test profile
+ :param prefs: Extra preferences to set in the profile
+ """
+ locations = mozprofile.permissions.ServerLocations()
+ locations.add_host(server, scheme="http", port=port)
+ locations.add_host(server, scheme="https", port=port)
+
+ sandbox_whitelist_paths = options.sandboxReadWhitelist
+ if platform.system() == "Linux" or platform.system() in (
+ "Windows",
+ "Microsoft",
+ ):
+ # Trailing slashes are needed to indicate directories on Linux and Windows
+ sandbox_whitelist_paths = map(
+ lambda p: os.path.join(p, ""), sandbox_whitelist_paths
+ )
+
+ addons = []
+ if not self.use_marionette:
+ addons.append(options.reftestExtensionPath)
+
+ if options.specialPowersExtensionPath is not None:
+ if not self.use_marionette:
+ addons.append(options.specialPowersExtensionPath)
+
+ # Install distributed extensions, if application has any.
+ distExtDir = os.path.join(
+ options.app[: options.app.rfind(os.sep)], "distribution", "extensions"
+ )
+ if os.path.isdir(distExtDir):
+ for f in os.listdir(distExtDir):
+ addons.append(os.path.join(distExtDir, f))
+
+ # Install custom extensions.
+ for f in options.extensionsToInstall:
+ addons.append(self.getFullPath(f))
+
+ kwargs = {
+ "addons": addons,
+ "locations": locations,
+ "whitelistpaths": sandbox_whitelist_paths,
+ }
+ if profile_to_clone:
+ profile = mozprofile.Profile.clone(profile_to_clone, **kwargs)
+ else:
+ profile = mozprofile.Profile(**kwargs)
+
+ # First set prefs from the base profiles under testing/profiles.
+
+ # In test packages used in CI, the profile_data directory is installed
+ # in the SCRIPT_DIRECTORY.
+ profile_data_dir = os.path.join(SCRIPT_DIRECTORY, "profile_data")
+ # If possible, read profile data from topsrcdir. This prevents us from
+ # requiring a re-build to pick up newly added extensions in the
+ # <profile>/extensions directory.
+ if build_obj:
+ path = os.path.join(build_obj.topsrcdir, "testing", "profiles")
+ if os.path.isdir(path):
+ profile_data_dir = path
+ # Still not found? Look for testing/profiles relative to layout/tools/reftest.
+ if not os.path.isdir(profile_data_dir):
+ path = os.path.abspath(
+ os.path.join(SCRIPT_DIRECTORY, "..", "..", "..", "testing", "profiles")
+ )
+ if os.path.isdir(path):
+ profile_data_dir = path
+
+ with open(os.path.join(profile_data_dir, "profiles.json"), "r") as fh:
+ base_profiles = json.load(fh)["reftest"]
+
+ for name in base_profiles:
+ path = os.path.join(profile_data_dir, name)
+ profile.merge(path)
+
+ # Second set preferences for communication between our command line
+ # arguments and the reftest harness. Preferences that are required for
+ # reftest to work should instead be set under srcdir/testing/profiles.
+ prefs = prefs or {}
+ prefs["reftest.timeout"] = options.timeout * 1000
+ if options.logFile:
+ prefs["reftest.logFile"] = options.logFile
+ if options.ignoreWindowSize:
+ prefs["reftest.ignoreWindowSize"] = True
+ if options.shuffle:
+ prefs["reftest.shuffle"] = True
+ if options.repeat:
+ prefs["reftest.repeat"] = options.repeat
+ if options.runUntilFailure:
+ prefs["reftest.runUntilFailure"] = True
+ if not options.repeat:
+ prefs["reftest.repeat"] = 30
+ if options.verify:
+ prefs["reftest.verify"] = True
+ if options.cleanupCrashes:
+ prefs["reftest.cleanupPendingCrashes"] = True
+ prefs["reftest.focusFilterMode"] = options.focusFilterMode
+ prefs["reftest.logLevel"] = options.log_tbpl_level or "info"
+ prefs["reftest.suite"] = options.suite
+ prefs["gfx.font_rendering.ahem_antialias_none"] = True
+ # Run the "deferred" font-loader immediately, because if it finishes
+ # mid-test, the extra reflow that is triggered can disrupt the test.
+ prefs["gfx.font_loader.delay"] = 0
+ # Ensure bundled fonts are activated, even if not enabled by default
+ # on the platform, so that tests can rely on them.
+ prefs["gfx.bundled-fonts.activate"] = 1
+ # Disable dark scrollbars because it's semi-transparent.
+ prefs["widget.disable-dark-scrollbar"] = True
+ prefs["reftest.isCoverageBuild"] = mozinfo.info.get("ccov", False)
+
+ # config specific flags
+ prefs["sandbox.apple_silicon"] = mozinfo.info.get("apple_silicon", False)
+
+ # Set tests to run or manifests to parse.
+ if tests:
+ testlist = os.path.join(profile.profile, "reftests.json")
+ with open(testlist, "w") as fh:
+ json.dump(tests, fh)
+ prefs["reftest.tests"] = testlist
+ elif manifests:
+ prefs["reftest.manifests"] = json.dumps(manifests)
+
+ # Unconditionally update the e10s pref, default True
+ prefs["browser.tabs.remote.autostart"] = True
+ if not options.e10s:
+ prefs["browser.tabs.remote.autostart"] = False
+
+ # default fission to True
+ prefs["fission.autostart"] = True
+ if options.disableFission:
+ prefs["fission.autostart"] = False
+
+ if not self.run_by_manifest:
+ if options.totalChunks:
+ prefs["reftest.totalChunks"] = options.totalChunks
+ if options.thisChunk:
+ prefs["reftest.thisChunk"] = options.thisChunk
+
+ # Bug 1262954: For winXP + e10s disable acceleration
+ if (
+ platform.system() in ("Windows", "Microsoft")
+ and "5.1" in platform.version()
+ and options.e10s
+ ):
+ prefs["layers.acceleration.disabled"] = True
+
+ # Bug 1300355: Disable canvas cache for win7 as it uses
+ # too much memory and causes OOMs.
+ if (
+ platform.system() in ("Windows", "Microsoft")
+ and "6.1" in platform.version()
+ ):
+ prefs["reftest.nocache"] = True
+
+ if options.marionette:
+ # options.marionette can specify host:port
+ port = options.marionette.split(":")[1]
+ prefs["marionette.port"] = int(port)
+
+ # Enable tracing output for detailed failures in case of
+ # failing connection attempts, and hangs (bug 1397201)
+ prefs["remote.log.level"] = "Trace"
+
+ # Third, set preferences passed in via the command line.
+ for v in options.extraPrefs:
+ thispref = v.split("=")
+ if len(thispref) < 2:
+ print("Error: syntax error in --setpref=" + v)
+ sys.exit(1)
+ prefs[thispref[0]] = thispref[1].strip()
+
+ for pref in prefs:
+ prefs[pref] = mozprofile.Preferences.cast(prefs[pref])
+ profile.set_preferences(prefs)
+
+ if os.path.join(here, "chrome") not in options.extraProfileFiles:
+ options.extraProfileFiles.append(os.path.join(here, "chrome"))
+
+ self.copyExtraFilesToProfile(options, profile)
+
+ self.log.info(
+ "Running with e10s: {}".format(prefs["browser.tabs.remote.autostart"])
+ )
+ self.log.info("Running with fission: {}".format(prefs["fission.autostart"]))
+
+ return profile
+
+ def environment(self, **kwargs):
+ kwargs["log"] = self.log
+ return test_environment(**kwargs)
+
+ def buildBrowserEnv(self, options, profileDir):
+ browserEnv = self.environment(
+ xrePath=options.xrePath, debugger=options.debugger
+ )
+ browserEnv["XPCOM_DEBUG_BREAK"] = "stack"
+ if options.topsrcdir:
+ browserEnv["MOZ_DEVELOPER_REPO_DIR"] = options.topsrcdir
+ if hasattr(options, "topobjdir"):
+ browserEnv["MOZ_DEVELOPER_OBJ_DIR"] = options.topobjdir
+
+ if mozinfo.info["asan"]:
+ # Disable leak checking for reftests for now
+ if "ASAN_OPTIONS" in browserEnv:
+ browserEnv["ASAN_OPTIONS"] += ":detect_leaks=0"
+ else:
+ browserEnv["ASAN_OPTIONS"] = "detect_leaks=0"
+
+ # Set environment defaults for jstestbrowser. Keep in sync with the
+ # defaults used in js/src/tests/lib/tests.py.
+ if options.suite == "jstestbrowser":
+ browserEnv["TZ"] = "PST8PDT"
+ browserEnv["LC_ALL"] = "en_US.UTF-8"
+
+ for v in options.environment:
+ ix = v.find("=")
+ if ix <= 0:
+ print("Error: syntax error in --setenv=" + v)
+ return None
+ browserEnv[v[:ix]] = v[ix + 1 :]
+
+ # Enable leaks detection to its own log file.
+ self.leakLogFile = os.path.join(profileDir, "runreftest_leaks.log")
+ browserEnv["XPCOM_MEM_BLOAT_LOG"] = self.leakLogFile
+
+ # TODO: this is always defined (as part of --enable-webrender which is default)
+ # can we make this default in the browser?
+ browserEnv["MOZ_ACCELERATED"] = "1"
+
+ if options.headless:
+ browserEnv["MOZ_HEADLESS"] = "1"
+
+ return browserEnv
+
+ def cleanup(self, profileDir):
+ if profileDir:
+ shutil.rmtree(profileDir, True)
+
+ def verifyTests(self, tests, options):
+ """
+ Support --verify mode: Run test(s) many times in a variety of
+ configurations/environments in an effort to find intermittent
+ failures.
+ """
+
+ self._populate_logger(options)
+
+ # Number of times to repeat test(s) when running with --repeat
+ VERIFY_REPEAT = 10
+ # Number of times to repeat test(s) when running test in separate browser
+ VERIFY_REPEAT_SINGLE_BROWSER = 5
+
+ def step1():
+ options.repeat = VERIFY_REPEAT
+ options.runUntilFailure = True
+ result = self.runTests(tests, options)
+ return result
+
+ def step2():
+ options.repeat = 0
+ options.runUntilFailure = False
+ for i in range(VERIFY_REPEAT_SINGLE_BROWSER):
+ result = self.runTests(tests, options)
+ if result != 0:
+ break
+ return result
+
+ def step3():
+ options.repeat = VERIFY_REPEAT
+ options.runUntilFailure = True
+ options.environment.append("MOZ_CHAOSMODE=0xfb")
+ result = self.runTests(tests, options)
+ options.environment.remove("MOZ_CHAOSMODE=0xfb")
+ return result
+
+ def step4():
+ options.repeat = 0
+ options.runUntilFailure = False
+ options.environment.append("MOZ_CHAOSMODE=0xfb")
+ for i in range(VERIFY_REPEAT_SINGLE_BROWSER):
+ result = self.runTests(tests, options)
+ if result != 0:
+ break
+ options.environment.remove("MOZ_CHAOSMODE=0xfb")
+ return result
+
+ steps = [
+ ("1. Run each test %d times in one browser." % VERIFY_REPEAT, step1),
+ (
+ "2. Run each test %d times in a new browser each time."
+ % VERIFY_REPEAT_SINGLE_BROWSER,
+ step2,
+ ),
+ (
+ "3. Run each test %d times in one browser, in chaos mode."
+ % VERIFY_REPEAT,
+ step3,
+ ),
+ (
+ "4. Run each test %d times in a new browser each time, in chaos mode."
+ % VERIFY_REPEAT_SINGLE_BROWSER,
+ step4,
+ ),
+ ]
+
+ stepResults = {}
+ for (descr, step) in steps:
+ stepResults[descr] = "not run / incomplete"
+
+ startTime = datetime.now()
+ maxTime = timedelta(seconds=options.verify_max_time)
+ finalResult = "PASSED"
+ for (descr, step) in steps:
+ if (datetime.now() - startTime) > maxTime:
+ self.log.info("::: Test verification is taking too long: Giving up!")
+ self.log.info(
+ "::: So far, all checks passed, but not all checks were run."
+ )
+ break
+ self.log.info(":::")
+ self.log.info('::: Running test verification step "%s"...' % descr)
+ self.log.info(":::")
+ result = step()
+ if result != 0:
+ stepResults[descr] = "FAIL"
+ finalResult = "FAILED!"
+ break
+ stepResults[descr] = "Pass"
+
+ self.log.info(":::")
+ self.log.info("::: Test verification summary for:")
+ self.log.info(":::")
+ for test in tests:
+ self.log.info("::: " + test)
+ self.log.info(":::")
+ for descr in sorted(stepResults.keys()):
+ self.log.info("::: %s : %s" % (descr, stepResults[descr]))
+ self.log.info(":::")
+ self.log.info("::: Test verification %s" % finalResult)
+ self.log.info(":::")
+
+ return result
+
+ def runTests(self, tests, options, cmdargs=None):
+ cmdargs = cmdargs or []
+ self._populate_logger(options)
+ self.outputHandler = OutputHandler(
+ self.log, options.utilityPath, options.symbolsPath
+ )
+
+ if options.cleanupCrashes:
+ mozcrash.cleanup_pending_crash_reports()
+
+ manifests = self.resolver.resolveManifests(options, tests)
+ if options.filter:
+ manifests[""] = (options.filter, None)
+
+ if not getattr(options, "runTestsInParallel", False):
+ return self.runSerialTests(manifests, options, cmdargs)
+
+ cpuCount = multiprocessing.cpu_count()
+
+ # We have the directive, technology, and machine to run multiple test instances.
+ # Experimentation says that reftests are not overly CPU-intensive, so we can run
+ # multiple jobs per CPU core.
+ #
+ # Our Windows machines in automation seem to get upset when we run a lot of
+ # simultaneous tests on them, so tone things down there.
+ if sys.platform == "win32":
+ jobsWithoutFocus = cpuCount
+ else:
+ jobsWithoutFocus = 2 * cpuCount
+
+ totalJobs = jobsWithoutFocus + 1
+ perProcessArgs = [sys.argv[:] for i in range(0, totalJobs)]
+
+ host = "localhost"
+ port = 2828
+ if options.marionette:
+ host, port = options.marionette.split(":")
+
+ # First job is only needs-focus tests. Remaining jobs are
+ # non-needs-focus and chunked.
+ perProcessArgs[0].insert(-1, "--focus-filter-mode=needs-focus")
+ for (chunkNumber, jobArgs) in enumerate(perProcessArgs[1:], start=1):
+ jobArgs[-1:-1] = [
+ "--focus-filter-mode=non-needs-focus",
+ "--total-chunks=%d" % jobsWithoutFocus,
+ "--this-chunk=%d" % chunkNumber,
+ "--marionette=%s:%d" % (host, port),
+ ]
+ port += 1
+
+ for jobArgs in perProcessArgs:
+ try:
+ jobArgs.remove("--run-tests-in-parallel")
+ except Exception:
+ pass
+ jobArgs[0:0] = [sys.executable, "-u"]
+
+ threads = [ReftestThread(args) for args in perProcessArgs[1:]]
+ for t in threads:
+ t.start()
+
+ while True:
+ # The test harness in each individual thread will be doing timeout
+ # handling on its own, so we shouldn't need to worry about any of
+ # the threads hanging for arbitrarily long.
+ for t in threads:
+ t.join(10)
+ if not any(t.is_alive() for t in threads):
+ break
+
+ # Run the needs-focus tests serially after the other ones, so we don't
+ # have to worry about races between the needs-focus tests *actually*
+ # needing focus and the dummy windows in the non-needs-focus tests
+ # trying to focus themselves.
+ focusThread = ReftestThread(perProcessArgs[0])
+ focusThread.start()
+ focusThread.join()
+
+ # Output the summaries that the ReftestThread filters suppressed.
+ summaryObjects = [defaultdict(int) for s in summaryLines]
+ for t in threads:
+ for (summaryObj, (text, categories)) in zip(summaryObjects, summaryLines):
+ threadMatches = t.summaryMatches[text]
+ for (attribute, description) in categories:
+ amount = int(threadMatches.group(attribute) if threadMatches else 0)
+ summaryObj[attribute] += amount
+ amount = int(threadMatches.group("total") if threadMatches else 0)
+ summaryObj["total"] += amount
+
+ print("REFTEST INFO | Result summary:")
+ for (summaryObj, (text, categories)) in zip(summaryObjects, summaryLines):
+ details = ", ".join(
+ [
+ "%d %s" % (summaryObj[attribute], description)
+ for (attribute, description) in categories
+ ]
+ )
+ print(
+ "REFTEST INFO | "
+ + text
+ + ": "
+ + str(summaryObj["total"])
+ + " ("
+ + details
+ + ")"
+ )
+
+ return int(any(t.retcode != 0 for t in threads))
+
+ def handleTimeout(self, timeout, proc, utilityPath, debuggerInfo):
+ """handle process output timeout"""
+ # TODO: bug 913975 : _processOutput should call self.processOutputLine
+ # one more time one timeout (I think)
+ self.log.error(
+ "%s | application timed out after %d seconds with no output"
+ % (self.lastTestSeen, int(timeout))
+ )
+ self.log.warning("Force-terminating active process(es).")
+ self.killAndGetStack(
+ proc, utilityPath, debuggerInfo, dump_screen=not debuggerInfo
+ )
+
+ def dumpScreen(self, utilityPath):
+ if self.haveDumpedScreen:
+ self.log.info(
+ "Not taking screenshot here: see the one that was previously logged"
+ )
+ return
+ self.haveDumpedScreen = True
+ dump_screen(utilityPath, self.log)
+
+ def killAndGetStack(self, process, utilityPath, debuggerInfo, dump_screen=False):
+ """
+ Kill the process, preferrably in a way that gets us a stack trace.
+ Also attempts to obtain a screenshot before killing the process
+ if specified.
+ """
+
+ if dump_screen:
+ self.dumpScreen(utilityPath)
+
+ if mozinfo.info.get("crashreporter", True) and not debuggerInfo:
+ if mozinfo.isWin:
+ # We should have a "crashinject" program in our utility path
+ crashinject = os.path.normpath(
+ os.path.join(utilityPath, "crashinject.exe")
+ )
+ if os.path.exists(crashinject):
+ status = subprocess.Popen([crashinject, str(process.pid)]).wait()
+ printstatus("crashinject", status)
+ if status == 0:
+ return
+ else:
+ try:
+ process.kill(sig=signal.SIGABRT)
+ except OSError:
+ # https://bugzilla.mozilla.org/show_bug.cgi?id=921509
+ self.log.info("Can't trigger Breakpad, process no longer exists")
+ return
+ self.log.info("Can't trigger Breakpad, just killing process")
+ process.kill()
+
+ def runApp(
+ self,
+ options,
+ cmdargs=None,
+ timeout=None,
+ debuggerInfo=None,
+ symbolsPath=None,
+ valgrindPath=None,
+ valgrindArgs=None,
+ valgrindSuppFiles=None,
+ **profileArgs
+ ):
+
+ if cmdargs is None:
+ cmdargs = []
+ cmdargs = cmdargs[:]
+
+ if self.use_marionette:
+ cmdargs.append("-marionette")
+
+ binary = options.app
+ profile = self.createReftestProfile(options, **profileArgs)
+
+ # browser environment
+ env = self.buildBrowserEnv(options, profile.profile)
+
+ def timeoutHandler():
+ self.handleTimeout(timeout, proc, options.utilityPath, debuggerInfo)
+
+ interactive = False
+ debug_args = None
+ if debuggerInfo:
+ interactive = debuggerInfo.interactive
+ debug_args = [debuggerInfo.path] + debuggerInfo.args
+
+ def record_last_test(message):
+ """Records the last test seen by this harness for the benefit of crash logging."""
+
+ def testid(test):
+ if " " in test:
+ return test.split(" ")[0]
+ return test
+
+ if message["action"] == "test_start":
+ self.lastTestSeen = testid(message["test"])
+ elif message["action"] == "test_end":
+ if self.lastTest and message["test"] == self.lastTest:
+ self.lastTestSeen = self.currentManifest
+ else:
+ self.lastTestSeen = "{} (finished)".format(testid(message["test"]))
+
+ self.log.add_handler(record_last_test)
+
+ kp_kwargs = {
+ "kill_on_timeout": False,
+ "cwd": SCRIPT_DIRECTORY,
+ "onTimeout": [timeoutHandler],
+ "processOutputLine": [self.outputHandler],
+ }
+
+ if mozinfo.isWin or mozinfo.isMac:
+ # Prevents log interleaving on Windows at the expense of losing
+ # true log order. See bug 798300 and bug 1324961 for more details.
+ kp_kwargs["processStderrLine"] = [self.outputHandler]
+
+ if interactive:
+ # If an interactive debugger is attached,
+ # don't use timeouts, and don't capture ctrl-c.
+ timeout = None
+ signal.signal(signal.SIGINT, lambda sigid, frame: None)
+
+ runner_cls = mozrunner.runners.get(
+ mozinfo.info.get("appname", "firefox"), mozrunner.Runner
+ )
+ runner = runner_cls(
+ profile=profile,
+ binary=binary,
+ process_class=mozprocess.ProcessHandlerMixin,
+ cmdargs=cmdargs,
+ env=env,
+ process_args=kp_kwargs,
+ )
+ runner.start(
+ debug_args=debug_args, interactive=interactive, outputTimeout=timeout
+ )
+ proc = runner.process_handler
+ self.outputHandler.proc_name = "GECKO({})".format(proc.pid)
+
+ # Used to defer a possible IOError exception from Marionette
+ marionette_exception = None
+
+ if self.use_marionette:
+ marionette_args = {
+ "socket_timeout": options.marionette_socket_timeout,
+ "startup_timeout": options.marionette_startup_timeout,
+ "symbols_path": options.symbolsPath,
+ }
+ if options.marionette:
+ host, port = options.marionette.split(":")
+ marionette_args["host"] = host
+ marionette_args["port"] = int(port)
+
+ try:
+ marionette = Marionette(**marionette_args)
+ marionette.start_session()
+
+ addons = Addons(marionette)
+ if options.specialPowersExtensionPath:
+ addons.install(options.specialPowersExtensionPath, temp=True)
+
+ addons.install(options.reftestExtensionPath, temp=True)
+
+ marionette.delete_session()
+ except IOError:
+ # Any IOError as thrown by Marionette means that something is
+ # wrong with the process, like a crash or the socket is no
+ # longer open. We defer raising this specific error so that
+ # post-test checks for leaks and crashes are performed and
+ # reported first.
+ marionette_exception = sys.exc_info()
+
+ status = runner.wait()
+ runner.process_handler = None
+ self.outputHandler.proc_name = None
+
+ crashed = mozcrash.log_crashes(
+ self.log,
+ os.path.join(profile.profile, "minidumps"),
+ options.symbolsPath,
+ test=self.lastTestSeen,
+ )
+
+ if crashed:
+ # log suite_end to wrap up, this is usually done with in in-browser harness
+ if not self.outputHandler.results:
+ # TODO: while .results is a defaultdict(int), it is proxied via log_actions as data, not type
+ self.outputHandler.results = {
+ "Pass": 0,
+ "LoadOnly": 0,
+ "Exception": 0,
+ "FailedLoad": 0,
+ "UnexpectedFail": 1,
+ "UnexpectedPass": 0,
+ "AssertionUnexpected": 0,
+ "AssertionUnexpectedFixed": 0,
+ "KnownFail": 0,
+ "AssertionKnown": 0,
+ "Random": 0,
+ "Skip": 0,
+ "Slow": 0,
+ }
+ self.log.suite_end(extra={"results": self.outputHandler.results})
+
+ if not status and crashed:
+ status = 1
+
+ if status and not crashed:
+ msg = (
+ "TEST-UNEXPECTED-FAIL | %s | application terminated with exit code %s"
+ % (self.lastTestSeen, status)
+ )
+ # use process_output so message is logged verbatim
+ self.log.process_output(None, msg)
+
+ runner.cleanup()
+ self.cleanup(profile.profile)
+
+ if marionette_exception is not None:
+ exc, value, tb = marionette_exception
+ raise reraise(exc, value, tb)
+
+ self.log.info("Process mode: {}".format("e10s" if options.e10s else "non-e10s"))
+ return status
+
+ def getActiveTests(self, manifests, options, testDumpFile=None):
+ # These prefs will cause reftest.jsm to parse the manifests,
+ # dump the resulting tests to a file, and exit.
+ prefs = {
+ "reftest.manifests": json.dumps(manifests),
+ "reftest.manifests.dumpTests": testDumpFile or self.testDumpFile,
+ }
+ cmdargs = []
+ self.runApp(options, cmdargs=cmdargs, prefs=prefs)
+
+ if not os.path.isfile(self.testDumpFile):
+ print("Error: parsing manifests failed!")
+ sys.exit(1)
+
+ with open(self.testDumpFile, "r") as fh:
+ tests = json.load(fh)
+
+ if os.path.isfile(self.testDumpFile):
+ mozfile.remove(self.testDumpFile)
+
+ for test in tests:
+ # Name and path are expected by manifestparser, but not used in reftest.
+ test["name"] = test["path"] = test["url1"]
+
+ mp = TestManifest(strict=False)
+ mp.tests = tests
+
+ filters = []
+ if options.totalChunks:
+ filters.append(
+ mpf.chunk_by_manifest(options.thisChunk, options.totalChunks)
+ )
+
+ tests = mp.active_tests(exists=False, filters=filters)
+ return tests
+
+ def runSerialTests(self, manifests, options, cmdargs=None):
+ debuggerInfo = None
+ if options.debugger:
+ debuggerInfo = mozdebug.get_debugger_info(
+ options.debugger, options.debuggerArgs, options.debuggerInteractive
+ )
+
+ def run(**kwargs):
+ if kwargs.get("tests"):
+ self.lastTest = kwargs["tests"][-1]["identifier"]
+ if not isinstance(self.lastTest, string_types):
+ self.lastTest = " ".join(self.lastTest)
+
+ status = self.runApp(
+ options,
+ manifests=manifests,
+ cmdargs=cmdargs,
+ # We generally want the JS harness or marionette
+ # to handle timeouts if they can.
+ # The default JS harness timeout is currently
+ # 300 seconds (default options.timeout).
+ # The default Marionette socket timeout is
+ # currently 360 seconds.
+ # Give the JS harness extra time to deal with
+ # its own timeouts and try to usually exceed
+ # the 360 second marionette socket timeout.
+ # See bug 479518 and bug 1414063.
+ timeout=options.timeout + 70.0,
+ debuggerInfo=debuggerInfo,
+ symbolsPath=options.symbolsPath,
+ **kwargs
+ )
+
+ # do not process leak log when we crash/assert
+ if status == 0:
+ mozleak.process_leak_log(
+ self.leakLogFile,
+ leak_thresholds=options.leakThresholds,
+ stack_fixer=get_stack_fixer_function(
+ options.utilityPath, options.symbolsPath
+ ),
+ )
+ return status
+
+ if not self.run_by_manifest:
+ return run()
+
+ tests = self.getActiveTests(manifests, options)
+ tests_by_manifest = defaultdict(list)
+ ids_by_manifest = defaultdict(list)
+ for t in tests:
+ tests_by_manifest[t["manifest"]].append(t)
+ test_id = t["identifier"]
+ if not isinstance(test_id, string_types):
+ test_id = " ".join(test_id)
+ ids_by_manifest[t["manifestID"]].append(test_id)
+
+ self.log.suite_start(ids_by_manifest, name=options.suite)
+
+ overall = 0
+ status = -1
+ for manifest, tests in tests_by_manifest.items():
+ self.log.info("Running tests in {}".format(manifest))
+ self.currentManifest = manifest
+ status = run(tests=tests)
+ overall = overall or status
+ if status == -1:
+ # we didn't run anything
+ overall = 1
+
+ self.log.suite_end(extra={"results": self.outputHandler.results})
+ return overall
+
+ def copyExtraFilesToProfile(self, options, profile):
+ "Copy extra files or dirs specified on the command line to the testing profile."
+ profileDir = profile.profile
+ for f in options.extraProfileFiles:
+ abspath = self.getFullPath(f)
+ if os.path.isfile(abspath):
+ if os.path.basename(abspath) == "user.js":
+ extra_prefs = mozprofile.Preferences.read_prefs(abspath)
+ profile.set_preferences(extra_prefs)
+ elif os.path.basename(abspath).endswith(".dic"):
+ hyphDir = os.path.join(profileDir, "hyphenation")
+ if not os.path.exists(hyphDir):
+ os.makedirs(hyphDir)
+ shutil.copy2(abspath, hyphDir)
+ else:
+ shutil.copy2(abspath, profileDir)
+ elif os.path.isdir(abspath):
+ dest = os.path.join(profileDir, os.path.basename(abspath))
+ shutil.copytree(abspath, dest)
+ else:
+ self.log.warning(
+ "runreftest.py | Failed to copy %s to profile" % abspath
+ )
+ continue
+
+
+def run_test_harness(parser, options):
+ reftest = RefTest(options.suite)
+ parser.validate(options, reftest)
+
+ # We have to validate options.app here for the case when the mach
+ # command is able to find it after argument parsing. This can happen
+ # when running from a tests archive.
+ if not options.app:
+ parser.error("could not find the application path, --appname must be specified")
+
+ options.app = reftest.getFullPath(options.app)
+ if not os.path.exists(options.app):
+ parser.error(
+ "Error: Path %(app)s doesn't exist. Are you executing "
+ "$objdir/_tests/reftest/runreftest.py?" % {"app": options.app}
+ )
+
+ if options.xrePath is None:
+ options.xrePath = os.path.dirname(options.app)
+
+ if options.verify:
+ result = reftest.verifyTests(options.tests, options)
+ else:
+ result = reftest.runTests(options.tests, options)
+
+ return result
+
+
+if __name__ == "__main__":
+ parser = reftestcommandline.DesktopArgumentsParser()
+ options = parser.parse_args()
+ sys.exit(run_test_harness(parser, options))
diff --git a/layout/tools/reftest/schema.json b/layout/tools/reftest/schema.json
new file mode 100644
index 0000000000..fe51488c70
--- /dev/null
+++ b/layout/tools/reftest/schema.json
@@ -0,0 +1 @@
+[]
diff --git a/layout/tools/reftest/selftest/conftest.py b/layout/tools/reftest/selftest/conftest.py
new file mode 100644
index 0000000000..1255caf8a7
--- /dev/null
+++ b/layout/tools/reftest/selftest/conftest.py
@@ -0,0 +1,147 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import json
+import os
+from argparse import Namespace
+
+try:
+ # Python2
+ from cStringIO import StringIO
+except ImportError:
+ # Python3
+ from io import StringIO
+
+import mozinfo
+import pytest
+from manifestparser import expression
+from moztest.selftest.fixtures import binary_fixture, setup_test_harness # noqa
+
+here = os.path.abspath(os.path.dirname(__file__))
+setup_args = [False, "reftest", "reftest"]
+
+
+@pytest.fixture(scope="module")
+def normalize():
+ """A function that can take a relative path and append it to the 'files'
+ directory which contains the data necessary to run these tests.
+ """
+
+ def inner(path):
+ if os.path.isabs(path):
+ return path
+ return os.path.join(here, "files", path)
+
+ return inner
+
+
+@pytest.fixture
+def parser(setup_test_harness):
+ setup_test_harness(*setup_args)
+ cmdline = pytest.importorskip("reftestcommandline")
+ return cmdline.DesktopArgumentsParser()
+
+
+@pytest.fixture
+def get_reftest(setup_test_harness, binary, parser):
+ setup_test_harness(*setup_args)
+ runreftest = pytest.importorskip("runreftest")
+ harness_root = runreftest.SCRIPT_DIRECTORY
+
+ build = parser.build_obj
+ options = vars(parser.parse_args([]))
+ options.update(
+ {
+ "app": binary,
+ "focusFilterMode": "non-needs-focus",
+ "suite": "reftest",
+ }
+ )
+
+ if not os.path.isdir(build.bindir):
+ package_root = os.path.dirname(harness_root)
+ options.update(
+ {
+ "extraProfileFiles": [os.path.join(package_root, "bin", "plugins")],
+ "reftestExtensionPath": os.path.join(harness_root, "reftest"),
+ "sandboxReadWhitelist": [here, os.environ["PYTHON_TEST_TMP"]],
+ "utilityPath": os.path.join(package_root, "bin"),
+ "specialPowersExtensionPath": os.path.join(
+ harness_root, "specialpowers"
+ ),
+ }
+ )
+
+ if "MOZ_FETCHES_DIR" in os.environ:
+ options["sandboxReadWhitelist"].append(os.environ["MOZ_FETCHES_DIR"])
+ else:
+ options.update(
+ {
+ "extraProfileFiles": [os.path.join(build.topobjdir, "dist", "plugins")],
+ "sandboxReadWhitelist": [build.topobjdir, build.topsrcdir],
+ "specialPowersExtensionPath": os.path.join(
+ build.distdir, "xpi-stage", "specialpowers"
+ ),
+ }
+ )
+
+ def inner(**opts):
+ options.update(opts)
+ config = Namespace(**options)
+
+ # This is pulled from `runreftest.run_test_harness` minus some error
+ # checking that isn't necessary in this context. It should stay roughly
+ # in sync.
+ reftest = runreftest.RefTest(config.suite)
+ parser.validate(config, reftest)
+
+ config.app = reftest.getFullPath(config.app)
+ assert os.path.exists(config.app)
+
+ if config.xrePath is None:
+ config.xrePath = os.path.dirname(config.app)
+
+ return reftest, config
+
+ return inner
+
+
+@pytest.fixture # noqa: F811
+def runtests(get_reftest, normalize):
+ def inner(*tests, **opts):
+ assert len(tests) > 0
+ opts["tests"] = map(normalize, tests)
+
+ buf = StringIO()
+ opts["log_raw"] = [buf]
+
+ reftest, options = get_reftest(**opts)
+ result = reftest.runTests(options.tests, options)
+
+ out = json.loads("[" + ",".join(buf.getvalue().splitlines()) + "]")
+ buf.close()
+ return result, out
+
+ return inner
+
+
+@pytest.fixture(autouse=True) # noqa: F811
+def skip_using_mozinfo(request, setup_test_harness):
+ """Gives tests the ability to skip based on values from mozinfo.
+
+ Example:
+ @pytest.mark.skip_mozinfo("!e10s || os == 'linux'")
+ def test_foo():
+ pass
+ """
+
+ setup_test_harness(*setup_args)
+ runreftest = pytest.importorskip("runreftest")
+ runreftest.update_mozinfo()
+
+ skip_mozinfo = request.node.get_closest_marker("skip_mozinfo")
+ if skip_mozinfo:
+ value = skip_mozinfo.args[0]
+ if expression.parse(value, **mozinfo.info):
+ pytest.skip("skipped due to mozinfo match: \n{}".format(value))
diff --git a/layout/tools/reftest/selftest/files/assert.html b/layout/tools/reftest/selftest/files/assert.html
new file mode 100644
index 0000000000..c9aedcf116
--- /dev/null
+++ b/layout/tools/reftest/selftest/files/assert.html
@@ -0,0 +1,7 @@
+<script>
+const Cc = SpecialPowers.Cc;
+const Ci = SpecialPowers.Ci;
+
+let debug = Cc["@mozilla.org/xpcom/debug;1"].getService(Ci.nsIDebug2);
+debug.assertion('failed assertion check', 'false', 'assert.html', 6);
+</script>
diff --git a/layout/tools/reftest/selftest/files/crash.html b/layout/tools/reftest/selftest/files/crash.html
new file mode 100644
index 0000000000..897863024b
--- /dev/null
+++ b/layout/tools/reftest/selftest/files/crash.html
@@ -0,0 +1,7 @@
+<script>
+const Cc = SpecialPowers.Cc;
+const Ci = SpecialPowers.Ci;
+
+let debug = Cc["@mozilla.org/xpcom/debug;1"].getService(Ci.nsIDebug2);
+debug.abort('crash.html', 6);
+</script>
diff --git a/layout/tools/reftest/selftest/files/defaults.list b/layout/tools/reftest/selftest/files/defaults.list
new file mode 100644
index 0000000000..d947eb8d1d
--- /dev/null
+++ b/layout/tools/reftest/selftest/files/defaults.list
@@ -0,0 +1,7 @@
+# test defaults
+defaults pref(foo.bar,true)
+== foo.html foo-ref.html
+
+# reset defaults
+defaults
+== bar.html bar-ref.html
diff --git a/layout/tools/reftest/selftest/files/failure-type-interactions.list b/layout/tools/reftest/selftest/files/failure-type-interactions.list
new file mode 100644
index 0000000000..c820e8f13f
--- /dev/null
+++ b/layout/tools/reftest/selftest/files/failure-type-interactions.list
@@ -0,0 +1,11 @@
+# interactions between skip and fail
+skip-if(true) fails == skip-if_fails.html ref.html
+skip-if(true) fails-if(true) == skip-if_fails-if.html ref.html
+skip fails == skip_fails.html ref.html
+skip-if(false) fails == fails.html ref.html
+fails skip-if(true) == fails_skip-if.html ref.html
+fails-if(true) skip-if(true) == fails-if_skip-if.html ref.html
+fails skip == fails_skip.html ref.html
+fails-if(false) skip == skip.html ref.html
+skip-if(true) fails skip-if(false) == skip-if-true_fails_skip-if-false ref.html
+skip-if(false) fails skip-if(true) == skip-if-false_fails_skip-if-true ref.html
diff --git a/layout/tools/reftest/selftest/files/green.html b/layout/tools/reftest/selftest/files/green.html
new file mode 100644
index 0000000000..d1695cb8b8
--- /dev/null
+++ b/layout/tools/reftest/selftest/files/green.html
@@ -0,0 +1,6 @@
+<!DOCTYPE html>
+<html>
+<body>
+<div style="color: green">Text</div>
+</body>
+</html>
diff --git a/layout/tools/reftest/selftest/files/invalid-defaults-include.list b/layout/tools/reftest/selftest/files/invalid-defaults-include.list
new file mode 100644
index 0000000000..408d3214f6
--- /dev/null
+++ b/layout/tools/reftest/selftest/files/invalid-defaults-include.list
@@ -0,0 +1,4 @@
+# can't use defaults prior to include
+defaults pref(foo.bar,1)
+== foo.html bar.html
+include defaults.list
diff --git a/layout/tools/reftest/selftest/files/invalid-defaults.list b/layout/tools/reftest/selftest/files/invalid-defaults.list
new file mode 100644
index 0000000000..7bb8d060da
--- /dev/null
+++ b/layout/tools/reftest/selftest/files/invalid-defaults.list
@@ -0,0 +1,3 @@
+# invalid tokens in defaults
+defaults skip-if(true) == foo.html bar.html
+== foo.html bar.html
diff --git a/layout/tools/reftest/selftest/files/invalid-include.list b/layout/tools/reftest/selftest/files/invalid-include.list
new file mode 100644
index 0000000000..cd1c1f0939
--- /dev/null
+++ b/layout/tools/reftest/selftest/files/invalid-include.list
@@ -0,0 +1,2 @@
+# non-skip items are not allowed with include
+pref(foo.bar,1) include defaults.list
diff --git a/layout/tools/reftest/selftest/files/leaks.log b/layout/tools/reftest/selftest/files/leaks.log
new file mode 100644
index 0000000000..af832f149d
--- /dev/null
+++ b/layout/tools/reftest/selftest/files/leaks.log
@@ -0,0 +1,73 @@
+== BloatView: ALL (cumulative) LEAK AND BLOAT STATISTICS, default process 1148
+ |<----------------Class--------------->|<-----Bytes------>|<----Objects---->|
+ | | Per-Inst Leaked| Total Rem|
+ 0 |TOTAL | 19 19915|16007885 378|
+ 68 |CacheEntry | 208 208| 521 1|
+ 71 |CacheEntryHandle | 16 16| 7692 1|
+ 72 |CacheFile | 264 264| 521 1|
+ 80 |CacheFileMetadata | 176 176| 521 1|
+ 81 |CacheFileOutputStream | 64 64| 463 1|
+ 90 |CacheStorageService | 256 256| 1 1|
+ 97 |CancelableRunnable | 24 24| 45614 1|
+ 103 |ChannelEventQueue | 96 96| 996 1|
+ 131 |CondVar | 36 360| 822 10|
+ 148 |ConsoleReportCollector | 60 120| 1380 2|
+ 150 |ContentParent | 1712 1712| 2 1|
+ 188 |DataStorage | 284 852| 3 3|
+ 320 |HttpBaseChannel | 1368 1368| 1143 1|
+ 321 |HttpChannelParent | 176 176| 996 1|
+ 322 |HttpChannelParentListener | 48 48| 961 1|
+ 342 |IdlePeriod | 12 36| 166 3|
+ 369 |InterceptedChannelBase | 240 240| 18 1|
+ 389 |LoadContext | 72 144| 1023 2|
+ 391 |LoadInfo | 144 144| 3903 1|
+ 427 |Mutex | 44 1012| 14660 23|
+ 439 |NullPrincipalURI | 80 80| 421 1|
+ 468 |PBrowserParent | 312 312| 21 1|
+ 479 |PContentParent | 1428 1428| 2 1|
+ 486 |PHttpChannelParent | 24 24| 996 1|
+ 527 |PollableEvent | 12 12| 1 1|
+ 576 |ReentrantMonitor | 24 72| 6922 3|
+ 577 |RefCountedMonitor | 84 84| 154 1|
+ 583 |RequestContextService | 60 60| 1 1|
+ 592 |Runnable | 20 40| 178102 2|
+ 627 |Service | 128 128| 1 1|
+ 646 |SharedMemory | 16 16| 1636 1|
+ 675 |StringAdopt | 1 3| 17087 3|
+ 688 |TabParent | 976 976| 21 1|
+ 699 |ThirdPartyUtil | 16 16| 1 1|
+ 875 |ipc::MessageChannel | 208 208| 166 1|
+ 920 |nsAuthURLParser | 12 12| 2 1|
+ 974 |nsCategoryObserver | 72 72| 8 1|
+ 976 |nsChannelClassifier | 28 28| 918 1|
+1004 |CookiePermission | 40 40| 1 1|
+1005 |CookieService | 80 80| 1 1|
+1010 |nsDNSService | 140 140| 1 1|
+1066 |nsEffectiveTLDService | 20 20| 1 1|
+1134 |nsHttpAuthCache::OriginClearObserver | 16 32| 2 2|
+1135 |nsHttpChannel | 1816 1816| 1143 1|
+1136 |nsHttpChannelAuthProvider | 148 148| 1012 1|
+1138 |nsHttpConnectionInfo | 128 128| 1021 1|
+1139 |nsHttpConnectionMgr | 304 304| 1 1|
+1141 |nsHttpHandler | 544 544| 1 1|
+1142 |nsHttpRequestHead | 92 92| 1190 1|
+1145 |nsIDNService | 56 56| 1 1|
+1146 |nsIOService | 176 176| 1 1|
+1176 |nsJSPrincipals | 16 64| 12583 4|
+1186 |nsLocalFile | 88 264| 13423 3|
+1192 |nsMainThreadPtrHolder<T> | 20 80| 2253 4|
+1222 |nsNodeWeakReference | 16 16| 919 1|
+1223 |nsNotifyAddrListener | 112 112| 1 1|
+1241 |PermissionManager | 136 136| 1 1|
+1248 |nsPrefBranch | 76 76| 63 1|
+1257 |nsProxyInfo | 72 72| 1098 1|
+1265 |nsRedirectHistoryEntry | 32 32| 69 1|
+1307 |nsSiteSecurityService | 56 56| 1 1|
+1311 |nsSocketTransportService | 208 208| 1 1|
+1313 |nsStandardURL | 196 1372| 59651 7|
+1319 |nsStreamConverterService | 48 48| 1 1|
+1324 |nsStringBuffer | 8 1688| 722245 211|
+1371 |nsTArray_base | 4 136| 3419841 34|
+1380 |nsThread | 304 912| 165 3|
+1416 |nsWeakReference | 20 180| 1388 9|
+nsTraceRefcnt::DumpStatistics: 1489 entries
diff --git a/layout/tools/reftest/selftest/files/red.html b/layout/tools/reftest/selftest/files/red.html
new file mode 100644
index 0000000000..a9db5be4df
--- /dev/null
+++ b/layout/tools/reftest/selftest/files/red.html
@@ -0,0 +1,6 @@
+<!DOCTYPE html>
+<html>
+<body>
+<div style="color: red">Text</div>
+</body>
+</html>
diff --git a/layout/tools/reftest/selftest/files/reftest-assert.list b/layout/tools/reftest/selftest/files/reftest-assert.list
new file mode 100644
index 0000000000..38fd3f57aa
--- /dev/null
+++ b/layout/tools/reftest/selftest/files/reftest-assert.list
@@ -0,0 +1 @@
+load assert.html
diff --git a/layout/tools/reftest/selftest/files/reftest-crash.list b/layout/tools/reftest/selftest/files/reftest-crash.list
new file mode 100644
index 0000000000..3e27bcba75
--- /dev/null
+++ b/layout/tools/reftest/selftest/files/reftest-crash.list
@@ -0,0 +1 @@
+load crash.html
diff --git a/layout/tools/reftest/selftest/files/reftest-fail.list b/layout/tools/reftest/selftest/files/reftest-fail.list
new file mode 100644
index 0000000000..c5259b6601
--- /dev/null
+++ b/layout/tools/reftest/selftest/files/reftest-fail.list
@@ -0,0 +1,3 @@
+== green.html red.html
+!= green.html green.html
+!= red.html red.html
diff --git a/layout/tools/reftest/selftest/files/reftest-pass.list b/layout/tools/reftest/selftest/files/reftest-pass.list
new file mode 100644
index 0000000000..51512c1202
--- /dev/null
+++ b/layout/tools/reftest/selftest/files/reftest-pass.list
@@ -0,0 +1,3 @@
+== green.html green.html
+== red.html red.html
+!= green.html red.html
diff --git a/layout/tools/reftest/selftest/files/scripttest-pass.html b/layout/tools/reftest/selftest/files/scripttest-pass.html
new file mode 100644
index 0000000000..e0371b3518
--- /dev/null
+++ b/layout/tools/reftest/selftest/files/scripttest-pass.html
@@ -0,0 +1,17 @@
+<!DOCTYPE html>
+<html>
+<head>
+<title>scripttest-pass</title>
+<script type="text/javascript">
+function getTestCases()
+{
+ return [
+ { testPassed: (function () { return true; }), testDescription: (function () { return "passed"; }) }
+ ];
+}
+</script>
+</head>
+<body>
+<h1>scripttest-pass</h1>
+</body>
+</html>
diff --git a/layout/tools/reftest/selftest/files/types.list b/layout/tools/reftest/selftest/files/types.list
new file mode 100644
index 0000000000..7622564527
--- /dev/null
+++ b/layout/tools/reftest/selftest/files/types.list
@@ -0,0 +1,5 @@
+== green.html green.html
+!= green.html red.html
+load green.html
+script scripttest-pass.html
+print green.html green.html
diff --git a/layout/tools/reftest/selftest/python.ini b/layout/tools/reftest/selftest/python.ini
new file mode 100644
index 0000000000..6c4dd9c713
--- /dev/null
+++ b/layout/tools/reftest/selftest/python.ini
@@ -0,0 +1,7 @@
+[DEFAULT]
+subsuite=reftest
+sequential=true
+
+[test_python_manifest_parser.py]
+[test_reftest_manifest_parser.py]
+[test_reftest_output.py]
diff --git a/layout/tools/reftest/selftest/test_python_manifest_parser.py b/layout/tools/reftest/selftest/test_python_manifest_parser.py
new file mode 100644
index 0000000000..5ec9ce324c
--- /dev/null
+++ b/layout/tools/reftest/selftest/test_python_manifest_parser.py
@@ -0,0 +1,37 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import mozunit
+import pytest
+
+
+@pytest.fixture
+def parse(normalize):
+ reftest = pytest.importorskip("reftest")
+
+ def inner(path):
+ mp = reftest.ReftestManifest()
+ mp.load(normalize(path))
+ return mp
+
+ return inner
+
+
+def test_parse_defaults(parse):
+ mp = parse("defaults.list")
+ assert len(mp.tests) == 4
+
+ for test in mp.tests:
+ if test["name"].startswith("foo"):
+ assert test["pref"] == "foo.bar,true"
+ else:
+ assert "pref" not in test
+
+ # invalid defaults
+ with pytest.raises(ValueError):
+ parse("invalid-defaults.list")
+
+
+if __name__ == "__main__":
+ mozunit.main()
diff --git a/layout/tools/reftest/selftest/test_reftest_manifest_parser.py b/layout/tools/reftest/selftest/test_reftest_manifest_parser.py
new file mode 100644
index 0000000000..009aa17a7f
--- /dev/null
+++ b/layout/tools/reftest/selftest/test_reftest_manifest_parser.py
@@ -0,0 +1,72 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import mozunit
+import pytest
+
+
+@pytest.fixture
+def parse(get_reftest, normalize):
+ output = pytest.importorskip("output")
+
+ reftest, options = get_reftest(tests=["dummy"])
+ reftest._populate_logger(options)
+ reftest.outputHandler = output.OutputHandler(
+ reftest.log, options.utilityPath, options.symbolsPath
+ )
+
+ def resolve(path):
+ path = normalize(path)
+ return "file://{}".format(path)
+
+ def inner(*manifests):
+ assert len(manifests) > 0
+ manifests = {m: (None, "id") for m in map(resolve, manifests)}
+ return reftest.getActiveTests(manifests, options)
+
+ return inner
+
+
+def test_parse_test_types(parse):
+ tests = parse("types.list")
+ assert tests[0]["type"] == "=="
+ assert tests[1]["type"] == "!="
+ assert tests[2]["type"] == "load"
+ assert tests[3]["type"] == "script"
+ assert tests[4]["type"] == "print"
+
+
+def test_parse_failure_type_interactions(parse):
+ """Tests interactions between skip and fails."""
+ tests = parse("failure-type-interactions.list")
+ for t in tests:
+ if "skip" in t["name"]:
+ assert t["skip"]
+ else:
+ assert not t["skip"]
+
+ # 0 => EXPECTED_PASS, 1 => EXPECTED_FAIL
+ if "fails" in t["name"]:
+ assert t["expected"] == 1
+ else:
+ assert t["expected"] == 0
+
+
+def test_parse_invalid_manifests(parse):
+ # XXX We should assert that the output contains the appropriate error
+ # message, but we seem to be hitting an issue in pytest that is preventing
+ # us from capturing the Gecko output with the capfd fixture. See:
+ # https://github.com/pytest-dev/pytest/issues/5997
+ with pytest.raises(SystemExit):
+ parse("invalid-defaults.list")
+
+ with pytest.raises(SystemExit):
+ parse("invalid-defaults-include.list")
+
+ with pytest.raises(SystemExit):
+ parse("invalid-include.list")
+
+
+if __name__ == "__main__":
+ mozunit.main()
diff --git a/layout/tools/reftest/selftest/test_reftest_output.py b/layout/tools/reftest/selftest/test_reftest_output.py
new file mode 100644
index 0000000000..ef343f754d
--- /dev/null
+++ b/layout/tools/reftest/selftest/test_reftest_output.py
@@ -0,0 +1,162 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import os
+
+try:
+ # Python2
+ from cStringIO import StringIO
+except ImportError:
+ # Python3
+ from io import StringIO
+
+from functools import partial
+
+import mozunit
+import pytest
+from mozharness.base.log import ERROR, INFO, WARNING
+from mozharness.mozilla.automation import TBPL_FAILURE, TBPL_SUCCESS, TBPL_WARNING
+from moztest.selftest.output import filter_action, get_mozharness_status
+
+here = os.path.abspath(os.path.dirname(__file__))
+get_mozharness_status = partial(get_mozharness_status, "reftest")
+
+
+def test_output_pass(runtests):
+ status, lines = runtests("reftest-pass.list")
+ assert status == 0
+
+ tbpl_status, log_level, summary = get_mozharness_status(lines, status)
+ assert tbpl_status == TBPL_SUCCESS
+ assert log_level in (INFO, WARNING)
+
+ test_status = filter_action("test_status", lines)
+ assert len(test_status) == 3
+ assert all(t["status"] == "PASS" for t in test_status)
+
+ test_end = filter_action("test_end", lines)
+ assert len(test_end) == 3
+ assert all(t["status"] == "OK" for t in test_end)
+
+
+def test_output_fail(runtests):
+ formatter = pytest.importorskip("output").ReftestFormatter()
+
+ status, lines = runtests("reftest-fail.list")
+ assert status == 0
+
+ buf = StringIO()
+ tbpl_status, log_level, summary = get_mozharness_status(
+ lines, status, formatter=formatter, buf=buf
+ )
+
+ assert tbpl_status == TBPL_WARNING
+ assert log_level == WARNING
+
+ test_status = filter_action("test_status", lines)
+ assert len(test_status) == 3
+ assert all(t["status"] == "FAIL" for t in test_status)
+ assert all("reftest_screenshots" in t["extra"] for t in test_status)
+
+ test_end = filter_action("test_end", lines)
+ assert len(test_end) == 3
+ assert all(t["status"] == "OK" for t in test_end)
+
+ # ensure screenshots were printed
+ formatted = buf.getvalue()
+ assert "REFTEST IMAGE 1" in formatted
+ assert "REFTEST IMAGE 2" in formatted
+
+
+@pytest.mark.skip_mozinfo("!crashreporter")
+def test_output_crash(runtests):
+ status, lines = runtests(
+ "reftest-crash.list", environment=["MOZ_CRASHREPORTER_SHUTDOWN=1"]
+ )
+ assert status == 245
+
+ tbpl_status, log_level, summary = get_mozharness_status(lines, status)
+ assert tbpl_status == TBPL_FAILURE
+ assert log_level == ERROR
+
+ crash = filter_action("crash", lines)
+ assert len(crash) == 1
+ assert crash[0]["action"] == "crash"
+ assert crash[0]["signature"]
+ assert crash[0]["minidump_path"]
+
+ lines = filter_action("test_end", lines)
+ assert len(lines) == 0
+
+
+@pytest.mark.skip_mozinfo("!asan")
+def test_output_asan(runtests):
+ status, lines = runtests(
+ "reftest-crash.list", environment=["MOZ_CRASHREPORTER_SHUTDOWN=1"]
+ )
+ assert status == 245
+
+ tbpl_status, log_level, summary = get_mozharness_status(lines, status)
+ assert tbpl_status == TBPL_FAILURE
+ assert log_level == ERROR
+
+ crash = filter_action("crash", lines)
+ assert len(crash) == 0
+
+ process_output = filter_action("process_output", lines)
+ assert any("ERROR: AddressSanitizer" in l["data"] for l in process_output)
+
+
+@pytest.mark.skip_mozinfo("!debug")
+def test_output_assertion(runtests):
+ status, lines = runtests("reftest-assert.list")
+ assert status == 0
+
+ tbpl_status, log_level, summary = get_mozharness_status(lines, status)
+ assert tbpl_status == TBPL_WARNING
+ assert log_level == WARNING
+
+ test_status = filter_action("test_status", lines)
+ assert len(test_status) == 1
+ assert test_status[0]["status"] == "PASS"
+
+ test_end = filter_action("test_end", lines)
+ assert len(test_end) == 1
+ assert test_end[0]["status"] == "OK"
+
+ assertions = filter_action("assertion_count", lines)
+ assert len(assertions) == 1
+ assert assertions[0]["count"] == 1
+
+
+@pytest.mark.skip_mozinfo("!debug")
+def test_output_leak(monkeypatch, runtests):
+ # Monkeypatch mozleak so we always process a failing leak log
+ # instead of the actual one.
+ import mozleak
+
+ old_process_leak_log = mozleak.process_leak_log
+
+ def process_leak_log(*args, **kwargs):
+ return old_process_leak_log(
+ os.path.join(here, "files", "leaks.log"), *args[1:], **kwargs
+ )
+
+ monkeypatch.setattr("mozleak.process_leak_log", process_leak_log)
+
+ status, lines = runtests("reftest-pass.list")
+ assert status == 0
+
+ tbpl_status, log_level, summary = get_mozharness_status(lines, status)
+ assert tbpl_status == TBPL_WARNING
+ assert log_level == WARNING
+
+ leaks = filter_action("mozleak_total", lines)
+ assert len(leaks) == 1
+ assert leaks[0]["process"] == "default"
+ assert leaks[0]["bytes"] == 19915
+
+
+if __name__ == "__main__":
+ mozunit.main()