/* This Source Code Form is subject to the terms of the Mozilla Public * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ "use strict"; // This is a UA widget. It runs in per-origin UA widget scope, // to be loaded by UAWidgetsChild.sys.mjs. this.TextRecognitionWidget = class { /** * @param {ShadowRoot} shadowRoot * @param {Record} _prefs */ constructor(shadowRoot, _prefs) { /** @type {ShadowRoot} */ this.shadowRoot = shadowRoot; /** @type {HTMLElement} */ this.element = shadowRoot.host; /** @type {Document} */ this.document = this.element.ownerDocument; /** @type {Window} */ this.window = this.document.defaultView; /** @type {ResizeObserver} */ this.resizeObserver = null; /** @type {Map { this.positionSpans(); }); this.resizeObserver.observe(this.element); } positionSpans() { if (!this.shadowRoot.firstChild) { return; } this.lazilyInitialize(); /** @type {HTMLDivElement} */ const div = this.shadowRoot.firstChild; const canvas = div.querySelector("canvas"); const spans = div.querySelectorAll("span"); // TODO Bug 1770438 - The element does not currently let child elements be // sized relative to the size of the containing element. It would be better // to teach the element how to do this. For the prototype, do the more expensive // operation of getting the bounding client rect, and handle the positioning manually. const imgRect = this.element.getBoundingClientRect(); div.style.width = imgRect.width + "px"; div.style.height = imgRect.height + "px"; canvas.style.width = imgRect.width + "px"; canvas.style.height = imgRect.height + "px"; // The ctx is only available when redrawing the canvas. This is operation is only // done when necessary, as it can be expensive. /** @type {null | CanvasRenderingContext2D} */ let ctx = null; if ( // The canvas hasn't been drawn to yet. this.lastCanvasStyleWidth === null || // Only redraw when the image has grown 25% larger. This percentage was chosen // as it visually seemed to work well, with the canvas never appearing blurry // when manually testing it. imgRect.width > this.lastCanvasStyleWidth * 1.25 ) { const dpr = this.window.devicePixelRatio; canvas.width = imgRect.width * dpr; canvas.height = imgRect.height * dpr; this.lastCanvasStyleWidth = imgRect.width; ctx = canvas.getContext("2d"); ctx.scale(dpr, dpr); ctx.fillStyle = "#00000088"; ctx.fillRect(0, 0, imgRect.width, imgRect.height); ctx.beginPath(); } for (const span of spans) { let spanRect = this.spanRects.get(span); if (!spanRect) { // This only needs to happen once. spanRect = span.getBoundingClientRect(); this.spanRects.set(span, spanRect); } const points = span.dataset.points.split(",").map(p => Number(p)); // Use the points in the string, e.g. // "0.0275349,0.14537,0.0275349,0.244662,0.176966,0.244565,0.176966,0.145273" // 0 1 2 3 4 5 6 7 // ^ bottomleft ^ topleft ^ topright ^ bottomright let [ bottomLeftX, bottomLeftY, topLeftX, topLeftY, topRightX, topRightY, bottomRightX, bottomRightY, ] = points; // Invert the Y. topLeftY = 1 - topLeftY; topRightY = 1 - topRightY; bottomLeftY = 1 - bottomLeftY; bottomRightY = 1 - bottomRightY; // Create a projection matrix to position the relative to the bounds. // prettier-ignore const mat4 = projectPoints( spanRect.width, spanRect.height, imgRect.width * topLeftX, imgRect.height * topLeftY, imgRect.width * topRightX, imgRect.height * topRightY, imgRect.width * bottomLeftX, imgRect.height * bottomLeftY, imgRect.width * bottomRightX, imgRect.height * bottomRightY ); span.style.transform = "matrix3d(" + mat4.join(", ") + ")"; if (ctx) { const inset = 3; ctx.moveTo( imgRect.width * bottomLeftX + inset, imgRect.height * bottomLeftY - inset ); ctx.lineTo( imgRect.width * topLeftX + inset, imgRect.height * topLeftY + inset ); ctx.lineTo( imgRect.width * topRightX - inset, imgRect.height * topRightY + inset ); ctx.lineTo( imgRect.width * bottomRightX - inset, imgRect.height * bottomRightY - inset ); ctx.closePath(); } } if (ctx) { // This composite operation will cut out the quads. The color is arbitrary. ctx.globalCompositeOperation = "destination-out"; ctx.fillStyle = "#ffffff"; ctx.fill(); // Creating a round line will grow the selection slightly, and round the corners. ctx.lineWidth = 10; ctx.lineJoin = "round"; ctx.strokeStyle = "#ffffff"; ctx.stroke(); } } teardown() { this.shadowRoot.firstChild.remove(); this.resizeObserver.disconnect(); this.spanRects.clear(); } lazilyInitialize() { if (this.isInitialized) { return; } this.isInitialized = true; const parser = new this.window.DOMParser(); let parserDoc = parser.parseFromString( `
`, "application/xml" ); if ( this.shadowRoot.children.length !== 1 || this.shadowRoot.firstChild.tagName !== "DIV" ) { throw new Error( "Expected the shadowRoot to have a single div as the root element." ); } const spansDiv = this.shadowRoot.firstChild; // Example layout of spansDiv: //
// // Text that has been recognized // // ... //
spansDiv.remove(); this.shadowRoot.importNodeAndAppendChildAt( this.shadowRoot, parserDoc.documentElement, true /* deep */ ); this.shadowRoot.importNodeAndAppendChildAt( this.shadowRoot.firstChild, spansDiv, true /* deep */ ); } }; /** * A three dimensional vector. * * @typedef {[number, number, number]} Vec3 */ /** * A 3x3 matrix. * * @typedef {[number, number, number, * number, number, number, * number, number, number]} Matrix3 */ /** * A 4x4 matrix. * * @typedef {[number, number, number, number, * number, number, number, number, * number, number, number, number, * number, number, number, number]} Matrix4 */ /** * Compute the adjugate matrix. * https://en.wikipedia.org/wiki/Adjugate_matrix * * @param {Matrix3} m * @returns {Matrix3} */ function computeAdjugate(m) { // prettier-ignore return [ m[4] * m[8] - m[5] * m[7], m[2] * m[7] - m[1] * m[8], m[1] * m[5] - m[2] * m[4], m[5] * m[6] - m[3] * m[8], m[0] * m[8] - m[2] * m[6], m[2] * m[3] - m[0] * m[5], m[3] * m[7] - m[4] * m[6], m[1] * m[6] - m[0] * m[7], m[0] * m[4] - m[1] * m[3], ]; } /** * @param {Matrix3} a * @param {Matrix3} b * @returns {Matrix3} */ function multiplyMat3(a, b) { let out = []; for (let i = 0; i < 3; i++) { for (let j = 0; j < 3; j++) { let sum = 0; for (let k = 0; k < 3; k++) { sum += a[3 * i + k] * b[3 * k + j]; } out[3 * i + j] = sum; } } return out; } /** * @param {Matrix3} m * @param {Vec3} v * @returns {Vec3} */ function multiplyMat3Vec3(m, v) { // prettier-ignore return [ m[0] * v[0] + m[1] * v[1] + m[2] * v[2], m[3] * v[0] + m[4] * v[1] + m[5] * v[2], m[6] * v[0] + m[7] * v[1] + m[8] * v[2], ]; } /** * @returns {Matrix3} */ function basisToPoints(x1, y1, x2, y2, x3, y3, x4, y4) { /** @type {Matrix3} */ let mat3 = [x1, x2, x3, y1, y2, y3, 1, 1, 1]; let vec3 = multiplyMat3Vec3(computeAdjugate(mat3), [x4, y4, 1]); // prettier-ignore return multiplyMat3( mat3, [ vec3[0], 0, 0, 0, vec3[1], 0, 0, 0, vec3[2] ] ); } /** * @type {(...Matrix4) => Matrix3} */ // prettier-ignore function general2DProjection( x1s, y1s, x1d, y1d, x2s, y2s, x2d, y2d, x3s, y3s, x3d, y3d, x4s, y4s, x4d, y4d ) { let s = basisToPoints(x1s, y1s, x2s, y2s, x3s, y3s, x4s, y4s); let d = basisToPoints(x1d, y1d, x2d, y2d, x3d, y3d, x4d, y4d); return multiplyMat3(d, computeAdjugate(s)); } /** * Given a width and height, compute a projection matrix to points 1-4. * * The points (x1,y1) through (x4, y4) use the following ordering: * * w * ┌─────┐ project 1 ─────── 2 * h │ │ --> │ / * └─────┘ │ / * 3 ──── 4 * * @returns {Matrix4} */ function projectPoints(w, h, x1, y1, x2, y2, x3, y3, x4, y4) { // prettier-ignore const mat3 = general2DProjection( 0, 0, x1, y1, w, 0, x2, y2, 0, h, x3, y3, w, h, x4, y4 ); for (let i = 0; i < 9; i++) { mat3[i] = mat3[i] / mat3[8]; } // prettier-ignore return [ mat3[0], mat3[3], 0, mat3[6], mat3[1], mat3[4], 0, mat3[7], 0, 0, 1, 0, mat3[2], mat3[5], 0, mat3[8], ]; }