firefox/browser/components/textrecognition/textrecognition.mjs

/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

window.docShell.chromeEventHandler.classList.add("textRecognitionDialogFrame");

window.addEventListener("DOMContentLoaded", () => {
  // The arguments are passed in as the final parameters to TabDialogBox.prototype.open.
  new TextRecognitionModal(...window.arguments);
});

/**
 * @typedef {object} TextRecognitionResult
 * @property {number} confidence
 * @property {string} string
 * @property {DOMQuad} quad
 */

class TextRecognitionModal {
  /**
   * @param {Promise<TextRecognitionResult[]>} resultsPromise
   * @param {Function} resizeVertically
   * @param {object} [openLinkIn]
   * @param {string} openLinkIn.url
   * @param {string} openLinkIn.where
   * @param {object} openLinkIn.params
   * @param {TimerId} timerId
   */
  constructor(resultsPromise, resizeVertically, openLinkIn, timerId) {
    /** @type {HTMLElement} */
    this.textEl = document.querySelector(".textRecognitionText");

    /** @type {NodeListOf<HTMLElement>} */
    this.headerEls = document.querySelectorAll(".textRecognitionHeader");

    /** @type {HTMLAnchorElement} */
    this.linkEl = document.querySelector(
      "#text-recognition-header-no-results a"
    );

    this.resizeVertically = resizeVertically;
    this.openLinkIn = openLinkIn;
    this.setupLink();
    this.setupCloseHandler();

    this.showHeaderByID("text-recognition-header-loading");

    resultsPromise.then(
      ({ results, direction }) => {
        if (results.length === 0) {
          // Update the UI to indicate that there were no results.
          this.showHeaderByID("text-recognition-header-no-results");
          // It's still worth recording telemetry times, as the API was still invoked.
          Glean.textRecognition.apiPerformance.stopAndAccumulate(timerId);
          return;
        }

        // There were results, cluster them into a nice presentation, and present
        // the results to the UI.
        this.runClusteringAndUpdateUI(results, direction);
        this.showHeaderByID("text-recognition-header-results");
        Glean.textRecognition.apiPerformance.stopAndAccumulate(timerId);

        TextRecognitionModal.recordInteractionTime();
      },
      error => {
        // There was an error in the text recognition call. Treat this as the same
        // as if there were no results, but report the error to the console and telemetry.
        this.showHeaderByID("text-recognition-header-no-results");

        console.error(
          "There was an error recognizing the text from an image.",
          error
        );
        Glean.browserUiInteraction.textrecognitionError.add(1);
        Glean.textRecognition.apiPerformance.cancel(timerId);
      }
    );
  }

  /**
   * After the results are shown, measure how long a user interacts with the modal.
   */
  static recordInteractionTime() {
    let timerId = Glean.textRecognition.interactionTiming.start();

    const finish = () => {
      Glean.textRecognition.interactionTiming.stopAndAccumulate(timerId);
      window.removeEventListener("blur", finish);
      window.removeEventListener("unload", finish);
    };

    // The user's focus went away, record this as the total interaction, even if they
    // go back and interact with it more. This can be triggered by doing actions like
    // clicking the URL bar, or by switching tabs.
    window.addEventListener("blur", finish);

    // The modal is closed.
    window.addEventListener("unload", finish);
  }

  /**
   * After the results are shown, measure how long a user interacts with the modal.
   *
   * @param {number} textLength
   */
  static recordTextLengthTelemetry(textLength) {
    Glean.textRecognition.textLength.accumulateSingleSample(textLength);
  }

  setupCloseHandler() {
    document
      .querySelector("#text-recognition-close")
      .addEventListener("click", () => {
        window.close();
      });
  }

  /**
   * Apply the variables for the support.mozilla.org URL.
   */
  setupLink() {
    this.linkEl.href = Services.urlFormatter.formatURL(this.linkEl.href);
    this.linkEl.addEventListener("click", event => {
      event.preventDefault();
      this.openLinkIn(this.linkEl.href, "tab", {
        forceForeground: true,
        triggeringPrincipal:
          Services.scriptSecurityManager.getSystemPrincipal(),
      });
    });
  }

  /**
   * A helper to only show the appropriate header.
   *
   * @param {string} id
   */
  showHeaderByID(id) {
    for (const header of this.headerEls) {
      header.style.display = "none";
    }

    document.getElementById(id).style.display = "";
    this.resizeVertically();
  }

  /**
   * @param {string} text
   */
  static copy(text) {
    const clipboard = Cc["@mozilla.org/widget/clipboardhelper;1"].getService(
      Ci.nsIClipboardHelper
    );
    clipboard.copyString(text);
  }

  /**
   * Cluster the text based on its visual position.
   *
   * @param {TextRecognitionResult[]} results
   * @param {"ltr" | "rtl"} direction
   */
  runClusteringAndUpdateUI(results, direction) {
    /** @type {Vec2[]} */
    const centers = [];

    for (const result of results) {
      const p = result.quad;

      // Pick either the left-most or right-most edge. This optimizes for
      // aligned text over centered text.
      const minOrMax = direction === "ltr" ? Math.min : Math.max;

      centers.push([
        minOrMax(p.p1.x, p.p2.x, p.p3.x, p.p4.x),
        (p.p1.y, p.p2.y, p.p3.y, p.p4.y) / 4,
      ]);
    }

    const distSq = new DistanceSquared(centers);

    // The values are ranged 0 - 1. This value might be able to be determined
    // algorithmically.
    const averageDistance = Math.sqrt(distSq.quantile(0.2));
    const clusters = densityCluster(
      centers,
      // Neighborhood radius:
      averageDistance,
      // Minimum points to form a cluster:
      2
    );

    let text = "";
    for (const cluster of clusters) {
      const pCluster = document.createElement("p");
      pCluster.className = "textRecognitionTextCluster";

      for (let i = 0; i < cluster.length; i++) {
        const index = cluster[i];
        const { string } = results[index];
        if (i + 1 === cluster.length) {
          // Each cluster could be a paragraph, so add newlines to the end
          // for better copying.
          text += string + "\n\n";
          // The paragraph tag automatically uses two newlines.
          pCluster.innerText += string;
        } else {
          // This text is assumed to be a newlines in a paragraph, so only needs
          // to be separated by a space.
          text += string + " ";
          pCluster.innerText += string + " ";
        }
      }
      this.textEl.appendChild(pCluster);
    }

    this.textEl.style.display = "block";

    text = text.trim();
    TextRecognitionModal.copy(text);
    TextRecognitionModal.recordTextLengthTelemetry(text.length);
  }
}

/**
 * A two dimensional vector.
 *
 * @typedef {number[]} Vec2
 */

/**
 * @typedef {number} PointIndex
 */

/**
 * An implementation of the DBSCAN clustering algorithm.
 *
 * https://en.wikipedia.org/wiki/DBSCAN#Algorithm
 *
 * @param {Vec2[]} points
 * @param {number} distance
 * @param {number} minPoints
 * @returns {Array<PointIndex[]>}
 */
function densityCluster(points, distance, minPoints) {
  /**
   * A flat of array of labels that match the index of the points array. The values have
   * the following meaning:
   *
   *   undefined := No label has been assigned
   *   "noise"   := Noise is a point that hasn't been clustered.
   *   number    := Cluster index
   *
   * @type {undefined | "noise" | Index}
   */
  const labels = Array(points.length);
  const noiseLabel = "noise";

  let nextClusterIndex = 0;

  // Every point must be visited at least once. Often they will be visited earlier
  // in the interior of the loop.
  for (let pointIndex = 0; pointIndex < points.length; pointIndex++) {
    if (labels[pointIndex] !== undefined) {
      // This point is already labeled from the interior logic.
      continue;
    }

    // Get the neighbors that are within the range of the epsilon value, includes
    // the current point.
    const neighbors = getNeighborsWithinDistance(points, distance, pointIndex);

    if (neighbors.length < minPoints) {
      labels[pointIndex] = noiseLabel;
      continue;
    }

    // Start a new cluster.
    const clusterIndex = nextClusterIndex++;
    labels[pointIndex] = clusterIndex;

    // Fill the cluster. The neighbors array grows.
    for (let i = 0; i < neighbors.length; i++) {
      const nextPointIndex = neighbors[i];
      if (typeof labels[nextPointIndex] === "number") {
        // This point was already claimed, ignore it.
        continue;
      }

      if (labels[nextPointIndex] === noiseLabel) {
        // Claim this point and move on since noise has no neighbors.
        labels[nextPointIndex] = clusterIndex;
        continue;
      }

      // Claim this point as part of this cluster.
      labels[nextPointIndex] = clusterIndex;

      const newNeighbors = getNeighborsWithinDistance(
        points,
        distance,
        nextPointIndex
      );

      if (newNeighbors.length >= minPoints) {
        // Add on to the neighbors if more are found.
        for (const newNeighbor of newNeighbors) {
          if (!neighbors.includes(newNeighbor)) {
            neighbors.push(newNeighbor);
          }
        }
      }
    }
  }

  const clusters = [];

  // Pre-populate the clusters.
  for (let i = 0; i < nextClusterIndex; i++) {
    clusters[i] = [];
  }

  // Turn the labels into clusters, adding the noise to the end.
  for (let pointIndex = 0; pointIndex < labels.length; pointIndex++) {
    const label = labels[pointIndex];
    if (typeof label === "number") {
      clusters[label].push(pointIndex);
    } else if (label === noiseLabel) {
      // Add a single cluster.
      clusters.push([pointIndex]);
    } else {
      throw new Error("Logic error. Expected every point to have a label.");
    }
  }

  clusters.sort((a, b) => points[b[0]][1] - points[a[0]][1]);

  return clusters;
}

/**
 * @param {Vec2[]} points
 * @param {number} distance
 * @param {number} index
 * @returns {Index[]}
 */
function getNeighborsWithinDistance(points, distance, index) {
  let neighbors = [index];
  // There is no reason to compute the square root here if we square the
  // original distance.
  const distanceSquared = distance * distance;

  for (let otherIndex = 0; otherIndex < points.length; otherIndex++) {
    if (otherIndex === index) {
      continue;
    }
    const a = points[index];
    const b = points[otherIndex];
    const dx = a[0] - b[0];
    const dy = a[1] - b[1];

    if (dx * dx + dy * dy < distanceSquared) {
      neighbors.push(otherIndex);
    }
  }

  return neighbors;
}

/**
 * This class pre-computes the squared distances to allow for efficient distance lookups,
 * and it provides a way to look up a distance quantile.
 */
class DistanceSquared {
  /** @type {Map<number>} */
  #distances = new Map();
  #list;
  #distancesSorted;

  /**
   * @param {Vec2[]} list
   */
  constructor(list) {
    this.#list = list;
    for (let aIndex = 0; aIndex < list.length; aIndex++) {
      for (let bIndex = aIndex + 1; bIndex < list.length; bIndex++) {
        const id = this.#getTupleID(aIndex, bIndex);
        const a = this.#list[aIndex];
        const b = this.#list[bIndex];
        const dx = a[0] - b[0];
        const dy = a[1] - b[1];
        this.#distances.set(id, dx * dx + dy * dy);
      }
    }
  }

  /**
   * Returns a unique tuple ID to identify the relationship between two vectors.
   */
  #getTupleID(aIndex, bIndex) {
    return aIndex < bIndex
      ? aIndex * this.#list.length + bIndex
      : bIndex * this.#list.length + aIndex;
  }

  /**
   * Returns the distance squared between two vectors.
   *
   * @param {Index} aIndex
   * @param {Index} bIndex
   * @returns {number} The distance squared
   */
  get(aIndex, bIndex) {
    return this.#distances.get(this.#getTupleID(aIndex, bIndex));
  }

  /**
   * Returns the quantile squared.
   *
   * @param {number} percentile - Ranged between 0 - 1
   * @returns {number}
   */
  quantile(percentile) {
    if (!this.#distancesSorted) {
      this.#distancesSorted = [...this.#distances.values()].sort(
        (a, b) => a - b
      );
    }
    const index = Math.max(
      0,
      Math.min(
        this.#distancesSorted.length - 1,
        Math.round(this.#distancesSorted.length * percentile)
      )
    );
    return this.#distancesSorted[index];
  }
}