path: root/devtools/client/netmonitor/src/har/har-collector.js

/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

"use strict";

const {
  getLongStringFullText,
} = require("resource://devtools/client/shared/string-utils.js");

// Helper tracer. Should be generic sharable by other modules (bug 1171927)
const trace = {
  log(...args) {},
};

/**
 * This object is responsible for collecting data related to all
 * HTTP requests executed by the page (including inner iframes).
 */
function HarCollector(options) {
  this.commands = options.commands;

  this.onResourceAvailable = this.onResourceAvailable.bind(this);
  this.onResourceUpdated = this.onResourceUpdated.bind(this);
  this.onRequestHeaders = this.onRequestHeaders.bind(this);
  this.onRequestCookies = this.onRequestCookies.bind(this);
  this.onRequestPostData = this.onRequestPostData.bind(this);
  this.onResponseHeaders = this.onResponseHeaders.bind(this);
  this.onResponseCookies = this.onResponseCookies.bind(this);
  this.onResponseContent = this.onResponseContent.bind(this);
  this.onEventTimings = this.onEventTimings.bind(this);

  this.clear();
}

HarCollector.prototype = {
  // Connection

  async start() {
    await this.commands.resourceCommand.watchResources(
      [this.commands.resourceCommand.TYPES.NETWORK_EVENT],
      {
        onAvailable: this.onResourceAvailable,
        onUpdated: this.onResourceUpdated,
      }
    );
  },

  async stop() {
    await this.commands.resourceCommand.unwatchResources(
      [this.commands.resourceCommand.TYPES.NETWORK_EVENT],
      {
        onAvailable: this.onResourceAvailable,
        onUpdated: this.onResourceUpdated,
      }
    );
  },

  clear() {
    // Any pending requests events will be ignored (they turn
    // into zombies, since not present in the files array).
    this.files = new Map();
    this.items = [];
    this.firstRequestStart = -1;
    this.lastRequestStart = -1;
    this.requests = [];
  },

  waitForHarLoad() {
    // There should be yet another timeout e.g.:
    // 'devtools.netmonitor.har.pageLoadTimeout'
    // that should force export even if page isn't fully loaded.
    return new Promise(resolve => {
      this.waitForResponses().then(() => {
        trace.log("HarCollector.waitForHarLoad; DONE HAR loaded!");
        resolve(this);
      });
    });
  },

  waitForResponses() {
    trace.log("HarCollector.waitForResponses; " + this.requests.length);

    // All requests for additional data must be received to have complete
    // HTTP info to generate the result HAR file. So, wait for all current
    // promises. Note that new promises (requests) can be generated during the
    // process of HTTP data collection.
    return waitForAll(this.requests).then(() => {
      // All responses are received from the backend now. We yet need to
      // wait for a little while to see if a new request appears. If yes,
      // lets's start gathering HTTP data again. If no, we can declare
      // the page loaded.
      // If some new requests appears in the meantime the promise will
      // be rejected and we need to wait for responses all over again.

      this.pageLoadDeferred = this.waitForTimeout().then(
        () => {
          // Page loaded!
        },
        () => {
          trace.log(
            "HarCollector.waitForResponses; NEW requests " +
              "appeared during page timeout!"
          );
          // New requests executed, let's wait again.
          return this.waitForResponses();
        }
      );
      return this.pageLoadDeferred;
    });
  },

  // Page Loaded Timeout

  /**
   * The page is loaded when there are no new requests within given period
   * of time. The time is set in preferences:
   * 'devtools.netmonitor.har.pageLoadedTimeout'
   */
  waitForTimeout() {
    // The auto-export is not done if the timeout is set to zero (or less).
    // This is useful in cases where the export is done manually through
    // API exposed to the content.
    const timeout = Services.prefs.getIntPref(
      "devtools.netmonitor.har.pageLoadedTimeout"
    );

    trace.log("HarCollector.waitForTimeout; " + timeout);

    return new Promise((resolve, reject) => {
      if (timeout <= 0) {
        resolve();
      }
      this.pageLoadReject = reject;
      this.pageLoadTimeout = setTimeout(() => {
        trace.log("HarCollector.onPageLoadTimeout;");
        resolve();
      }, timeout);
    });
  },

  resetPageLoadTimeout() {
    // Remove the current timeout.
    if (this.pageLoadTimeout) {
      trace.log("HarCollector.resetPageLoadTimeout;");

      clearTimeout(this.pageLoadTimeout);
      this.pageLoadTimeout = null;
    }

    // Reject the current page load promise
    if (this.pageLoadReject) {
      this.pageLoadReject();
      this.pageLoadReject = null;
    }
  },

  // Collected Data

  getFile(actorId) {
    return this.files.get(actorId);
  },

  getItems() {
    return this.items;
  },

  // Event Handlers

  onResourceAvailable(resources) {
    for (const resource of resources) {
      trace.log("HarCollector.onNetworkEvent; ", resource);

      const { actor, startedDateTime, method, url, isXHR } = resource;
      const startTime = Date.parse(startedDateTime);

      if (this.firstRequestStart == -1) {
        this.firstRequestStart = startTime;
      }

      if (this.lastRequestEnd < startTime) {
        this.lastRequestEnd = startTime;
      }

      let file = this.getFile(actor);
      if (file) {
        console.error(
          "HarCollector.onNetworkEvent; ERROR " + "existing file conflict!"
        );
        continue;
      }

      file = {
        id: actor,
        startedDeltaMs: startTime - this.firstRequestStart,
        startedMs: startTime,
        method,
        url,
        isXHR,
      };

      this.files.set(actor, file);

      // Mimic the Net panel data structure
      this.items.push(file);
    }
  },

  onResourceUpdated(updates) {
    for (const { resource } of updates) {
      // Skip events from unknown actors (not in the list).
      // It can happen when there are zombie requests received after
      // the target is closed or multiple tabs are attached through
      // one connection (one DevToolsClient object).
      const file = this.getFile(resource.actor);
      if (!file) {
        return;
      }

      const includeResponseBodies = Services.prefs.getBoolPref(
        "devtools.netmonitor.har.includeResponseBodies"
      );

      [
        {
          type: "eventTimings",
          method: "getEventTimings",
          callbackName: "onEventTimings",
        },
        {
          type: "requestHeaders",
          method: "getRequestHeaders",
          callbackName: "onRequestHeaders",
        },
        {
          type: "requestPostData",
          method: "getRequestPostData",
          callbackName: "onRequestPostData",
        },
        {
          type: "responseHeaders",
          method: "getResponseHeaders",
          callbackName: "onResponseHeaders",
        },
        { type: "responseStart" },
        {
          type: "responseContent",
          method: "getResponseContent",
          callbackName: "onResponseContent",
        },
        {
          type: "requestCookies",
          method: "getRequestCookies",
          callbackName: "onRequestCookies",
        },
        {
          type: "responseCookies",
          method: "getResponseCookies",
          callbackName: "onResponseCookies",
        },
      ].forEach(updateType => {
        trace.log(
          "HarCollector.onNetworkEventUpdate; " + updateType.type,
          resource
        );

        let request;
        if (resource[`${updateType.type}Available`]) {
          if (updateType.type == "responseStart") {
            file.httpVersion = resource.httpVersion;
            file.status = resource.status;
            file.statusText = resource.statusText;
          } else if (updateType.type == "responseContent") {
            file.contentSize = resource.contentSize;
            file.mimeType = resource.mimeType;
            file.transferredSize = resource.transferredSize;
            if (includeResponseBodies) {
              request = this.getData(
                resource.actor,
                updateType.method,
                this[updateType.callbackName]
              );
            }
          } else {
            request = this.getData(
              resource.actor,
              updateType.method,
              this[updateType.callbackName]
            );
          }
        }

        if (request) {
          this.requests.push(request);
        }
        this.resetPageLoadTimeout();
      });
    }
  },

  async getData(actor, method, callback) {
    const file = this.getFile(actor);

    trace.log(
      "HarCollector.getData; REQUEST " + method + ", " + file.url,
      file
    );

    // Bug 1519082: We don't create fronts for NetworkEvent actors,
    // so that we have to do the request manually via DevToolsClient.request()
    const packet = {
      to: actor,
      type: method,
    };
    const response = await this.commands.client.request(packet);

    trace.log(
      "HarCollector.getData; RESPONSE " + method + ", " + file.url,
      response
    );
    callback(response);
    return response;
  },

  /**
   * Handles additional information received for a "requestHeaders" packet.
   *
   * @param object response
   *        The message received from the server.
   */
  onRequestHeaders(response) {
    const file = this.getFile(response.from);
    file.requestHeaders = response;

    this.getLongHeaders(response.headers);
  },

  /**
   * Handles additional information received for a "requestCookies" packet.
   *
   * @param object response
   *        The message received from the server.
   */
  onRequestCookies(response) {
    const file = this.getFile(response.from);
    file.requestCookies = response;

    this.getLongHeaders(response.cookies);
  },

  /**
   * Handles additional information received for a "requestPostData" packet.
   *
   * @param object response
   *        The message received from the server.
   */
  onRequestPostData(response) {
    trace.log("HarCollector.onRequestPostData;", response);

    const file = this.getFile(response.from);
    file.requestPostData = response;

    // Resolve long string
    const { text } = response.postData;
    if (typeof text == "object") {
      this.getString(text).then(value => {
        response.postData.text = value;
      });
    }
  },

  /**
   * Handles additional information received for a "responseHeaders" packet.
   *
   * @param object response
   *        The message received from the server.
   */
  onResponseHeaders(response) {
    const file = this.getFile(response.from);
    file.responseHeaders = response;

    this.getLongHeaders(response.headers);
  },

  /**
   * Handles additional information received for a "responseCookies" packet.
   *
   * @param object response
   *        The message received from the server.
   */
  onResponseCookies(response) {
    const file = this.getFile(response.from);
    file.responseCookies = response;

    this.getLongHeaders(response.cookies);
  },

  /**
   * Handles additional information received for a "responseContent" packet.
   *
   * @param object response
   *        The message received from the server.
   */
  onResponseContent(response) {
    const file = this.getFile(response.from);
    file.responseContent = response;

    // Resolve long string
    const { text } = response.content;
    if (typeof text == "object") {
      this.getString(text).then(value => {
        response.content.text = value;
      });
    }
  },

  /**
   * Handles additional information received for a "eventTimings" packet.
   *
   * @param object response
   *        The message received from the server.
   */
  onEventTimings(response) {
    const file = this.getFile(response.from);
    file.eventTimings = response;
    file.totalTime = response.totalTime;
  },

  // Helpers

  getLongHeaders(headers) {
    for (const header of headers) {
      if (typeof header.value == "object") {
        try {
          this.getString(header.value).then(value => {
            header.value = value;
          });
        } catch (error) {
          trace.log("HarCollector.getLongHeaders; ERROR when getString", error);
        }
      }
    }
  },

  /**
   * Fetches the full text of a string.
   *
   * @param object | string stringGrip
   *        The long string grip containing the corresponding actor.
   *        If you pass in a plain string (by accident or because you're lazy),
   *        then a promise of the same string is simply returned.
   * @return object Promise
   *         A promise that is resolved when the full string contents
   *         are available, or rejected if something goes wrong.
   */
  async getString(stringGrip) {
    const promise = getLongStringFullText(this.commands.client, stringGrip);
    this.requests.push(promise);
    return promise;
  },
};

// Helpers

/**
 * Helper function that allows to wait for array of promises. It is
 * possible to dynamically add new promises in the provided array.
 * The function will wait even for the newly added promises.
 * (this isn't possible with the default Promise.all);
 */
function waitForAll(promises) {
  // Remove all from the original array and get clone of it.
  const clone = promises.splice(0, promises.length);

  // Wait for all promises in the given array.
  return Promise.all(clone).then(() => {
    // If there are new promises (in the original array)
    // to wait for - chain them!
    if (promises.length) {
      return waitForAll(promises);
    }

    return undefined;
  });
}

// Exports from this module
exports.HarCollector = HarCollector;