Adding upstream version 1:115.7.0.upstream/1%115.7.0upstream

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

4 files changed, 1481 insertions, 0 deletions

diff --git a/devtools/client/netmonitor/src/connector/firefox-data-provider.js b/devtools/client/netmonitor/src/connector/firefox-data-provider.js
new file mode 100644
index 0000000000..9cdf6fc1d7
--- /dev/null
+++ b/devtools/client/netmonitor/src/connector/firefox-data-provider.js
@@ -0,0 +1,832 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+/* eslint-disable block-scoped-var */
+
+"use strict";
+
+const {
+  EVENTS,
+  TEST_EVENTS,
+} = require("resource://devtools/client/netmonitor/src/constants.js");
+const { CurlUtils } = require("resource://devtools/client/shared/curl.js");
+const {
+  fetchHeaders,
+} = require("resource://devtools/client/netmonitor/src/utils/request-utils.js");
+
+const {
+  getLongStringFullText,
+} = require("resource://devtools/client/shared/string-utils.js");
+
+/**
+ * This object is responsible for fetching additional HTTP
+ * data from the backend over RDP protocol.
+ *
+ * The object also keeps track of RDP requests in-progress,
+ * so it's possible to determine whether all has been fetched
+ * or not.
+ */
+class FirefoxDataProvider {
+  /**
+   * Constructor for data provider
+   *
+   * @param {Object} commands Object defined from devtools/shared/commands to interact with the devtools backend
+   * @param {Object} actions set of actions fired during data fetching process.
+   * @param {Object} owner all events are fired on this object.
+   */
+  constructor({ commands, actions, owner }) {
+    // Options
+    this.commands = commands;
+    this.actions = actions || {};
+    this.actionsEnabled = true;
+
+    // Allow requesting of on-demand network data, this would be `false` when requests
+    // are cleared (as we clear also on the backend), and will be flipped back
+    // to true on the next `onNetworkResourceAvailable` call.
+    this._requestDataEnabled = true;
+
+    // `_requestDataEnabled` can only be used to prevent new calls to
+    // requestData. For pending/already started calls, we need to check if
+    // clear() was called during the call, which is the purpose of this counter.
+    this._lastRequestDataClearId = 0;
+
+    this.owner = owner;
+
+    // This holds stacktraces infomation temporarily. Stacktrace resources
+    // can come before or after (out of order) their related network events.
+    // This will hold stacktrace related info from the NETWORK_EVENT_STACKTRACE resource
+    // for the NETWORK_EVENT resource and vice versa.
+    this.stackTraces = new Map();
+    // Map of the stacktrace information keyed by the actor id's
+    this.stackTraceRequestInfoByActorID = new Map();
+
+    // For tracking unfinished requests
+    this.pendingRequests = new Set();
+
+    // Map[key string => Promise] used by `requestData` to prevent requesting the same
+    // request data twice.
+    this.lazyRequestData = new Map();
+
+    // Fetching data from the backend
+    this.getLongString = this.getLongString.bind(this);
+
+    // Event handlers
+    this.onNetworkResourceAvailable =
+      this.onNetworkResourceAvailable.bind(this);
+    this.onNetworkResourceUpdated = this.onNetworkResourceUpdated.bind(this);
+
+    this.onWebSocketOpened = this.onWebSocketOpened.bind(this);
+    this.onWebSocketClosed = this.onWebSocketClosed.bind(this);
+    this.onFrameSent = this.onFrameSent.bind(this);
+    this.onFrameReceived = this.onFrameReceived.bind(this);
+
+    this.onEventSourceConnectionClosed =
+      this.onEventSourceConnectionClosed.bind(this);
+    this.onEventReceived = this.onEventReceived.bind(this);
+    this.setEventStreamFlag = this.setEventStreamFlag.bind(this);
+  }
+
+  /*
+   * Cleans up all the internal states, this usually done before navigation
+   * (without the persist flag on).
+   */
+  clear() {
+    this.stackTraces.clear();
+    this.pendingRequests.clear();
+    this.lazyRequestData.clear();
+    this.stackTraceRequestInfoByActorID.clear();
+    this._requestDataEnabled = false;
+    this._lastRequestDataClearId++;
+  }
+
+  destroy() {
+    // TODO: clear() is called in the middle of the lifecycle of the
+    // FirefoxDataProvider, for clarity we are exposing it as a separate method.
+    // `destroy` should be updated to nullify relevant instance properties.
+    this.clear();
+  }
+
+  /**
+   * Enable/disable firing redux actions (enabled by default).
+   *
+   * @param {boolean} enable Set to true to fire actions.
+   */
+  enableActions(enable) {
+    this.actionsEnabled = enable;
+  }
+
+  /**
+   * Add a new network request to application state.
+   *
+   * @param {string} id request id
+   * @param {object} resource resource payload will be added to application state
+   */
+  async addRequest(id, resource) {
+    // Add to the pending requests which helps when deciding if the request is complete.
+    this.pendingRequests.add(id);
+
+    if (this.actionsEnabled && this.actions.addRequest) {
+      await this.actions.addRequest(id, resource, true);
+    }
+
+    this.emit(EVENTS.REQUEST_ADDED, id);
+  }
+
+  /**
+   * Update a network request if it already exists in application state.
+   *
+   * @param {string} id request id
+   * @param {object} data data payload will be updated to application state
+   */
+  async updateRequest(id, data) {
+    const {
+      responseContent,
+      responseCookies,
+      responseHeaders,
+      requestCookies,
+      requestHeaders,
+      requestPostData,
+      responseCache,
+    } = data;
+
+    // fetch request detail contents in parallel
+    const [
+      responseContentObj,
+      requestHeadersObj,
+      responseHeadersObj,
+      postDataObj,
+      requestCookiesObj,
+      responseCookiesObj,
+      responseCacheObj,
+    ] = await Promise.all([
+      this.fetchResponseContent(responseContent),
+      this.fetchRequestHeaders(requestHeaders),
+      this.fetchResponseHeaders(responseHeaders),
+      this.fetchPostData(requestPostData),
+      this.fetchRequestCookies(requestCookies),
+      this.fetchResponseCookies(responseCookies),
+      this.fetchResponseCache(responseCache),
+    ]);
+
+    const payload = Object.assign(
+      {},
+      data,
+      responseContentObj,
+      requestHeadersObj,
+      responseHeadersObj,
+      postDataObj,
+      requestCookiesObj,
+      responseCookiesObj,
+      responseCacheObj
+    );
+
+    if (this.actionsEnabled && this.actions.updateRequest) {
+      await this.actions.updateRequest(id, payload, true);
+    }
+
+    return payload;
+  }
+
+  async fetchResponseContent(responseContent) {
+    const payload = {};
+    if (responseContent?.content) {
+      const { text } = responseContent.content;
+      const response = await this.getLongString(text);
+      responseContent.content.text = response;
+      payload.responseContent = responseContent;
+    }
+    return payload;
+  }
+
+  async fetchRequestHeaders(requestHeaders) {
+    const payload = {};
+    if (requestHeaders?.headers?.length) {
+      const headers = await fetchHeaders(requestHeaders, this.getLongString);
+      if (headers) {
+        payload.requestHeaders = headers;
+      }
+    }
+    return payload;
+  }
+
+  async fetchResponseHeaders(responseHeaders) {
+    const payload = {};
+    if (responseHeaders?.headers?.length) {
+      const headers = await fetchHeaders(responseHeaders, this.getLongString);
+      if (headers) {
+        payload.responseHeaders = headers;
+      }
+    }
+    return payload;
+  }
+
+  async fetchPostData(requestPostData) {
+    const payload = {};
+    if (requestPostData?.postData) {
+      const { text } = requestPostData.postData;
+      const postData = await this.getLongString(text);
+      const headers = CurlUtils.getHeadersFromMultipartText(postData);
+
+      // Calculate total header size and don't forget to include
+      // two new-line characters at the end.
+      const headersSize = headers.reduce((acc, { name, value }) => {
+        return acc + name.length + value.length + 2;
+      }, 0);
+
+      requestPostData.postData.text = postData;
+      payload.requestPostData = {
+        ...requestPostData,
+        uploadHeaders: { headers, headersSize },
+      };
+    }
+    return payload;
+  }
+
+  async fetchRequestCookies(requestCookies) {
+    const payload = {};
+    if (requestCookies) {
+      const reqCookies = [];
+      // request store cookies in requestCookies or requestCookies.cookies
+      const cookies = requestCookies.cookies
+        ? requestCookies.cookies
+        : requestCookies;
+      // make sure cookies is iterable
+      if (typeof cookies[Symbol.iterator] === "function") {
+        for (const cookie of cookies) {
+          reqCookies.push(
+            Object.assign({}, cookie, {
+              value: await this.getLongString(cookie.value),
+            })
+          );
+        }
+        if (reqCookies.length) {
+          payload.requestCookies = reqCookies;
+        }
+      }
+    }
+    return payload;
+  }
+
+  async fetchResponseCookies(responseCookies) {
+    const payload = {};
+    if (responseCookies) {
+      const resCookies = [];
+      // response store cookies in responseCookies or responseCookies.cookies
+      const cookies = responseCookies.cookies
+        ? responseCookies.cookies
+        : responseCookies;
+      // make sure cookies is iterable
+      if (typeof cookies[Symbol.iterator] === "function") {
+        for (const cookie of cookies) {
+          resCookies.push(
+            Object.assign({}, cookie, {
+              value: await this.getLongString(cookie.value),
+            })
+          );
+        }
+        if (resCookies.length) {
+          payload.responseCookies = resCookies;
+        }
+      }
+    }
+    return payload;
+  }
+
+  async fetchResponseCache(responseCache) {
+    const payload = {};
+    if (responseCache) {
+      payload.responseCache = await responseCache;
+      payload.responseCacheAvailable = false;
+    }
+    return payload;
+  }
+
+  /**
+   * Public API used by the Toolbox: Tells if there is still any pending request.
+   *
+   * @return {boolean} returns true if pending requests still exist in the queue.
+   */
+  hasPendingRequests() {
+    return this.pendingRequests.size > 0;
+  }
+
+  /**
+   * Fetches the full text of a LongString.
+   *
+   * @param {object|string} stringGrip
+   *        The long string grip containing the corresponding actor.
+   *        If you pass in a plain string (by accident or because you're lazy),
+   *        then a promise of the same string is simply returned.
+   * @return {object}
+   *         A promise that is resolved when the full string contents
+   *         are available, or rejected if something goes wrong.
+   */
+  async getLongString(stringGrip) {
+    const payload = await getLongStringFullText(
+      this.commands.client,
+      stringGrip
+    );
+    this.emitForTests(TEST_EVENTS.LONGSTRING_RESOLVED, { payload });
+    return payload;
+  }
+
+  /**
+   * Retrieve the stack-trace information for the given StackTracesActor.
+   *
+   * @param object actor
+   *        - {Object} targetFront: the target front.
+   *
+   *        - {String} resourceId: the resource id for the network request".
+   * @return {object}
+   */
+  async _getStackTraceFromWatcher(actor) {
+    // If we request the stack trace for the navigation request,
+    // t was coming from previous page content process, which may no longer be around.
+    // In any case, the previous target is destroyed and we can't fetch the stack anymore.
+    let stacktrace = [];
+    if (!actor.targetFront.isDestroyed()) {
+      const networkContentFront = await actor.targetFront.getFront(
+        "networkContent"
+      );
+      stacktrace = await networkContentFront.getStackTrace(
+        actor.stacktraceResourceId
+      );
+    }
+    return { stacktrace };
+  }
+
+  /**
+   * The handler for when the network event stacktrace resource is available.
+   * The resource contains basic info, the actual stacktrace is fetched lazily
+   * using requestData.
+   * @param {object} resource The network event stacktrace resource
+   */
+  async onStackTraceAvailable(resource) {
+    if (!this.stackTraces.has(resource.resourceId)) {
+      // If no stacktrace info exists, this means the network event
+      // has not fired yet, lets store useful info for the NETWORK_EVENT
+      // resource.
+      this.stackTraces.set(resource.resourceId, resource);
+    } else {
+      // If stacktrace info exists, this means the network event has already
+      // fired, so lets just update the reducer with the necessary stacktrace info.
+      const request = this.stackTraces.get(resource.resourceId);
+      request.cause.stacktraceAvailable = resource.stacktraceAvailable;
+      request.cause.lastFrame = resource.lastFrame;
+
+      this.stackTraces.delete(resource.resourceId);
+
+      this.stackTraceRequestInfoByActorID.set(request.actor, {
+        targetFront: resource.targetFront,
+        stacktraceResourceId: resource.resourceId,
+      });
+
+      if (this.actionsEnabled && this.actions.updateRequest) {
+        await this.actions.updateRequest(request.actor, request, true);
+      }
+    }
+  }
+
+  /**
+   * The handler for when the network event resource is available.
+   *
+   * @param {object} resource The network event resource
+   */
+  async onNetworkResourceAvailable(resource) {
+    const { actor, stacktraceResourceId, cause } = resource;
+
+    if (!this._requestDataEnabled) {
+      this._requestDataEnabled = true;
+    }
+
+    // Check if a stacktrace resource already exists for this network resource.
+    if (this.stackTraces.has(stacktraceResourceId)) {
+      const { stacktraceAvailable, lastFrame, targetFront } =
+        this.stackTraces.get(stacktraceResourceId);
+
+      resource.cause.stacktraceAvailable = stacktraceAvailable;
+      resource.cause.lastFrame = lastFrame;
+
+      this.stackTraces.delete(stacktraceResourceId);
+      // We retrieve preliminary information about the stacktrace from the
+      // NETWORK_EVENT_STACKTRACE resource via `this.stackTraces` Map,
+      // The actual stacktrace is fetched lazily based on the actor id, using
+      // the targetFront and the stacktrace resource id therefore we
+      // map these for easy access.
+      this.stackTraceRequestInfoByActorID.set(actor, {
+        targetFront,
+        stacktraceResourceId,
+      });
+    } else if (cause) {
+      // If the stacktrace for this request is not available, and we
+      // expect that this request should have a stacktrace, lets store
+      // some useful info for when the NETWORK_EVENT_STACKTRACE resource
+      // finally comes.
+      this.stackTraces.set(stacktraceResourceId, { actor, cause });
+    }
+    await this.addRequest(actor, resource);
+    this.emitForTests(TEST_EVENTS.NETWORK_EVENT, resource);
+  }
+
+  /**
+   * The handler for when the network event resource is updated.
+   *
+   * @param {object} resource The updated network event resource.
+   */
+  async onNetworkResourceUpdated(resource) {
+    // Identify the channel as SSE if mimeType is event-stream.
+    if (resource?.mimeType?.includes("text/event-stream")) {
+      await this.setEventStreamFlag(resource.actor);
+    }
+
+    this.pendingRequests.delete(resource.actor);
+    if (this.actionsEnabled && this.actions.updateRequest) {
+      await this.actions.updateRequest(resource.actor, resource, true);
+    }
+
+    // This event is fired only once per request, once all the properties are fetched
+    // from `onNetworkResourceUpdated`. There should be no more RDP requests after this.
+    // Note that this event might be consumed by extension so, emit it in production
+    // release as well.
+    this.emitForTests(TEST_EVENTS.NETWORK_EVENT_UPDATED, resource.actor);
+    this.emit(EVENTS.PAYLOAD_READY, resource);
+  }
+
+  /**
+   * The "webSocketOpened" message type handler.
+   *
+   * @param {number} httpChannelId the channel ID
+   * @param {string} effectiveURI the effective URI of the page
+   * @param {string} protocols webSocket protocols
+   * @param {string} extensions
+   */
+  async onWebSocketOpened(httpChannelId, effectiveURI, protocols, extensions) {}
+
+  /**
+   * The "webSocketClosed" message type handler.
+   *
+   * @param {number} httpChannelId
+   * @param {boolean} wasClean
+   * @param {number} code
+   * @param {string} reason
+   */
+  async onWebSocketClosed(httpChannelId, wasClean, code, reason) {
+    if (this.actionsEnabled && this.actions.closeConnection) {
+      await this.actions.closeConnection(httpChannelId, wasClean, code, reason);
+    }
+  }
+
+  /**
+   * The "frameSent" message type handler.
+   *
+   * @param {number} httpChannelId the channel ID
+   * @param {object} data websocket frame information
+   */
+  async onFrameSent(httpChannelId, data) {
+    this.addMessage(httpChannelId, data);
+  }
+
+  /**
+   * The "frameReceived" message type handler.
+   *
+   * @param {number} httpChannelId the channel ID
+   * @param {object} data websocket frame information
+   */
+  async onFrameReceived(httpChannelId, data) {
+    this.addMessage(httpChannelId, data);
+  }
+
+  /**
+   * Add a new WebSocket frame to application state.
+   *
+   * @param {number} httpChannelId the channel ID
+   * @param {object} data websocket frame information
+   */
+  async addMessage(httpChannelId, data) {
+    if (this.actionsEnabled && this.actions.addMessage) {
+      await this.actions.addMessage(httpChannelId, data, true);
+    }
+    // TODO: Emit an event for test here
+  }
+
+  /**
+   * Public connector API to lazily request HTTP details from the backend.
+   *
+   * The method focus on:
+   * - calling the right actor method,
+   * - emitting an event to tell we start fetching some request data,
+   * - call data processing method.
+   *
+   * @param {string} actor actor id (used as request id)
+   * @param {string} method identifier of the data we want to fetch
+   *
+   * @return {Promise} return a promise resolved when data is received.
+   */
+  requestData(actor, method) {
+    // if this is `false`, do not try to request data as requests on the backend
+    // might no longer exist (usually `false` after requests are cleared).
+    if (!this._requestDataEnabled) {
+      return Promise.resolve();
+    }
+    // Key string used in `lazyRequestData`. We use this Map to prevent requesting
+    // the same data twice at the same time.
+    const key = `${actor}-${method}`;
+    let promise = this.lazyRequestData.get(key);
+    // If a request is pending, reuse it.
+    if (promise) {
+      return promise;
+    }
+    // Fetch the data
+    promise = this._requestData(actor, method).then(async payload => {
+      // Remove the request from the cache, any new call to requestData will fetch the
+      // data again.
+      this.lazyRequestData.delete(key);
+
+      if (this.actionsEnabled && this.actions.updateRequest) {
+        await this.actions.updateRequest(
+          actor,
+          {
+            ...payload,
+            // Lockdown *Available property once we fetch data from back-end.
+            // Using this as a flag to prevent fetching arrived data again.
+            [`${method}Available`]: false,
+          },
+          true
+        );
+      }
+
+      return payload;
+    });
+
+    this.lazyRequestData.set(key, promise);
+
+    return promise;
+  }
+
+  /**
+   * Internal helper used to request HTTP details from the backend.
+   *
+   * This is internal method that focus on:
+   * - calling the right actor method,
+   * - emitting an event to tell we start fetching some request data,
+   * - call data processing method.
+   *
+   * @param {string} actor actor id (used as request id)
+   * @param {string} method identifier of the data we want to fetch
+   *
+   * @return {Promise} return a promise resolved when data is received.
+   */
+  async _requestData(actor, method) {
+    // Backup the lastRequestDataClearId before doing any async processing.
+    const lastRequestDataClearId = this._lastRequestDataClearId;
+
+    // Calculate real name of the client getter.
+    const clientMethodName = `get${method
+      .charAt(0)
+      .toUpperCase()}${method.slice(1)}`;
+    // The name of the callback that processes request response
+    const callbackMethodName = `on${method
+      .charAt(0)
+      .toUpperCase()}${method.slice(1)}`;
+    // And the event to fire before updating this data
+    const updatingEventName = `UPDATING_${method
+      .replace(/([A-Z])/g, "_$1")
+      .toUpperCase()}`;
+
+    // Emit event that tell we just start fetching some data
+    this.emitForTests(EVENTS[updatingEventName], actor);
+
+    // Make sure we fetch the real actor data instead of cloned actor
+    // e.g. CustomRequestPanel will clone a request with additional '-clone' actor id
+    const actorID = actor.replace("-clone", "");
+
+    // 'getStackTrace' is the only one to be fetched via the NetworkContent actor in content process
+    // while all other attributes are fetched from the NetworkEvent actors, running in the parent process
+    let response;
+    if (
+      clientMethodName == "getStackTrace" &&
+      this.commands.resourceCommand.hasResourceCommandSupport(
+        this.commands.resourceCommand.TYPES.NETWORK_EVENT_STACKTRACE
+      )
+    ) {
+      const requestInfo = this.stackTraceRequestInfoByActorID.get(actorID);
+      const { stacktrace } = await this._getStackTraceFromWatcher(requestInfo);
+      this.stackTraceRequestInfoByActorID.delete(actorID);
+      response = { from: actor, stacktrace };
+    } else {
+      // We don't create fronts for NetworkEvent actors,
+      // so that we have to do the request manually via DevToolsClient.request()
+      try {
+        const packet = {
+          to: actorID,
+          type: clientMethodName,
+        };
+        response = await this.commands.client.request(packet);
+      } catch (e) {
+        if (this._lastRequestDataClearId !== lastRequestDataClearId) {
+          // If lastRequestDataClearId was updated, FirefoxDataProvider:clear()
+          // was called and all network event actors have been destroyed.
+          // Swallow errors to avoid unhandled promise rejections in tests.
+          console.warn(
+            `Firefox Data Provider destroyed while requesting data: ${e.message}`
+          );
+          // Return an empty response packet to avoid too many callback errors.
+          response = { from: actor };
+        } else {
+          throw new Error(
+            `Error while calling method ${clientMethodName}: ${e.message}`
+          );
+        }
+      }
+    }
+
+    // Restore clone actor id
+    if (actor.includes("-clone")) {
+      // Because response's properties are read-only, we create a new response
+      response = { ...response, from: `${response.from}-clone` };
+    }
+
+    // Call data processing method.
+    return this[callbackMethodName](response);
+  }
+
+  /**
+   * Handles additional information received for a "requestHeaders" packet.
+   *
+   * @param {object} response the message received from the server.
+   */
+  async onRequestHeaders(response) {
+    const payload = await this.updateRequest(response.from, {
+      requestHeaders: response,
+    });
+    this.emitForTests(TEST_EVENTS.RECEIVED_REQUEST_HEADERS, response);
+    return payload.requestHeaders;
+  }
+
+  /**
+   * Handles additional information received for a "responseHeaders" packet.
+   *
+   * @param {object} response the message received from the server.
+   */
+  async onResponseHeaders(response) {
+    const payload = await this.updateRequest(response.from, {
+      responseHeaders: response,
+    });
+    this.emitForTests(TEST_EVENTS.RECEIVED_RESPONSE_HEADERS, response);
+    return payload.responseHeaders;
+  }
+
+  /**
+   * Handles additional information received for a "requestCookies" packet.
+   *
+   * @param {object} response the message received from the server.
+   */
+  async onRequestCookies(response) {
+    const payload = await this.updateRequest(response.from, {
+      requestCookies: response,
+    });
+    this.emitForTests(TEST_EVENTS.RECEIVED_REQUEST_COOKIES, response);
+    return payload.requestCookies;
+  }
+
+  /**
+   * Handles additional information received for a "requestPostData" packet.
+   *
+   * @param {object} response the message received from the server.
+   */
+  async onRequestPostData(response) {
+    const payload = await this.updateRequest(response.from, {
+      requestPostData: response,
+    });
+    this.emitForTests(TEST_EVENTS.RECEIVED_REQUEST_POST_DATA, response);
+    return payload.requestPostData;
+  }
+
+  /**
+   * Handles additional information received for a "securityInfo" packet.
+   *
+   * @param {object} response the message received from the server.
+   */
+  async onSecurityInfo(response) {
+    const payload = await this.updateRequest(response.from, {
+      securityInfo: response.securityInfo,
+    });
+    this.emitForTests(TEST_EVENTS.RECEIVED_SECURITY_INFO, response);
+    return payload.securityInfo;
+  }
+
+  /**
+   * Handles additional information received for a "responseCookies" packet.
+   *
+   * @param {object} response the message received from the server.
+   */
+  async onResponseCookies(response) {
+    const payload = await this.updateRequest(response.from, {
+      responseCookies: response,
+    });
+    this.emitForTests(TEST_EVENTS.RECEIVED_RESPONSE_COOKIES, response);
+    return payload.responseCookies;
+  }
+
+  /**
+   * Handles additional information received for a "responseCache" packet.
+   * @param {object} response the message received from the server.
+   */
+  async onResponseCache(response) {
+    const payload = await this.updateRequest(response.from, {
+      responseCache: response,
+    });
+    this.emitForTests(TEST_EVENTS.RECEIVED_RESPONSE_CACHE, response);
+    return payload.responseCache;
+  }
+
+  /**
+   * Handles additional information received via "getResponseContent" request.
+   *
+   * @param {object} response the message received from the server.
+   */
+  async onResponseContent(response) {
+    const payload = await this.updateRequest(response.from, {
+      // We have to ensure passing mimeType as fetchResponseContent needs it from
+      // updateRequest. It will convert the LongString in `response.content.text` to a
+      // string.
+      mimeType: response.content.mimeType,
+      responseContent: response,
+    });
+    this.emitForTests(TEST_EVENTS.RECEIVED_RESPONSE_CONTENT, response);
+    return payload.responseContent;
+  }
+
+  /**
+   * Handles additional information received for a "eventTimings" packet.
+   *
+   * @param {object} response the message received from the server.
+   */
+  async onEventTimings(response) {
+    const payload = await this.updateRequest(response.from, {
+      eventTimings: response,
+    });
+
+    // This event is utilized only in tests but, DAMP is using it too
+    // and running DAMP test doesn't set the `devtools.testing` flag.
+    // So, emit this event even in the production mode.
+    // See also: https://bugzilla.mozilla.org/show_bug.cgi?id=1578215
+    this.emit(EVENTS.RECEIVED_EVENT_TIMINGS, response);
+    return payload.eventTimings;
+  }
+
+  /**
+   * Handles information received for a "stackTrace" packet.
+   *
+   * @param {object} response the message received from the server.
+   */
+  async onStackTrace(response) {
+    const payload = await this.updateRequest(response.from, {
+      stacktrace: response.stacktrace,
+    });
+    this.emitForTests(TEST_EVENTS.RECEIVED_EVENT_STACKTRACE, response);
+    return payload.stacktrace;
+  }
+
+  /**
+   * Handle EventSource events.
+   */
+
+  async onEventSourceConnectionClosed(httpChannelId) {
+    if (this.actionsEnabled && this.actions.closeConnection) {
+      await this.actions.closeConnection(httpChannelId);
+    }
+  }
+
+  async onEventReceived(httpChannelId, data) {
+    // Dispatch the same action used by websocket inspector.
+    this.addMessage(httpChannelId, data);
+  }
+
+  async setEventStreamFlag(actorId) {
+    if (this.actionsEnabled && this.actions.setEventStreamFlag) {
+      await this.actions.setEventStreamFlag(actorId, true);
+    }
+  }
+
+  /**
+   * Fire events for the owner object.
+   */
+  emit(type, data) {
+    if (this.owner) {
+      this.owner.emit(type, data);
+    }
+  }
+
+  /**
+   * Fire test events for the owner object. These events are
+   * emitted only when tests are running.
+   */
+  emitForTests(type, data) {
+    if (this.owner) {
+      this.owner.emitForTests(type, data);
+    }
+  }
+}
+
+module.exports = FirefoxDataProvider;
diff --git a/devtools/client/netmonitor/src/connector/har-metadata-collector.js b/devtools/client/netmonitor/src/connector/har-metadata-collector.js
new file mode 100644
index 0000000000..c0227065a5
--- /dev/null
+++ b/devtools/client/netmonitor/src/connector/har-metadata-collector.js
@@ -0,0 +1,97 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+"use strict";
+
+const {
+  TYPES,
+} = require("resource://devtools/shared/commands/resource/resource-command.js");
+
+/**
+ * This collector class is dedicated to recording additional metadata necessary
+ * to build HAR files. The actual request data will be provided by the
+ * netmonitor which is already monitoring for requests.
+ *
+ * The only purpose of this class is to record additional document and network
+ * events, which will help to assign requests to individual pages.
+ *
+ * It should be created and destroyed by the main netmonitor data collector.
+ */
+class HarMetadataCollector {
+  #commands;
+  #initialTargetTitle;
+  #navigationRequests;
+  #targetTitlesPerURL;
+
+  constructor(commands) {
+    this.#commands = commands;
+  }
+
+  /**
+   * Stop recording and clear the state.
+   */
+  destroy() {
+    this.clear();
+
+    this.#commands.resourceCommand.unwatchResources(
+      [TYPES.DOCUMENT_EVENT, TYPES.NETWORK_EVENT],
+      {
+        onAvailable: this.#onResourceAvailable,
+      }
+    );
+  }
+
+  /**
+   * Reset the current state.
+   */
+  clear() {
+    this.#navigationRequests = [];
+    this.#targetTitlesPerURL = new Map();
+    this.#initialTargetTitle = this.#commands.targetCommand.targetFront.title;
+  }
+
+  /**
+   * Start recording additional events for HAR files building.
+   */
+  async connect() {
+    this.clear();
+
+    await this.#commands.resourceCommand.watchResources(
+      [TYPES.DOCUMENT_EVENT, TYPES.NETWORK_EVENT],
+      {
+        onAvailable: this.#onResourceAvailable,
+      }
+    );
+  }
+
+  getHarData() {
+    return {
+      initialTargetTitle: this.#initialTargetTitle,
+      navigationRequests: this.#navigationRequests,
+      targetTitlesPerURL: this.#targetTitlesPerURL,
+    };
+  }
+
+  #onResourceAvailable = resources => {
+    for (const resource of resources) {
+      if (resource.resourceType === TYPES.DOCUMENT_EVENT) {
+        if (
+          resource.name === "dom-complete" &&
+          resource.targetFront.isTopLevel
+        ) {
+          this.#targetTitlesPerURL.set(
+            resource.targetFront.url,
+            resource.targetFront.title
+          );
+        }
+      } else if (resource.resourceType === TYPES.NETWORK_EVENT) {
+        if (resource.isNavigationRequest) {
+          this.#navigationRequests.push(resource);
+        }
+      }
+    }
+  };
+}
+
+exports.HarMetadataCollector = HarMetadataCollector;
diff --git a/devtools/client/netmonitor/src/connector/index.js b/devtools/client/netmonitor/src/connector/index.js
new file mode 100644
index 0000000000..b1f23a2269
--- /dev/null
+++ b/devtools/client/netmonitor/src/connector/index.js
@@ -0,0 +1,543 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+"use strict";
+
+const {
+  ACTIVITY_TYPE,
+  EVENTS,
+  TEST_EVENTS,
+} = require("resource://devtools/client/netmonitor/src/constants.js");
+const FirefoxDataProvider = require("resource://devtools/client/netmonitor/src/connector/firefox-data-provider.js");
+const {
+  getDisplayedTimingMarker,
+} = require("resource://devtools/client/netmonitor/src/selectors/index.js");
+
+const {
+  TYPES,
+} = require("resource://devtools/shared/commands/resource/resource-command.js");
+
+// Network throttling
+loader.lazyRequireGetter(
+  this,
+  "throttlingProfiles",
+  "resource://devtools/client/shared/components/throttling/profiles.js"
+);
+
+loader.lazyRequireGetter(
+  this,
+  "HarMetadataCollector",
+  "resource://devtools/client/netmonitor/src/connector/har-metadata-collector.js",
+  true
+);
+
+const DEVTOOLS_ENABLE_PERSISTENT_LOG_PREF = "devtools.netmonitor.persistlog";
+
+/**
+ * Connector to Firefox backend.
+ */
+class Connector {
+  constructor() {
+    // Public methods
+    this.connect = this.connect.bind(this);
+    this.disconnect = this.disconnect.bind(this);
+    this.willNavigate = this.willNavigate.bind(this);
+    this.navigate = this.navigate.bind(this);
+    this.triggerActivity = this.triggerActivity.bind(this);
+    this.viewSourceInDebugger = this.viewSourceInDebugger.bind(this);
+    this.requestData = this.requestData.bind(this);
+    this.getTimingMarker = this.getTimingMarker.bind(this);
+    this.updateNetworkThrottling = this.updateNetworkThrottling.bind(this);
+
+    // Internals
+    this.getLongString = this.getLongString.bind(this);
+    this.onResourceAvailable = this.onResourceAvailable.bind(this);
+    this.onResourceUpdated = this.onResourceUpdated.bind(this);
+    this.updatePersist = this.updatePersist.bind(this);
+
+    this.networkFront = null;
+  }
+
+  static NETWORK_RESOURCES = [
+    TYPES.NETWORK_EVENT,
+    TYPES.NETWORK_EVENT_STACKTRACE,
+    TYPES.WEBSOCKET,
+    TYPES.SERVER_SENT_EVENT,
+  ];
+
+  get currentTarget() {
+    return this.commands.targetCommand.targetFront;
+  }
+
+  /**
+   * Connect to the backend.
+   *
+   * @param {Object} connection object with e.g. reference to the Toolbox.
+   * @param {Object} actions (optional) is used to fire Redux actions to update store.
+   * @param {Object} getState (optional) is used to get access to the state.
+   */
+  async connect(connection, actions, getState) {
+    this.actions = actions;
+    this.getState = getState;
+    this.toolbox = connection.toolbox;
+    this.commands = this.toolbox.commands;
+    this.networkCommand = this.commands.networkCommand;
+
+    // The owner object (NetMonitorAPI) received all events.
+    this.owner = connection.owner;
+
+    this.networkFront =
+      await this.commands.watcherFront.getNetworkParentActor();
+
+    this.dataProvider = new FirefoxDataProvider({
+      commands: this.commands,
+      actions: this.actions,
+      owner: this.owner,
+    });
+
+    this._harMetadataCollector = new HarMetadataCollector(this.commands);
+    await this._harMetadataCollector.connect();
+
+    await this.commands.resourceCommand.watchResources([TYPES.DOCUMENT_EVENT], {
+      onAvailable: this.onResourceAvailable,
+    });
+
+    await this.resume(false);
+
+    // Server side persistance of the data across reload is disabled by default.
+    // Ensure enabling it, if the related frontend pref is true.
+    if (Services.prefs.getBoolPref(DEVTOOLS_ENABLE_PERSISTENT_LOG_PREF)) {
+      await this.updatePersist();
+    }
+    Services.prefs.addObserver(
+      DEVTOOLS_ENABLE_PERSISTENT_LOG_PREF,
+      this.updatePersist
+    );
+  }
+
+  disconnect() {
+    // As this function might be called twice, we need to guard if already called.
+    if (this._destroyed) {
+      return;
+    }
+
+    this._destroyed = true;
+
+    this.commands.resourceCommand.unwatchResources([TYPES.DOCUMENT_EVENT], {
+      onAvailable: this.onResourceAvailable,
+    });
+
+    this.pause();
+
+    Services.prefs.removeObserver(
+      DEVTOOLS_ENABLE_PERSISTENT_LOG_PREF,
+      this.updatePersist
+    );
+
+    if (this.actions) {
+      this.actions.batchReset();
+    }
+
+    this.dataProvider.destroy();
+    this.dataProvider = null;
+    this._harMetadataCollector.destroy();
+  }
+
+  clear() {
+    // Clear all the caches in the data provider
+    this.dataProvider.clear();
+
+    this._harMetadataCollector.clear();
+
+    this.commands.resourceCommand.clearResources(Connector.NETWORK_RESOURCES);
+    this.emitForTests("clear-network-resources");
+
+    // Disable the realted network logs in the webconsole
+    this.toolbox.disableAllConsoleNetworkLogs();
+  }
+
+  pause() {
+    return this.commands.resourceCommand.unwatchResources(
+      Connector.NETWORK_RESOURCES,
+      {
+        onAvailable: this.onResourceAvailable,
+        onUpdated: this.onResourceUpdated,
+      }
+    );
+  }
+
+  resume(ignoreExistingResources = true) {
+    return this.commands.resourceCommand.watchResources(
+      Connector.NETWORK_RESOURCES,
+      {
+        onAvailable: this.onResourceAvailable,
+        onUpdated: this.onResourceUpdated,
+        ignoreExistingResources,
+      }
+    );
+  }
+
+  async onResourceAvailable(resources, { areExistingResources }) {
+    for (const resource of resources) {
+      if (resource.resourceType === TYPES.DOCUMENT_EVENT) {
+        this.onDocEvent(resource, { areExistingResources });
+        continue;
+      }
+
+      if (resource.resourceType === TYPES.NETWORK_EVENT) {
+        this.dataProvider.onNetworkResourceAvailable(resource);
+        continue;
+      }
+
+      if (resource.resourceType === TYPES.NETWORK_EVENT_STACKTRACE) {
+        this.dataProvider.onStackTraceAvailable(resource);
+        continue;
+      }
+
+      if (resource.resourceType === TYPES.WEBSOCKET) {
+        const { wsMessageType } = resource;
+
+        switch (wsMessageType) {
+          case "webSocketOpened": {
+            this.dataProvider.onWebSocketOpened(
+              resource.httpChannelId,
+              resource.effectiveURI,
+              resource.protocols,
+              resource.extensions
+            );
+            break;
+          }
+          case "webSocketClosed": {
+            this.dataProvider.onWebSocketClosed(
+              resource.httpChannelId,
+              resource.wasClean,
+              resource.code,
+              resource.reason
+            );
+            break;
+          }
+          case "frameReceived": {
+            this.dataProvider.onFrameReceived(
+              resource.httpChannelId,
+              resource.data
+            );
+            break;
+          }
+          case "frameSent": {
+            this.dataProvider.onFrameSent(
+              resource.httpChannelId,
+              resource.data
+            );
+            break;
+          }
+        }
+        continue;
+      }
+
+      if (resource.resourceType === TYPES.SERVER_SENT_EVENT) {
+        const { messageType, httpChannelId, data } = resource;
+        switch (messageType) {
+          case "eventSourceConnectionClosed": {
+            this.dataProvider.onEventSourceConnectionClosed(httpChannelId);
+            break;
+          }
+          case "eventReceived": {
+            this.dataProvider.onEventReceived(httpChannelId, data);
+            break;
+          }
+        }
+      }
+    }
+  }
+
+  async onResourceUpdated(updates) {
+    for (const { resource, update } of updates) {
+      this.dataProvider.onNetworkResourceUpdated(resource, update);
+    }
+  }
+
+  enableActions(enable) {
+    this.dataProvider.enableActions(enable);
+  }
+
+  willNavigate() {
+    if (this.actions) {
+      if (!Services.prefs.getBoolPref(DEVTOOLS_ENABLE_PERSISTENT_LOG_PREF)) {
+        this.actions.batchReset();
+        this.actions.clearRequests();
+      } else {
+        // If the log is persistent, just clear all accumulated timing markers.
+        this.actions.clearTimingMarkers();
+      }
+    }
+
+    if (this.actions && this.getState) {
+      const state = this.getState();
+      // Resume is done automatically on page reload/navigation.
+      if (!state.requests.recording) {
+        this.actions.toggleRecording();
+      }
+
+      // Stop any ongoing search.
+      if (state.search.ongoingSearch) {
+        this.actions.stopOngoingSearch();
+      }
+    }
+  }
+
+  navigate() {
+    if (!this.dataProvider.hasPendingRequests()) {
+      this.onReloaded();
+      return;
+    }
+    const listener = () => {
+      if (this.dataProvider && this.dataProvider.hasPendingRequests()) {
+        return;
+      }
+      if (this.owner) {
+        this.owner.off(EVENTS.PAYLOAD_READY, listener);
+      }
+      // Netmonitor may already be destroyed,
+      // so do not try to notify the listeners
+      if (this.dataProvider) {
+        this.onReloaded();
+      }
+    };
+    if (this.owner) {
+      this.owner.on(EVENTS.PAYLOAD_READY, listener);
+    }
+  }
+
+  onReloaded() {
+    const panel = this.toolbox.getPanel("netmonitor");
+    if (panel) {
+      panel.emit("reloaded");
+    }
+  }
+
+  /**
+   * The "DOMContentLoaded" and "Load" events sent by the console actor.
+   *
+   * @param {object} resource The DOCUMENT_EVENT resource
+   */
+  onDocEvent(resource, { areExistingResources }) {
+    if (!resource.targetFront.isTopLevel) {
+      // Only consider top level document, and ignore remote iframes top document
+      return;
+    }
+
+    // Netmonitor does not support dom-loading
+    if (
+      resource.name != "dom-interactive" &&
+      resource.name != "dom-complete" &&
+      resource.name != "will-navigate"
+    ) {
+      return;
+    }
+
+    if (resource.name == "will-navigate") {
+      // When we open the netmonitor while the page already started loading,
+      // we don't want to clear it. So here, we ignore will-navigate events
+      // which were stored in the ResourceCommand cache and only consider
+      // the live one coming straight from the server.
+      if (!areExistingResources) {
+        this.willNavigate();
+      }
+      return;
+    }
+
+    if (this.actions) {
+      this.actions.addTimingMarker(resource);
+    }
+
+    if (resource.name === "dom-complete") {
+      this.navigate();
+    }
+
+    this.emitForTests(TEST_EVENTS.TIMELINE_EVENT, resource);
+  }
+
+  async updatePersist() {
+    const enabled = Services.prefs.getBoolPref(
+      DEVTOOLS_ENABLE_PERSISTENT_LOG_PREF
+    );
+
+    await this.networkFront.setPersist(enabled);
+
+    this.emitForTests(TEST_EVENTS.PERSIST_CHANGED, enabled);
+  }
+
+  /**
+   * Triggers a specific "activity" to be performed by the frontend.
+   * This can be, for example, triggering reloads or enabling/disabling cache.
+   *
+   * @param {number} type The activity type. See the ACTIVITY_TYPE const.
+   * @return {object} A promise resolved once the activity finishes and the frontend
+   *                  is back into "standby" mode.
+   */
+  triggerActivity(type) {
+    // Puts the frontend into "standby" (when there's no particular activity).
+    const standBy = () => {
+      this.currentActivity = ACTIVITY_TYPE.NONE;
+    };
+
+    // Reconfigures the tab, optionally triggering a reload.
+    const reconfigureTab = async options => {
+      await this.commands.targetConfigurationCommand.updateConfiguration(
+        options
+      );
+    };
+
+    // Reconfigures the tab and waits for the target to finish navigating.
+    const reconfigureTabAndReload = async options => {
+      await reconfigureTab(options);
+      await this.commands.targetCommand.reloadTopLevelTarget();
+    };
+
+    switch (type) {
+      case ACTIVITY_TYPE.RELOAD.WITH_CACHE_DEFAULT:
+        return reconfigureTabAndReload({}).then(standBy);
+      case ACTIVITY_TYPE.RELOAD.WITH_CACHE_ENABLED:
+        this.currentActivity = ACTIVITY_TYPE.ENABLE_CACHE;
+        this.commands.resourceCommand
+          .waitForNextResource(
+            this.commands.resourceCommand.TYPES.DOCUMENT_EVENT,
+            {
+              ignoreExistingResources: true,
+              predicate(resource) {
+                return resource.name == "will-navigate";
+              },
+            }
+          )
+          .then(() => {
+            this.currentActivity = type;
+          });
+        return reconfigureTabAndReload({
+          cacheDisabled: false,
+        }).then(standBy);
+      case ACTIVITY_TYPE.RELOAD.WITH_CACHE_DISABLED:
+        this.currentActivity = ACTIVITY_TYPE.DISABLE_CACHE;
+        this.commands.resourceCommand
+          .waitForNextResource(
+            this.commands.resourceCommand.TYPES.DOCUMENT_EVENT,
+            {
+              ignoreExistingResources: true,
+              predicate(resource) {
+                return resource.name == "will-navigate";
+              },
+            }
+          )
+          .then(() => {
+            this.currentActivity = type;
+          });
+        return reconfigureTabAndReload({
+          cacheDisabled: true,
+        }).then(standBy);
+      case ACTIVITY_TYPE.ENABLE_CACHE:
+        this.currentActivity = type;
+        return reconfigureTab({
+          cacheDisabled: false,
+        }).then(standBy);
+      case ACTIVITY_TYPE.DISABLE_CACHE:
+        this.currentActivity = type;
+        return reconfigureTab({
+          cacheDisabled: true,
+        }).then(standBy);
+    }
+    this.currentActivity = ACTIVITY_TYPE.NONE;
+    return Promise.reject(new Error("Invalid activity type"));
+  }
+
+  /**
+   * Fetches the full text of a LongString.
+   *
+   * @param {object|string} stringGrip
+   *        The long string grip containing the corresponding actor.
+   *        If you pass in a plain string (by accident or because you're lazy),
+   *        then a promise of the same string is simply returned.
+   * @return {object}
+   *         A promise that is resolved when the full string contents
+   *         are available, or rejected if something goes wrong.
+   */
+  getLongString(stringGrip) {
+    return this.dataProvider.getLongString(stringGrip);
+  }
+
+  /**
+   * Used for HAR generation.
+   */
+  getHarData() {
+    return this._harMetadataCollector.getHarData();
+  }
+
+  /**
+   * Getter that returns the current toolbox instance.
+   * @return {Toolbox} toolbox instance
+   */
+  getToolbox() {
+    return this.toolbox;
+  }
+
+  /**
+   * Open a given source in Debugger
+   * @param {string} sourceURL source url
+   * @param {number} sourceLine source line number
+   */
+  viewSourceInDebugger(sourceURL, sourceLine, sourceColumn) {
+    if (this.toolbox) {
+      this.toolbox.viewSourceInDebugger(sourceURL, sourceLine, sourceColumn);
+    }
+  }
+
+  /**
+   * Fetch networkEventUpdate websocket message from back-end when
+   * data provider is connected.
+   * @param {object} request network request instance
+   * @param {string} type NetworkEventUpdate type
+   */
+  requestData(request, type) {
+    return this.dataProvider.requestData(request, type);
+  }
+
+  getTimingMarker(name) {
+    if (!this.getState) {
+      return -1;
+    }
+
+    const state = this.getState();
+    return getDisplayedTimingMarker(state, name);
+  }
+
+  async updateNetworkThrottling(enabled, profile) {
+    if (!enabled) {
+      this.networkFront.clearNetworkThrottling();
+    } else {
+      // The profile can be either a profile id which is used to
+      // search the predefined throttle profiles or a profile object
+      // as defined in the trottle tests.
+      if (typeof profile === "string") {
+        profile = throttlingProfiles.find(({ id }) => id == profile);
+      }
+      const { download, upload, latency } = profile;
+      await this.networkFront.setNetworkThrottling({
+        downloadThroughput: download,
+        uploadThroughput: upload,
+        latency,
+      });
+    }
+
+    this.emitForTests(TEST_EVENTS.THROTTLING_CHANGED, { profile });
+  }
+
+  /**
+   * Fire events for the owner object. These events are only
+   * used in tests so, don't fire them in production release.
+   */
+  emitForTests(type, data) {
+    if (this.owner) {
+      this.owner.emitForTests(type, data);
+    }
+  }
+}
+module.exports.Connector = Connector;
diff --git a/devtools/client/netmonitor/src/connector/moz.build b/devtools/client/netmonitor/src/connector/moz.build
new file mode 100644
index 0000000000..4f65ddea0e
--- /dev/null
+++ b/devtools/client/netmonitor/src/connector/moz.build
@@ -0,0 +1,9 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+DevToolsModules(
+    "firefox-data-provider.js",
+    "har-metadata-collector.js",
+    "index.js",
+)