From 6bf0a5cb5034a7e684dcc3500e841785237ce2dd Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Sun, 7 Apr 2024 19:32:43 +0200 Subject: Adding upstream version 1:115.7.0. Signed-off-by: Daniel Baumann --- .../src/connector/firefox-data-provider.js | 832 +++++++++++++++++++++ .../src/connector/har-metadata-collector.js | 97 +++ devtools/client/netmonitor/src/connector/index.js | 543 ++++++++++++++ devtools/client/netmonitor/src/connector/moz.build | 9 + 4 files changed, 1481 insertions(+) create mode 100644 devtools/client/netmonitor/src/connector/firefox-data-provider.js create mode 100644 devtools/client/netmonitor/src/connector/har-metadata-collector.js create mode 100644 devtools/client/netmonitor/src/connector/index.js create mode 100644 devtools/client/netmonitor/src/connector/moz.build (limited to 'devtools/client/netmonitor/src/connector') diff --git a/devtools/client/netmonitor/src/connector/firefox-data-provider.js b/devtools/client/netmonitor/src/connector/firefox-data-provider.js new file mode 100644 index 0000000000..9cdf6fc1d7 --- /dev/null +++ b/devtools/client/netmonitor/src/connector/firefox-data-provider.js @@ -0,0 +1,832 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +/* eslint-disable block-scoped-var */ + +"use strict"; + +const { + EVENTS, + TEST_EVENTS, +} = require("resource://devtools/client/netmonitor/src/constants.js"); +const { CurlUtils } = require("resource://devtools/client/shared/curl.js"); +const { + fetchHeaders, +} = require("resource://devtools/client/netmonitor/src/utils/request-utils.js"); + +const { + getLongStringFullText, +} = require("resource://devtools/client/shared/string-utils.js"); + +/** + * This object is responsible for fetching additional HTTP + * data from the backend over RDP protocol. + * + * The object also keeps track of RDP requests in-progress, + * so it's possible to determine whether all has been fetched + * or not. + */ +class FirefoxDataProvider { + /** + * Constructor for data provider + * + * @param {Object} commands Object defined from devtools/shared/commands to interact with the devtools backend + * @param {Object} actions set of actions fired during data fetching process. + * @param {Object} owner all events are fired on this object. + */ + constructor({ commands, actions, owner }) { + // Options + this.commands = commands; + this.actions = actions || {}; + this.actionsEnabled = true; + + // Allow requesting of on-demand network data, this would be `false` when requests + // are cleared (as we clear also on the backend), and will be flipped back + // to true on the next `onNetworkResourceAvailable` call. + this._requestDataEnabled = true; + + // `_requestDataEnabled` can only be used to prevent new calls to + // requestData. For pending/already started calls, we need to check if + // clear() was called during the call, which is the purpose of this counter. + this._lastRequestDataClearId = 0; + + this.owner = owner; + + // This holds stacktraces infomation temporarily. Stacktrace resources + // can come before or after (out of order) their related network events. + // This will hold stacktrace related info from the NETWORK_EVENT_STACKTRACE resource + // for the NETWORK_EVENT resource and vice versa. + this.stackTraces = new Map(); + // Map of the stacktrace information keyed by the actor id's + this.stackTraceRequestInfoByActorID = new Map(); + + // For tracking unfinished requests + this.pendingRequests = new Set(); + + // Map[key string => Promise] used by `requestData` to prevent requesting the same + // request data twice. + this.lazyRequestData = new Map(); + + // Fetching data from the backend + this.getLongString = this.getLongString.bind(this); + + // Event handlers + this.onNetworkResourceAvailable = + this.onNetworkResourceAvailable.bind(this); + this.onNetworkResourceUpdated = this.onNetworkResourceUpdated.bind(this); + + this.onWebSocketOpened = this.onWebSocketOpened.bind(this); + this.onWebSocketClosed = this.onWebSocketClosed.bind(this); + this.onFrameSent = this.onFrameSent.bind(this); + this.onFrameReceived = this.onFrameReceived.bind(this); + + this.onEventSourceConnectionClosed = + this.onEventSourceConnectionClosed.bind(this); + this.onEventReceived = this.onEventReceived.bind(this); + this.setEventStreamFlag = this.setEventStreamFlag.bind(this); + } + + /* + * Cleans up all the internal states, this usually done before navigation + * (without the persist flag on). + */ + clear() { + this.stackTraces.clear(); + this.pendingRequests.clear(); + this.lazyRequestData.clear(); + this.stackTraceRequestInfoByActorID.clear(); + this._requestDataEnabled = false; + this._lastRequestDataClearId++; + } + + destroy() { + // TODO: clear() is called in the middle of the lifecycle of the + // FirefoxDataProvider, for clarity we are exposing it as a separate method. + // `destroy` should be updated to nullify relevant instance properties. + this.clear(); + } + + /** + * Enable/disable firing redux actions (enabled by default). + * + * @param {boolean} enable Set to true to fire actions. + */ + enableActions(enable) { + this.actionsEnabled = enable; + } + + /** + * Add a new network request to application state. + * + * @param {string} id request id + * @param {object} resource resource payload will be added to application state + */ + async addRequest(id, resource) { + // Add to the pending requests which helps when deciding if the request is complete. + this.pendingRequests.add(id); + + if (this.actionsEnabled && this.actions.addRequest) { + await this.actions.addRequest(id, resource, true); + } + + this.emit(EVENTS.REQUEST_ADDED, id); + } + + /** + * Update a network request if it already exists in application state. + * + * @param {string} id request id + * @param {object} data data payload will be updated to application state + */ + async updateRequest(id, data) { + const { + responseContent, + responseCookies, + responseHeaders, + requestCookies, + requestHeaders, + requestPostData, + responseCache, + } = data; + + // fetch request detail contents in parallel + const [ + responseContentObj, + requestHeadersObj, + responseHeadersObj, + postDataObj, + requestCookiesObj, + responseCookiesObj, + responseCacheObj, + ] = await Promise.all([ + this.fetchResponseContent(responseContent), + this.fetchRequestHeaders(requestHeaders), + this.fetchResponseHeaders(responseHeaders), + this.fetchPostData(requestPostData), + this.fetchRequestCookies(requestCookies), + this.fetchResponseCookies(responseCookies), + this.fetchResponseCache(responseCache), + ]); + + const payload = Object.assign( + {}, + data, + responseContentObj, + requestHeadersObj, + responseHeadersObj, + postDataObj, + requestCookiesObj, + responseCookiesObj, + responseCacheObj + ); + + if (this.actionsEnabled && this.actions.updateRequest) { + await this.actions.updateRequest(id, payload, true); + } + + return payload; + } + + async fetchResponseContent(responseContent) { + const payload = {}; + if (responseContent?.content) { + const { text } = responseContent.content; + const response = await this.getLongString(text); + responseContent.content.text = response; + payload.responseContent = responseContent; + } + return payload; + } + + async fetchRequestHeaders(requestHeaders) { + const payload = {}; + if (requestHeaders?.headers?.length) { + const headers = await fetchHeaders(requestHeaders, this.getLongString); + if (headers) { + payload.requestHeaders = headers; + } + } + return payload; + } + + async fetchResponseHeaders(responseHeaders) { + const payload = {}; + if (responseHeaders?.headers?.length) { + const headers = await fetchHeaders(responseHeaders, this.getLongString); + if (headers) { + payload.responseHeaders = headers; + } + } + return payload; + } + + async fetchPostData(requestPostData) { + const payload = {}; + if (requestPostData?.postData) { + const { text } = requestPostData.postData; + const postData = await this.getLongString(text); + const headers = CurlUtils.getHeadersFromMultipartText(postData); + + // Calculate total header size and don't forget to include + // two new-line characters at the end. + const headersSize = headers.reduce((acc, { name, value }) => { + return acc + name.length + value.length + 2; + }, 0); + + requestPostData.postData.text = postData; + payload.requestPostData = { + ...requestPostData, + uploadHeaders: { headers, headersSize }, + }; + } + return payload; + } + + async fetchRequestCookies(requestCookies) { + const payload = {}; + if (requestCookies) { + const reqCookies = []; + // request store cookies in requestCookies or requestCookies.cookies + const cookies = requestCookies.cookies + ? requestCookies.cookies + : requestCookies; + // make sure cookies is iterable + if (typeof cookies[Symbol.iterator] === "function") { + for (const cookie of cookies) { + reqCookies.push( + Object.assign({}, cookie, { + value: await this.getLongString(cookie.value), + }) + ); + } + if (reqCookies.length) { + payload.requestCookies = reqCookies; + } + } + } + return payload; + } + + async fetchResponseCookies(responseCookies) { + const payload = {}; + if (responseCookies) { + const resCookies = []; + // response store cookies in responseCookies or responseCookies.cookies + const cookies = responseCookies.cookies + ? responseCookies.cookies + : responseCookies; + // make sure cookies is iterable + if (typeof cookies[Symbol.iterator] === "function") { + for (const cookie of cookies) { + resCookies.push( + Object.assign({}, cookie, { + value: await this.getLongString(cookie.value), + }) + ); + } + if (resCookies.length) { + payload.responseCookies = resCookies; + } + } + } + return payload; + } + + async fetchResponseCache(responseCache) { + const payload = {}; + if (responseCache) { + payload.responseCache = await responseCache; + payload.responseCacheAvailable = false; + } + return payload; + } + + /** + * Public API used by the Toolbox: Tells if there is still any pending request. + * + * @return {boolean} returns true if pending requests still exist in the queue. + */ + hasPendingRequests() { + return this.pendingRequests.size > 0; + } + + /** + * Fetches the full text of a LongString. + * + * @param {object|string} stringGrip + * The long string grip containing the corresponding actor. + * If you pass in a plain string (by accident or because you're lazy), + * then a promise of the same string is simply returned. + * @return {object} + * A promise that is resolved when the full string contents + * are available, or rejected if something goes wrong. + */ + async getLongString(stringGrip) { + const payload = await getLongStringFullText( + this.commands.client, + stringGrip + ); + this.emitForTests(TEST_EVENTS.LONGSTRING_RESOLVED, { payload }); + return payload; + } + + /** + * Retrieve the stack-trace information for the given StackTracesActor. + * + * @param object actor + * - {Object} targetFront: the target front. + * + * - {String} resourceId: the resource id for the network request". + * @return {object} + */ + async _getStackTraceFromWatcher(actor) { + // If we request the stack trace for the navigation request, + // t was coming from previous page content process, which may no longer be around. + // In any case, the previous target is destroyed and we can't fetch the stack anymore. + let stacktrace = []; + if (!actor.targetFront.isDestroyed()) { + const networkContentFront = await actor.targetFront.getFront( + "networkContent" + ); + stacktrace = await networkContentFront.getStackTrace( + actor.stacktraceResourceId + ); + } + return { stacktrace }; + } + + /** + * The handler for when the network event stacktrace resource is available. + * The resource contains basic info, the actual stacktrace is fetched lazily + * using requestData. + * @param {object} resource The network event stacktrace resource + */ + async onStackTraceAvailable(resource) { + if (!this.stackTraces.has(resource.resourceId)) { + // If no stacktrace info exists, this means the network event + // has not fired yet, lets store useful info for the NETWORK_EVENT + // resource. + this.stackTraces.set(resource.resourceId, resource); + } else { + // If stacktrace info exists, this means the network event has already + // fired, so lets just update the reducer with the necessary stacktrace info. + const request = this.stackTraces.get(resource.resourceId); + request.cause.stacktraceAvailable = resource.stacktraceAvailable; + request.cause.lastFrame = resource.lastFrame; + + this.stackTraces.delete(resource.resourceId); + + this.stackTraceRequestInfoByActorID.set(request.actor, { + targetFront: resource.targetFront, + stacktraceResourceId: resource.resourceId, + }); + + if (this.actionsEnabled && this.actions.updateRequest) { + await this.actions.updateRequest(request.actor, request, true); + } + } + } + + /** + * The handler for when the network event resource is available. + * + * @param {object} resource The network event resource + */ + async onNetworkResourceAvailable(resource) { + const { actor, stacktraceResourceId, cause } = resource; + + if (!this._requestDataEnabled) { + this._requestDataEnabled = true; + } + + // Check if a stacktrace resource already exists for this network resource. + if (this.stackTraces.has(stacktraceResourceId)) { + const { stacktraceAvailable, lastFrame, targetFront } = + this.stackTraces.get(stacktraceResourceId); + + resource.cause.stacktraceAvailable = stacktraceAvailable; + resource.cause.lastFrame = lastFrame; + + this.stackTraces.delete(stacktraceResourceId); + // We retrieve preliminary information about the stacktrace from the + // NETWORK_EVENT_STACKTRACE resource via `this.stackTraces` Map, + // The actual stacktrace is fetched lazily based on the actor id, using + // the targetFront and the stacktrace resource id therefore we + // map these for easy access. + this.stackTraceRequestInfoByActorID.set(actor, { + targetFront, + stacktraceResourceId, + }); + } else if (cause) { + // If the stacktrace for this request is not available, and we + // expect that this request should have a stacktrace, lets store + // some useful info for when the NETWORK_EVENT_STACKTRACE resource + // finally comes. + this.stackTraces.set(stacktraceResourceId, { actor, cause }); + } + await this.addRequest(actor, resource); + this.emitForTests(TEST_EVENTS.NETWORK_EVENT, resource); + } + + /** + * The handler for when the network event resource is updated. + * + * @param {object} resource The updated network event resource. + */ + async onNetworkResourceUpdated(resource) { + // Identify the channel as SSE if mimeType is event-stream. + if (resource?.mimeType?.includes("text/event-stream")) { + await this.setEventStreamFlag(resource.actor); + } + + this.pendingRequests.delete(resource.actor); + if (this.actionsEnabled && this.actions.updateRequest) { + await this.actions.updateRequest(resource.actor, resource, true); + } + + // This event is fired only once per request, once all the properties are fetched + // from `onNetworkResourceUpdated`. There should be no more RDP requests after this. + // Note that this event might be consumed by extension so, emit it in production + // release as well. + this.emitForTests(TEST_EVENTS.NETWORK_EVENT_UPDATED, resource.actor); + this.emit(EVENTS.PAYLOAD_READY, resource); + } + + /** + * The "webSocketOpened" message type handler. + * + * @param {number} httpChannelId the channel ID + * @param {string} effectiveURI the effective URI of the page + * @param {string} protocols webSocket protocols + * @param {string} extensions + */ + async onWebSocketOpened(httpChannelId, effectiveURI, protocols, extensions) {} + + /** + * The "webSocketClosed" message type handler. + * + * @param {number} httpChannelId + * @param {boolean} wasClean + * @param {number} code + * @param {string} reason + */ + async onWebSocketClosed(httpChannelId, wasClean, code, reason) { + if (this.actionsEnabled && this.actions.closeConnection) { + await this.actions.closeConnection(httpChannelId, wasClean, code, reason); + } + } + + /** + * The "frameSent" message type handler. + * + * @param {number} httpChannelId the channel ID + * @param {object} data websocket frame information + */ + async onFrameSent(httpChannelId, data) { + this.addMessage(httpChannelId, data); + } + + /** + * The "frameReceived" message type handler. + * + * @param {number} httpChannelId the channel ID + * @param {object} data websocket frame information + */ + async onFrameReceived(httpChannelId, data) { + this.addMessage(httpChannelId, data); + } + + /** + * Add a new WebSocket frame to application state. + * + * @param {number} httpChannelId the channel ID + * @param {object} data websocket frame information + */ + async addMessage(httpChannelId, data) { + if (this.actionsEnabled && this.actions.addMessage) { + await this.actions.addMessage(httpChannelId, data, true); + } + // TODO: Emit an event for test here + } + + /** + * Public connector API to lazily request HTTP details from the backend. + * + * The method focus on: + * - calling the right actor method, + * - emitting an event to tell we start fetching some request data, + * - call data processing method. + * + * @param {string} actor actor id (used as request id) + * @param {string} method identifier of the data we want to fetch + * + * @return {Promise} return a promise resolved when data is received. + */ + requestData(actor, method) { + // if this is `false`, do not try to request data as requests on the backend + // might no longer exist (usually `false` after requests are cleared). + if (!this._requestDataEnabled) { + return Promise.resolve(); + } + // Key string used in `lazyRequestData`. We use this Map to prevent requesting + // the same data twice at the same time. + const key = `${actor}-${method}`; + let promise = this.lazyRequestData.get(key); + // If a request is pending, reuse it. + if (promise) { + return promise; + } + // Fetch the data + promise = this._requestData(actor, method).then(async payload => { + // Remove the request from the cache, any new call to requestData will fetch the + // data again. + this.lazyRequestData.delete(key); + + if (this.actionsEnabled && this.actions.updateRequest) { + await this.actions.updateRequest( + actor, + { + ...payload, + // Lockdown *Available property once we fetch data from back-end. + // Using this as a flag to prevent fetching arrived data again. + [`${method}Available`]: false, + }, + true + ); + } + + return payload; + }); + + this.lazyRequestData.set(key, promise); + + return promise; + } + + /** + * Internal helper used to request HTTP details from the backend. + * + * This is internal method that focus on: + * - calling the right actor method, + * - emitting an event to tell we start fetching some request data, + * - call data processing method. + * + * @param {string} actor actor id (used as request id) + * @param {string} method identifier of the data we want to fetch + * + * @return {Promise} return a promise resolved when data is received. + */ + async _requestData(actor, method) { + // Backup the lastRequestDataClearId before doing any async processing. + const lastRequestDataClearId = this._lastRequestDataClearId; + + // Calculate real name of the client getter. + const clientMethodName = `get${method + .charAt(0) + .toUpperCase()}${method.slice(1)}`; + // The name of the callback that processes request response + const callbackMethodName = `on${method + .charAt(0) + .toUpperCase()}${method.slice(1)}`; + // And the event to fire before updating this data + const updatingEventName = `UPDATING_${method + .replace(/([A-Z])/g, "_$1") + .toUpperCase()}`; + + // Emit event that tell we just start fetching some data + this.emitForTests(EVENTS[updatingEventName], actor); + + // Make sure we fetch the real actor data instead of cloned actor + // e.g. CustomRequestPanel will clone a request with additional '-clone' actor id + const actorID = actor.replace("-clone", ""); + + // 'getStackTrace' is the only one to be fetched via the NetworkContent actor in content process + // while all other attributes are fetched from the NetworkEvent actors, running in the parent process + let response; + if ( + clientMethodName == "getStackTrace" && + this.commands.resourceCommand.hasResourceCommandSupport( + this.commands.resourceCommand.TYPES.NETWORK_EVENT_STACKTRACE + ) + ) { + const requestInfo = this.stackTraceRequestInfoByActorID.get(actorID); + const { stacktrace } = await this._getStackTraceFromWatcher(requestInfo); + this.stackTraceRequestInfoByActorID.delete(actorID); + response = { from: actor, stacktrace }; + } else { + // We don't create fronts for NetworkEvent actors, + // so that we have to do the request manually via DevToolsClient.request() + try { + const packet = { + to: actorID, + type: clientMethodName, + }; + response = await this.commands.client.request(packet); + } catch (e) { + if (this._lastRequestDataClearId !== lastRequestDataClearId) { + // If lastRequestDataClearId was updated, FirefoxDataProvider:clear() + // was called and all network event actors have been destroyed. + // Swallow errors to avoid unhandled promise rejections in tests. + console.warn( + `Firefox Data Provider destroyed while requesting data: ${e.message}` + ); + // Return an empty response packet to avoid too many callback errors. + response = { from: actor }; + } else { + throw new Error( + `Error while calling method ${clientMethodName}: ${e.message}` + ); + } + } + } + + // Restore clone actor id + if (actor.includes("-clone")) { + // Because response's properties are read-only, we create a new response + response = { ...response, from: `${response.from}-clone` }; + } + + // Call data processing method. + return this[callbackMethodName](response); + } + + /** + * Handles additional information received for a "requestHeaders" packet. + * + * @param {object} response the message received from the server. + */ + async onRequestHeaders(response) { + const payload = await this.updateRequest(response.from, { + requestHeaders: response, + }); + this.emitForTests(TEST_EVENTS.RECEIVED_REQUEST_HEADERS, response); + return payload.requestHeaders; + } + + /** + * Handles additional information received for a "responseHeaders" packet. + * + * @param {object} response the message received from the server. + */ + async onResponseHeaders(response) { + const payload = await this.updateRequest(response.from, { + responseHeaders: response, + }); + this.emitForTests(TEST_EVENTS.RECEIVED_RESPONSE_HEADERS, response); + return payload.responseHeaders; + } + + /** + * Handles additional information received for a "requestCookies" packet. + * + * @param {object} response the message received from the server. + */ + async onRequestCookies(response) { + const payload = await this.updateRequest(response.from, { + requestCookies: response, + }); + this.emitForTests(TEST_EVENTS.RECEIVED_REQUEST_COOKIES, response); + return payload.requestCookies; + } + + /** + * Handles additional information received for a "requestPostData" packet. + * + * @param {object} response the message received from the server. + */ + async onRequestPostData(response) { + const payload = await this.updateRequest(response.from, { + requestPostData: response, + }); + this.emitForTests(TEST_EVENTS.RECEIVED_REQUEST_POST_DATA, response); + return payload.requestPostData; + } + + /** + * Handles additional information received for a "securityInfo" packet. + * + * @param {object} response the message received from the server. + */ + async onSecurityInfo(response) { + const payload = await this.updateRequest(response.from, { + securityInfo: response.securityInfo, + }); + this.emitForTests(TEST_EVENTS.RECEIVED_SECURITY_INFO, response); + return payload.securityInfo; + } + + /** + * Handles additional information received for a "responseCookies" packet. + * + * @param {object} response the message received from the server. + */ + async onResponseCookies(response) { + const payload = await this.updateRequest(response.from, { + responseCookies: response, + }); + this.emitForTests(TEST_EVENTS.RECEIVED_RESPONSE_COOKIES, response); + return payload.responseCookies; + } + + /** + * Handles additional information received for a "responseCache" packet. + * @param {object} response the message received from the server. + */ + async onResponseCache(response) { + const payload = await this.updateRequest(response.from, { + responseCache: response, + }); + this.emitForTests(TEST_EVENTS.RECEIVED_RESPONSE_CACHE, response); + return payload.responseCache; + } + + /** + * Handles additional information received via "getResponseContent" request. + * + * @param {object} response the message received from the server. + */ + async onResponseContent(response) { + const payload = await this.updateRequest(response.from, { + // We have to ensure passing mimeType as fetchResponseContent needs it from + // updateRequest. It will convert the LongString in `response.content.text` to a + // string. + mimeType: response.content.mimeType, + responseContent: response, + }); + this.emitForTests(TEST_EVENTS.RECEIVED_RESPONSE_CONTENT, response); + return payload.responseContent; + } + + /** + * Handles additional information received for a "eventTimings" packet. + * + * @param {object} response the message received from the server. + */ + async onEventTimings(response) { + const payload = await this.updateRequest(response.from, { + eventTimings: response, + }); + + // This event is utilized only in tests but, DAMP is using it too + // and running DAMP test doesn't set the `devtools.testing` flag. + // So, emit this event even in the production mode. + // See also: https://bugzilla.mozilla.org/show_bug.cgi?id=1578215 + this.emit(EVENTS.RECEIVED_EVENT_TIMINGS, response); + return payload.eventTimings; + } + + /** + * Handles information received for a "stackTrace" packet. + * + * @param {object} response the message received from the server. + */ + async onStackTrace(response) { + const payload = await this.updateRequest(response.from, { + stacktrace: response.stacktrace, + }); + this.emitForTests(TEST_EVENTS.RECEIVED_EVENT_STACKTRACE, response); + return payload.stacktrace; + } + + /** + * Handle EventSource events. + */ + + async onEventSourceConnectionClosed(httpChannelId) { + if (this.actionsEnabled && this.actions.closeConnection) { + await this.actions.closeConnection(httpChannelId); + } + } + + async onEventReceived(httpChannelId, data) { + // Dispatch the same action used by websocket inspector. + this.addMessage(httpChannelId, data); + } + + async setEventStreamFlag(actorId) { + if (this.actionsEnabled && this.actions.setEventStreamFlag) { + await this.actions.setEventStreamFlag(actorId, true); + } + } + + /** + * Fire events for the owner object. + */ + emit(type, data) { + if (this.owner) { + this.owner.emit(type, data); + } + } + + /** + * Fire test events for the owner object. These events are + * emitted only when tests are running. + */ + emitForTests(type, data) { + if (this.owner) { + this.owner.emitForTests(type, data); + } + } +} + +module.exports = FirefoxDataProvider; diff --git a/devtools/client/netmonitor/src/connector/har-metadata-collector.js b/devtools/client/netmonitor/src/connector/har-metadata-collector.js new file mode 100644 index 0000000000..c0227065a5 --- /dev/null +++ b/devtools/client/netmonitor/src/connector/har-metadata-collector.js @@ -0,0 +1,97 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +"use strict"; + +const { + TYPES, +} = require("resource://devtools/shared/commands/resource/resource-command.js"); + +/** + * This collector class is dedicated to recording additional metadata necessary + * to build HAR files. The actual request data will be provided by the + * netmonitor which is already monitoring for requests. + * + * The only purpose of this class is to record additional document and network + * events, which will help to assign requests to individual pages. + * + * It should be created and destroyed by the main netmonitor data collector. + */ +class HarMetadataCollector { + #commands; + #initialTargetTitle; + #navigationRequests; + #targetTitlesPerURL; + + constructor(commands) { + this.#commands = commands; + } + + /** + * Stop recording and clear the state. + */ + destroy() { + this.clear(); + + this.#commands.resourceCommand.unwatchResources( + [TYPES.DOCUMENT_EVENT, TYPES.NETWORK_EVENT], + { + onAvailable: this.#onResourceAvailable, + } + ); + } + + /** + * Reset the current state. + */ + clear() { + this.#navigationRequests = []; + this.#targetTitlesPerURL = new Map(); + this.#initialTargetTitle = this.#commands.targetCommand.targetFront.title; + } + + /** + * Start recording additional events for HAR files building. + */ + async connect() { + this.clear(); + + await this.#commands.resourceCommand.watchResources( + [TYPES.DOCUMENT_EVENT, TYPES.NETWORK_EVENT], + { + onAvailable: this.#onResourceAvailable, + } + ); + } + + getHarData() { + return { + initialTargetTitle: this.#initialTargetTitle, + navigationRequests: this.#navigationRequests, + targetTitlesPerURL: this.#targetTitlesPerURL, + }; + } + + #onResourceAvailable = resources => { + for (const resource of resources) { + if (resource.resourceType === TYPES.DOCUMENT_EVENT) { + if ( + resource.name === "dom-complete" && + resource.targetFront.isTopLevel + ) { + this.#targetTitlesPerURL.set( + resource.targetFront.url, + resource.targetFront.title + ); + } + } else if (resource.resourceType === TYPES.NETWORK_EVENT) { + if (resource.isNavigationRequest) { + this.#navigationRequests.push(resource); + } + } + } + }; +} + +exports.HarMetadataCollector = HarMetadataCollector; diff --git a/devtools/client/netmonitor/src/connector/index.js b/devtools/client/netmonitor/src/connector/index.js new file mode 100644 index 0000000000..b1f23a2269 --- /dev/null +++ b/devtools/client/netmonitor/src/connector/index.js @@ -0,0 +1,543 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +"use strict"; + +const { + ACTIVITY_TYPE, + EVENTS, + TEST_EVENTS, +} = require("resource://devtools/client/netmonitor/src/constants.js"); +const FirefoxDataProvider = require("resource://devtools/client/netmonitor/src/connector/firefox-data-provider.js"); +const { + getDisplayedTimingMarker, +} = require("resource://devtools/client/netmonitor/src/selectors/index.js"); + +const { + TYPES, +} = require("resource://devtools/shared/commands/resource/resource-command.js"); + +// Network throttling +loader.lazyRequireGetter( + this, + "throttlingProfiles", + "resource://devtools/client/shared/components/throttling/profiles.js" +); + +loader.lazyRequireGetter( + this, + "HarMetadataCollector", + "resource://devtools/client/netmonitor/src/connector/har-metadata-collector.js", + true +); + +const DEVTOOLS_ENABLE_PERSISTENT_LOG_PREF = "devtools.netmonitor.persistlog"; + +/** + * Connector to Firefox backend. + */ +class Connector { + constructor() { + // Public methods + this.connect = this.connect.bind(this); + this.disconnect = this.disconnect.bind(this); + this.willNavigate = this.willNavigate.bind(this); + this.navigate = this.navigate.bind(this); + this.triggerActivity = this.triggerActivity.bind(this); + this.viewSourceInDebugger = this.viewSourceInDebugger.bind(this); + this.requestData = this.requestData.bind(this); + this.getTimingMarker = this.getTimingMarker.bind(this); + this.updateNetworkThrottling = this.updateNetworkThrottling.bind(this); + + // Internals + this.getLongString = this.getLongString.bind(this); + this.onResourceAvailable = this.onResourceAvailable.bind(this); + this.onResourceUpdated = this.onResourceUpdated.bind(this); + this.updatePersist = this.updatePersist.bind(this); + + this.networkFront = null; + } + + static NETWORK_RESOURCES = [ + TYPES.NETWORK_EVENT, + TYPES.NETWORK_EVENT_STACKTRACE, + TYPES.WEBSOCKET, + TYPES.SERVER_SENT_EVENT, + ]; + + get currentTarget() { + return this.commands.targetCommand.targetFront; + } + + /** + * Connect to the backend. + * + * @param {Object} connection object with e.g. reference to the Toolbox. + * @param {Object} actions (optional) is used to fire Redux actions to update store. + * @param {Object} getState (optional) is used to get access to the state. + */ + async connect(connection, actions, getState) { + this.actions = actions; + this.getState = getState; + this.toolbox = connection.toolbox; + this.commands = this.toolbox.commands; + this.networkCommand = this.commands.networkCommand; + + // The owner object (NetMonitorAPI) received all events. + this.owner = connection.owner; + + this.networkFront = + await this.commands.watcherFront.getNetworkParentActor(); + + this.dataProvider = new FirefoxDataProvider({ + commands: this.commands, + actions: this.actions, + owner: this.owner, + }); + + this._harMetadataCollector = new HarMetadataCollector(this.commands); + await this._harMetadataCollector.connect(); + + await this.commands.resourceCommand.watchResources([TYPES.DOCUMENT_EVENT], { + onAvailable: this.onResourceAvailable, + }); + + await this.resume(false); + + // Server side persistance of the data across reload is disabled by default. + // Ensure enabling it, if the related frontend pref is true. + if (Services.prefs.getBoolPref(DEVTOOLS_ENABLE_PERSISTENT_LOG_PREF)) { + await this.updatePersist(); + } + Services.prefs.addObserver( + DEVTOOLS_ENABLE_PERSISTENT_LOG_PREF, + this.updatePersist + ); + } + + disconnect() { + // As this function might be called twice, we need to guard if already called. + if (this._destroyed) { + return; + } + + this._destroyed = true; + + this.commands.resourceCommand.unwatchResources([TYPES.DOCUMENT_EVENT], { + onAvailable: this.onResourceAvailable, + }); + + this.pause(); + + Services.prefs.removeObserver( + DEVTOOLS_ENABLE_PERSISTENT_LOG_PREF, + this.updatePersist + ); + + if (this.actions) { + this.actions.batchReset(); + } + + this.dataProvider.destroy(); + this.dataProvider = null; + this._harMetadataCollector.destroy(); + } + + clear() { + // Clear all the caches in the data provider + this.dataProvider.clear(); + + this._harMetadataCollector.clear(); + + this.commands.resourceCommand.clearResources(Connector.NETWORK_RESOURCES); + this.emitForTests("clear-network-resources"); + + // Disable the realted network logs in the webconsole + this.toolbox.disableAllConsoleNetworkLogs(); + } + + pause() { + return this.commands.resourceCommand.unwatchResources( + Connector.NETWORK_RESOURCES, + { + onAvailable: this.onResourceAvailable, + onUpdated: this.onResourceUpdated, + } + ); + } + + resume(ignoreExistingResources = true) { + return this.commands.resourceCommand.watchResources( + Connector.NETWORK_RESOURCES, + { + onAvailable: this.onResourceAvailable, + onUpdated: this.onResourceUpdated, + ignoreExistingResources, + } + ); + } + + async onResourceAvailable(resources, { areExistingResources }) { + for (const resource of resources) { + if (resource.resourceType === TYPES.DOCUMENT_EVENT) { + this.onDocEvent(resource, { areExistingResources }); + continue; + } + + if (resource.resourceType === TYPES.NETWORK_EVENT) { + this.dataProvider.onNetworkResourceAvailable(resource); + continue; + } + + if (resource.resourceType === TYPES.NETWORK_EVENT_STACKTRACE) { + this.dataProvider.onStackTraceAvailable(resource); + continue; + } + + if (resource.resourceType === TYPES.WEBSOCKET) { + const { wsMessageType } = resource; + + switch (wsMessageType) { + case "webSocketOpened": { + this.dataProvider.onWebSocketOpened( + resource.httpChannelId, + resource.effectiveURI, + resource.protocols, + resource.extensions + ); + break; + } + case "webSocketClosed": { + this.dataProvider.onWebSocketClosed( + resource.httpChannelId, + resource.wasClean, + resource.code, + resource.reason + ); + break; + } + case "frameReceived": { + this.dataProvider.onFrameReceived( + resource.httpChannelId, + resource.data + ); + break; + } + case "frameSent": { + this.dataProvider.onFrameSent( + resource.httpChannelId, + resource.data + ); + break; + } + } + continue; + } + + if (resource.resourceType === TYPES.SERVER_SENT_EVENT) { + const { messageType, httpChannelId, data } = resource; + switch (messageType) { + case "eventSourceConnectionClosed": { + this.dataProvider.onEventSourceConnectionClosed(httpChannelId); + break; + } + case "eventReceived": { + this.dataProvider.onEventReceived(httpChannelId, data); + break; + } + } + } + } + } + + async onResourceUpdated(updates) { + for (const { resource, update } of updates) { + this.dataProvider.onNetworkResourceUpdated(resource, update); + } + } + + enableActions(enable) { + this.dataProvider.enableActions(enable); + } + + willNavigate() { + if (this.actions) { + if (!Services.prefs.getBoolPref(DEVTOOLS_ENABLE_PERSISTENT_LOG_PREF)) { + this.actions.batchReset(); + this.actions.clearRequests(); + } else { + // If the log is persistent, just clear all accumulated timing markers. + this.actions.clearTimingMarkers(); + } + } + + if (this.actions && this.getState) { + const state = this.getState(); + // Resume is done automatically on page reload/navigation. + if (!state.requests.recording) { + this.actions.toggleRecording(); + } + + // Stop any ongoing search. + if (state.search.ongoingSearch) { + this.actions.stopOngoingSearch(); + } + } + } + + navigate() { + if (!this.dataProvider.hasPendingRequests()) { + this.onReloaded(); + return; + } + const listener = () => { + if (this.dataProvider && this.dataProvider.hasPendingRequests()) { + return; + } + if (this.owner) { + this.owner.off(EVENTS.PAYLOAD_READY, listener); + } + // Netmonitor may already be destroyed, + // so do not try to notify the listeners + if (this.dataProvider) { + this.onReloaded(); + } + }; + if (this.owner) { + this.owner.on(EVENTS.PAYLOAD_READY, listener); + } + } + + onReloaded() { + const panel = this.toolbox.getPanel("netmonitor"); + if (panel) { + panel.emit("reloaded"); + } + } + + /** + * The "DOMContentLoaded" and "Load" events sent by the console actor. + * + * @param {object} resource The DOCUMENT_EVENT resource + */ + onDocEvent(resource, { areExistingResources }) { + if (!resource.targetFront.isTopLevel) { + // Only consider top level document, and ignore remote iframes top document + return; + } + + // Netmonitor does not support dom-loading + if ( + resource.name != "dom-interactive" && + resource.name != "dom-complete" && + resource.name != "will-navigate" + ) { + return; + } + + if (resource.name == "will-navigate") { + // When we open the netmonitor while the page already started loading, + // we don't want to clear it. So here, we ignore will-navigate events + // which were stored in the ResourceCommand cache and only consider + // the live one coming straight from the server. + if (!areExistingResources) { + this.willNavigate(); + } + return; + } + + if (this.actions) { + this.actions.addTimingMarker(resource); + } + + if (resource.name === "dom-complete") { + this.navigate(); + } + + this.emitForTests(TEST_EVENTS.TIMELINE_EVENT, resource); + } + + async updatePersist() { + const enabled = Services.prefs.getBoolPref( + DEVTOOLS_ENABLE_PERSISTENT_LOG_PREF + ); + + await this.networkFront.setPersist(enabled); + + this.emitForTests(TEST_EVENTS.PERSIST_CHANGED, enabled); + } + + /** + * Triggers a specific "activity" to be performed by the frontend. + * This can be, for example, triggering reloads or enabling/disabling cache. + * + * @param {number} type The activity type. See the ACTIVITY_TYPE const. + * @return {object} A promise resolved once the activity finishes and the frontend + * is back into "standby" mode. + */ + triggerActivity(type) { + // Puts the frontend into "standby" (when there's no particular activity). + const standBy = () => { + this.currentActivity = ACTIVITY_TYPE.NONE; + }; + + // Reconfigures the tab, optionally triggering a reload. + const reconfigureTab = async options => { + await this.commands.targetConfigurationCommand.updateConfiguration( + options + ); + }; + + // Reconfigures the tab and waits for the target to finish navigating. + const reconfigureTabAndReload = async options => { + await reconfigureTab(options); + await this.commands.targetCommand.reloadTopLevelTarget(); + }; + + switch (type) { + case ACTIVITY_TYPE.RELOAD.WITH_CACHE_DEFAULT: + return reconfigureTabAndReload({}).then(standBy); + case ACTIVITY_TYPE.RELOAD.WITH_CACHE_ENABLED: + this.currentActivity = ACTIVITY_TYPE.ENABLE_CACHE; + this.commands.resourceCommand + .waitForNextResource( + this.commands.resourceCommand.TYPES.DOCUMENT_EVENT, + { + ignoreExistingResources: true, + predicate(resource) { + return resource.name == "will-navigate"; + }, + } + ) + .then(() => { + this.currentActivity = type; + }); + return reconfigureTabAndReload({ + cacheDisabled: false, + }).then(standBy); + case ACTIVITY_TYPE.RELOAD.WITH_CACHE_DISABLED: + this.currentActivity = ACTIVITY_TYPE.DISABLE_CACHE; + this.commands.resourceCommand + .waitForNextResource( + this.commands.resourceCommand.TYPES.DOCUMENT_EVENT, + { + ignoreExistingResources: true, + predicate(resource) { + return resource.name == "will-navigate"; + }, + } + ) + .then(() => { + this.currentActivity = type; + }); + return reconfigureTabAndReload({ + cacheDisabled: true, + }).then(standBy); + case ACTIVITY_TYPE.ENABLE_CACHE: + this.currentActivity = type; + return reconfigureTab({ + cacheDisabled: false, + }).then(standBy); + case ACTIVITY_TYPE.DISABLE_CACHE: + this.currentActivity = type; + return reconfigureTab({ + cacheDisabled: true, + }).then(standBy); + } + this.currentActivity = ACTIVITY_TYPE.NONE; + return Promise.reject(new Error("Invalid activity type")); + } + + /** + * Fetches the full text of a LongString. + * + * @param {object|string} stringGrip + * The long string grip containing the corresponding actor. + * If you pass in a plain string (by accident or because you're lazy), + * then a promise of the same string is simply returned. + * @return {object} + * A promise that is resolved when the full string contents + * are available, or rejected if something goes wrong. + */ + getLongString(stringGrip) { + return this.dataProvider.getLongString(stringGrip); + } + + /** + * Used for HAR generation. + */ + getHarData() { + return this._harMetadataCollector.getHarData(); + } + + /** + * Getter that returns the current toolbox instance. + * @return {Toolbox} toolbox instance + */ + getToolbox() { + return this.toolbox; + } + + /** + * Open a given source in Debugger + * @param {string} sourceURL source url + * @param {number} sourceLine source line number + */ + viewSourceInDebugger(sourceURL, sourceLine, sourceColumn) { + if (this.toolbox) { + this.toolbox.viewSourceInDebugger(sourceURL, sourceLine, sourceColumn); + } + } + + /** + * Fetch networkEventUpdate websocket message from back-end when + * data provider is connected. + * @param {object} request network request instance + * @param {string} type NetworkEventUpdate type + */ + requestData(request, type) { + return this.dataProvider.requestData(request, type); + } + + getTimingMarker(name) { + if (!this.getState) { + return -1; + } + + const state = this.getState(); + return getDisplayedTimingMarker(state, name); + } + + async updateNetworkThrottling(enabled, profile) { + if (!enabled) { + this.networkFront.clearNetworkThrottling(); + } else { + // The profile can be either a profile id which is used to + // search the predefined throttle profiles or a profile object + // as defined in the trottle tests. + if (typeof profile === "string") { + profile = throttlingProfiles.find(({ id }) => id == profile); + } + const { download, upload, latency } = profile; + await this.networkFront.setNetworkThrottling({ + downloadThroughput: download, + uploadThroughput: upload, + latency, + }); + } + + this.emitForTests(TEST_EVENTS.THROTTLING_CHANGED, { profile }); + } + + /** + * Fire events for the owner object. These events are only + * used in tests so, don't fire them in production release. + */ + emitForTests(type, data) { + if (this.owner) { + this.owner.emitForTests(type, data); + } + } +} +module.exports.Connector = Connector; diff --git a/devtools/client/netmonitor/src/connector/moz.build b/devtools/client/netmonitor/src/connector/moz.build new file mode 100644 index 0000000000..4f65ddea0e --- /dev/null +++ b/devtools/client/netmonitor/src/connector/moz.build @@ -0,0 +1,9 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +DevToolsModules( + "firefox-data-provider.js", + "har-metadata-collector.js", + "index.js", +) -- cgit v1.2.3