diff options
Diffstat (limited to 'devtools/client/netmonitor/src/har')
27 files changed, 3241 insertions, 0 deletions
diff --git a/devtools/client/netmonitor/src/har/README.md b/devtools/client/netmonitor/src/har/README.md new file mode 100644 index 0000000000..fe3e362aaf --- /dev/null +++ b/devtools/client/netmonitor/src/har/README.md @@ -0,0 +1,42 @@ +# HAR +HAR stands for `HTTP Archive` format used by various HTTP monitoring tools +to export collected data. This format is based on JSON and is supported by +many tools on the market including all main browsers (Firefox/Chrome/IE/Edge etc.) + +HAR spec: +* http://www.softwareishard.com/blog/har-12-spec/ +* https://dvcs.w3.org/hg/webperf/raw-file/tip/specs/HAR/Overview.html + +HAR adopters: +* http://www.softwareishard.com/blog/har-adopters/ + +# Netmonitor +Network monitor tool (in Firefox) supports exporting data in HAR format and +the implementation consists from the following objects. + +## HarAutomation +This object is responsible for automated HAR export. It listens for Network +activity and triggers HAR export when the page is loaded. + +The user needs to enable `devtools.netmonitor.har.enableAutoExportToFile` pref +and restart Firefox to switch this automation tool on. + +## HarBuilder +This object is responsible for building HAR object (JSON). It gets all +HTTP requests currently displayed in the Network panel and builds valid HAR. +This object lazy loads all necessary data from the backend if needed, +so the result structure is complete. + +## HarCollector +This object is responsible for collecting data related to all HTTP requests +executed by the page (including inner iframes). The final export is triggered +by HarAutomation at the right time. + +Note: this object is using it's own logic to fetch data from the backend. +It should reuse the Netmonitor Connector (src/connector/index), +so we don't have to maintain two code paths. + +## HarExporter +This object represents the main public API designed to access export logic. +Clients, such as the Network panel itself, or WebExtensions API should use +this object to trigger exporting of collected HTTP data from the panel. diff --git a/devtools/client/netmonitor/src/har/har-automation.js b/devtools/client/netmonitor/src/har/har-automation.js new file mode 100644 index 0000000000..b3c4153d1e --- /dev/null +++ b/devtools/client/netmonitor/src/har/har-automation.js @@ -0,0 +1,253 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +"use strict"; + +const { + HarCollector, +} = require("resource://devtools/client/netmonitor/src/har/har-collector.js"); +const { + HarExporter, +} = require("resource://devtools/client/netmonitor/src/har/har-exporter.js"); +const { + HarUtils, +} = require("resource://devtools/client/netmonitor/src/har/har-utils.js"); +const { + getLongStringFullText, +} = require("resource://devtools/client/shared/string-utils.js"); + +const prefDomain = "devtools.netmonitor.har."; + +// Helper tracer. Should be generic sharable by other modules (bug 1171927) +const trace = { + log(...args) {}, +}; + +/** + * This object is responsible for automated HAR export. It listens + * for Network activity, collects all HTTP data and triggers HAR + * export when the page is loaded. + * + * The user needs to enable the following preference to make the + * auto-export work: devtools.netmonitor.har.enableAutoExportToFile + * + * HAR files are stored within directory that is specified in this + * preference: devtools.netmonitor.har.defaultLogDir + * + * If the default log directory preference isn't set the following + * directory is used by default: <profile>/har/logs + */ +function HarAutomation() {} + +HarAutomation.prototype = { + // Initialization + + async initialize(toolbox) { + this.toolbox = toolbox; + this.commands = toolbox.commands; + + await this.startMonitoring(); + }, + + destroy() { + if (this.collector) { + this.collector.stop(); + } + + if (this.tabWatcher) { + this.tabWatcher.disconnect(); + } + }, + + // Automation + + async startMonitoring() { + await this.toolbox.resourceCommand.watchResources( + [this.toolbox.resourceCommand.TYPES.DOCUMENT_EVENT], + { + onAvailable: resources => { + // Only consider top level document, and ignore remote iframes top document + if ( + resources.find( + r => r.name == "will-navigate" && r.targetFront.isTopLevel + ) + ) { + this.pageLoadBegin(); + } + if ( + resources.find( + r => r.name == "dom-complete" && r.targetFront.isTopLevel + ) + ) { + this.pageLoadDone(); + } + }, + ignoreExistingResources: true, + } + ); + }, + + pageLoadBegin(response) { + this.resetCollector(); + }, + + resetCollector() { + if (this.collector) { + this.collector.stop(); + } + + // A page is about to be loaded, start collecting HTTP + // data from events sent from the backend. + this.collector = new HarCollector({ + commands: this.commands, + }); + + this.collector.start(); + }, + + /** + * A page is done loading, export collected data. Note that + * some requests for additional page resources might be pending, + * so export all after all has been properly received from the backend. + * + * This collector still works and collects any consequent HTTP + * traffic (e.g. XHRs) happening after the page is loaded and + * The additional traffic can be exported by executing + * triggerExport on this object. + */ + pageLoadDone(response) { + trace.log("HarAutomation.pageLoadDone; ", response); + + if (this.collector) { + this.collector.waitForHarLoad().then(collector => { + return this.autoExport(); + }); + } + }, + + autoExport() { + const autoExport = Services.prefs.getBoolPref( + prefDomain + "enableAutoExportToFile" + ); + + if (!autoExport) { + return Promise.resolve(); + } + + // Auto export to file is enabled, so save collected data + // into a file and use all the default options. + const data = { + fileName: Services.prefs.getCharPref(prefDomain + "defaultFileName"), + }; + + return this.executeExport(data); + }, + + // Public API + + /** + * Export all what is currently collected. + */ + triggerExport(data) { + if (!data.fileName) { + data.fileName = Services.prefs.getCharPref( + prefDomain + "defaultFileName" + ); + } + + return this.executeExport(data); + }, + + /** + * Clear currently collected data. + */ + clear() { + this.resetCollector(); + }, + + // HAR Export + + /** + * Execute HAR export. This method fetches all data from the + * Network panel (asynchronously) and saves it into a file. + */ + async executeExport(data) { + const items = this.collector.getItems(); + const { title } = this.commands.targetCommand.targetFront; + + const netMonitor = await this.toolbox.getNetMonitorAPI(); + const connector = await netMonitor.getHarExportConnector(); + + const options = { + connector, + requestData: null, + getTimingMarker: null, + getString: this.getString.bind(this), + view: this, + items, + }; + + options.defaultFileName = data.fileName; + options.compress = data.compress; + options.title = data.title || title; + options.id = data.id; + options.jsonp = data.jsonp; + options.includeResponseBodies = data.includeResponseBodies; + options.jsonpCallback = data.jsonpCallback; + options.forceExport = data.forceExport; + + trace.log("HarAutomation.executeExport; " + data.fileName, options); + + const jsonString = await HarExporter.fetchHarData(options); + + // Save the HAR file if the file name is provided. + if (jsonString && options.defaultFileName) { + const file = getDefaultTargetFile(options); + if (file) { + HarUtils.saveToFile(file, jsonString, options.compress); + } + } + + return jsonString; + }, + + /** + * Fetches the full text of a string. + */ + async getString(stringGrip) { + const fullText = await getLongStringFullText( + this.commands.client, + stringGrip + ); + return fullText; + }, +}; + +// Protocol Helpers + +/** + * Returns target file for exported HAR data. + */ +function getDefaultTargetFile(options) { + const path = + options.defaultLogDir || + Services.prefs.getCharPref("devtools.netmonitor.har.defaultLogDir"); + const folder = HarUtils.getLocalDirectory(path); + + const host = new URL(options.connector.currentTarget.url); + const fileName = HarUtils.getHarFileName( + options.defaultFileName, + options.jsonp, + options.compress, + host.hostname + ); + + folder.append(fileName); + folder.createUnique(Ci.nsIFile.NORMAL_FILE_TYPE, parseInt("0666", 8)); + + return folder; +} + +// Exports from this module +exports.HarAutomation = HarAutomation; diff --git a/devtools/client/netmonitor/src/har/har-builder-utils.js b/devtools/client/netmonitor/src/har/har-builder-utils.js new file mode 100644 index 0000000000..e669263681 --- /dev/null +++ b/devtools/client/netmonitor/src/har/har-builder-utils.js @@ -0,0 +1,30 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +"use strict"; + +/** + * Currently supported HAR version. + */ +const HAR_VERSION = "1.2"; + +function buildHarLog(appInfo) { + return { + log: { + version: HAR_VERSION, + creator: { + name: appInfo.name, + version: appInfo.version, + }, + browser: { + name: appInfo.name, + version: appInfo.version, + }, + pages: [], + entries: [], + }, + }; +} + +exports.buildHarLog = buildHarLog; diff --git a/devtools/client/netmonitor/src/har/har-builder.js b/devtools/client/netmonitor/src/har/har-builder.js new file mode 100644 index 0000000000..9361df7f03 --- /dev/null +++ b/devtools/client/netmonitor/src/har/har-builder.js @@ -0,0 +1,656 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +"use strict"; + +const appInfo = Services.appinfo; +const { LocalizationHelper } = require("resource://devtools/shared/l10n.js"); +const { CurlUtils } = require("resource://devtools/client/shared/curl.js"); +const { + getFormDataSections, + getUrlQuery, + parseQueryString, +} = require("resource://devtools/client/netmonitor/src/utils/request-utils.js"); +const { + buildHarLog, +} = require("resource://devtools/client/netmonitor/src/har/har-builder-utils.js"); +const L10N = new LocalizationHelper("devtools/client/locales/har.properties"); +const { + TIMING_KEYS, +} = require("resource://devtools/client/netmonitor/src/constants.js"); + +/** + * This object is responsible for building HAR file. See HAR spec: + * https://dvcs.w3.org/hg/webperf/raw-file/tip/specs/HAR/Overview.html + * http://www.softwareishard.com/blog/har-12-spec/ + * + * @param {Object} options + * configuration object + * @param {Boolean} options.connector + * Set to true to include HTTP response bodies in the result data + * structure. + * @param {String} options.id + * ID of the exported page. + * @param {Boolean} options.includeResponseBodies + * Set to true to include HTTP response bodies in the result data + * structure. + * @param {Array} options.items + * List of network events to be exported. + * @param {Boolean} options.supportsMultiplePages + * Set to true to create distinct page entries for each navigation. + */ +var HarBuilder = function (options) { + this._connector = options.connector; + this._id = options.id; + this._includeResponseBodies = options.includeResponseBodies; + this._items = options.items; + // Page id counter, only used when options.supportsMultiplePages is true. + this._pageId = options.supportsMultiplePages ? 0 : options.id; + this._pageMap = []; + this._supportsMultiplePages = options.supportsMultiplePages; + this._title = this._connector.currentTarget.title; +}; + +HarBuilder.prototype = { + // Public API + + /** + * This is the main method used to build the entire result HAR data. + * The process is asynchronous since it can involve additional RDP + * communication (e.g. resolving long strings). + * + * @returns {Promise} A promise that resolves to the HAR object when + * the entire build process is done. + */ + async build() { + this.promises = []; + + // Build basic structure for data. + const harLog = buildHarLog(appInfo); + + // Build pages. + this.buildPages(harLog.log); + + // Build entries. + for (const request of this._items) { + const entry = await this.buildEntry(harLog.log, request); + if (entry) { + harLog.log.entries.push(entry); + } + } + + // Some data needs to be fetched from the backend during the + // build process, so wait till all is done. + await Promise.all(this.promises); + + return harLog; + }, + + // Helpers + + buildPages(log) { + if (this._supportsMultiplePages) { + this.buildPagesFromTargetTitles(log); + } else if (this._items.length) { + const firstRequest = this._items[0]; + const page = this.buildPage(this._title, firstRequest); + log.pages.push(page); + this._pageMap[this._id] = page; + } + }, + + buildPagesFromTargetTitles(log) { + // Retrieve the additional HAR data collected by the connector. + const { initialTargetTitle, navigationRequests, targetTitlesPerURL } = + this._connector.getHarData(); + const firstNavigationRequest = navigationRequests[0]; + const firstRequest = this._items[0]; + + if ( + !firstNavigationRequest || + firstRequest.resourceId !== firstNavigationRequest.resourceId + ) { + // If the first request is not a navigation request, it must be related + // to the initial page. Create a first page entry for such early requests. + const initialPage = this.buildPage(initialTargetTitle, firstRequest); + log.pages.push(initialPage); + } + + for (const request of navigationRequests) { + if (targetTitlesPerURL.has(request.url)) { + const title = targetTitlesPerURL.get(request.url); + const page = this.buildPage(title, request); + log.pages.push(page); + } else { + console.warn( + `Could not find any page corresponding to a navigation to ${request.url}` + ); + } + } + }, + + buildPage(title, networkEvent) { + const page = {}; + + page.id = "page_" + this._pageId; + page.pageTimings = this.buildPageTimings(page, networkEvent); + page.startedDateTime = dateToHarString(new Date(networkEvent.startedMs)); + page.title = title; + + // Increase the pageId, for upcoming calls to buildPage. + // If supportsMultiplePages is disabled this method is only called once. + this._pageId++; + + return page; + }, + + getPage(log, entry) { + const existingPage = log.pages.findLast( + ({ startedDateTime }) => startedDateTime <= entry.startedDateTime + ); + + if (!existingPage) { + throw new Error( + "Could not find a page for request: " + entry.request.url + ); + } + + return existingPage; + }, + + async buildEntry(log, networkEvent) { + const entry = {}; + entry.startedDateTime = dateToHarString(new Date(networkEvent.startedMs)); + + let { eventTimings, id } = networkEvent; + try { + if (!eventTimings && this._connector.requestData) { + eventTimings = await this._connector.requestData(id, "eventTimings"); + } + + entry.request = await this.buildRequest(networkEvent); + entry.response = await this.buildResponse(networkEvent); + entry.cache = await this.buildCache(networkEvent); + } catch (e) { + // Ignore any request for which we can't retrieve lazy data + // The request has most likely been destroyed on the server side, + // either because persist is disabled or the request's target/WindowGlobal/process + // has been destroyed. + console.warn("HAR builder failed on", networkEvent.url, e, e.stack); + return null; + } + entry.timings = eventTimings ? eventTimings.timings : {}; + + // Calculate total time by summing all timings. Note that + // `networkEvent.totalTime` can't be used since it doesn't have to + // correspond to plain summary of individual timings. + // With TCP Fast Open and TLS early data sending data can + // start at the same time as connect (we can send data on + // TCP syn packet). Also TLS handshake can carry application + // data thereby overlapping a sending data period and TLS + // handshake period. + entry.time = TIMING_KEYS.reduce((sum, type) => { + const time = entry.timings[type]; + return typeof time != "undefined" && time != -1 ? sum + time : sum; + }, 0); + + // Security state isn't part of HAR spec, and so create + // custom field that needs to use '_' prefix. + entry._securityState = networkEvent.securityState; + + if (networkEvent.remoteAddress) { + entry.serverIPAddress = networkEvent.remoteAddress; + } + + if (networkEvent.remotePort) { + entry.connection = networkEvent.remotePort + ""; + } + + const page = this.getPage(log, entry); + entry.pageref = page.id; + + return entry; + }, + + buildPageTimings(page, networkEvent) { + // Event timing info isn't available + const timings = { + onContentLoad: -1, + onLoad: -1, + }; + + // TODO: This method currently ignores the networkEvent and always retrieves + // the same timing markers for all pages. Seee Bug 1833806. + if (this._connector.getTimingMarker) { + timings.onContentLoad = this._connector.getTimingMarker( + "firstDocumentDOMContentLoadedTimestamp" + ); + timings.onLoad = this._connector.getTimingMarker( + "firstDocumentLoadTimestamp" + ); + } + + return timings; + }, + + async buildRequest(networkEvent) { + // When using HarAutomation, HarCollector will automatically fetch requestHeaders + // and requestCookies, but when we use it from netmonitor, FirefoxDataProvider + // should fetch it itself lazily, via requestData. + + let { id, requestHeaders } = networkEvent; + if (!requestHeaders && this._connector.requestData) { + requestHeaders = await this._connector.requestData(id, "requestHeaders"); + } + + let { requestCookies } = networkEvent; + if (!requestCookies && this._connector.requestData) { + requestCookies = await this._connector.requestData(id, "requestCookies"); + } + + const request = { + bodySize: 0, + }; + request.method = networkEvent.method; + request.url = networkEvent.url; + request.httpVersion = networkEvent.httpVersion || ""; + request.headers = this.buildHeaders(requestHeaders); + request.headers = this.appendHeadersPostData(request.headers, networkEvent); + request.cookies = this.buildCookies(requestCookies); + request.queryString = parseQueryString(getUrlQuery(networkEvent.url)) || []; + request.headersSize = requestHeaders.headersSize; + request.postData = await this.buildPostData(networkEvent); + + if (request.postData?.text) { + request.bodySize = request.postData.text.length; + } + + return request; + }, + + /** + * Fetch all header values from the backend (if necessary) and + * build the result HAR structure. + * + * @param {Object} input Request or response header object. + */ + buildHeaders(input) { + if (!input) { + return []; + } + + return this.buildNameValuePairs(input.headers); + }, + + appendHeadersPostData(input = [], networkEvent) { + if (!networkEvent.requestPostData) { + return input; + } + + this.fetchData(networkEvent.requestPostData.postData.text).then(value => { + const multipartHeaders = CurlUtils.getHeadersFromMultipartText(value); + for (const header of multipartHeaders) { + input.push(header); + } + }); + + return input; + }, + + buildCookies(input) { + if (!input) { + return []; + } + + return this.buildNameValuePairs(input.cookies || input); + }, + + buildNameValuePairs(entries) { + const result = []; + + // HAR requires headers array to be presented, so always + // return at least an empty array. + if (!entries) { + return result; + } + + // Make sure header values are fully fetched from the server. + entries.forEach(entry => { + this.fetchData(entry.value).then(value => { + result.push({ + name: entry.name, + value, + }); + }); + }); + + return result; + }, + + async buildPostData(networkEvent) { + // When using HarAutomation, HarCollector will automatically fetch requestPostData + // and requestHeaders, but when we use it from netmonitor, FirefoxDataProvider + // should fetch it itself lazily, via requestData. + let { id, requestHeaders, requestPostData } = networkEvent; + let requestHeadersFromUploadStream; + + if (!requestPostData && this._connector.requestData) { + requestPostData = await this._connector.requestData( + id, + "requestPostData" + ); + requestHeadersFromUploadStream = requestPostData.uploadHeaders; + } + + if (!requestPostData.postData.text) { + return undefined; + } + + if (!requestHeaders && this._connector.requestData) { + requestHeaders = await this._connector.requestData(id, "requestHeaders"); + } + + const postData = { + mimeType: findValue(requestHeaders.headers, "content-type"), + params: [], + text: requestPostData.postData.text, + }; + + if (requestPostData.postDataDiscarded) { + postData.comment = L10N.getStr("har.requestBodyNotIncluded"); + return postData; + } + + // If we are dealing with URL encoded body, parse parameters. + if ( + CurlUtils.isUrlEncodedRequest({ + headers: requestHeaders.headers, + postDataText: postData.text, + }) + ) { + postData.mimeType = "application/x-www-form-urlencoded"; + // Extract form parameters and produce nice HAR array. + const formDataSections = await getFormDataSections( + requestHeaders, + requestHeadersFromUploadStream, + requestPostData, + this._connector.getLongString + ); + + formDataSections.forEach(section => { + const paramsArray = parseQueryString(section); + if (paramsArray) { + postData.params = [...postData.params, ...paramsArray]; + } + }); + } + + return postData; + }, + + async buildResponse(networkEvent) { + // When using HarAutomation, HarCollector will automatically fetch responseHeaders + // and responseCookies, but when we use it from netmonitor, FirefoxDataProvider + // should fetch it itself lazily, via requestData. + + let { id, responseCookies, responseHeaders } = networkEvent; + if (!responseHeaders && this._connector.requestData) { + responseHeaders = await this._connector.requestData( + id, + "responseHeaders" + ); + } + + if (!responseCookies && this._connector.requestData) { + responseCookies = await this._connector.requestData( + id, + "responseCookies" + ); + } + + const response = { + status: 0, + }; + + // Arbitrary value if it's aborted to make sure status has a number + if (networkEvent.status) { + response.status = parseInt(networkEvent.status, 10); + } + response.statusText = networkEvent.statusText || ""; + response.httpVersion = networkEvent.httpVersion || ""; + + response.headers = this.buildHeaders(responseHeaders); + response.cookies = this.buildCookies(responseCookies); + response.content = await this.buildContent(networkEvent); + + const headers = responseHeaders ? responseHeaders.headers : null; + const headersSize = responseHeaders ? responseHeaders.headersSize : -1; + + response.redirectURL = findValue(headers, "Location"); + response.headersSize = headersSize; + + // 'bodySize' is size of the received response body in bytes. + // Set to zero in case of responses coming from the cache (304). + // Set to -1 if the info is not available. + if (typeof networkEvent.transferredSize != "number") { + response.bodySize = response.status == 304 ? 0 : -1; + } else { + response.bodySize = networkEvent.transferredSize; + } + + return response; + }, + + async buildContent(networkEvent) { + const content = { + mimeType: networkEvent.mimeType, + size: -1, + }; + + // When using HarAutomation, HarCollector will automatically fetch responseContent, + // but when we use it from netmonitor, FirefoxDataProvider should fetch it itself + // lazily, via requestData. + let { responseContent } = networkEvent; + if (!responseContent && this._connector.requestData) { + responseContent = await this._connector.requestData( + networkEvent.id, + "responseContent" + ); + } + if (responseContent?.content) { + content.size = responseContent.content.size; + content.encoding = responseContent.content.encoding; + } + + const includeBodies = this._includeResponseBodies; + const contentDiscarded = responseContent + ? responseContent.contentDiscarded + : false; + + // The comment is appended only if the response content + // is explicitly discarded. + if (!includeBodies || contentDiscarded) { + content.comment = L10N.getStr("har.responseBodyNotIncluded"); + return content; + } + + if (responseContent) { + const { text } = responseContent.content; + this.fetchData(text).then(value => { + content.text = value; + }); + } + + return content; + }, + + async buildCache(networkEvent) { + const cache = {}; + + // if resource has changed, return early + if (networkEvent.status != "304") { + return cache; + } + + if (networkEvent.responseCacheAvailable && this._connector.requestData) { + const responseCache = await this._connector.requestData( + networkEvent.id, + "responseCache" + ); + if (responseCache.cache) { + cache.afterRequest = this.buildCacheEntry(responseCache.cache); + } + } else if (networkEvent.responseCache?.cache) { + cache.afterRequest = this.buildCacheEntry( + networkEvent.responseCache.cache + ); + } else { + cache.afterRequest = null; + } + + return cache; + }, + + buildCacheEntry(cacheEntry) { + const cache = {}; + + if (typeof cacheEntry !== "undefined") { + cache.expires = findKeys(cacheEntry, ["expirationTime", "expires"]); + cache.lastFetched = findKeys(cacheEntry, ["lastFetched"]); + + // TODO: eTag support + // Har format expects cache entries to provide information about eTag, + // however this is not currently exposed on nsICacheEntry. + // This should be stored under cache.eTag. See Bug 1799844. + + cache.fetchCount = findKeys(cacheEntry, ["fetchCount"]); + + // har-importer.js, along with other files, use buildCacheEntry + // initial value comes from properties without underscores. + // this checks for both in appropriate order. + cache._dataSize = findKeys(cacheEntry, ["storageDataSize", "_dataSize"]); + cache._lastModified = findKeys(cacheEntry, [ + "lastModified", + "_lastModified", + ]); + cache._device = findKeys(cacheEntry, ["deviceID", "_device"]); + } + + return cache; + }, + + // RDP Helpers + + fetchData(string) { + const promise = this._connector.getLongString(string).then(value => { + return value; + }); + + // Building HAR is asynchronous and not done till all + // collected promises are resolved. + this.promises.push(promise); + + return promise; + }, +}; + +// Helpers + +/** + * Find specified keys within an object. + * Searches object for keys passed in, returns first value returned, + * or an empty string. + * + * @param obj (object) + * @param keys (array) + * @returns {string} + */ +function findKeys(obj, keys) { + if (!keys) { + return ""; + } + + const keyFound = keys.filter(key => obj[key]); + if (!keys.length) { + return ""; + } + + const value = obj[keyFound[0]]; + if (typeof value === "undefined" || typeof value === "object") { + return ""; + } + + return String(value); +} + +/** + * Find specified value within an array of name-value pairs + * (used for headers, cookies and cache entries) + */ +function findValue(arr, name) { + if (!arr) { + return ""; + } + + name = name.toLowerCase(); + const result = arr.find(entry => entry.name.toLowerCase() == name); + return result ? result.value : ""; +} + +/** + * Generate HAR representation of a date. + * (YYYY-MM-DDThh:mm:ss.sTZD, e.g. 2009-07-24T19:20:30.45+01:00) + * See also HAR Schema: http://janodvarko.cz/har/viewer/ + * + * Note: it would be great if we could utilize Date.toJSON(), but + * it doesn't return proper time zone offset. + * + * An example: + * This helper returns: 2015-05-29T16:10:30.424+02:00 + * Date.toJSON() returns: 2015-05-29T14:10:30.424Z + * + * @param date {Date} The date object we want to convert. + */ +function dateToHarString(date) { + function f(n, c) { + if (!c) { + c = 2; + } + let s = String(n); + while (s.length < c) { + s = "0" + s; + } + return s; + } + + const result = + date.getFullYear() + + "-" + + f(date.getMonth() + 1) + + "-" + + f(date.getDate()) + + "T" + + f(date.getHours()) + + ":" + + f(date.getMinutes()) + + ":" + + f(date.getSeconds()) + + "." + + f(date.getMilliseconds(), 3); + + let offset = date.getTimezoneOffset(); + const positive = offset > 0; + + // Convert to positive number before using Math.floor (see issue 5512) + offset = Math.abs(offset); + const offsetHours = Math.floor(offset / 60); + const offsetMinutes = Math.floor(offset % 60); + const prettyOffset = + (positive > 0 ? "-" : "+") + f(offsetHours) + ":" + f(offsetMinutes); + + return result + prettyOffset; +} + +// Exports from this module +exports.HarBuilder = HarBuilder; diff --git a/devtools/client/netmonitor/src/har/har-collector.js b/devtools/client/netmonitor/src/har/har-collector.js new file mode 100644 index 0000000000..c5a4ae959d --- /dev/null +++ b/devtools/client/netmonitor/src/har/har-collector.js @@ -0,0 +1,488 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +"use strict"; + +const { + getLongStringFullText, +} = require("resource://devtools/client/shared/string-utils.js"); + +// Helper tracer. Should be generic sharable by other modules (bug 1171927) +const trace = { + log(...args) {}, +}; + +/** + * This object is responsible for collecting data related to all + * HTTP requests executed by the page (including inner iframes). + */ +function HarCollector(options) { + this.commands = options.commands; + + this.onResourceAvailable = this.onResourceAvailable.bind(this); + this.onResourceUpdated = this.onResourceUpdated.bind(this); + this.onRequestHeaders = this.onRequestHeaders.bind(this); + this.onRequestCookies = this.onRequestCookies.bind(this); + this.onRequestPostData = this.onRequestPostData.bind(this); + this.onResponseHeaders = this.onResponseHeaders.bind(this); + this.onResponseCookies = this.onResponseCookies.bind(this); + this.onResponseContent = this.onResponseContent.bind(this); + this.onEventTimings = this.onEventTimings.bind(this); + + this.clear(); +} + +HarCollector.prototype = { + // Connection + + async start() { + await this.commands.resourceCommand.watchResources( + [this.commands.resourceCommand.TYPES.NETWORK_EVENT], + { + onAvailable: this.onResourceAvailable, + onUpdated: this.onResourceUpdated, + } + ); + }, + + async stop() { + await this.commands.resourceCommand.unwatchResources( + [this.commands.resourceCommand.TYPES.NETWORK_EVENT], + { + onAvailable: this.onResourceAvailable, + onUpdated: this.onResourceUpdated, + } + ); + }, + + clear() { + // Any pending requests events will be ignored (they turn + // into zombies, since not present in the files array). + this.files = new Map(); + this.items = []; + this.firstRequestStart = -1; + this.lastRequestStart = -1; + this.requests = []; + }, + + waitForHarLoad() { + // There should be yet another timeout e.g.: + // 'devtools.netmonitor.har.pageLoadTimeout' + // that should force export even if page isn't fully loaded. + return new Promise(resolve => { + this.waitForResponses().then(() => { + trace.log("HarCollector.waitForHarLoad; DONE HAR loaded!"); + resolve(this); + }); + }); + }, + + waitForResponses() { + trace.log("HarCollector.waitForResponses; " + this.requests.length); + + // All requests for additional data must be received to have complete + // HTTP info to generate the result HAR file. So, wait for all current + // promises. Note that new promises (requests) can be generated during the + // process of HTTP data collection. + return waitForAll(this.requests).then(() => { + // All responses are received from the backend now. We yet need to + // wait for a little while to see if a new request appears. If yes, + // lets's start gathering HTTP data again. If no, we can declare + // the page loaded. + // If some new requests appears in the meantime the promise will + // be rejected and we need to wait for responses all over again. + + this.pageLoadDeferred = this.waitForTimeout().then( + () => { + // Page loaded! + }, + () => { + trace.log( + "HarCollector.waitForResponses; NEW requests " + + "appeared during page timeout!" + ); + // New requests executed, let's wait again. + return this.waitForResponses(); + } + ); + return this.pageLoadDeferred; + }); + }, + + // Page Loaded Timeout + + /** + * The page is loaded when there are no new requests within given period + * of time. The time is set in preferences: + * 'devtools.netmonitor.har.pageLoadedTimeout' + */ + waitForTimeout() { + // The auto-export is not done if the timeout is set to zero (or less). + // This is useful in cases where the export is done manually through + // API exposed to the content. + const timeout = Services.prefs.getIntPref( + "devtools.netmonitor.har.pageLoadedTimeout" + ); + + trace.log("HarCollector.waitForTimeout; " + timeout); + + return new Promise((resolve, reject) => { + if (timeout <= 0) { + resolve(); + } + this.pageLoadReject = reject; + this.pageLoadTimeout = setTimeout(() => { + trace.log("HarCollector.onPageLoadTimeout;"); + resolve(); + }, timeout); + }); + }, + + resetPageLoadTimeout() { + // Remove the current timeout. + if (this.pageLoadTimeout) { + trace.log("HarCollector.resetPageLoadTimeout;"); + + clearTimeout(this.pageLoadTimeout); + this.pageLoadTimeout = null; + } + + // Reject the current page load promise + if (this.pageLoadReject) { + this.pageLoadReject(); + this.pageLoadReject = null; + } + }, + + // Collected Data + + getFile(actorId) { + return this.files.get(actorId); + }, + + getItems() { + return this.items; + }, + + // Event Handlers + + onResourceAvailable(resources) { + for (const resource of resources) { + trace.log("HarCollector.onNetworkEvent; ", resource); + + const { actor, startedDateTime, method, url, isXHR } = resource; + const startTime = Date.parse(startedDateTime); + + if (this.firstRequestStart == -1) { + this.firstRequestStart = startTime; + } + + if (this.lastRequestEnd < startTime) { + this.lastRequestEnd = startTime; + } + + let file = this.getFile(actor); + if (file) { + console.error( + "HarCollector.onNetworkEvent; ERROR " + "existing file conflict!" + ); + continue; + } + + file = { + id: actor, + startedDeltaMs: startTime - this.firstRequestStart, + startedMs: startTime, + method, + url, + isXHR, + }; + + this.files.set(actor, file); + + // Mimic the Net panel data structure + this.items.push(file); + } + }, + + onResourceUpdated(updates) { + for (const { resource } of updates) { + // Skip events from unknown actors (not in the list). + // It can happen when there are zombie requests received after + // the target is closed or multiple tabs are attached through + // one connection (one DevToolsClient object). + const file = this.getFile(resource.actor); + if (!file) { + return; + } + + const includeResponseBodies = Services.prefs.getBoolPref( + "devtools.netmonitor.har.includeResponseBodies" + ); + + [ + { + type: "eventTimings", + method: "getEventTimings", + callbackName: "onEventTimings", + }, + { + type: "requestHeaders", + method: "getRequestHeaders", + callbackName: "onRequestHeaders", + }, + { + type: "requestPostData", + method: "getRequestPostData", + callbackName: "onRequestPostData", + }, + { + type: "responseHeaders", + method: "getResponseHeaders", + callbackName: "onResponseHeaders", + }, + { type: "responseStart" }, + { + type: "responseContent", + method: "getResponseContent", + callbackName: "onResponseContent", + }, + { + type: "requestCookies", + method: "getRequestCookies", + callbackName: "onRequestCookies", + }, + { + type: "responseCookies", + method: "getResponseCookies", + callbackName: "onResponseCookies", + }, + ].forEach(updateType => { + trace.log( + "HarCollector.onNetworkEventUpdate; " + updateType.type, + resource + ); + + let request; + if (resource[`${updateType.type}Available`]) { + if (updateType.type == "responseStart") { + file.httpVersion = resource.httpVersion; + file.status = resource.status; + file.statusText = resource.statusText; + } else if (updateType.type == "responseContent") { + file.contentSize = resource.contentSize; + file.mimeType = resource.mimeType; + file.transferredSize = resource.transferredSize; + if (includeResponseBodies) { + request = this.getData( + resource.actor, + updateType.method, + this[updateType.callbackName] + ); + } + } else { + request = this.getData( + resource.actor, + updateType.method, + this[updateType.callbackName] + ); + } + } + + if (request) { + this.requests.push(request); + } + this.resetPageLoadTimeout(); + }); + } + }, + + async getData(actor, method, callback) { + const file = this.getFile(actor); + + trace.log( + "HarCollector.getData; REQUEST " + method + ", " + file.url, + file + ); + + // Bug 1519082: We don't create fronts for NetworkEvent actors, + // so that we have to do the request manually via DevToolsClient.request() + const packet = { + to: actor, + type: method, + }; + const response = await this.commands.client.request(packet); + + trace.log( + "HarCollector.getData; RESPONSE " + method + ", " + file.url, + response + ); + callback(response); + return response; + }, + + /** + * Handles additional information received for a "requestHeaders" packet. + * + * @param object response + * The message received from the server. + */ + onRequestHeaders(response) { + const file = this.getFile(response.from); + file.requestHeaders = response; + + this.getLongHeaders(response.headers); + }, + + /** + * Handles additional information received for a "requestCookies" packet. + * + * @param object response + * The message received from the server. + */ + onRequestCookies(response) { + const file = this.getFile(response.from); + file.requestCookies = response; + + this.getLongHeaders(response.cookies); + }, + + /** + * Handles additional information received for a "requestPostData" packet. + * + * @param object response + * The message received from the server. + */ + onRequestPostData(response) { + trace.log("HarCollector.onRequestPostData;", response); + + const file = this.getFile(response.from); + file.requestPostData = response; + + // Resolve long string + const { text } = response.postData; + if (typeof text == "object") { + this.getString(text).then(value => { + response.postData.text = value; + }); + } + }, + + /** + * Handles additional information received for a "responseHeaders" packet. + * + * @param object response + * The message received from the server. + */ + onResponseHeaders(response) { + const file = this.getFile(response.from); + file.responseHeaders = response; + + this.getLongHeaders(response.headers); + }, + + /** + * Handles additional information received for a "responseCookies" packet. + * + * @param object response + * The message received from the server. + */ + onResponseCookies(response) { + const file = this.getFile(response.from); + file.responseCookies = response; + + this.getLongHeaders(response.cookies); + }, + + /** + * Handles additional information received for a "responseContent" packet. + * + * @param object response + * The message received from the server. + */ + onResponseContent(response) { + const file = this.getFile(response.from); + file.responseContent = response; + + // Resolve long string + const { text } = response.content; + if (typeof text == "object") { + this.getString(text).then(value => { + response.content.text = value; + }); + } + }, + + /** + * Handles additional information received for a "eventTimings" packet. + * + * @param object response + * The message received from the server. + */ + onEventTimings(response) { + const file = this.getFile(response.from); + file.eventTimings = response; + file.totalTime = response.totalTime; + }, + + // Helpers + + getLongHeaders(headers) { + for (const header of headers) { + if (typeof header.value == "object") { + try { + this.getString(header.value).then(value => { + header.value = value; + }); + } catch (error) { + trace.log("HarCollector.getLongHeaders; ERROR when getString", error); + } + } + } + }, + + /** + * Fetches the full text of a string. + * + * @param object | string stringGrip + * The long string grip containing the corresponding actor. + * If you pass in a plain string (by accident or because you're lazy), + * then a promise of the same string is simply returned. + * @return object Promise + * A promise that is resolved when the full string contents + * are available, or rejected if something goes wrong. + */ + async getString(stringGrip) { + const promise = getLongStringFullText(this.commands.client, stringGrip); + this.requests.push(promise); + return promise; + }, +}; + +// Helpers + +/** + * Helper function that allows to wait for array of promises. It is + * possible to dynamically add new promises in the provided array. + * The function will wait even for the newly added promises. + * (this isn't possible with the default Promise.all); + */ +function waitForAll(promises) { + // Remove all from the original array and get clone of it. + const clone = promises.splice(0, promises.length); + + // Wait for all promises in the given array. + return Promise.all(clone).then(() => { + // If there are new promises (in the original array) + // to wait for - chain them! + if (promises.length) { + return waitForAll(promises); + } + + return undefined; + }); +} + +// Exports from this module +exports.HarCollector = HarCollector; diff --git a/devtools/client/netmonitor/src/har/har-exporter.js b/devtools/client/netmonitor/src/har/har-exporter.js new file mode 100644 index 0000000000..fb401c2737 --- /dev/null +++ b/devtools/client/netmonitor/src/har/har-exporter.js @@ -0,0 +1,230 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +"use strict"; + +const DevToolsUtils = require("resource://devtools/shared/DevToolsUtils.js"); +const JSZip = require("resource://devtools/client/shared/vendor/jszip.js"); +const clipboardHelper = require("resource://devtools/shared/platform/clipboard.js"); +const { + HarUtils, +} = require("resource://devtools/client/netmonitor/src/har/har-utils.js"); +const { + HarBuilder, +} = require("resource://devtools/client/netmonitor/src/har/har-builder.js"); + +var uid = 1; + +// Helper tracer. Should be generic sharable by other modules (bug 1171927) +const trace = { + log(...args) {}, +}; + +/** + * This object represents the main public API designed to access + * Network export logic. Clients, such as the Network panel itself, + * should use this API to export collected HTTP data from the panel. + */ +const HarExporter = { + // Public API + + /** + * Save collected HTTP data from the Network panel into HAR file. + * + * @param Object options + * Configuration object + * + * The following options are supported: + * + * - includeResponseBodies {Boolean}: If set to true, HTTP response bodies + * are also included in the HAR file (can produce significantly bigger + * amount of data). + * + * - items {Array}: List of Network requests to be exported. + * + * - jsonp {Boolean}: If set to true the export format is HARP (support + * for JSONP syntax). + * + * - jsonpCallback {String}: Default name of JSONP callback (used for + * HARP format). + * + * - compress {Boolean}: If set to true the final HAR file is zipped. + * This represents great disk-space optimization. + * + * - defaultFileName {String}: Default name of the target HAR file. + * The default file name supports the format specifier %date to output the + * current date/time. + * + * - defaultLogDir {String}: Default log directory for automated logs. + * + * - id {String}: ID of the page (used in the HAR file). + * + * - title {String}: Title of the page (used in the HAR file). + * + * - forceExport {Boolean}: The result HAR file is created even if + * there are no HTTP entries. + */ + async save(options) { + // Set default options related to save operation. + const defaultFileName = Services.prefs.getCharPref( + "devtools.netmonitor.har.defaultFileName" + ); + const compress = Services.prefs.getBoolPref( + "devtools.netmonitor.har.compress" + ); + + trace.log("HarExporter.save; " + defaultFileName, options); + + let data = await this.fetchHarData(options); + + const host = new URL(options.connector.currentTarget.url); + + const fileName = HarUtils.getHarFileName( + defaultFileName, + options.jsonp, + compress, + host.hostname + ); + + if (compress) { + data = await JSZip() + .file(fileName, data) + .generateAsync({ + compression: "DEFLATE", + platform: Services.appinfo.OS === "WINNT" ? "DOS" : "UNIX", + type: "uint8array", + }); + } else { + data = new TextEncoder().encode(data); + } + + DevToolsUtils.saveAs(window, data, fileName); + }, + + /** + * Copy HAR string into the clipboard. + * + * @param Object options + * Configuration object, see save() for detailed description. + */ + copy(options) { + return this.fetchHarData(options).then(jsonString => { + clipboardHelper.copyString(jsonString); + return jsonString; + }); + }, + + /** + * Get HAR data as JSON object. + * + * @param Object options + * Configuration object, see save() for detailed description. + */ + getHar(options) { + return this.fetchHarData(options).then(data => { + return data ? JSON.parse(data) : null; + }); + }, + + // Helpers + + fetchHarData(options) { + // Generate page ID + options.id = options.id || uid++; + + // Set default generic HAR export options. + if (typeof options.jsonp != "boolean") { + options.jsonp = Services.prefs.getBoolPref( + "devtools.netmonitor.har.jsonp" + ); + } + if (typeof options.includeResponseBodies != "boolean") { + options.includeResponseBodies = Services.prefs.getBoolPref( + "devtools.netmonitor.har.includeResponseBodies" + ); + } + if (typeof options.jsonpCallback != "boolean") { + options.jsonpCallback = Services.prefs.getCharPref( + "devtools.netmonitor.har.jsonpCallback" + ); + } + if (typeof options.forceExport != "boolean") { + options.forceExport = Services.prefs.getBoolPref( + "devtools.netmonitor.har.forceExport" + ); + } + if (typeof options.supportsMultiplePages != "boolean") { + options.supportsMultiplePages = Services.prefs.getBoolPref( + "devtools.netmonitor.har.multiple-pages" + ); + } + + // Build HAR object. + return this.buildHarData(options) + .then(har => { + // Do not export an empty HAR file, unless the user + // explicitly says so (using the forceExport option). + if (!har.log.entries.length && !options.forceExport) { + return Promise.resolve(); + } + + let jsonString = this.stringify(har); + if (!jsonString) { + return Promise.resolve(); + } + + // If JSONP is wanted, wrap the string in a function call + if (options.jsonp) { + // This callback name is also used in HAR Viewer by default. + // http://www.softwareishard.com/har/viewer/ + const callbackName = options.jsonpCallback || "onInputData"; + jsonString = callbackName + "(" + jsonString + ");"; + } + + return jsonString; + }) + .catch(function onError(err) { + console.error(err); + }); + }, + + /** + * Build HAR data object. This object contains all HTTP data + * collected by the Network panel. The process is asynchronous + * since it can involve additional RDP communication (e.g. resolving + * long strings). + */ + async buildHarData(options) { + // Disconnect from redux actions/store. + options.connector.enableActions(false); + + // Build HAR object from collected data. + const builder = new HarBuilder(options); + const result = await builder.build(); + + // Connect to redux actions again. + options.connector.enableActions(true); + + return result; + }, + + /** + * Build JSON string from the HAR data object. + */ + stringify(har) { + if (!har) { + return null; + } + + try { + return JSON.stringify(har, null, " "); + } catch (err) { + console.error(err); + return undefined; + } + }, +}; + +// Exports from this module +exports.HarExporter = HarExporter; diff --git a/devtools/client/netmonitor/src/har/har-importer.js b/devtools/client/netmonitor/src/har/har-importer.js new file mode 100644 index 0000000000..2246a29086 --- /dev/null +++ b/devtools/client/netmonitor/src/har/har-importer.js @@ -0,0 +1,166 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +"use strict"; + +const { + TIMING_KEYS, +} = require("resource://devtools/client/netmonitor/src/constants.js"); +const { + getUrlDetails, +} = require("resource://devtools/client/netmonitor/src/utils/request-utils.js"); + +var guid = 0; + +/** + * This object is responsible for importing HAR file. See HAR spec: + * https://dvcs.w3.org/hg/webperf/raw-file/tip/specs/HAR/Overview.html + * http://www.softwareishard.com/blog/har-12-spec/ + */ +var HarImporter = function (actions) { + this.actions = actions; +}; + +HarImporter.prototype = { + /** + * This is the main method used to import HAR data. + */ + import(har) { + const json = JSON.parse(har); + this.doImport(json); + }, + + doImport(har) { + this.actions.clearRequests(); + + // Helper map for pages. + const pages = new Map(); + har.log.pages.forEach(page => { + pages.set(page.id, page); + }); + + // Iterate all entries/requests and generate state. + har.log.entries.forEach(entry => { + const requestId = String(++guid); + const startedMs = Date.parse(entry.startedDateTime); + + // Add request + this.actions.addRequest( + requestId, + { + startedMs, + method: entry.request.method, + url: entry.request.url, + urlDetails: getUrlDetails(entry.request.url), + isXHR: false, + cause: { + loadingDocumentUri: "", + stackTraceAvailable: false, + type: "", + }, + fromCache: false, + fromServiceWorker: false, + }, + false + ); + + // Update request + const data = { + requestHeaders: { + headers: entry.request.headers, + headersSize: entry.request.headersSize, + rawHeaders: "", + }, + responseHeaders: { + headers: entry.response.headers, + headersSize: entry.response.headersSize, + rawHeaders: "", + }, + requestCookies: entry.request.cookies, + responseCookies: entry.response.cookies, + requestPostData: { + postData: entry.request.postData || {}, + postDataDiscarded: false, + }, + responseContent: { + content: entry.response.content, + contentDiscarded: false, + }, + eventTimings: { + timings: entry.timings, + }, + totalTime: TIMING_KEYS.reduce((sum, type) => { + const time = entry.timings[type]; + return typeof time != "undefined" && time != -1 ? sum + time : sum; + }, 0), + + httpVersion: entry.request.httpVersion, + contentSize: entry.response.content.size, + mimeType: entry.response.content.mimeType, + remoteAddress: entry.serverIPAddress, + remotePort: entry.connection, + status: entry.response.status, + statusText: entry.response.statusText, + transferredSize: entry.response.bodySize, + securityState: entry._securityState, + + // Avoid auto-fetching data from the backend + eventTimingsAvailable: false, + requestCookiesAvailable: false, + requestHeadersAvailable: false, + responseContentAvailable: false, + responseStartAvailable: false, + responseCookiesAvailable: false, + responseHeadersAvailable: false, + securityInfoAvailable: false, + requestPostDataAvailable: false, + }; + + if (entry.cache.afterRequest) { + const { afterRequest } = entry.cache; + data.responseCache = { + cache: { + expires: afterRequest.expires, + fetchCount: afterRequest.fetchCount, + lastFetched: afterRequest.lastFetched, + // TODO: eTag support, see Bug 1799844. + // eTag: afterRequest.eTag, + _dataSize: afterRequest._dataSize, + _lastModified: afterRequest._lastModified, + _device: afterRequest._device, + }, + }; + } + + this.actions.updateRequest(requestId, data, false); + + // Page timing markers + const pageTimings = pages.get(entry.pageref)?.pageTimings; + let onContentLoad = (pageTimings && pageTimings.onContentLoad) || 0; + let onLoad = (pageTimings && pageTimings.onLoad) || 0; + + // Set 0 as the default value + onContentLoad = onContentLoad != -1 ? onContentLoad : 0; + onLoad = onLoad != -1 ? onLoad : 0; + + // Add timing markers + if (onContentLoad > 0) { + this.actions.addTimingMarker({ + name: "dom-interactive", + time: startedMs + onContentLoad, + }); + } + + if (onLoad > 0) { + this.actions.addTimingMarker({ + name: "dom-complete", + time: startedMs + onLoad, + }); + } + }); + }, +}; + +// Exports from this module +exports.HarImporter = HarImporter; diff --git a/devtools/client/netmonitor/src/har/har-menu-utils.js b/devtools/client/netmonitor/src/har/har-menu-utils.js new file mode 100644 index 0000000000..756d9d9f96 --- /dev/null +++ b/devtools/client/netmonitor/src/har/har-menu-utils.js @@ -0,0 +1,118 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +"use strict"; + +const { + L10N, +} = require("resource://devtools/client/netmonitor/src/utils/l10n.js"); +const EventEmitter = require("resource://devtools/shared/event-emitter.js"); + +loader.lazyRequireGetter( + this, + "HarExporter", + "resource://devtools/client/netmonitor/src/har/har-exporter.js", + true +); + +loader.lazyGetter(this, "HarImporter", function () { + return require("resource://devtools/client/netmonitor/src/har/har-importer.js") + .HarImporter; +}); + +/** + * Helper object with HAR related context menu actions. + */ +var HarMenuUtils = { + /** + * Copy HAR from the network panel content to the clipboard. + */ + async copyAllAsHar(requests, connector) { + const har = await HarExporter.copy( + this.getDefaultHarOptions(requests, connector) + ); + + // We cannot easily expect the clipboard content from tests, instead we emit + // a test event. + HarMenuUtils.emitForTests("copy-all-as-har-done", har); + + return har; + }, + + /** + * Save HAR from the network panel content to a file. + */ + saveAllAsHar(requests, connector) { + // This will not work in launchpad + // document.execCommand(‘cut’/‘copy’) was denied because it was not called from + // inside a short running user-generated event handler. + // https://developer.mozilla.org/en-US/Add-ons/WebExtensions/Interact_with_the_clipboard + return HarExporter.save(this.getDefaultHarOptions(requests, connector)); + }, + + /** + * Import HAR file and preview its content in the Network panel. + */ + openHarFile(actions, openSplitConsole) { + const fp = Cc["@mozilla.org/filepicker;1"].createInstance(Ci.nsIFilePicker); + fp.init( + window, + L10N.getStr("netmonitor.har.importHarDialogTitle"), + Ci.nsIFilePicker.modeOpen + ); + + // Append file filters + fp.appendFilter( + L10N.getStr("netmonitor.har.importDialogHARFilter"), + "*.har" + ); + fp.appendFilter(L10N.getStr("netmonitor.har.importDialogAllFilter"), "*.*"); + + fp.open(rv => { + if (rv == Ci.nsIFilePicker.returnOK) { + const file = Cc["@mozilla.org/file/local;1"].createInstance(Ci.nsIFile); + file.initWithPath(fp.file.path); + readFile(file).then(har => { + if (har) { + this.appendPreview(har, actions, openSplitConsole); + } + }); + } + }); + }, + + appendPreview(har, actions, openSplitConsole) { + try { + const importer = new HarImporter(actions); + importer.import(har); + } catch (err) { + if (openSplitConsole) { + openSplitConsole("Error while processing HAR file: " + err.message); + } + } + }, + + getDefaultHarOptions(requests, connector) { + return { + connector, + items: requests, + }; + }, +}; + +// Helpers + +function readFile(file) { + return new Promise(resolve => { + IOUtils.read(file.path).then(data => { + const decoder = new TextDecoder(); + resolve(decoder.decode(data)); + }); + }); +} + +EventEmitter.decorate(HarMenuUtils); + +// Exports from this module +exports.HarMenuUtils = HarMenuUtils; diff --git a/devtools/client/netmonitor/src/har/har-utils.js b/devtools/client/netmonitor/src/har/har-utils.js new file mode 100644 index 0000000000..fc7ae533f1 --- /dev/null +++ b/devtools/client/netmonitor/src/har/har-utils.js @@ -0,0 +1,167 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +"use strict"; + +const { XPCOMUtils } = ChromeUtils.importESModule( + "resource://gre/modules/XPCOMUtils.sys.mjs" +); + +XPCOMUtils.defineLazyGetter(this, "ZipWriter", function () { + return Components.Constructor("@mozilla.org/zipwriter;1", "nsIZipWriter"); +}); + +const OPEN_FLAGS = { + RDONLY: parseInt("0x01", 16), + WRONLY: parseInt("0x02", 16), + CREATE_FILE: parseInt("0x08", 16), + APPEND: parseInt("0x10", 16), + TRUNCATE: parseInt("0x20", 16), + EXCL: parseInt("0x80", 16), +}; + +function formatDate(date) { + const year = String(date.getFullYear() % 100).padStart(2, "0"); + const month = String(date.getMonth() + 1).padStart(2, "0"); + const day = String(date.getDate()).padStart(2, "0"); + const hour = String(date.getHours()).padStart(2, "0"); + const minutes = String(date.getMinutes()).padStart(2, "0"); + const seconds = String(date.getSeconds()).padStart(2, "0"); + + return `${year}-${month}-${day} ${hour}-${minutes}-${seconds}`; +} + +/** + * Helper API for HAR export features. + */ +var HarUtils = { + getHarFileName(defaultFileName, jsonp, compress, hostname) { + const extension = jsonp ? ".harp" : ".har"; + + const now = new Date(); + let name = defaultFileName.replace(/%date/g, formatDate(now)); + name = name.replace(/%hostname/g, hostname); + name = name.replace(/\:/gm, "-", ""); + name = name.replace(/\//gm, "_", ""); + + let fileName = name + extension; + + // Default file extension is zip if compressing is on. + if (compress) { + fileName += ".zip"; + } + + return fileName; + }, + + /** + * Save HAR string into a given file. The file might be compressed + * if specified in the options. + * + * @param {File} file Target file where the HAR string (JSON) + * should be stored. + * @param {String} jsonString HAR data (JSON or JSONP) + * @param {Boolean} compress The result file is zipped if set to true. + */ + saveToFile(file, jsonString, compress) { + const openFlags = + OPEN_FLAGS.WRONLY | OPEN_FLAGS.CREATE_FILE | OPEN_FLAGS.TRUNCATE; + + try { + const foStream = Cc[ + "@mozilla.org/network/file-output-stream;1" + ].createInstance(Ci.nsIFileOutputStream); + + const permFlags = parseInt("0666", 8); + foStream.init(file, openFlags, permFlags, 0); + + const convertor = Cc[ + "@mozilla.org/intl/converter-output-stream;1" + ].createInstance(Ci.nsIConverterOutputStream); + convertor.init(foStream, "UTF-8"); + + // The entire jsonString can be huge so, write the data in chunks. + const chunkLength = 1024 * 1024; + for (let i = 0; i <= jsonString.length; i++) { + const data = jsonString.substr(i, chunkLength + 1); + if (data) { + convertor.writeString(data); + } + + i = i + chunkLength; + } + + // this closes foStream + convertor.close(); + } catch (err) { + console.error(err); + return false; + } + + // If no compressing then bail out. + if (!compress) { + return true; + } + + // Remember name of the original file, it'll be replaced by a zip file. + const originalFilePath = file.path; + const originalFileName = file.leafName; + + try { + // Rename using unique name (the file is going to be removed). + file.moveTo(null, "temp" + new Date().getTime() + "temphar"); + + // Create compressed file with the original file path name. + const zipFile = Cc["@mozilla.org/file/local;1"].createInstance( + Ci.nsIFile + ); + zipFile.initWithPath(originalFilePath); + + // The file within the zipped file doesn't use .zip extension. + let fileName = originalFileName; + if (fileName.indexOf(".zip") == fileName.length - 4) { + fileName = fileName.substr(0, fileName.indexOf(".zip")); + } + + const zip = new ZipWriter(); + zip.open(zipFile, openFlags); + zip.addEntryFile( + fileName, + Ci.nsIZipWriter.COMPRESSION_DEFAULT, + file, + false + ); + zip.close(); + + // Remove the original file (now zipped). + file.remove(true); + return true; + } catch (err) { + console.error(err); + + // Something went wrong (disk space?) rename the original file back. + file.moveTo(null, originalFileName); + } + + return false; + }, + + getLocalDirectory(path) { + let dir; + + if (!path) { + dir = Services.dirsvc.get("ProfD", Ci.nsIFile); + dir.append("har"); + dir.append("logs"); + } else { + dir = Cc["@mozilla.org/file/local;1"].createInstance(Ci.nsIFile); + dir.initWithPath(path); + } + + return dir; + }, +}; + +// Exports from this module +exports.HarUtils = HarUtils; diff --git a/devtools/client/netmonitor/src/har/moz.build b/devtools/client/netmonitor/src/har/moz.build new file mode 100644 index 0000000000..3a73373c43 --- /dev/null +++ b/devtools/client/netmonitor/src/har/moz.build @@ -0,0 +1,19 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +DevToolsModules( + "har-automation.js", + "har-builder-utils.js", + "har-builder.js", + "har-collector.js", + "har-exporter.js", + "har-importer.js", + "har-menu-utils.js", + "har-utils.js", +) + +BROWSER_CHROME_MANIFESTS += [ + "test/browser-harautomation.ini", + "test/browser.ini", +] diff --git a/devtools/client/netmonitor/src/har/test/browser-harautomation.ini b/devtools/client/netmonitor/src/har/test/browser-harautomation.ini new file mode 100644 index 0000000000..61ee56be44 --- /dev/null +++ b/devtools/client/netmonitor/src/har/test/browser-harautomation.ini @@ -0,0 +1,16 @@ +[DEFAULT] +tags = devtools +subsuite = devtools +prefs= + # This preference needs to be set before starting Firefox, so we use a + # dedicated browser.ini + devtools.netmonitor.har.enableAutoExportToFile=true + +support-files = + head.js + !/devtools/client/netmonitor/test/head.js + !/devtools/client/netmonitor/test/html_simple-test-page.html + !/devtools/client/shared/test/shared-head.js + !/devtools/client/shared/test/telemetry-test-helpers.js + +[browser_harautomation_simple.js] diff --git a/devtools/client/netmonitor/src/har/test/browser.ini b/devtools/client/netmonitor/src/har/test/browser.ini new file mode 100644 index 0000000000..c45b2b0358 --- /dev/null +++ b/devtools/client/netmonitor/src/har/test/browser.ini @@ -0,0 +1,29 @@ +[DEFAULT] +tags = devtools +subsuite = devtools +skip-if = http3 # Bug 1829298 +support-files = + head.js + html_har_import-test-page.html + html_har_multipage_iframe.html + html_har_multipage_page.html + html_har_post-data-test-page.html + sjs_cache-test-server.sjs + sjs_cookies-test-server.sjs + !/devtools/client/netmonitor/test/head.js + !/devtools/client/netmonitor/test/html_simple-test-page.html + !/devtools/client/shared/test/shared-head.js + !/devtools/client/shared/test/telemetry-test-helpers.js + +[browser_net_har_copy_all_as_har.js] +skip-if = + !debug && os == "mac" #Bug 1622925 + !debug && os == "linux" #Bug 1622925 + win10_2004 # Bug 1723573 + win11_2009 # Bug 1797751 +[browser_net_har_import.js] +[browser_net_har_import_no-mime.js] +[browser_net_har_multipage.js] +[browser_net_har_post_data.js] +[browser_net_har_post_data_on_get.js] +[browser_net_har_throttle_upload.js] diff --git a/devtools/client/netmonitor/src/har/test/browser_harautomation_simple.js b/devtools/client/netmonitor/src/har/test/browser_harautomation_simple.js new file mode 100644 index 0000000000..9da746d28f --- /dev/null +++ b/devtools/client/netmonitor/src/har/test/browser_harautomation_simple.js @@ -0,0 +1,35 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +const HAR_FILENAME = "test_filename.har"; + +// We expect the HAR file to be created on reload in profD/har/logs +const HAR_PATH = ["har", "logs", HAR_FILENAME]; + +/** + * Smoke test for automated HAR export. + * Note that the `enableAutoExportToFile` is set from browser-harautomation.ini + * because the preference needs to be set before starting the browser. + */ +add_task(async function () { + // Set a simple test filename for the exported HAR. + await pushPref("devtools.netmonitor.har.defaultFileName", "test_filename"); + + const tab = await addTab(SIMPLE_URL); + const toolbox = await gDevTools.showToolboxForTab(tab, { + toolId: "inspector", + }); + + await reloadBrowser(); + + info("Wait until the HAR file is created in the profile directory"); + await waitUntil(() => FileUtils.getFile("ProfD", HAR_PATH).exists()); + + const harFile = FileUtils.getFile("ProfD", HAR_PATH); + ok(harFile.exists(), "HAR file was automatically created"); + + await toolbox.destroy(); + await removeTab(tab); +}); diff --git a/devtools/client/netmonitor/src/har/test/browser_net_har_copy_all_as_har.js b/devtools/client/netmonitor/src/har/test/browser_net_har_copy_all_as_har.js new file mode 100644 index 0000000000..ab4302883c --- /dev/null +++ b/devtools/client/netmonitor/src/har/test/browser_net_har_copy_all_as_har.js @@ -0,0 +1,220 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +/** + * Basic tests for exporting Network panel content into HAR format. + */ + +const EXPECTED_REQUEST_HEADER_COUNT = 9; +const EXPECTED_RESPONSE_HEADER_COUNT = 6; + +add_task(async function () { + // Disable tcp fast open, because it is setting a response header indicator + // (bug 1352274). TCP Fast Open is not present on all platforms therefore the + // number of response headers will vary depending on the platform. + await pushPref("network.tcp.tcp_fastopen_enable", false); + const { tab, monitor, toolbox } = await initNetMonitor(SIMPLE_URL, { + requestCount: 1, + }); + + info("Starting test... "); + + await testSimpleReload({ tab, monitor, toolbox }); + await testResponseBodyLimits({ tab, monitor, toolbox }); + await testManyReloads({ tab, monitor, toolbox }); + await testClearedRequests({ tab, monitor, toolbox }); + + // Do not use teardown(monitor) as testClearedRequests register broken requests + // which never complete and would block on waitForAllNetworkUpdateEvents + await closeTabAndToolbox(); +}); + +async function testSimpleReload({ tab, monitor, toolbox }) { + info("Test with a simple page reload"); + + const har = await reloadAndCopyAllAsHar({ tab, monitor, toolbox }); + + // Check out HAR log + isnot(har.log, null, "The HAR log must exist"); + is(har.log.creator.name, "Firefox", "The creator field must be set"); + is(har.log.browser.name, "Firefox", "The browser field must be set"); + is(har.log.pages.length, 1, "There must be one page"); + is(har.log.entries.length, 1, "There must be one request"); + + const page = har.log.pages[0]; + + is(page.title, "Network Monitor test page", "There must be some page title"); + ok("onContentLoad" in page.pageTimings, "There must be onContentLoad time"); + ok("onLoad" in page.pageTimings, "There must be onLoad time"); + + const entry = har.log.entries[0]; + assertNavigationRequestEntry(entry); + + info("We get the response content and timings when doing a simple reload"); + isnot(entry.response.content.text, undefined, "Check response body"); + is(entry.response.content.text.length, 465, "Response body is complete"); + isnot(entry.timings, undefined, "Check timings"); +} + +async function testResponseBodyLimits({ tab, monitor, toolbox }) { + info("Test response body limit (non zero)."); + await pushPref("devtools.netmonitor.responseBodyLimit", 10); + let har = await reloadAndCopyAllAsHar({ tab, monitor, toolbox }); + let entry = har.log.entries[0]; + is(entry.response.content.text.length, 10, "Response body must be truncated"); + + info("Test response body limit (zero)."); + await pushPref("devtools.netmonitor.responseBodyLimit", 0); + har = await reloadAndCopyAllAsHar({ tab, monitor, toolbox }); + entry = har.log.entries[0]; + is( + entry.response.content.text.length, + 465, + "Response body must not be truncated" + ); +} + +async function testManyReloads({ tab, monitor, toolbox }) { + const har = await reloadAndCopyAllAsHar({ + tab, + monitor, + toolbox, + reloadTwice: true, + }); + // In most cases, we will have two requests, but sometimes, + // the first one might be missing as we couldn't fetch any lazy data for it. + ok(har.log.entries.length >= 1, "There must be at least one request"); + info( + "Assert the first navigation request which has been cancelled by the second reload" + ); + // Requests may come out of order, so try to find the bogus cancelled request + let entry = har.log.entries.find(e => e.response.status == 0); + if (entry) { + ok(entry, "Found the cancelled request"); + is(entry.request.method, "GET", "Method is set"); + is(entry.request.url, SIMPLE_URL, "URL is set"); + // We always get the following headers: + // "Host", "User-agent", "Accept", "Accept-Language", "Accept-Encoding", "Connection" + // but are missing the three last headers: + // "Upgrade-Insecure-Requests", "Pragma", "Cache-Control" + is(entry.request.headers.length, 6, "But headers are partialy populated"); + is(entry.response.status, 0, "And status is set to 0"); + } + + entry = har.log.entries.find(e => e.response.status != 0); + assertNavigationRequestEntry(entry); +} + +async function testClearedRequests({ tab, monitor, toolbox }) { + info("Navigate to an empty page"); + const topDocumentURL = + "https://example.org/document-builder.sjs?html=empty-document"; + const iframeURL = + "https://example.org/document-builder.sjs?html=" + + encodeURIComponent( + `iframe<script>fetch("/document-builder.sjs?html=iframe-request")</script>` + ); + + await waitForAllNetworkUpdateEvents(); + await navigateTo(topDocumentURL); + + info("Create an iframe doing a request and remove the iframe."); + info( + "Doing this, should notify a network request that is destroyed on the server side" + ); + const onNetworkEvents = waitForNetworkEvents(monitor, 2); + await SpecialPowers.spawn( + tab.linkedBrowser, + [iframeURL], + async function (_iframeURL) { + const iframe = content.document.createElement("iframe"); + iframe.setAttribute("src", _iframeURL); + content.document.body.appendChild(iframe); + } + ); + // Wait for the two request to be processed (iframe doc + fetch requests) + // before removing the iframe so that the netmonitor is able to fetch + // all lazy data without throwing + await onNetworkEvents; + await waitForAllNetworkUpdateEvents(); + + info("Remove the iframe so that lazy request data are freed"); + await SpecialPowers.spawn(tab.linkedBrowser, [], async function () { + content.document.querySelector("iframe").remove(); + }); + + // HAR will try to re-fetch lazy data and may throw on the iframe fetch request. + // This subtest is meants to verify we aren't throwing here and HAR export + // works fine, even if some requests can't be fetched. + const har = await copyAllAsHARWithContextMenu(monitor); + is(har.log.entries.length, 2, "There must be two requests"); + is( + har.log.entries[0].request.url, + topDocumentURL, + "First request is for the top level document" + ); + is( + har.log.entries[1].request.url, + iframeURL, + "Second request is for the iframe" + ); + info( + "The fetch request doesn't appear in HAR export, because its lazy data is freed and we completely ignore the request." + ); +} + +function assertNavigationRequestEntry(entry) { + info("Assert that the entry relates to the navigation request"); + ok(entry.time > 0, "Check the total time"); + is(entry.request.method, "GET", "Check the method"); + is(entry.request.url, SIMPLE_URL, "Check the URL"); + is( + entry.request.headers.length, + EXPECTED_REQUEST_HEADER_COUNT, + "Check number of request headers" + ); + is(entry.response.status, 200, "Check response status"); + is(entry.response.statusText, "OK", "Check response status text"); + is( + entry.response.headers.length, + EXPECTED_RESPONSE_HEADER_COUNT, + "Check number of response headers" + ); + is( + entry.response.content.mimeType, + "text/html", + "Check response content type" + ); +} +/** + * Reload the page and copy all as HAR. + */ +async function reloadAndCopyAllAsHar({ + tab, + monitor, + toolbox, + reloadTwice = false, +}) { + const { store, windowRequire } = monitor.panelWin; + const Actions = windowRequire("devtools/client/netmonitor/src/actions/index"); + + store.dispatch(Actions.batchEnable(false)); + + const onNetworkEvent = waitForNetworkEvents(monitor, 1); + const { onDomCompleteResource } = + await waitForNextTopLevelDomCompleteResource(toolbox.commands); + + if (reloadTwice) { + reloadBrowser(); + } + await reloadBrowser(); + + info("Waiting for network events"); + await onNetworkEvent; + info("Waiting for DOCUMENT_EVENT dom-complete resource"); + await onDomCompleteResource; + + return copyAllAsHARWithContextMenu(monitor); +} diff --git a/devtools/client/netmonitor/src/har/test/browser_net_har_import.js b/devtools/client/netmonitor/src/har/test/browser_net_har_import.js new file mode 100644 index 0000000000..879966e653 --- /dev/null +++ b/devtools/client/netmonitor/src/har/test/browser_net_har_import.js @@ -0,0 +1,149 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +/** + * Tests for importing HAR data. + */ +add_task(async () => { + const { tab, monitor } = await initNetMonitor( + HAR_EXAMPLE_URL + "html_har_import-test-page.html", + { requestCount: 1 } + ); + + info("Starting test... "); + + const { actions, store, windowRequire } = monitor.panelWin; + const Actions = windowRequire("devtools/client/netmonitor/src/actions/index"); + const { HarImporter } = windowRequire( + "devtools/client/netmonitor/src/har/har-importer" + ); + + store.dispatch(Actions.batchEnable(false)); + + // Execute one POST request on the page and wait till its done. + const wait = waitForNetworkEvents(monitor, 3); + await SpecialPowers.spawn(tab.linkedBrowser, [], async () => { + await content.wrappedJSObject.executeTest(); + }); + await wait; + + // Copy HAR into the clipboard + const json1 = await copyAllAsHARWithContextMenu(monitor, { asString: true }); + + // Import HAR string + const importer = new HarImporter(actions); + importer.import(json1); + + // Copy HAR into the clipboard again + const json2 = await copyAllAsHARWithContextMenu(monitor, { asString: true }); + + // Compare exported HAR data + const har1 = JSON.parse(json1); + const har2 = JSON.parse(json2); + + // Explicit tests + is(har2.log.entries.length, 3, "There must be expected number of requests"); + ok( + har2.log.pages[0].title.endsWith("Network Monitor Test Page"), + "There must be some page title" + ); + ok( + !!har2.log.entries[0].request.headers.length, + "There must be some request headers" + ); + ok( + !!har2.log.entries[0].response.headers.length, + "There must be some response headers" + ); + is( + har2.log.entries[1].response.cookies.length, + 3, + "There must be expected number of cookies" + ); + is( + har2.log.entries[1]._securityState, + "insecure", + "There must be expected security state" + ); + is(har2.log.entries[2].response.status, 304, "There must be expected status"); + + // Complex test comparing exported & imported HARs. + ok(compare(har1.log, har2.log, ["log"]), "Exported HAR must be the same"); + + // Clean up + return teardown(monitor); +}); + +/** + * Check equality of HAR files. + */ +function compare(obj1, obj2, path) { + const keys1 = Object.getOwnPropertyNames(obj1).sort(); + const keys2 = Object.getOwnPropertyNames(obj2).sort(); + + const name = path.join("/"); + + is( + keys1.length, + keys2.length, + "There must be the same number of keys for: " + name + ); + if (keys1.length != keys2.length) { + return false; + } + + is(keys1.join(), keys2.join(), "There must be the same keys for: " + name); + if (keys1.join() != keys2.join()) { + return false; + } + + // Page IDs are generated and don't have to be the same after import. + const ignore = [ + "log/entries/0/pageref", + "log/entries/1/pageref", + "log/entries/2/pageref", + "log/pages/0/id", + "log/pages/1/id", + "log/pages/2/id", + ]; + + let result = true; + for (let i = 0; i < keys1.length; i++) { + const key = keys1[i]; + const prop1 = obj1[key]; + const prop2 = obj2[key]; + + if (prop1 instanceof Array) { + if (!(prop2 instanceof Array)) { + ok(false, "Arrays are not the same " + name); + result = false; + break; + } + if (!compare(prop1, prop2, path.concat(key))) { + result = false; + break; + } + } else if (prop1 instanceof Object) { + if (!(prop2 instanceof Object)) { + ok(false, "Objects are not the same in: " + name); + result = false; + break; + } + if (!compare(prop1, prop2, path.concat(key))) { + result = false; + break; + } + } else if (prop1 !== prop2) { + const propName = name + "/" + key; + if (!ignore.includes(propName)) { + is(prop1, prop2, "Values are not the same: " + propName); + result = false; + break; + } + } + } + + return result; +} diff --git a/devtools/client/netmonitor/src/har/test/browser_net_har_import_no-mime.js b/devtools/client/netmonitor/src/har/test/browser_net_har_import_no-mime.js new file mode 100644 index 0000000000..91c5160217 --- /dev/null +++ b/devtools/client/netmonitor/src/har/test/browser_net_har_import_no-mime.js @@ -0,0 +1,78 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +/** + * Tests importing HAR with missing `response.content.mimeType` does not crash the netmonitor. + */ +add_task(async () => { + const { monitor } = await initNetMonitor(SIMPLE_URL, { + requestCount: 1, + }); + + info("Starting test... "); + + const { document, actions, store, windowRequire } = monitor.panelWin; + const Actions = windowRequire("devtools/client/netmonitor/src/actions/index"); + + const { HarImporter } = windowRequire( + "devtools/client/netmonitor/src/har/har-importer" + ); + + store.dispatch(Actions.batchEnable(false)); + + // Invalid HAR json which should contain `entries[0].response.content.mimeType` + const invalidHarJSON = { + log: { + version: "1.2", + pages: [ + { + title: "bla", + }, + ], + entries: [ + { + request: { + method: "POST", + url: "https://bla.com", + httpVersion: "", + headers: [], + cookies: [], + queryString: [], + }, + response: { + content: { + size: 1231, + text: '{"requests":[{"uri":"https://bla.com"}]}', + }, + headers: [], + }, + timings: {}, + cache: {}, + }, + ], + }, + }; + + // Import invalid Har file + const importer = new HarImporter(actions); + importer.import(JSON.stringify(invalidHarJSON)); + + const waitForResponsePanelOpen = waitUntil(() => + document.querySelector("#response-panel") + ); + + // Open the response details panel + EventUtils.sendMouseEvent( + { type: "mousedown" }, + document.querySelector(".request-list-item") + ); + clickOnSidebarTab(document, "response"); + + await waitForResponsePanelOpen; + ok(true, "The response panel opened"); + + // Clean up + return teardown(monitor); +}); diff --git a/devtools/client/netmonitor/src/har/test/browser_net_har_multipage.js b/devtools/client/netmonitor/src/har/test/browser_net_har_multipage.js new file mode 100644 index 0000000000..38936f73fe --- /dev/null +++ b/devtools/client/netmonitor/src/har/test/browser_net_har_multipage.js @@ -0,0 +1,153 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +requestLongerTimeout(2); + +const MULTIPAGE_IFRAME_URL = HAR_EXAMPLE_URL + "html_har_multipage_iframe.html"; +const MULTIPAGE_PAGE_URL = HAR_EXAMPLE_URL + "html_har_multipage_page.html"; + +/** + * Tests HAR export with navigations and multipage support + */ +add_task(async function () { + await testHARWithNavigation({ enableMultipage: false, filter: false }); + await testHARWithNavigation({ enableMultipage: true, filter: false }); + await testHARWithNavigation({ enableMultipage: false, filter: true }); + await testHARWithNavigation({ enableMultipage: true, filter: true }); +}); + +async function testHARWithNavigation({ enableMultipage, filter }) { + await pushPref("devtools.netmonitor.persistlog", true); + await pushPref("devtools.netmonitor.har.multiple-pages", enableMultipage); + + const { tab, monitor } = await initNetMonitor(MULTIPAGE_PAGE_URL + "?page1", { + requestCount: 1, + }); + + info("Starting test... "); + + const { store, windowRequire } = monitor.panelWin; + const Actions = windowRequire("devtools/client/netmonitor/src/actions/index"); + + store.dispatch(Actions.batchEnable(false)); + + info("Perform 3 additional requests"); + let onNetworkEvents = waitForNetworkEvents(monitor, 3); + await SpecialPowers.spawn(tab.linkedBrowser, [], async function () { + content.wrappedJSObject.sendRequests(3); + }); + await onNetworkEvents; + + info("Navigate to a second page where we will not perform any extra request"); + onNetworkEvents = waitForNetworkEvents(monitor, 1); + await navigateTo(MULTIPAGE_PAGE_URL + "?page2"); + await onNetworkEvents; + + info("Navigate to a third page where we will not perform any extra request"); + onNetworkEvents = waitForNetworkEvents(monitor, 1); + await navigateTo(MULTIPAGE_PAGE_URL + "?page3"); + await onNetworkEvents; + + info("Perform 2 additional requests"); + onNetworkEvents = waitForNetworkEvents(monitor, 2); + await SpecialPowers.spawn(tab.linkedBrowser, [], async function () { + content.wrappedJSObject.sendRequests(2); + }); + await onNetworkEvents; + + info("Create an iframe which will perform 2 additional requests"); + onNetworkEvents = waitForNetworkEvents(monitor, 2); + await SpecialPowers.spawn( + tab.linkedBrowser, + [MULTIPAGE_IFRAME_URL], + async function (url) { + const iframe = content.document.createElement("iframe"); + const onLoad = new Promise(resolve => + iframe.addEventListener("load", resolve, { once: true }) + ); + content.content.document.body.appendChild(iframe); + iframe.setAttribute("src", url); + await onLoad; + } + ); + await onNetworkEvents; + + if (filter) { + info("Start filtering requests"); + store.dispatch(Actions.setRequestFilterText("?request")); + } + + info("Trigger Copy All As HAR from the context menu"); + const har = await copyAllAsHARWithContextMenu(monitor); + + // Check out the HAR log. + isnot(har.log, null, "The HAR log must exist"); + + if (enableMultipage) { + is(har.log.pages.length, 3, "There must be three pages"); + } else { + is(har.log.pages.length, 1, "There must be one page"); + } + + if (!filter) { + // Expect 9 requests: + // - 3 requests performed with sendRequests on the first page + // - 1 navigation request to the second page + // - 1 navigation request to the third page + // - 2 requests performed with sendRequests on the third page + // - 1 request to load an iframe on the third page + // - 1 request from the iframe on the third page + is(har.log.entries.length, 9, "There must be 9 requests"); + } else { + // Same but we only expect the fetch requests + is(har.log.entries.length, 6, "There must be 6 requests"); + } + + if (enableMultipage) { + // With multipage enabled, check that the page entries are valid and that + // requests are referencing the expected page id. + assertPageDetails(har.log.pages[0], "page_0", "HAR Multipage test page"); + assertPageRequests(har.log.entries, 0, 2, har.log.pages[0].id); + + assertPageDetails(har.log.pages[1], "page_1", "HAR Multipage test page"); + if (filter) { + // When filtering, we don't expect any request to match page_1 + } else { + assertPageRequests(har.log.entries, 3, 3, har.log.pages[1].id); + } + + assertPageDetails(har.log.pages[2], "page_2", "HAR Multipage test page"); + if (filter) { + assertPageRequests(har.log.entries, 3, 5, har.log.pages[2].id); + } else { + assertPageRequests(har.log.entries, 4, 8, har.log.pages[2].id); + } + } else { + is(har.log.pages[0].id, "page_1"); + // Without multipage, all requests are associated with the only page entry. + for (const entry of har.log.entries) { + is(entry.pageref, "page_1"); + } + } + + // Clean up + return teardown(monitor); +} + +function assertPageDetails(page, expectedId, expectedTitle) { + is(page.id, expectedId, "Page has the expected id"); + is(page.title, expectedTitle, "Page has the expected title"); +} + +function assertPageRequests(entries, startIndex, endIndex, expectedPageId) { + for (let i = startIndex; i < endIndex + 1; i++) { + const entry = entries[i]; + is( + entry.pageref, + expectedPageId, + `Entry ${i} is attached to page id: ${expectedPageId}` + ); + } +} diff --git a/devtools/client/netmonitor/src/har/test/browser_net_har_post_data.js b/devtools/client/netmonitor/src/har/test/browser_net_har_post_data.js new file mode 100644 index 0000000000..0640364a39 --- /dev/null +++ b/devtools/client/netmonitor/src/har/test/browser_net_har_post_data.js @@ -0,0 +1,51 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +/** + * Tests for exporting POST data into HAR format. + */ +add_task(async function () { + const { tab, monitor } = await initNetMonitor( + HAR_EXAMPLE_URL + "html_har_post-data-test-page.html", + { requestCount: 1 } + ); + + info("Starting test... "); + + const { store, windowRequire } = monitor.panelWin; + const Actions = windowRequire("devtools/client/netmonitor/src/actions/index"); + + store.dispatch(Actions.batchEnable(false)); + + // Execute one POST request on the page and wait till its done. + const wait = waitForNetworkEvents(monitor, 1); + await SpecialPowers.spawn(tab.linkedBrowser, [], async function () { + content.wrappedJSObject.executeTest(); + }); + await wait; + + // Copy HAR into the clipboard (asynchronous). + const har = await copyAllAsHARWithContextMenu(monitor); + + // Check out the HAR log. + isnot(har.log, null, "The HAR log must exist"); + is(har.log.pages.length, 1, "There must be one page"); + is(har.log.entries.length, 1, "There must be one request"); + + const entry = har.log.entries[0]; + is( + entry.request.postData.mimeType, + "application/json", + "Check post data content type" + ); + is( + entry.request.postData.text, + "{'first': 'John', 'last': 'Doe'}", + "Check post data payload" + ); + + // Clean up + return teardown(monitor); +}); diff --git a/devtools/client/netmonitor/src/har/test/browser_net_har_post_data_on_get.js b/devtools/client/netmonitor/src/har/test/browser_net_har_post_data_on_get.js new file mode 100644 index 0000000000..206fc43da6 --- /dev/null +++ b/devtools/client/netmonitor/src/har/test/browser_net_har_post_data_on_get.js @@ -0,0 +1,43 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +/** + * Tests for exporting POST data into HAR format. + */ +add_task(async function () { + const { tab, monitor } = await initNetMonitor( + HAR_EXAMPLE_URL + "html_har_post-data-test-page.html", + { requestCount: 1 } + ); + + info("Starting test... "); + + const { store, windowRequire } = monitor.panelWin; + const Actions = windowRequire("devtools/client/netmonitor/src/actions/index"); + + store.dispatch(Actions.batchEnable(false)); + + // Execute one GET request on the page and wait till its done. + const wait = waitForNetworkEvents(monitor, 1); + await SpecialPowers.spawn(tab.linkedBrowser, [], async function () { + content.wrappedJSObject.executeTest3(); + }); + await wait; + + // Copy HAR into the clipboard (asynchronous). + const har = await copyAllAsHARWithContextMenu(monitor); + + // Check out the HAR log. + isnot(har.log, null, "The HAR log must exist"); + is(har.log.pages.length, 1, "There must be one page"); + is(har.log.entries.length, 1, "There must be one request"); + + const entry = har.log.entries[0]; + + is(entry.request.postData, undefined, "Check post data is not present"); + + // Clean up + return teardown(monitor); +}); diff --git a/devtools/client/netmonitor/src/har/test/browser_net_har_throttle_upload.js b/devtools/client/netmonitor/src/har/test/browser_net_har_throttle_upload.js new file mode 100644 index 0000000000..24f7d482ca --- /dev/null +++ b/devtools/client/netmonitor/src/har/test/browser_net_har_throttle_upload.js @@ -0,0 +1,69 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +// Test timing of upload when throttling. + +"use strict"; + +add_task(async function () { + await throttleUploadTest(true); + await throttleUploadTest(false); +}); + +async function throttleUploadTest(actuallyThrottle) { + const { tab, monitor } = await initNetMonitor( + HAR_EXAMPLE_URL + "html_har_post-data-test-page.html", + { requestCount: 1 } + ); + + info("Starting test... (actuallyThrottle = " + actuallyThrottle + ")"); + + const { connector, store, windowRequire } = monitor.panelWin; + const Actions = windowRequire("devtools/client/netmonitor/src/actions/index"); + + store.dispatch(Actions.batchEnable(false)); + + const size = 4096; + const uploadSize = actuallyThrottle ? size / 3 : 0; + + const throttleProfile = { + latency: 0, + download: 200000, + upload: uploadSize, + }; + + info("sending throttle request"); + await connector.updateNetworkThrottling(true, throttleProfile); + + // Execute one POST request on the page and wait till its done. + const wait = waitForNetworkEvents(monitor, 1); + await SpecialPowers.spawn( + tab.linkedBrowser, + [{ size }], + async function (args) { + content.wrappedJSObject.executeTest2(args.size); + } + ); + await wait; + + // Copy HAR into the clipboard (asynchronous). + const har = await copyAllAsHARWithContextMenu(monitor); + + // Check out the HAR log. + isnot(har.log, null, "The HAR log must exist"); + is(har.log.pages.length, 1, "There must be one page"); + is(har.log.entries.length, 1, "There must be one request"); + + const entry = har.log.entries[0]; + is(entry.request.postData.text, "x".repeat(size), "Check post data payload"); + + const wasTwoSeconds = entry.timings.send >= 2000; + if (actuallyThrottle) { + ok(wasTwoSeconds, "upload should have taken more than 2 seconds"); + } else { + ok(!wasTwoSeconds, "upload should not have taken more than 2 seconds"); + } + + // Clean up + await teardown(monitor); +} diff --git a/devtools/client/netmonitor/src/har/test/head.js b/devtools/client/netmonitor/src/har/test/head.js new file mode 100644 index 0000000000..b41ea580fd --- /dev/null +++ b/devtools/client/netmonitor/src/har/test/head.js @@ -0,0 +1,45 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +/* eslint no-unused-vars: [2, {"vars": "local", "args": "none"}] */ +/* import-globals-from ../../../test/head.js */ + +// Load the NetMonitor head.js file to share its API. +var netMonitorHead = + "chrome://mochitests/content/browser/devtools/client/netmonitor/test/head.js"; +Services.scriptloader.loadSubScript(netMonitorHead, this); + +// Directory with HAR related test files. +const HAR_EXAMPLE_URL = + "http://example.com/browser/devtools/client/netmonitor/src/har/test/"; + +/** + * Trigger a "copy all as har" from the context menu of the requests list. + + * @param {Object} monitor + * The network monitor object + */ +async function copyAllAsHARWithContextMenu(monitor, { asString = false } = {}) { + const { HarMenuUtils } = monitor.panelWin.windowRequire( + "devtools/client/netmonitor/src/har/har-menu-utils" + ); + + info("Open the context menu on the first visible request."); + const firstRequest = + monitor.panelWin.document.querySelectorAll(".request-list-item")[0]; + + EventUtils.sendMouseEvent({ type: "mousedown" }, firstRequest); + EventUtils.sendMouseEvent({ type: "contextmenu" }, firstRequest); + + info("Trigger Copy All As HAR from the context menu"); + const onHarCopyDone = HarMenuUtils.once("copy-all-as-har-done"); + await selectContextMenuItem(monitor, "request-list-context-copy-all-as-har"); + const jsonString = await onHarCopyDone; + + if (asString) { + return jsonString; + } + return JSON.parse(jsonString); +} diff --git a/devtools/client/netmonitor/src/har/test/html_har_import-test-page.html b/devtools/client/netmonitor/src/har/test/html_har_import-test-page.html new file mode 100644 index 0000000000..04d5ec33ba --- /dev/null +++ b/devtools/client/netmonitor/src/har/test/html_har_import-test-page.html @@ -0,0 +1,51 @@ +<!-- Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ --> + +<!doctype html> + +<html> + <head> + <meta charset="utf-8"/> + <meta http-equiv="Cache-Control" content="no-cache, no-store, must-revalidate" /> + <meta http-equiv="Pragma" content="no-cache" /> + <meta http-equiv="Expires" content="0" /> + <title>Network Monitor Test Page</title> + </head> + + <body> + <p>HAR import test</p> + + <script type="text/javascript"> + /* exported executeTest, executeTest2, executeTest3 */ + "use strict"; + + function post(address, data) { + return new Promise(resolve => { + const xhr = new XMLHttpRequest(); + xhr.open("POST", address, true); + xhr.setRequestHeader("Content-Type", "application/json"); + xhr.onload = resolve; + xhr.send(data); + }); + } + + function get(address) { + return new Promise(resolve => { + const xhr = new XMLHttpRequest(); + xhr.open("GET", address); + xhr.onload = resolve; + xhr.send(); + }); + } + + async function executeTest() { + const url = "html_har_import-test-page.html"; + const data = "{'first': 'John', 'last': 'Doe'}"; + await post(url, data); + await get("sjs_cookies-test-server.sjs"); + await get("sjs_cache-test-server.sjs"); + } + </script> + </body> + +</html> diff --git a/devtools/client/netmonitor/src/har/test/html_har_multipage_iframe.html b/devtools/client/netmonitor/src/har/test/html_har_multipage_iframe.html new file mode 100644 index 0000000000..4e0fc96344 --- /dev/null +++ b/devtools/client/netmonitor/src/har/test/html_har_multipage_iframe.html @@ -0,0 +1,24 @@ +<!-- Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ --> + +<!doctype html> + +<html> + <head> + <meta charset="utf-8"/> + <meta http-equiv="Cache-Control" content="no-cache, no-store, must-revalidate" /> + <meta http-equiv="Pragma" content="no-cache" /> + <meta http-equiv="Expires" content="0" /> + <title>Network Monitor HAR Multipage test iframe</title> + </head> + + <body> + <p>HAR Multipage test iframe</p> + + <script type="text/javascript"> + "use strict"; + fetch("?request-from-iframe"); + </script> + </body> + +</html> diff --git a/devtools/client/netmonitor/src/har/test/html_har_multipage_page.html b/devtools/client/netmonitor/src/har/test/html_har_multipage_page.html new file mode 100644 index 0000000000..d36fbca52b --- /dev/null +++ b/devtools/client/netmonitor/src/har/test/html_har_multipage_page.html @@ -0,0 +1,30 @@ +<!-- Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ --> + +<!doctype html> + +<html> + <head> + <meta charset="utf-8"/> + <meta http-equiv="Cache-Control" content="no-cache, no-store, must-revalidate" /> + <meta http-equiv="Pragma" content="no-cache" /> + <meta http-equiv="Expires" content="0" /> + <title>HAR Multipage test page</title> + </head> + + <body> + <p>HAR Multipage test page</p> + + <script type="text/javascript"> + /* exported sendRequests */ + "use strict"; + + async function sendRequests(requestsCount) { + for (let i = 0; i < requestsCount; i++) { + fetch("?request" + i); + } + } + </script> + </body> + +</html> diff --git a/devtools/client/netmonitor/src/har/test/html_har_post-data-test-page.html b/devtools/client/netmonitor/src/har/test/html_har_post-data-test-page.html new file mode 100644 index 0000000000..5e42c6139d --- /dev/null +++ b/devtools/client/netmonitor/src/har/test/html_har_post-data-test-page.html @@ -0,0 +1,55 @@ +<!-- Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ --> + +<!doctype html> + +<html> + <head> + <meta charset="utf-8"/> + <meta http-equiv="Cache-Control" content="no-cache, no-store, must-revalidate" /> + <meta http-equiv="Pragma" content="no-cache" /> + <meta http-equiv="Expires" content="0" /> + <title>Network Monitor Test Page</title> + </head> + + <body> + <p>HAR POST data test</p> + + <script type="text/javascript"> + /* exported executeTest, executeTest2, executeTest3 */ + "use strict"; + + function post(address, data) { + const xhr = new XMLHttpRequest(); + xhr.open("POST", address, true); + xhr.setRequestHeader("Content-Type", "application/json"); + xhr.send(data); + } + + function get(address) { + const xhr = new XMLHttpRequest(); + xhr.open("GET", address); + xhr.send(); + } + + function executeTest() { + const url = "html_har_post-data-test-page.html"; + const data = "{'first': 'John', 'last': 'Doe'}"; + post(url, data); + } + + function executeTest2(size) { + const url = "html_har_post-data-test-page.html"; + const data = "x".repeat(size); + post(url, data); + } + + function executeTest3(size) { + const url = "html_har_post-data-test-page.html"; + get(url); + } + + </script> + </body> + +</html> diff --git a/devtools/client/netmonitor/src/har/test/sjs_cache-test-server.sjs b/devtools/client/netmonitor/src/har/test/sjs_cache-test-server.sjs new file mode 100644 index 0000000000..66081fe1bb --- /dev/null +++ b/devtools/client/netmonitor/src/har/test/sjs_cache-test-server.sjs @@ -0,0 +1,14 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ +"use strict"; + +function handleRequest(request, response) { + response.setStatusLine(request.httpVersion, 304, "Not Modified"); + response.setHeader( + "Cache-Control", + "no-transform,public,max-age=300,s-maxage=900" + ); + response.setHeader("Expires", "Thu, 01 Dec 2100 20:00:00 GMT"); + response.setHeader("Content-Type", "text/plain; charset=utf-8", false); + response.write("Hello from cache!"); +} diff --git a/devtools/client/netmonitor/src/har/test/sjs_cookies-test-server.sjs b/devtools/client/netmonitor/src/har/test/sjs_cookies-test-server.sjs new file mode 100644 index 0000000000..a86a0f13cd --- /dev/null +++ b/devtools/client/netmonitor/src/har/test/sjs_cookies-test-server.sjs @@ -0,0 +1,10 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ +"use strict"; + +function handleRequest(request, response) { + response.setHeader("Set-Cookie", "tom=cool; Max-Age=10; HttpOnly", true); + response.setHeader("Set-Cookie", "bob=true; Max-Age=10; HttpOnly", true); + response.setHeader("Set-Cookie", "foo=bar; Max-Age=10; HttpOnly", true); + response.write("Hello world!"); +} |