diff options
Diffstat (limited to 'devtools/client/netmonitor/src/har/har-builder.js')
| -rw-r--r-- | devtools/client/netmonitor/src/har/har-builder.js | 656 |
1 files changed, 656 insertions, 0 deletions
diff --git a/devtools/client/netmonitor/src/har/har-builder.js b/devtools/client/netmonitor/src/har/har-builder.js new file mode 100644 index 0000000000..9361df7f03 --- /dev/null +++ b/devtools/client/netmonitor/src/har/har-builder.js @@ -0,0 +1,656 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +"use strict"; + +const appInfo = Services.appinfo; +const { LocalizationHelper } = require("resource://devtools/shared/l10n.js"); +const { CurlUtils } = require("resource://devtools/client/shared/curl.js"); +const { + getFormDataSections, + getUrlQuery, + parseQueryString, +} = require("resource://devtools/client/netmonitor/src/utils/request-utils.js"); +const { + buildHarLog, +} = require("resource://devtools/client/netmonitor/src/har/har-builder-utils.js"); +const L10N = new LocalizationHelper("devtools/client/locales/har.properties"); +const { + TIMING_KEYS, +} = require("resource://devtools/client/netmonitor/src/constants.js"); + +/** + * This object is responsible for building HAR file. See HAR spec: + * https://dvcs.w3.org/hg/webperf/raw-file/tip/specs/HAR/Overview.html + * http://www.softwareishard.com/blog/har-12-spec/ + * + * @param {Object} options + * configuration object + * @param {Boolean} options.connector + * Set to true to include HTTP response bodies in the result data + * structure. + * @param {String} options.id + * ID of the exported page. + * @param {Boolean} options.includeResponseBodies + * Set to true to include HTTP response bodies in the result data + * structure. + * @param {Array} options.items + * List of network events to be exported. + * @param {Boolean} options.supportsMultiplePages + * Set to true to create distinct page entries for each navigation. + */ +var HarBuilder = function (options) { + this._connector = options.connector; + this._id = options.id; + this._includeResponseBodies = options.includeResponseBodies; + this._items = options.items; + // Page id counter, only used when options.supportsMultiplePages is true. + this._pageId = options.supportsMultiplePages ? 0 : options.id; + this._pageMap = []; + this._supportsMultiplePages = options.supportsMultiplePages; + this._title = this._connector.currentTarget.title; +}; + +HarBuilder.prototype = { + // Public API + + /** + * This is the main method used to build the entire result HAR data. + * The process is asynchronous since it can involve additional RDP + * communication (e.g. resolving long strings). + * + * @returns {Promise} A promise that resolves to the HAR object when + * the entire build process is done. + */ + async build() { + this.promises = []; + + // Build basic structure for data. + const harLog = buildHarLog(appInfo); + + // Build pages. + this.buildPages(harLog.log); + + // Build entries. + for (const request of this._items) { + const entry = await this.buildEntry(harLog.log, request); + if (entry) { + harLog.log.entries.push(entry); + } + } + + // Some data needs to be fetched from the backend during the + // build process, so wait till all is done. + await Promise.all(this.promises); + + return harLog; + }, + + // Helpers + + buildPages(log) { + if (this._supportsMultiplePages) { + this.buildPagesFromTargetTitles(log); + } else if (this._items.length) { + const firstRequest = this._items[0]; + const page = this.buildPage(this._title, firstRequest); + log.pages.push(page); + this._pageMap[this._id] = page; + } + }, + + buildPagesFromTargetTitles(log) { + // Retrieve the additional HAR data collected by the connector. + const { initialTargetTitle, navigationRequests, targetTitlesPerURL } = + this._connector.getHarData(); + const firstNavigationRequest = navigationRequests[0]; + const firstRequest = this._items[0]; + + if ( + !firstNavigationRequest || + firstRequest.resourceId !== firstNavigationRequest.resourceId + ) { + // If the first request is not a navigation request, it must be related + // to the initial page. Create a first page entry for such early requests. + const initialPage = this.buildPage(initialTargetTitle, firstRequest); + log.pages.push(initialPage); + } + + for (const request of navigationRequests) { + if (targetTitlesPerURL.has(request.url)) { + const title = targetTitlesPerURL.get(request.url); + const page = this.buildPage(title, request); + log.pages.push(page); + } else { + console.warn( + `Could not find any page corresponding to a navigation to ${request.url}` + ); + } + } + }, + + buildPage(title, networkEvent) { + const page = {}; + + page.id = "page_" + this._pageId; + page.pageTimings = this.buildPageTimings(page, networkEvent); + page.startedDateTime = dateToHarString(new Date(networkEvent.startedMs)); + page.title = title; + + // Increase the pageId, for upcoming calls to buildPage. + // If supportsMultiplePages is disabled this method is only called once. + this._pageId++; + + return page; + }, + + getPage(log, entry) { + const existingPage = log.pages.findLast( + ({ startedDateTime }) => startedDateTime <= entry.startedDateTime + ); + + if (!existingPage) { + throw new Error( + "Could not find a page for request: " + entry.request.url + ); + } + + return existingPage; + }, + + async buildEntry(log, networkEvent) { + const entry = {}; + entry.startedDateTime = dateToHarString(new Date(networkEvent.startedMs)); + + let { eventTimings, id } = networkEvent; + try { + if (!eventTimings && this._connector.requestData) { + eventTimings = await this._connector.requestData(id, "eventTimings"); + } + + entry.request = await this.buildRequest(networkEvent); + entry.response = await this.buildResponse(networkEvent); + entry.cache = await this.buildCache(networkEvent); + } catch (e) { + // Ignore any request for which we can't retrieve lazy data + // The request has most likely been destroyed on the server side, + // either because persist is disabled or the request's target/WindowGlobal/process + // has been destroyed. + console.warn("HAR builder failed on", networkEvent.url, e, e.stack); + return null; + } + entry.timings = eventTimings ? eventTimings.timings : {}; + + // Calculate total time by summing all timings. Note that + // `networkEvent.totalTime` can't be used since it doesn't have to + // correspond to plain summary of individual timings. + // With TCP Fast Open and TLS early data sending data can + // start at the same time as connect (we can send data on + // TCP syn packet). Also TLS handshake can carry application + // data thereby overlapping a sending data period and TLS + // handshake period. + entry.time = TIMING_KEYS.reduce((sum, type) => { + const time = entry.timings[type]; + return typeof time != "undefined" && time != -1 ? sum + time : sum; + }, 0); + + // Security state isn't part of HAR spec, and so create + // custom field that needs to use '_' prefix. + entry._securityState = networkEvent.securityState; + + if (networkEvent.remoteAddress) { + entry.serverIPAddress = networkEvent.remoteAddress; + } + + if (networkEvent.remotePort) { + entry.connection = networkEvent.remotePort + ""; + } + + const page = this.getPage(log, entry); + entry.pageref = page.id; + + return entry; + }, + + buildPageTimings(page, networkEvent) { + // Event timing info isn't available + const timings = { + onContentLoad: -1, + onLoad: -1, + }; + + // TODO: This method currently ignores the networkEvent and always retrieves + // the same timing markers for all pages. Seee Bug 1833806. + if (this._connector.getTimingMarker) { + timings.onContentLoad = this._connector.getTimingMarker( + "firstDocumentDOMContentLoadedTimestamp" + ); + timings.onLoad = this._connector.getTimingMarker( + "firstDocumentLoadTimestamp" + ); + } + + return timings; + }, + + async buildRequest(networkEvent) { + // When using HarAutomation, HarCollector will automatically fetch requestHeaders + // and requestCookies, but when we use it from netmonitor, FirefoxDataProvider + // should fetch it itself lazily, via requestData. + + let { id, requestHeaders } = networkEvent; + if (!requestHeaders && this._connector.requestData) { + requestHeaders = await this._connector.requestData(id, "requestHeaders"); + } + + let { requestCookies } = networkEvent; + if (!requestCookies && this._connector.requestData) { + requestCookies = await this._connector.requestData(id, "requestCookies"); + } + + const request = { + bodySize: 0, + }; + request.method = networkEvent.method; + request.url = networkEvent.url; + request.httpVersion = networkEvent.httpVersion || ""; + request.headers = this.buildHeaders(requestHeaders); + request.headers = this.appendHeadersPostData(request.headers, networkEvent); + request.cookies = this.buildCookies(requestCookies); + request.queryString = parseQueryString(getUrlQuery(networkEvent.url)) || []; + request.headersSize = requestHeaders.headersSize; + request.postData = await this.buildPostData(networkEvent); + + if (request.postData?.text) { + request.bodySize = request.postData.text.length; + } + + return request; + }, + + /** + * Fetch all header values from the backend (if necessary) and + * build the result HAR structure. + * + * @param {Object} input Request or response header object. + */ + buildHeaders(input) { + if (!input) { + return []; + } + + return this.buildNameValuePairs(input.headers); + }, + + appendHeadersPostData(input = [], networkEvent) { + if (!networkEvent.requestPostData) { + return input; + } + + this.fetchData(networkEvent.requestPostData.postData.text).then(value => { + const multipartHeaders = CurlUtils.getHeadersFromMultipartText(value); + for (const header of multipartHeaders) { + input.push(header); + } + }); + + return input; + }, + + buildCookies(input) { + if (!input) { + return []; + } + + return this.buildNameValuePairs(input.cookies || input); + }, + + buildNameValuePairs(entries) { + const result = []; + + // HAR requires headers array to be presented, so always + // return at least an empty array. + if (!entries) { + return result; + } + + // Make sure header values are fully fetched from the server. + entries.forEach(entry => { + this.fetchData(entry.value).then(value => { + result.push({ + name: entry.name, + value, + }); + }); + }); + + return result; + }, + + async buildPostData(networkEvent) { + // When using HarAutomation, HarCollector will automatically fetch requestPostData + // and requestHeaders, but when we use it from netmonitor, FirefoxDataProvider + // should fetch it itself lazily, via requestData. + let { id, requestHeaders, requestPostData } = networkEvent; + let requestHeadersFromUploadStream; + + if (!requestPostData && this._connector.requestData) { + requestPostData = await this._connector.requestData( + id, + "requestPostData" + ); + requestHeadersFromUploadStream = requestPostData.uploadHeaders; + } + + if (!requestPostData.postData.text) { + return undefined; + } + + if (!requestHeaders && this._connector.requestData) { + requestHeaders = await this._connector.requestData(id, "requestHeaders"); + } + + const postData = { + mimeType: findValue(requestHeaders.headers, "content-type"), + params: [], + text: requestPostData.postData.text, + }; + + if (requestPostData.postDataDiscarded) { + postData.comment = L10N.getStr("har.requestBodyNotIncluded"); + return postData; + } + + // If we are dealing with URL encoded body, parse parameters. + if ( + CurlUtils.isUrlEncodedRequest({ + headers: requestHeaders.headers, + postDataText: postData.text, + }) + ) { + postData.mimeType = "application/x-www-form-urlencoded"; + // Extract form parameters and produce nice HAR array. + const formDataSections = await getFormDataSections( + requestHeaders, + requestHeadersFromUploadStream, + requestPostData, + this._connector.getLongString + ); + + formDataSections.forEach(section => { + const paramsArray = parseQueryString(section); + if (paramsArray) { + postData.params = [...postData.params, ...paramsArray]; + } + }); + } + + return postData; + }, + + async buildResponse(networkEvent) { + // When using HarAutomation, HarCollector will automatically fetch responseHeaders + // and responseCookies, but when we use it from netmonitor, FirefoxDataProvider + // should fetch it itself lazily, via requestData. + + let { id, responseCookies, responseHeaders } = networkEvent; + if (!responseHeaders && this._connector.requestData) { + responseHeaders = await this._connector.requestData( + id, + "responseHeaders" + ); + } + + if (!responseCookies && this._connector.requestData) { + responseCookies = await this._connector.requestData( + id, + "responseCookies" + ); + } + + const response = { + status: 0, + }; + + // Arbitrary value if it's aborted to make sure status has a number + if (networkEvent.status) { + response.status = parseInt(networkEvent.status, 10); + } + response.statusText = networkEvent.statusText || ""; + response.httpVersion = networkEvent.httpVersion || ""; + + response.headers = this.buildHeaders(responseHeaders); + response.cookies = this.buildCookies(responseCookies); + response.content = await this.buildContent(networkEvent); + + const headers = responseHeaders ? responseHeaders.headers : null; + const headersSize = responseHeaders ? responseHeaders.headersSize : -1; + + response.redirectURL = findValue(headers, "Location"); + response.headersSize = headersSize; + + // 'bodySize' is size of the received response body in bytes. + // Set to zero in case of responses coming from the cache (304). + // Set to -1 if the info is not available. + if (typeof networkEvent.transferredSize != "number") { + response.bodySize = response.status == 304 ? 0 : -1; + } else { + response.bodySize = networkEvent.transferredSize; + } + + return response; + }, + + async buildContent(networkEvent) { + const content = { + mimeType: networkEvent.mimeType, + size: -1, + }; + + // When using HarAutomation, HarCollector will automatically fetch responseContent, + // but when we use it from netmonitor, FirefoxDataProvider should fetch it itself + // lazily, via requestData. + let { responseContent } = networkEvent; + if (!responseContent && this._connector.requestData) { + responseContent = await this._connector.requestData( + networkEvent.id, + "responseContent" + ); + } + if (responseContent?.content) { + content.size = responseContent.content.size; + content.encoding = responseContent.content.encoding; + } + + const includeBodies = this._includeResponseBodies; + const contentDiscarded = responseContent + ? responseContent.contentDiscarded + : false; + + // The comment is appended only if the response content + // is explicitly discarded. + if (!includeBodies || contentDiscarded) { + content.comment = L10N.getStr("har.responseBodyNotIncluded"); + return content; + } + + if (responseContent) { + const { text } = responseContent.content; + this.fetchData(text).then(value => { + content.text = value; + }); + } + + return content; + }, + + async buildCache(networkEvent) { + const cache = {}; + + // if resource has changed, return early + if (networkEvent.status != "304") { + return cache; + } + + if (networkEvent.responseCacheAvailable && this._connector.requestData) { + const responseCache = await this._connector.requestData( + networkEvent.id, + "responseCache" + ); + if (responseCache.cache) { + cache.afterRequest = this.buildCacheEntry(responseCache.cache); + } + } else if (networkEvent.responseCache?.cache) { + cache.afterRequest = this.buildCacheEntry( + networkEvent.responseCache.cache + ); + } else { + cache.afterRequest = null; + } + + return cache; + }, + + buildCacheEntry(cacheEntry) { + const cache = {}; + + if (typeof cacheEntry !== "undefined") { + cache.expires = findKeys(cacheEntry, ["expirationTime", "expires"]); + cache.lastFetched = findKeys(cacheEntry, ["lastFetched"]); + + // TODO: eTag support + // Har format expects cache entries to provide information about eTag, + // however this is not currently exposed on nsICacheEntry. + // This should be stored under cache.eTag. See Bug 1799844. + + cache.fetchCount = findKeys(cacheEntry, ["fetchCount"]); + + // har-importer.js, along with other files, use buildCacheEntry + // initial value comes from properties without underscores. + // this checks for both in appropriate order. + cache._dataSize = findKeys(cacheEntry, ["storageDataSize", "_dataSize"]); + cache._lastModified = findKeys(cacheEntry, [ + "lastModified", + "_lastModified", + ]); + cache._device = findKeys(cacheEntry, ["deviceID", "_device"]); + } + + return cache; + }, + + // RDP Helpers + + fetchData(string) { + const promise = this._connector.getLongString(string).then(value => { + return value; + }); + + // Building HAR is asynchronous and not done till all + // collected promises are resolved. + this.promises.push(promise); + + return promise; + }, +}; + +// Helpers + +/** + * Find specified keys within an object. + * Searches object for keys passed in, returns first value returned, + * or an empty string. + * + * @param obj (object) + * @param keys (array) + * @returns {string} + */ +function findKeys(obj, keys) { + if (!keys) { + return ""; + } + + const keyFound = keys.filter(key => obj[key]); + if (!keys.length) { + return ""; + } + + const value = obj[keyFound[0]]; + if (typeof value === "undefined" || typeof value === "object") { + return ""; + } + + return String(value); +} + +/** + * Find specified value within an array of name-value pairs + * (used for headers, cookies and cache entries) + */ +function findValue(arr, name) { + if (!arr) { + return ""; + } + + name = name.toLowerCase(); + const result = arr.find(entry => entry.name.toLowerCase() == name); + return result ? result.value : ""; +} + +/** + * Generate HAR representation of a date. + * (YYYY-MM-DDThh:mm:ss.sTZD, e.g. 2009-07-24T19:20:30.45+01:00) + * See also HAR Schema: http://janodvarko.cz/har/viewer/ + * + * Note: it would be great if we could utilize Date.toJSON(), but + * it doesn't return proper time zone offset. + * + * An example: + * This helper returns: 2015-05-29T16:10:30.424+02:00 + * Date.toJSON() returns: 2015-05-29T14:10:30.424Z + * + * @param date {Date} The date object we want to convert. + */ +function dateToHarString(date) { + function f(n, c) { + if (!c) { + c = 2; + } + let s = String(n); + while (s.length < c) { + s = "0" + s; + } + return s; + } + + const result = + date.getFullYear() + + "-" + + f(date.getMonth() + 1) + + "-" + + f(date.getDate()) + + "T" + + f(date.getHours()) + + ":" + + f(date.getMinutes()) + + ":" + + f(date.getSeconds()) + + "." + + f(date.getMilliseconds(), 3); + + let offset = date.getTimezoneOffset(); + const positive = offset > 0; + + // Convert to positive number before using Math.floor (see issue 5512) + offset = Math.abs(offset); + const offsetHours = Math.floor(offset / 60); + const offsetMinutes = Math.floor(offset % 60); + const prettyOffset = + (positive > 0 ? "-" : "+") + f(offsetHours) + ":" + f(offsetMinutes); + + return result + prettyOffset; +} + +// Exports from this module +exports.HarBuilder = HarBuilder; |