diff options
Diffstat (limited to 'devtools/shared/network-observer')
30 files changed, 5685 insertions, 0 deletions
diff --git a/devtools/shared/network-observer/ChannelMap.sys.mjs b/devtools/shared/network-observer/ChannelMap.sys.mjs new file mode 100644 index 0000000000..3c54b1171a --- /dev/null +++ b/devtools/shared/network-observer/ChannelMap.sys.mjs @@ -0,0 +1,129 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** + * FinalizationRegistry callback, see + * https://searchfox.org/mozilla-central/source/js/src/builtin/FinalizationRegistryObject.h + * + * Will be invoked when the channel corresponding to the weak reference is + * "destroyed", at which point we can cleanup the corresponding entry in our + * regular map. + */ +function deleteIdFromRefMap({ refMap, id }) { + refMap.delete(id); +} + +/** + * This object implements iterable weak map for HTTP channels tracked by + * the network observer. + * + * We can't use Map() for storing HTTP channel references since we don't + * know when we should remove the entry in it (it's wrong to do it in + * 'onTransactionClose' since it doesn't have to be the last platform + * notification for a given channel). We want the map to auto update + * when the channel is garbage collected. + * + * We can't use WeakMap() since searching for a value by the channel object + * isn't reliable (there might be different objects representing the same + * channel). We need to search by channel ID, but ID can't be used as key + * in WeakMap(). + * + * So, this custom map solves aforementioned issues. + */ +export class ChannelMap { + #finalizationRegistry; + #refMap; + #weakMap; + + constructor() { + // See https://searchfox.org/mozilla-central/source/js/src/builtin/FinalizationRegistryObject.h + this.#finalizationRegistry = new FinalizationRegistry(deleteIdFromRefMap); + + // Map of channel id to a channel weak reference. + this.#refMap = new Map(); + + /** + * WeakMap from nsIChannel instances to objects which encapsulate ChannelMap + * values with the following structure: + * @property {Object} value + * The actual value stored in this ChannelMap entry, which should relate + * to this channel. + * @property {WeakRef} ref + * Weak reference for the channel object which is the key of the entry. + */ + this.#weakMap = new WeakMap(); + } + + /** + * Remove all entries from the ChannelMap. + */ + clear() { + this.#refMap.clear(); + } + + /** + * Delete the entry for the provided channel from the underlying maps, if any. + * Note that this will only delete entries which were set for the exact same + * nsIChannel object, and will not attempt to look up entries by channel id. + * + * @param {nsIChannel} channel + * The key to delete from the ChannelMap. + * + * @return {boolean} + * True if an entry was deleted, false otherwise. + */ + delete(channel) { + const entry = this.#weakMap.get(channel); + if (!entry) { + return false; + } + + this.#weakMap.delete(channel); + this.#refMap.delete(channel.channelId); + this.#finalizationRegistry.unregister(entry.ref); + return true; + } + + /** + * Retrieve a value stored in the ChannelMap by the provided channel. + * + * @param {nsIChannel} channel + * The key to delete from the ChannelMap. + * + * @return {Object|null} + * The value held for the provided channel. + * Null if the channel did not match any known key. + */ + get(channel) { + const ref = this.#refMap.get(channel.channelId); + const key = ref ? ref.deref() : null; + if (!key) { + return null; + } + const channelInfo = this.#weakMap.get(key); + return channelInfo ? channelInfo.value : null; + } + + /** + * Adds or updates an entry in the ChannelMap for the provided channel. + * + * @param {nsIChannel} channel + * The key of the entry to add or update. + * @param {Object} value + * The value to add or update. + */ + set(channel, value) { + const ref = new WeakRef(channel); + this.#weakMap.set(channel, { value, ref }); + this.#refMap.set(channel.channelId, ref); + this.#finalizationRegistry.register( + channel, + { + refMap: this.#refMap, + id: channel.channelId, + }, + ref + ); + } +} diff --git a/devtools/shared/network-observer/NetworkHelper.sys.mjs b/devtools/shared/network-observer/NetworkHelper.sys.mjs new file mode 100644 index 0000000000..3523c0854e --- /dev/null +++ b/devtools/shared/network-observer/NetworkHelper.sys.mjs @@ -0,0 +1,917 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* + * Software License Agreement (BSD License) + * + * Copyright (c) 2007, Parakey Inc. + * All rights reserved. + * + * Redistribution and use of this software in source and binary forms, + * with or without modification, are permitted provided that the + * following conditions are met: + * + * * Redistributions of source code must retain the above + * copyright notice, this list of conditions and the + * following disclaimer. + * + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the + * following disclaimer in the documentation and/or other + * materials provided with the distribution. + * + * * Neither the name of Parakey Inc. nor the names of its + * contributors may be used to endorse or promote products + * derived from this software without specific prior + * written permission of Parakey Inc. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED + * OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +/* + * Creator: + * Joe Hewitt + * Contributors + * John J. Barton (IBM Almaden) + * Jan Odvarko (Mozilla Corp.) + * Max Stepanov (Aptana Inc.) + * Rob Campbell (Mozilla Corp.) + * Hans Hillen (Paciello Group, Mozilla) + * Curtis Bartley (Mozilla Corp.) + * Mike Collins (IBM Almaden) + * Kevin Decker + * Mike Ratcliffe (Comartis AG) + * Hernan Rodríguez Colmeiro + * Austin Andrews + * Christoph Dorn + * Steven Roussey (AppCenter Inc, Network54) + * Mihai Sucan (Mozilla Corp.) + */ + +import { XPCOMUtils } from "resource://gre/modules/XPCOMUtils.sys.mjs"; + +const lazy = {}; + +ChromeUtils.defineESModuleGetters(lazy, { + DevToolsInfaillibleUtils: + "resource://devtools/shared/DevToolsInfaillibleUtils.sys.mjs", +}); + +XPCOMUtils.defineLazyModuleGetters(lazy, { + NetUtil: "resource://gre/modules/NetUtil.jsm", +}); + +// It would make sense to put this in the above +// ChromeUtils.defineESModuleGetters, but that doesn't seem to work. +XPCOMUtils.defineLazyGetter(lazy, "certDecoder", () => { + const { parse, pemToDER } = ChromeUtils.importESModule( + "chrome://global/content/certviewer/certDecoder.mjs" + ); + return { parse, pemToDER }; +}); + +// "Lax", "Strict" and "None" are special values of the SameSite cookie +// attribute that should not be translated. +const COOKIE_SAMESITE = { + LAX: "Lax", + STRICT: "Strict", + NONE: "None", +}; + +/** + * Helper object for networking stuff. + * + * Most of the following functions have been taken from the Firebug source. They + * have been modified to match the Firefox coding rules. + */ +export var NetworkHelper = { + /** + * Converts text with a given charset to unicode. + * + * @param string text + * Text to convert. + * @param string charset + * Charset to convert the text to. + * @returns string + * Converted text. + */ + convertToUnicode(text, charset) { + // FIXME: We need to throw when text can't be converted e.g. the contents of + // an image. Until we have a way to do so with TextEncoder and TextDecoder + // we need to use nsIScriptableUnicodeConverter instead. + const conv = Cc[ + "@mozilla.org/intl/scriptableunicodeconverter" + ].createInstance(Ci.nsIScriptableUnicodeConverter); + try { + conv.charset = charset || "UTF-8"; + return conv.ConvertToUnicode(text); + } catch (ex) { + return text; + } + }, + + /** + * Reads all available bytes from stream and converts them to charset. + * + * @param nsIInputStream stream + * @param string charset + * @returns string + * UTF-16 encoded string based on the content of stream and charset. + */ + readAndConvertFromStream(stream, charset) { + let text = null; + try { + text = lazy.NetUtil.readInputStreamToString(stream, stream.available()); + return this.convertToUnicode(text, charset); + } catch (err) { + return text; + } + }, + + /** + * Reads the posted text from request. + * + * @param nsIHttpChannel request + * @param string charset + * The content document charset, used when reading the POSTed data. + * @returns string or null + * Returns the posted string if it was possible to read from request + * otherwise null. + */ + readPostTextFromRequest(request, charset) { + if (request instanceof Ci.nsIUploadChannel) { + const iStream = request.uploadStream; + + let isSeekableStream = false; + if (iStream instanceof Ci.nsISeekableStream) { + isSeekableStream = true; + } + + let prevOffset; + if (isSeekableStream) { + prevOffset = iStream.tell(); + iStream.seek(Ci.nsISeekableStream.NS_SEEK_SET, 0); + } + + // Read data from the stream. + const text = this.readAndConvertFromStream(iStream, charset); + + // Seek locks the file, so seek to the beginning only if necko hasn't + // read it yet, since necko doesn't seek to 0 before reading (at lest + // not till 459384 is fixed). + if (isSeekableStream && prevOffset == 0) { + iStream.seek(Ci.nsISeekableStream.NS_SEEK_SET, 0); + } + return text; + } + return null; + }, + + /** + * Reads the posted text from the page's cache. + * + * @param nsIDocShell docShell + * @param string charset + * @returns string or null + * Returns the posted string if it was possible to read from + * docShell otherwise null. + */ + readPostTextFromPage(docShell, charset) { + const webNav = docShell.QueryInterface(Ci.nsIWebNavigation); + return this.readPostTextFromPageViaWebNav(webNav, charset); + }, + + /** + * Reads the posted text from the page's cache, given an nsIWebNavigation + * object. + * + * @param nsIWebNavigation webNav + * @param string charset + * @returns string or null + * Returns the posted string if it was possible to read from + * webNav, otherwise null. + */ + readPostTextFromPageViaWebNav(webNav, charset) { + if (webNav instanceof Ci.nsIWebPageDescriptor) { + const descriptor = webNav.currentDescriptor; + + if ( + descriptor instanceof Ci.nsISHEntry && + descriptor.postData && + descriptor instanceof Ci.nsISeekableStream + ) { + descriptor.seek(Ci.nsISeekableStream.NS_SEEK_SET, 0); + + return this.readAndConvertFromStream(descriptor, charset); + } + } + return null; + }, + + /** + * Gets the topFrameElement that is associated with request. This + * works in single-process and multiprocess contexts. It may cross + * the content/chrome boundary. + * + * @param nsIHttpChannel request + * @returns Element|null + * The top frame element for the given request. + */ + getTopFrameForRequest(request) { + try { + return this.getRequestLoadContext(request).topFrameElement; + } catch (ex) { + // request loadContext is not always available. + } + return null; + }, + + /** + * Gets the nsIDOMWindow that is associated with request. + * + * @param nsIHttpChannel request + * @returns nsIDOMWindow or null + */ + getWindowForRequest(request) { + try { + return this.getRequestLoadContext(request).associatedWindow; + } catch (ex) { + // On some request notificationCallbacks and loadGroup are both null, + // so that we can't retrieve any nsILoadContext interface. + // Fallback on nsILoadInfo to try to retrieve the request's window. + // (this is covered by test_network_get.html and its CSS request) + return request.loadInfo.loadingDocument?.defaultView; + } + }, + + /** + * Gets the nsILoadContext that is associated with request. + * + * @param nsIHttpChannel request + * @returns nsILoadContext or null + */ + getRequestLoadContext(request) { + try { + if (request.loadInfo.workerAssociatedBrowsingContext) { + return request.loadInfo.workerAssociatedBrowsingContext; + } + } catch (ex) { + // Ignore. + } + try { + return request.notificationCallbacks.getInterface(Ci.nsILoadContext); + } catch (ex) { + // Ignore. + } + + try { + return request.loadGroup.notificationCallbacks.getInterface( + Ci.nsILoadContext + ); + } catch (ex) { + // Ignore. + } + + return null; + }, + + /** + * Determines whether the request has been made for the top level document. + * + * @param nsIHttpChannel request + * @returns Boolean True if the request represents the top level document. + */ + isTopLevelLoad(request) { + if (request instanceof Ci.nsIChannel) { + const loadInfo = request.loadInfo; + if (loadInfo?.isTopLevelLoad) { + return request.loadFlags & Ci.nsIChannel.LOAD_DOCUMENT_URI; + } + } + + return false; + }, + + /** + * Loads the content of url from the cache. + * + * @param string url + * URL to load the cached content for. + * @param string charset + * Assumed charset of the cached content. Used if there is no charset + * on the channel directly. + * @param function callback + * Callback that is called with the loaded cached content if available + * or null if something failed while getting the cached content. + */ + loadFromCache(url, charset, callback) { + const channel = lazy.NetUtil.newChannel({ + uri: url, + loadUsingSystemPrincipal: true, + }); + + // Ensure that we only read from the cache and not the server. + channel.loadFlags = + Ci.nsIRequest.LOAD_FROM_CACHE | + Ci.nsICachingChannel.LOAD_ONLY_FROM_CACHE | + Ci.nsICachingChannel.LOAD_BYPASS_LOCAL_CACHE_IF_BUSY; + + lazy.NetUtil.asyncFetch(channel, (inputStream, statusCode, request) => { + if (!Components.isSuccessCode(statusCode)) { + callback(null); + return; + } + + // Try to get the encoding from the channel. If there is none, then use + // the passed assumed charset. + const requestChannel = request.QueryInterface(Ci.nsIChannel); + const contentCharset = requestChannel.contentCharset || charset; + + // Read the content of the stream using contentCharset as encoding. + callback(this.readAndConvertFromStream(inputStream, contentCharset)); + }); + }, + + /** + * Parse a raw Cookie header value. + * + * @param string header + * The raw Cookie header value. + * @return array + * Array holding an object for each cookie. Each object holds the + * following properties: name and value. + */ + parseCookieHeader(header) { + const cookies = header.split(";"); + const result = []; + + cookies.forEach(function (cookie) { + const equal = cookie.indexOf("="); + const name = cookie.substr(0, equal); + const value = cookie.substr(equal + 1); + result.push({ + name: unescape(name.trim()), + value: unescape(value.trim()), + }); + }); + + return result; + }, + + /** + * Parse a raw Set-Cookie header value. + * + * @param array headers + * Array of raw Set-Cookie header values. + * @return array + * Array holding an object for each cookie. Each object holds the + * following properties: name, value, secure (boolean), httpOnly + * (boolean), path, domain, samesite and expires (ISO date string). + */ + parseSetCookieHeaders(headers) { + function parseSameSiteAttribute(attribute) { + attribute = attribute.toLowerCase(); + switch (attribute) { + case COOKIE_SAMESITE.LAX.toLowerCase(): + return COOKIE_SAMESITE.LAX; + case COOKIE_SAMESITE.STRICT.toLowerCase(): + return COOKIE_SAMESITE.STRICT; + default: + return COOKIE_SAMESITE.NONE; + } + } + + const cookies = []; + + for (const header of headers) { + const rawCookies = header.split(/\r\n|\n|\r/); + + rawCookies.forEach(function (cookie) { + const equal = cookie.indexOf("="); + const name = unescape(cookie.substr(0, equal).trim()); + const parts = cookie.substr(equal + 1).split(";"); + const value = unescape(parts.shift().trim()); + + cookie = { name, value }; + + parts.forEach(function (part) { + part = part.trim(); + if (part.toLowerCase() == "secure") { + cookie.secure = true; + } else if (part.toLowerCase() == "httponly") { + cookie.httpOnly = true; + } else if (part.indexOf("=") > -1) { + const pair = part.split("="); + pair[0] = pair[0].toLowerCase(); + if (pair[0] == "path" || pair[0] == "domain") { + cookie[pair[0]] = pair[1]; + } else if (pair[0] == "samesite") { + cookie[pair[0]] = parseSameSiteAttribute(pair[1]); + } else if (pair[0] == "expires") { + try { + pair[1] = pair[1].replace(/-/g, " "); + cookie.expires = new Date(pair[1]).toISOString(); + } catch (ex) { + // Ignore. + } + } + } + }); + + cookies.push(cookie); + }); + } + + return cookies; + }, + + // This is a list of all the mime category maps jviereck could find in the + // firebug code base. + mimeCategoryMap: { + "text/plain": "txt", + "text/html": "html", + "text/xml": "xml", + "text/xsl": "txt", + "text/xul": "txt", + "text/css": "css", + "text/sgml": "txt", + "text/rtf": "txt", + "text/x-setext": "txt", + "text/richtext": "txt", + "text/javascript": "js", + "text/jscript": "txt", + "text/tab-separated-values": "txt", + "text/rdf": "txt", + "text/xif": "txt", + "text/ecmascript": "js", + "text/vnd.curl": "txt", + "text/x-json": "json", + "text/x-js": "txt", + "text/js": "txt", + "text/vbscript": "txt", + "view-source": "txt", + "view-fragment": "txt", + "application/xml": "xml", + "application/xhtml+xml": "xml", + "application/atom+xml": "xml", + "application/rss+xml": "xml", + "application/vnd.mozilla.maybe.feed": "xml", + "application/javascript": "js", + "application/x-javascript": "js", + "application/x-httpd-php": "txt", + "application/rdf+xml": "xml", + "application/ecmascript": "js", + "application/http-index-format": "txt", + "application/json": "json", + "application/x-js": "txt", + "application/x-mpegurl": "txt", + "application/vnd.apple.mpegurl": "txt", + "multipart/mixed": "txt", + "multipart/x-mixed-replace": "txt", + "image/svg+xml": "svg", + "application/octet-stream": "bin", + "image/jpeg": "image", + "image/jpg": "image", + "image/gif": "image", + "image/png": "image", + "image/bmp": "image", + "application/x-shockwave-flash": "flash", + "video/x-flv": "flash", + "audio/mpeg3": "media", + "audio/x-mpeg-3": "media", + "video/mpeg": "media", + "video/x-mpeg": "media", + "video/vnd.mpeg.dash.mpd": "xml", + "audio/ogg": "media", + "application/ogg": "media", + "application/x-ogg": "media", + "application/x-midi": "media", + "audio/midi": "media", + "audio/x-mid": "media", + "audio/x-midi": "media", + "music/crescendo": "media", + "audio/wav": "media", + "audio/x-wav": "media", + "text/json": "json", + "application/x-json": "json", + "application/json-rpc": "json", + "application/x-web-app-manifest+json": "json", + "application/manifest+json": "json", + }, + + /** + * Check if the given MIME type is a text-only MIME type. + * + * @param string mimeType + * @return boolean + */ + isTextMimeType(mimeType) { + if (mimeType.indexOf("text/") == 0) { + return true; + } + + // XML and JSON often come with custom MIME types, so in addition to the + // standard "application/xml" and "application/json", we also look for + // variants like "application/x-bigcorp+xml". For JSON we allow "+json" and + // "-json" as suffixes. + if (/^application\/\w+(?:[\.-]\w+)*(?:\+xml|[-+]json)$/.test(mimeType)) { + return true; + } + + const category = this.mimeCategoryMap[mimeType] || null; + switch (category) { + case "txt": + case "js": + case "json": + case "css": + case "html": + case "svg": + case "xml": + return true; + + default: + return false; + } + }, + + /** + * Takes a securityInfo object of nsIRequest, the nsIRequest itself and + * extracts security information from them. + * + * @param object securityInfo + * The securityInfo object of a request. If null channel is assumed + * to be insecure. + * @param object originAttributes + * The OriginAttributes of the request. + * @param object httpActivity + * The httpActivity object for the request with at least members + * { private, hostname }. + * @param Map decodedCertificateCache + * A Map of certificate fingerprints to decoded certificates, to avoid + * repeatedly decoding previously-seen certificates. + * + * @return object + * Returns an object containing following members: + * - state: The security of the connection used to fetch this + * request. Has one of following string values: + * * "insecure": the connection was not secure (only http) + * * "weak": the connection has minor security issues + * * "broken": secure connection failed (e.g. expired cert) + * * "secure": the connection was properly secured. + * If state == broken: + * - errorMessage: error code string. + * If state == secure: + * - protocolVersion: one of TLSv1, TLSv1.1, TLSv1.2, TLSv1.3. + * - cipherSuite: the cipher suite used in this connection. + * - cert: information about certificate used in this connection. + * See parseCertificateInfo for the contents. + * - hsts: true if host uses Strict Transport Security, + * false otherwise + * - hpkp: true if host uses Public Key Pinning, false otherwise + * If state == weak: Same as state == secure and + * - weaknessReasons: list of reasons that cause the request to be + * considered weak. See getReasonsForWeakness. + */ + async parseSecurityInfo( + securityInfo, + originAttributes, + httpActivity, + decodedCertificateCache + ) { + const info = { + state: "insecure", + }; + + // The request did not contain any security info. + if (!securityInfo) { + return info; + } + + /** + * Different scenarios to consider here and how they are handled: + * - request is HTTP, the connection is not secure + * => securityInfo is null + * => state === "insecure" + * + * - request is HTTPS, the connection is secure + * => .securityState has STATE_IS_SECURE flag + * => state === "secure" + * + * - request is HTTPS, the connection has security issues + * => .securityState has STATE_IS_INSECURE flag + * => .errorCode is an NSS error code. + * => state === "broken" + * + * - request is HTTPS, the connection was terminated before the security + * could be validated + * => .securityState has STATE_IS_INSECURE flag + * => .errorCode is NOT an NSS error code. + * => .errorMessage is not available. + * => state === "insecure" + * + * - request is HTTPS but it uses a weak cipher or old protocol, see + * https://hg.mozilla.org/mozilla-central/annotate/def6ed9d1c1a/ + * security/manager/ssl/nsNSSCallbacks.cpp#l1233 + * - request is mixed content (which makes no sense whatsoever) + * => .securityState has STATE_IS_BROKEN flag + * => .errorCode is NOT an NSS error code + * => .errorMessage is not available + * => state === "weak" + */ + + const wpl = Ci.nsIWebProgressListener; + const NSSErrorsService = Cc["@mozilla.org/nss_errors_service;1"].getService( + Ci.nsINSSErrorsService + ); + if (!NSSErrorsService.isNSSErrorCode(securityInfo.errorCode)) { + const state = securityInfo.securityState; + + let uri = null; + if (httpActivity.channel?.URI) { + uri = httpActivity.channel.URI; + } + if (uri && !uri.schemeIs("https") && !uri.schemeIs("wss")) { + // it is not enough to look at the transport security info - + // schemes other than https and wss are subject to + // downgrade/etc at the scheme level and should always be + // considered insecure + info.state = "insecure"; + } else if (state & wpl.STATE_IS_SECURE) { + // The connection is secure if the scheme is sufficient + info.state = "secure"; + } else if (state & wpl.STATE_IS_BROKEN) { + // The connection is not secure, there was no error but there's some + // minor security issues. + info.state = "weak"; + info.weaknessReasons = this.getReasonsForWeakness(state); + } else if (state & wpl.STATE_IS_INSECURE) { + // This was most likely an https request that was aborted before + // validation. Return info as info.state = insecure. + return info; + } else { + lazy.DevToolsInfaillibleUtils.reportException( + "NetworkHelper.parseSecurityInfo", + "Security state " + state + " has no known STATE_IS_* flags." + ); + return info; + } + + // Cipher suite. + info.cipherSuite = securityInfo.cipherName; + + // Key exchange group name. + info.keaGroupName = securityInfo.keaGroupName; + + // Certificate signature scheme. + info.signatureSchemeName = securityInfo.signatureSchemeName; + + // Protocol version. + info.protocolVersion = this.formatSecurityProtocol( + securityInfo.protocolVersion + ); + + // Certificate. + info.cert = await this.parseCertificateInfo( + securityInfo.serverCert, + decodedCertificateCache + ); + + // Certificate transparency status. + info.certificateTransparency = securityInfo.certificateTransparencyStatus; + + // HSTS and HPKP if available. + if (httpActivity.hostname) { + const sss = Cc["@mozilla.org/ssservice;1"].getService( + Ci.nsISiteSecurityService + ); + const pkps = Cc[ + "@mozilla.org/security/publickeypinningservice;1" + ].getService(Ci.nsIPublicKeyPinningService); + + if (!uri) { + // isSecureURI only cares about the host, not the scheme. + const host = httpActivity.hostname; + uri = Services.io.newURI("https://" + host); + } + + info.hsts = sss.isSecureURI(uri, originAttributes); + info.hpkp = pkps.hostHasPins(uri); + } else { + lazy.DevToolsInfaillibleUtils.reportException( + "NetworkHelper.parseSecurityInfo", + "Could not get HSTS/HPKP status as hostname is not available." + ); + info.hsts = false; + info.hpkp = false; + } + } else { + // The connection failed. + info.state = "broken"; + info.errorMessage = securityInfo.errorCodeString; + } + + // These values can be unset in rare cases, e.g. when stashed connection + // data is deseralized from an older version of Firefox. + try { + info.usedEch = securityInfo.isAcceptedEch; + } catch { + info.usedEch = false; + } + try { + info.usedDelegatedCredentials = securityInfo.isDelegatedCredential; + } catch { + info.usedDelegatedCredentials = false; + } + info.usedOcsp = securityInfo.madeOCSPRequests; + info.usedPrivateDns = securityInfo.usedPrivateDNS; + + return info; + }, + + /** + * Takes an nsIX509Cert and returns an object with certificate information. + * + * @param nsIX509Cert cert + * The certificate to extract the information from. + * @param Map decodedCertificateCache + * A Map of certificate fingerprints to decoded certificates, to avoid + * repeatedly decoding previously-seen certificates. + * @return object + * An object with following format: + * { + * subject: { commonName, organization, organizationalUnit }, + * issuer: { commonName, organization, organizationUnit }, + * validity: { start, end }, + * fingerprint: { sha1, sha256 } + * } + */ + async parseCertificateInfo(cert, decodedCertificateCache) { + function getDNComponent(dn, componentType) { + for (const [type, value] of dn.entries) { + if (type == componentType) { + return value; + } + } + return undefined; + } + + const info = {}; + if (cert) { + const certHash = cert.sha256Fingerprint; + let parsedCert = decodedCertificateCache.get(certHash); + if (!parsedCert) { + parsedCert = await lazy.certDecoder.parse( + lazy.certDecoder.pemToDER(cert.getBase64DERString()) + ); + decodedCertificateCache.set(certHash, parsedCert); + } + info.subject = { + commonName: getDNComponent(parsedCert.subject, "Common Name"), + organization: getDNComponent(parsedCert.subject, "Organization"), + organizationalUnit: getDNComponent( + parsedCert.subject, + "Organizational Unit" + ), + }; + + info.issuer = { + commonName: getDNComponent(parsedCert.issuer, "Common Name"), + organization: getDNComponent(parsedCert.issuer, "Organization"), + organizationUnit: getDNComponent( + parsedCert.issuer, + "Organizational Unit" + ), + }; + + info.validity = { + start: parsedCert.notBeforeUTC, + end: parsedCert.notAfterUTC, + }; + + info.fingerprint = { + sha1: parsedCert.fingerprint.sha1, + sha256: parsedCert.fingerprint.sha256, + }; + } else { + lazy.DevToolsInfaillibleUtils.reportException( + "NetworkHelper.parseCertificateInfo", + "Secure connection established without certificate." + ); + } + + return info; + }, + + /** + * Takes protocolVersion of TransportSecurityInfo object and returns + * human readable description. + * + * @param Number version + * One of nsITransportSecurityInfo version constants. + * @return string + * One of TLSv1, TLSv1.1, TLSv1.2, TLSv1.3 if @param version + * is valid, Unknown otherwise. + */ + formatSecurityProtocol(version) { + switch (version) { + case Ci.nsITransportSecurityInfo.TLS_VERSION_1: + return "TLSv1"; + case Ci.nsITransportSecurityInfo.TLS_VERSION_1_1: + return "TLSv1.1"; + case Ci.nsITransportSecurityInfo.TLS_VERSION_1_2: + return "TLSv1.2"; + case Ci.nsITransportSecurityInfo.TLS_VERSION_1_3: + return "TLSv1.3"; + default: + lazy.DevToolsInfaillibleUtils.reportException( + "NetworkHelper.formatSecurityProtocol", + "protocolVersion " + version + " is unknown." + ); + return "Unknown"; + } + }, + + /** + * Takes the securityState bitfield and returns reasons for weak connection + * as an array of strings. + * + * @param Number state + * nsITransportSecurityInfo.securityState. + * + * @return Array[String] + * List of weakness reasons. A subset of { cipher } where + * * cipher: The cipher suite is consireded to be weak (RC4). + */ + getReasonsForWeakness(state) { + const wpl = Ci.nsIWebProgressListener; + + // If there's non-fatal security issues the request has STATE_IS_BROKEN + // flag set. See https://hg.mozilla.org/mozilla-central/file/44344099d119 + // /security/manager/ssl/nsNSSCallbacks.cpp#l1233 + const reasons = []; + + if (state & wpl.STATE_IS_BROKEN) { + const isCipher = state & wpl.STATE_USES_WEAK_CRYPTO; + + if (isCipher) { + reasons.push("cipher"); + } + + if (!isCipher) { + lazy.DevToolsInfaillibleUtils.reportException( + "NetworkHelper.getReasonsForWeakness", + "STATE_IS_BROKEN without a known reason. Full state was: " + state + ); + } + } + + return reasons; + }, + + /** + * Parse a url's query string into its components + * + * @param string queryString + * The query part of a url + * @return array + * Array of query params {name, value} + */ + parseQueryString(queryString) { + // Make sure there's at least one param available. + // Be careful here, params don't necessarily need to have values, so + // no need to verify the existence of a "=". + if (!queryString) { + return null; + } + + // Turn the params string into an array containing { name: value } tuples. + const paramsArray = queryString + .replace(/^[?&]/, "") + .split("&") + .map(e => { + const param = e.split("="); + return { + name: param[0] + ? NetworkHelper.convertToUnicode(unescape(param[0])) + : "", + value: param[1] + ? NetworkHelper.convertToUnicode(unescape(param[1])) + : "", + }; + }); + + return paramsArray; + }, +}; diff --git a/devtools/shared/network-observer/NetworkObserver.sys.mjs b/devtools/shared/network-observer/NetworkObserver.sys.mjs new file mode 100644 index 0000000000..7137345393 --- /dev/null +++ b/devtools/shared/network-observer/NetworkObserver.sys.mjs @@ -0,0 +1,1397 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** + * NetworkObserver is the main class in DevTools to observe network requests + * out of many events fired by the platform code. + */ + +// Enable logging all platform events this module listen to +const DEBUG_PLATFORM_EVENTS = false; + +const lazy = {}; + +import { DevToolsInfaillibleUtils } from "resource://devtools/shared/DevToolsInfaillibleUtils.sys.mjs"; + +ChromeUtils.defineESModuleGetters(lazy, { + ChannelMap: "resource://devtools/shared/network-observer/ChannelMap.sys.mjs", + NetworkHelper: + "resource://devtools/shared/network-observer/NetworkHelper.sys.mjs", + NetworkOverride: + "resource://devtools/shared/network-observer/NetworkOverride.sys.mjs", + NetworkResponseListener: + "resource://devtools/shared/network-observer/NetworkResponseListener.sys.mjs", + NetworkThrottleManager: + "resource://devtools/shared/network-observer/NetworkThrottleManager.sys.mjs", + NetworkUtils: + "resource://devtools/shared/network-observer/NetworkUtils.sys.mjs", + wildcardToRegExp: + "resource://devtools/shared/network-observer/WildcardToRegexp.sys.mjs", +}); + +const gActivityDistributor = Cc[ + "@mozilla.org/network/http-activity-distributor;1" +].getService(Ci.nsIHttpActivityDistributor); + +function logPlatformEvent(eventName, channel, message = "") { + if (!DEBUG_PLATFORM_EVENTS) { + return; + } + dump(`[netmonitor] ${channel.channelId} - ${eventName} ${message}\n`); +} + +// The maximum uint32 value. +const PR_UINT32_MAX = 4294967295; + +const HTTP_TRANSACTION_CODES = { + 0x5001: "REQUEST_HEADER", + 0x5002: "REQUEST_BODY_SENT", + 0x5003: "RESPONSE_START", + 0x5004: "RESPONSE_HEADER", + 0x5005: "RESPONSE_COMPLETE", + 0x5006: "TRANSACTION_CLOSE", + + 0x4b0003: "STATUS_RESOLVING", + 0x4b000b: "STATUS_RESOLVED", + 0x4b0007: "STATUS_CONNECTING_TO", + 0x4b0004: "STATUS_CONNECTED_TO", + 0x4b0005: "STATUS_SENDING_TO", + 0x4b000a: "STATUS_WAITING_FOR", + 0x4b0006: "STATUS_RECEIVING_FROM", + 0x4b000c: "STATUS_TLS_STARTING", + 0x4b000d: "STATUS_TLS_ENDING", +}; + +const HTTP_DOWNLOAD_ACTIVITIES = [ + gActivityDistributor.ACTIVITY_SUBTYPE_RESPONSE_START, + gActivityDistributor.ACTIVITY_SUBTYPE_RESPONSE_HEADER, + gActivityDistributor.ACTIVITY_SUBTYPE_RESPONSE_COMPLETE, + gActivityDistributor.ACTIVITY_SUBTYPE_TRANSACTION_CLOSE, +]; + +/** + * The network monitor uses the nsIHttpActivityDistributor to monitor network + * requests. The nsIObserverService is also used for monitoring + * http-on-examine-response notifications. All network request information is + * routed to the remote Web Console. + * + * @constructor + * @param {Object} options + * @param {Function(nsIChannel): boolean} options.ignoreChannelFunction + * This function will be called for every detected channel to decide if it + * should be monitored or not. + * @param {Function(NetworkEvent): owner} options.onNetworkEvent + * This method is invoked once for every new network request with two + * arguments: + * - {Object} networkEvent: object created by NetworkUtils:createNetworkEvent, + * containing initial network request information as an argument. + * - {nsIChannel} channel: the channel for which the request was detected + * + * `onNetworkEvent()` must return an "owner" object which holds several add*() + * methods which are used to add further network request/response information. + */ +export class NetworkObserver { + /** + * Map of URL patterns to RegExp + * + * @type {Map} + */ + #blockedURLs = new Map(); + + /** + * Map of URL to local file path in order to redirect URL + * to local file overrides. + * + * This will replace the content of some request with the content of local files. + */ + #overrides = new Map(); + + /** + * Used by NetworkHelper.parseSecurityInfo to skip decoding known certificates. + * + * @type {Map} + */ + #decodedCertificateCache = new Map(); + /** + * See constructor argument of the same name. + * + * @type {Function} + */ + #ignoreChannelFunction; + /** + * Used to store channels intercepted for service-worker requests. + * + * @type {WeakSet} + */ + #interceptedChannels = new WeakSet(); + /** + * Explicit flag to check if this observer was already destroyed. + * + * @type {boolean} + */ + #isDestroyed = false; + /** + * See constructor argument of the same name. + * + * @type {Function} + */ + #onNetworkEvent; + /** + * Object that holds the HTTP activity objects for ongoing requests. + * + * @type {ChannelMap} + */ + #openRequests = new lazy.ChannelMap(); + /** + * Network response bodies are piped through a buffer of the given size + * (in bytes). + * + * @type {Number} + */ + #responsePipeSegmentSize = Services.prefs.getIntPref( + "network.buffer.cache.size" + ); + /** + * Whether to save the bodies of network requests and responses. + * + * @type {boolean} + */ + #saveRequestAndResponseBodies = true; + /** + * Throttling configuration, see constructor of NetworkThrottleManager + * + * @type {Object} + */ + #throttleData = null; + /** + * NetworkThrottleManager instance, created when a valid throttleData is set. + * @type {NetworkThrottleManager} + */ + #throttler = null; + + constructor(options = {}) { + const { ignoreChannelFunction, onNetworkEvent } = options; + if (typeof ignoreChannelFunction !== "function") { + throw new Error( + `Expected "ignoreChannelFunction" to be a function, got ${ignoreChannelFunction} (${typeof ignoreChannelFunction})` + ); + } + + if (typeof onNetworkEvent !== "function") { + throw new Error( + `Expected "onNetworkEvent" to be a function, got ${onNetworkEvent} (${typeof onNetworkEvent})` + ); + } + + this.#ignoreChannelFunction = ignoreChannelFunction; + this.#onNetworkEvent = onNetworkEvent; + + // Start all platform observers. + if (Services.appinfo.processType != Ci.nsIXULRuntime.PROCESS_TYPE_CONTENT) { + gActivityDistributor.addObserver(this); + Services.obs.addObserver( + this.#httpResponseExaminer, + "http-on-examine-response" + ); + Services.obs.addObserver( + this.#httpResponseExaminer, + "http-on-examine-cached-response" + ); + Services.obs.addObserver( + this.#httpModifyExaminer, + "http-on-modify-request" + ); + Services.obs.addObserver(this.#httpStopRequest, "http-on-stop-request"); + } else { + Services.obs.addObserver( + this.#httpFailedOpening, + "http-on-failed-opening-request" + ); + } + // In child processes, only watch for service worker requests + // everything else only happens in the parent process + Services.obs.addObserver( + this.#serviceWorkerRequest, + "service-worker-synthesized-response" + ); + } + + setSaveRequestAndResponseBodies(save) { + this.#saveRequestAndResponseBodies = save; + } + + getThrottleData() { + return this.#throttleData; + } + + setThrottleData(value) { + this.#throttleData = value; + // Clear out any existing throttlers + this.#throttler = null; + } + + #getThrottler() { + if (this.#throttleData !== null && this.#throttler === null) { + this.#throttler = new lazy.NetworkThrottleManager(this.#throttleData); + } + return this.#throttler; + } + + #serviceWorkerRequest = DevToolsInfaillibleUtils.makeInfallible( + (subject, topic, data) => { + const channel = subject.QueryInterface(Ci.nsIHttpChannel); + + if (this.#ignoreChannelFunction(channel)) { + return; + } + + logPlatformEvent(topic, channel); + + this.#interceptedChannels.add(subject); + + // Service workers never fire http-on-examine-cached-response, so fake one. + this.#httpResponseExaminer(channel, "http-on-examine-cached-response"); + } + ); + + /** + * Observes for http-on-failed-opening-request notification to catch any + * channels for which asyncOpen has synchronously failed. This is the only + * place to catch early security check failures. + */ + #httpFailedOpening = DevToolsInfaillibleUtils.makeInfallible( + (subject, topic) => { + if ( + this.#isDestroyed || + topic != "http-on-failed-opening-request" || + !(subject instanceof Ci.nsIHttpChannel) + ) { + return; + } + + const channel = subject.QueryInterface(Ci.nsIHttpChannel); + if (this.#ignoreChannelFunction(channel)) { + return; + } + + logPlatformEvent(topic, channel); + + // Ignore preload requests to avoid duplicity request entries in + // the Network panel. If a preload fails (for whatever reason) + // then the platform kicks off another 'real' request. + if (lazy.NetworkUtils.isPreloadRequest(channel)) { + return; + } + + this.#httpResponseExaminer(subject, topic); + } + ); + + #httpStopRequest = DevToolsInfaillibleUtils.makeInfallible( + (subject, topic) => { + if ( + this.#isDestroyed || + topic != "http-on-stop-request" || + !(subject instanceof Ci.nsIHttpChannel) + ) { + return; + } + + const channel = subject.QueryInterface(Ci.nsIHttpChannel); + if (this.#ignoreChannelFunction(channel)) { + return; + } + + logPlatformEvent(topic, channel); + + const httpActivity = this.#createOrGetActivityObject(channel); + const serverTimings = this.#extractServerTimings(channel); + + if (httpActivity.owner) { + // Try extracting server timings. Note that they will be sent to the client + // in the `_onTransactionClose` method together with network event timings. + httpActivity.owner.addServerTimings(serverTimings); + + // If the owner isn't set we need to create the network event and send + // it to the client. This happens in case where: + // - the request has been blocked (e.g. CORS) and "http-on-stop-request" is the first notification. + // - the NetworkObserver is start *after* the request started and we only receive the http-stop notification, + // but that doesn't mean the request is blocked, so check for its status. + } else if (Components.isSuccessCode(channel.status)) { + // Do not pass any blocked reason, as this request is just fine. + // Bug 1489217 - Prevent watching for this request response content, + // as this request is already running, this is too late to watch for it. + this.#createNetworkEvent(subject, { inProgressRequest: true }); + } else { + // Handles any early blockings e.g by Web Extensions or by CORS + const { blockingExtension, blockedReason } = + lazy.NetworkUtils.getBlockedReason(channel); + this.#createNetworkEvent(subject, { blockedReason, blockingExtension }); + } + } + ); + + /** + * Check if the current channel has its content being overriden + * by the content of some local file. + */ + #checkForContentOverride(channel) { + const overridePath = this.#overrides.get(channel.URI.spec); + if (!overridePath) { + return false; + } + + dump(" Override " + channel.URI.spec + " to " + overridePath + "\n"); + try { + lazy.NetworkOverride.overrideChannelWithFilePath(channel, overridePath); + } catch (e) { + dump("Exception while trying to override request content: " + e + "\n"); + } + + return true; + } + + /** + * Observe notifications for the http-on-examine-response topic, coming from + * the nsIObserverService. + * + * @private + * @param nsIHttpChannel subject + * @param string topic + * @returns void + */ + #httpResponseExaminer = DevToolsInfaillibleUtils.makeInfallible( + (subject, topic) => { + // The httpResponseExaminer is used to retrieve the uncached response + // headers. + if ( + this.#isDestroyed || + (topic != "http-on-examine-response" && + topic != "http-on-examine-cached-response" && + topic != "http-on-failed-opening-request") || + !(subject instanceof Ci.nsIHttpChannel) || + !(subject instanceof Ci.nsIClassifiedChannel) + ) { + return; + } + + const blockedOrFailed = topic === "http-on-failed-opening-request"; + + subject.QueryInterface(Ci.nsIClassifiedChannel); + const channel = subject.QueryInterface(Ci.nsIHttpChannel); + + if (this.#ignoreChannelFunction(channel)) { + return; + } + + logPlatformEvent( + topic, + subject, + blockedOrFailed + ? "blockedOrFailed:" + channel.loadInfo.requestBlockingReason + : channel.responseStatus + ); + + this.#checkForContentOverride(channel); + + channel.QueryInterface(Ci.nsIHttpChannelInternal); + + let httpActivity = this.#createOrGetActivityObject(channel); + if (topic === "http-on-examine-cached-response") { + // Service worker requests emits cached-response notification on non-e10s, + // and we fake one on e10s. + const fromServiceWorker = this.#interceptedChannels.has(channel); + this.#interceptedChannels.delete(channel); + + // If this is a cached response (which are also emitted by service worker requests), + // there never was a request event so we need to construct one here + // so the frontend gets all the expected events. + if (!httpActivity.owner) { + httpActivity = this.#createNetworkEvent(channel, { + fromCache: !fromServiceWorker, + fromServiceWorker, + }); + } + + // We need to send the request body to the frontend for + // the faked (cached/service worker request) event. + this.#prepareRequestBody(httpActivity); + this.#sendRequestBody(httpActivity); + + // There also is never any timing events, so we can fire this + // event with zeroed out values. + const timings = this.#setupHarTimings(httpActivity); + + const serverTimings = this.#extractServerTimings(httpActivity.channel); + httpActivity.owner.addEventTimings( + timings.total, + timings.timings, + timings.offsets, + serverTimings + ); + } else if (topic === "http-on-failed-opening-request") { + const { blockedReason } = lazy.NetworkUtils.getBlockedReason(channel); + this.#createNetworkEvent(channel, { blockedReason }); + } + + if (httpActivity.owner) { + httpActivity.owner.addResponseStart({ + channel: httpActivity.channel, + fromCache: httpActivity.fromCache || httpActivity.fromServiceWorker, + rawHeaders: httpActivity.responseRawHeaders, + }); + } + } + ); + + /** + * Observe notifications for the http-on-modify-request topic, coming from + * the nsIObserverService. + * + * @private + * @param nsIHttpChannel aSubject + * @returns void + */ + #httpModifyExaminer = DevToolsInfaillibleUtils.makeInfallible(subject => { + const throttler = this.#getThrottler(); + if (throttler) { + const channel = subject.QueryInterface(Ci.nsIHttpChannel); + if (this.#ignoreChannelFunction(channel)) { + return; + } + logPlatformEvent("http-on-modify-request", channel); + + // Read any request body here, before it is throttled. + const httpActivity = this.#createOrGetActivityObject(channel); + this.#prepareRequestBody(httpActivity); + throttler.manageUpload(channel); + } + }); + + /** + * A helper function for observeActivity. This does whatever work + * is required by a particular http activity event. Arguments are + * the same as for observeActivity. + */ + #dispatchActivity( + httpActivity, + channel, + activityType, + activitySubtype, + timestamp, + extraSizeData, + extraStringData + ) { + // Store the time information for this activity subtype. + if (activitySubtype in HTTP_TRANSACTION_CODES) { + const stage = HTTP_TRANSACTION_CODES[activitySubtype]; + if (stage in httpActivity.timings) { + httpActivity.timings[stage].last = timestamp; + } else { + httpActivity.timings[stage] = { + first: timestamp, + last: timestamp, + }; + } + } + + switch (activitySubtype) { + case gActivityDistributor.ACTIVITY_SUBTYPE_REQUEST_BODY_SENT: + this.#prepareRequestBody(httpActivity); + this.#sendRequestBody(httpActivity); + break; + case gActivityDistributor.ACTIVITY_SUBTYPE_RESPONSE_HEADER: + httpActivity.responseRawHeaders = extraStringData; + httpActivity.headersSize = extraStringData.length; + break; + case gActivityDistributor.ACTIVITY_SUBTYPE_TRANSACTION_CLOSE: + this.#onTransactionClose(httpActivity); + break; + default: + break; + } + } + + getActivityTypeString(activityType, activitySubtype) { + if ( + activityType === Ci.nsIHttpActivityObserver.ACTIVITY_TYPE_SOCKET_TRANSPORT + ) { + for (const name in Ci.nsISocketTransport) { + if (Ci.nsISocketTransport[name] === activitySubtype) { + return "SOCKET_TRANSPORT:" + name; + } + } + } else if ( + activityType === Ci.nsIHttpActivityObserver.ACTIVITY_TYPE_HTTP_TRANSACTION + ) { + for (const name in Ci.nsIHttpActivityObserver) { + if (Ci.nsIHttpActivityObserver[name] === activitySubtype) { + return "HTTP_TRANSACTION:" + name.replace("ACTIVITY_SUBTYPE_", ""); + } + } + } + return "unexpected-activity-types:" + activityType + ":" + activitySubtype; + } + + /** + * Begin observing HTTP traffic that originates inside the current tab. + * + * @see https://developer.mozilla.org/en/XPCOM_Interface_Reference/nsIHttpActivityObserver + * + * @param nsIHttpChannel channel + * @param number activityType + * @param number activitySubtype + * @param number timestamp + * @param number extraSizeData + * @param string extraStringData + */ + observeActivity = DevToolsInfaillibleUtils.makeInfallible(function ( + channel, + activityType, + activitySubtype, + timestamp, + extraSizeData, + extraStringData + ) { + if ( + this.#isDestroyed || + (activityType != gActivityDistributor.ACTIVITY_TYPE_HTTP_TRANSACTION && + activityType != gActivityDistributor.ACTIVITY_TYPE_SOCKET_TRANSPORT) + ) { + return; + } + + if ( + !(channel instanceof Ci.nsIHttpChannel) || + !(channel instanceof Ci.nsIClassifiedChannel) + ) { + return; + } + + channel = channel.QueryInterface(Ci.nsIHttpChannel); + channel = channel.QueryInterface(Ci.nsIClassifiedChannel); + + if (DEBUG_PLATFORM_EVENTS) { + logPlatformEvent( + this.getActivityTypeString(activityType, activitySubtype), + channel + ); + } + + if ( + activitySubtype == gActivityDistributor.ACTIVITY_SUBTYPE_REQUEST_HEADER + ) { + this.#onRequestHeader(channel, timestamp, extraStringData); + return; + } + + // Iterate over all currently ongoing requests. If channel can't + // be found within them, then exit this function. + const httpActivity = this.#findActivityObject(channel); + if (!httpActivity) { + return; + } + + // If we're throttling, we must not report events as they arrive + // from platform, but instead let the throttler emit the events + // after some time has elapsed. + if ( + httpActivity.downloadThrottle && + HTTP_DOWNLOAD_ACTIVITIES.includes(activitySubtype) + ) { + const callback = this.#dispatchActivity.bind(this); + httpActivity.downloadThrottle.addActivityCallback( + callback, + httpActivity, + channel, + activityType, + activitySubtype, + timestamp, + extraSizeData, + extraStringData + ); + } else { + this.#dispatchActivity( + httpActivity, + channel, + activityType, + activitySubtype, + timestamp, + extraSizeData, + extraStringData + ); + } + }); + + /** + * Craft the "event" object passed to the Watcher class in order + * to instantiate the NetworkEventActor. + * + * /!\ This method does many other important things: + * - Cancel requests blocked by DevTools + * - Fetch request headers/cookies + * - Set a few attributes on http activity object + * - Register listener to record response content + */ + #createNetworkEvent( + channel, + { + timestamp, + rawHeaders, + fromCache, + fromServiceWorker, + blockedReason, + blockingExtension, + inProgressRequest, + } + ) { + const httpActivity = this.#createOrGetActivityObject(channel); + + if (timestamp) { + httpActivity.timings.REQUEST_HEADER = { + first: timestamp, + last: timestamp, + }; + } + + if (blockedReason === undefined && this.#shouldBlockChannel(channel)) { + // Check the request URL with ones manually blocked by the user in DevTools. + // If it's meant to be blocked, we cancel the request and annotate the event. + channel.cancel(Cr.NS_BINDING_ABORTED); + blockedReason = "devtools"; + } + + httpActivity.owner = this.#onNetworkEvent( + { + timestamp, + fromCache, + fromServiceWorker, + rawHeaders, + blockedReason, + blockingExtension, + discardRequestBody: !this.#saveRequestAndResponseBodies, + discardResponseBody: !this.#saveRequestAndResponseBodies, + }, + channel + ); + httpActivity.fromCache = fromCache || fromServiceWorker; + + // Bug 1489217 - Avoid watching for response content for blocked or in-progress requests + // as it can't be observed and would throw if we try. + if (blockedReason === undefined && !inProgressRequest) { + this.#setupResponseListener(httpActivity, { + fromCache, + fromServiceWorker, + }); + } + + return httpActivity; + } + + /** + * Handler for ACTIVITY_SUBTYPE_REQUEST_HEADER. When a request starts the + * headers are sent to the server. This method creates the |httpActivity| + * object where we store the request and response information that is + * collected through its lifetime. + * + * @private + * @param nsIHttpChannel channel + * @param number timestamp + * @param string rawHeaders + * @return void + */ + #onRequestHeader(channel, timestamp, rawHeaders) { + if (this.#ignoreChannelFunction(channel)) { + return; + } + + this.#createNetworkEvent(channel, { + timestamp, + rawHeaders, + }); + } + + /** + * Check if the provided channel should be blocked given the current + * blocked URLs configured for this network observer. + */ + #shouldBlockChannel(channel) { + for (const regexp of this.#blockedURLs.values()) { + if (regexp.test(channel.URI.spec)) { + return true; + } + } + return false; + } + + /** + * Find an HTTP activity object for the channel. + * + * @param nsIHttpChannel channel + * The HTTP channel whose activity object we want to find. + * @return object + * The HTTP activity object, or null if it is not found. + */ + #findActivityObject(channel) { + return this.#openRequests.get(channel); + } + + /** + * Find an existing HTTP activity object, or create a new one. This + * object is used for storing all the request and response + * information. + * + * This is a HAR-like object. Conformance to the spec is not guaranteed at + * this point. + * + * @see http://www.softwareishard.com/blog/har-12-spec + * @param nsIHttpChannel channel + * The HTTP channel for which the HTTP activity object is created. + * @return object + * The new HTTP activity object. + */ + #createOrGetActivityObject(channel) { + let httpActivity = this.#findActivityObject(channel); + if (!httpActivity) { + const win = lazy.NetworkHelper.getWindowForRequest(channel); + const charset = win ? win.document.characterSet : null; + + // Most of the data needed from the channel is only available via the + // nsIHttpChannelInternal interface. + channel.QueryInterface(Ci.nsIHttpChannelInternal); + + httpActivity = { + // The nsIChannel for which this activity object was created. + channel, + // See #prepareRequestBody() + charset, + // The postData sent by this request. + sentBody: null, + // The URL for the current channel. + url: channel.URI.spec, + // The encoded response body size. + bodySize: 0, + // The response headers size. + headersSize: 0, + // needed for host specific security info + hostname: channel.URI.host, + discardRequestBody: !this.#saveRequestAndResponseBodies, + discardResponseBody: !this.#saveRequestAndResponseBodies, + // internal timing information, see observeActivity() + timings: {}, + // the activity owner which is notified when changes happen + owner: null, + }; + + this.#openRequests.set(channel, httpActivity); + } + + return httpActivity; + } + + /** + * Block a request based on certain filtering options. + * + * Currently, exact URL match or URL patterns are supported. + */ + blockRequest(filter) { + if (!filter || !filter.url) { + // In the future, there may be other types of filters, such as domain. + // For now, ignore anything other than URL. + return; + } + + this.#addBlockedUrl(filter.url); + } + + /** + * Unblock a request based on certain filtering options. + * + * Currently, exact URL match or URL patterns are supported. + */ + unblockRequest(filter) { + if (!filter || !filter.url) { + // In the future, there may be other types of filters, such as domain. + // For now, ignore anything other than URL. + return; + } + + this.#blockedURLs.delete(filter.url); + } + + /** + * Updates the list of blocked request strings + * + * This match will be a (String).includes match, not an exact URL match + */ + setBlockedUrls(urls) { + urls = urls || []; + this.#blockedURLs = new Map(); + urls.forEach(url => this.#addBlockedUrl(url)); + } + + #addBlockedUrl(url) { + this.#blockedURLs.set(url, lazy.wildcardToRegExp(url)); + } + + /** + * Returns a list of blocked requests + * Useful as blockedURLs is mutated by both console & netmonitor + */ + getBlockedUrls() { + return this.#blockedURLs.keys(); + } + + override(url, path) { + this.#overrides.set(url, path); + } + + removeOverride(url) { + this.#overrides.delete(url); + } + + /** + * Setup the network response listener for the given HTTP activity. The + * NetworkResponseListener is responsible for storing the response body. + * + * @private + * @param object httpActivity + * The HTTP activity object we are tracking. + */ + #setupResponseListener(httpActivity, { fromCache, fromServiceWorker }) { + const channel = httpActivity.channel; + channel.QueryInterface(Ci.nsITraceableChannel); + + if (!fromCache) { + const throttler = this.#getThrottler(); + if (throttler) { + httpActivity.downloadThrottle = throttler.manage(channel); + } + } + + // The response will be written into the outputStream of this pipe. + // This allows us to buffer the data we are receiving and read it + // asynchronously. + // Both ends of the pipe must be blocking. + const sink = Cc["@mozilla.org/pipe;1"].createInstance(Ci.nsIPipe); + + // The streams need to be blocking because this is required by the + // stream tee. + sink.init(false, false, this.#responsePipeSegmentSize, PR_UINT32_MAX, null); + + // Add listener for the response body. + const newListener = new lazy.NetworkResponseListener( + httpActivity, + this.#decodedCertificateCache, + fromServiceWorker + ); + + // Remember the input stream, so it isn't released by GC. + newListener.inputStream = sink.inputStream; + newListener.sink = sink; + + const tee = Cc["@mozilla.org/network/stream-listener-tee;1"].createInstance( + Ci.nsIStreamListenerTee + ); + + const originalListener = channel.setNewListener(tee); + + tee.init(originalListener, sink.outputStream, newListener); + } + + /** + * Handler for ACTIVITY_SUBTYPE_REQUEST_BODY_SENT. Read and record the request + * body here. It will be available in addResponseStart. + * + * @private + * @param object httpActivity + * The HTTP activity object we are working with. + */ + #prepareRequestBody(httpActivity) { + // Return early if we don't need the request body, or if we've + // already found it. + if (httpActivity.discardRequestBody || httpActivity.sentBody !== null) { + return; + } + + let sentBody = lazy.NetworkHelper.readPostTextFromRequest( + httpActivity.channel, + httpActivity.charset + ); + + if ( + sentBody !== null && + this.window && + httpActivity.url == this.window.location.href + ) { + // If the request URL is the same as the current page URL, then + // we can try to get the posted text from the page directly. + // This check is necessary as otherwise the + // lazy.NetworkHelper.readPostTextFromPageViaWebNav() + // function is called for image requests as well but these + // are not web pages and as such don't store the posted text + // in the cache of the webpage. + const webNav = this.window.docShell.QueryInterface(Ci.nsIWebNavigation); + sentBody = lazy.NetworkHelper.readPostTextFromPageViaWebNav( + webNav, + httpActivity.charset + ); + } + + if (sentBody !== null) { + httpActivity.sentBody = sentBody; + } + } + + /** + * Handler for ACTIVITY_SUBTYPE_TRANSACTION_CLOSE. This method updates the HAR + * timing information on the HTTP activity object and clears the request + * from the list of known open requests. + * + * @private + * @param object httpActivity + * The HTTP activity object we work with. + */ + #onTransactionClose(httpActivity) { + if (httpActivity.owner) { + const result = this.#setupHarTimings(httpActivity); + const serverTimings = this.#extractServerTimings(httpActivity.channel); + + httpActivity.owner.addEventTimings( + result.total, + result.timings, + result.offsets, + serverTimings + ); + } + } + + #getBlockedTiming(timings) { + if (timings.STATUS_RESOLVING && timings.STATUS_CONNECTING_TO) { + return timings.STATUS_RESOLVING.first - timings.REQUEST_HEADER.first; + } else if (timings.STATUS_SENDING_TO) { + return timings.STATUS_SENDING_TO.first - timings.REQUEST_HEADER.first; + } + + return -1; + } + + #getDnsTiming(timings) { + if (timings.STATUS_RESOLVING && timings.STATUS_RESOLVED) { + return timings.STATUS_RESOLVED.last - timings.STATUS_RESOLVING.first; + } + + return -1; + } + + #getConnectTiming(timings) { + if (timings.STATUS_CONNECTING_TO && timings.STATUS_CONNECTED_TO) { + return ( + timings.STATUS_CONNECTED_TO.last - timings.STATUS_CONNECTING_TO.first + ); + } + + return -1; + } + + #getReceiveTiming(timings) { + if (timings.RESPONSE_START && timings.RESPONSE_COMPLETE) { + return timings.RESPONSE_COMPLETE.last - timings.RESPONSE_START.first; + } + + return -1; + } + + #getWaitTiming(timings) { + if (timings.RESPONSE_START) { + return ( + timings.RESPONSE_START.first - + (timings.REQUEST_BODY_SENT || timings.STATUS_SENDING_TO).last + ); + } + + return -1; + } + + #getSslTiming(timings) { + if (timings.STATUS_TLS_STARTING && timings.STATUS_TLS_ENDING) { + return timings.STATUS_TLS_ENDING.last - timings.STATUS_TLS_STARTING.first; + } + + return -1; + } + + #getSendTiming(timings) { + if (timings.STATUS_SENDING_TO) { + return timings.STATUS_SENDING_TO.last - timings.STATUS_SENDING_TO.first; + } else if (timings.REQUEST_HEADER && timings.REQUEST_BODY_SENT) { + return timings.REQUEST_BODY_SENT.last - timings.REQUEST_HEADER.first; + } + + return -1; + } + + #getDataFromTimedChannel(timedChannel) { + const lookUpArr = [ + "tcpConnectEndTime", + "connectStartTime", + "connectEndTime", + "secureConnectionStartTime", + "domainLookupEndTime", + "domainLookupStartTime", + ]; + + return lookUpArr.reduce((prev, prop) => { + const propName = prop + "Tc"; + return { + ...prev, + [propName]: (() => { + if (!timedChannel) { + return 0; + } + + const value = timedChannel[prop]; + + if ( + value != 0 && + timedChannel.asyncOpenTime && + value < timedChannel.asyncOpenTime + ) { + return 0; + } + + return value; + })(), + }; + }, {}); + } + + #getSecureConnectionStartTimeInfo(timings) { + let secureConnectionStartTime = 0; + let secureConnectionStartTimeRelative = false; + + if (timings.STATUS_TLS_STARTING && timings.STATUS_TLS_ENDING) { + if (timings.STATUS_CONNECTING_TO) { + secureConnectionStartTime = + timings.STATUS_TLS_STARTING.first - + timings.STATUS_CONNECTING_TO.first; + } + + if (secureConnectionStartTime < 0) { + secureConnectionStartTime = 0; + } + secureConnectionStartTimeRelative = true; + } + + return { + secureConnectionStartTime, + secureConnectionStartTimeRelative, + }; + } + + #getStartSendingTimeInfo(timings, connectStartTimeTc) { + let startSendingTime = 0; + let startSendingTimeRelative = false; + + if (timings.STATUS_SENDING_TO) { + if (timings.STATUS_CONNECTING_TO) { + startSendingTime = + timings.STATUS_SENDING_TO.first - timings.STATUS_CONNECTING_TO.first; + startSendingTimeRelative = true; + } else if (connectStartTimeTc != 0) { + startSendingTime = timings.STATUS_SENDING_TO.first - connectStartTimeTc; + startSendingTimeRelative = true; + } + + if (startSendingTime < 0) { + startSendingTime = 0; + } + } + return { startSendingTime, startSendingTimeRelative }; + } + + /** + * Update the HTTP activity object to include timing information as in the HAR + * spec. The HTTP activity object holds the raw timing information in + * |timings| - these are timings stored for each activity notification. The + * HAR timing information is constructed based on these lower level + * data. + * + * @param {Object} httpActivity + * The HTTP activity object we are working with. + * @return {Object} + * This object holds three properties: + * - {Object} offsets: the timings computed as offsets from the initial + * request start time. + * - {Object} timings: the HAR timings object + * - {number} total: the total time for all of the request and response + */ + #setupHarTimings(httpActivity) { + if (httpActivity.fromCache) { + // If it came from the browser cache, we have no timing + // information and these should all be 0 + return { + total: 0, + timings: { + blocked: 0, + dns: 0, + ssl: 0, + connect: 0, + send: 0, + wait: 0, + receive: 0, + }, + offsets: { + blocked: 0, + dns: 0, + ssl: 0, + connect: 0, + send: 0, + wait: 0, + receive: 0, + }, + }; + } + + const timings = httpActivity.timings; + const harTimings = {}; + // If the TCP Fast Open option or tls1.3 0RTT is used tls and data can + // be dispatched in SYN packet and not after tcp socket is connected. + // To demostrate this properly we will calculated TLS and send start time + // relative to CONNECTING_TO. + // Similary if 0RTT is used, data can be sent as soon as a TLS handshake + // starts. + + harTimings.blocked = this.#getBlockedTiming(timings); + // DNS timing information is available only in when the DNS record is not + // cached. + harTimings.dns = this.#getDnsTiming(timings); + harTimings.connect = this.#getConnectTiming(timings); + harTimings.ssl = this.#getSslTiming(timings); + + let { secureConnectionStartTime, secureConnectionStartTimeRelative } = + this.#getSecureConnectionStartTimeInfo(timings); + + // sometimes the connection information events are attached to a speculative + // channel instead of this one, but necko might glue them back together in the + // nsITimedChannel interface used by Resource and Navigation Timing + const timedChannel = httpActivity.channel.QueryInterface( + Ci.nsITimedChannel + ); + + const { + tcpConnectEndTimeTc, + connectStartTimeTc, + connectEndTimeTc, + secureConnectionStartTimeTc, + domainLookupEndTimeTc, + domainLookupStartTimeTc, + } = this.#getDataFromTimedChannel(timedChannel); + + if ( + harTimings.connect <= 0 && + timedChannel && + tcpConnectEndTimeTc != 0 && + connectStartTimeTc != 0 + ) { + harTimings.connect = tcpConnectEndTimeTc - connectStartTimeTc; + if (secureConnectionStartTimeTc != 0) { + harTimings.ssl = connectEndTimeTc - secureConnectionStartTimeTc; + secureConnectionStartTime = + secureConnectionStartTimeTc - connectStartTimeTc; + secureConnectionStartTimeRelative = true; + } else { + harTimings.ssl = -1; + } + } else if ( + timedChannel && + timings.STATUS_TLS_STARTING && + secureConnectionStartTimeTc != 0 + ) { + // It can happen that TCP Fast Open actually have not sent any data and + // timings.STATUS_TLS_STARTING.first value will be corrected in + // timedChannel.secureConnectionStartTime + if (secureConnectionStartTimeTc > timings.STATUS_TLS_STARTING.first) { + // TCP Fast Open actually did not sent any data. + harTimings.ssl = connectEndTimeTc - secureConnectionStartTimeTc; + secureConnectionStartTimeRelative = false; + } + } + + if ( + harTimings.dns <= 0 && + timedChannel && + domainLookupEndTimeTc != 0 && + domainLookupStartTimeTc != 0 + ) { + harTimings.dns = domainLookupEndTimeTc - domainLookupStartTimeTc; + } + + harTimings.send = this.#getSendTiming(timings); + harTimings.wait = this.#getWaitTiming(timings); + harTimings.receive = this.#getReceiveTiming(timings); + let { startSendingTime, startSendingTimeRelative } = + this.#getStartSendingTimeInfo(timings, connectStartTimeTc); + + if (secureConnectionStartTimeRelative) { + const time = Math.max(Math.round(secureConnectionStartTime / 1000), -1); + secureConnectionStartTime = time; + } + if (startSendingTimeRelative) { + const time = Math.max(Math.round(startSendingTime / 1000), -1); + startSendingTime = time; + } + + const ot = this.#calculateOffsetAndTotalTime( + harTimings, + secureConnectionStartTime, + startSendingTimeRelative, + secureConnectionStartTimeRelative, + startSendingTime + ); + return { + total: ot.total, + timings: harTimings, + offsets: ot.offsets, + }; + } + + #extractServerTimings(channel) { + if (!channel || !channel.serverTiming) { + return null; + } + + const serverTimings = new Array(channel.serverTiming.length); + + for (let i = 0; i < channel.serverTiming.length; ++i) { + const { name, duration, description } = + channel.serverTiming.queryElementAt(i, Ci.nsIServerTiming); + serverTimings[i] = { name, duration, description }; + } + + return serverTimings; + } + + #convertTimeToMs(timing) { + return Math.max(Math.round(timing / 1000), -1); + } + + #calculateOffsetAndTotalTime( + harTimings, + secureConnectionStartTime, + startSendingTimeRelative, + secureConnectionStartTimeRelative, + startSendingTime + ) { + let totalTime = 0; + for (const timing in harTimings) { + const time = this.#convertTimeToMs(harTimings[timing]); + harTimings[timing] = time; + if (time > -1 && timing != "connect" && timing != "ssl") { + totalTime += time; + } + } + + // connect, ssl and send times can be overlapped. + if (startSendingTimeRelative) { + totalTime += startSendingTime; + } else if (secureConnectionStartTimeRelative) { + totalTime += secureConnectionStartTime; + totalTime += harTimings.ssl; + } + + const offsets = {}; + offsets.blocked = 0; + offsets.dns = harTimings.blocked; + offsets.connect = offsets.dns + harTimings.dns; + if (secureConnectionStartTimeRelative) { + offsets.ssl = offsets.connect + secureConnectionStartTime; + } else { + offsets.ssl = offsets.connect + harTimings.connect; + } + if (startSendingTimeRelative) { + offsets.send = offsets.connect + startSendingTime; + if (!secureConnectionStartTimeRelative) { + offsets.ssl = offsets.send - harTimings.ssl; + } + } else { + offsets.send = offsets.ssl + harTimings.ssl; + } + offsets.wait = offsets.send + harTimings.send; + offsets.receive = offsets.wait + harTimings.wait; + + return { + total: totalTime, + offsets, + }; + } + + #sendRequestBody(httpActivity) { + if (httpActivity.sentBody !== null) { + const limit = Services.prefs.getIntPref( + "devtools.netmonitor.requestBodyLimit" + ); + const size = httpActivity.sentBody.length; + if (size > limit && limit > 0) { + httpActivity.sentBody = httpActivity.sentBody.substr(0, limit); + } + httpActivity.owner.addRequestPostData({ + text: httpActivity.sentBody, + size, + }); + httpActivity.sentBody = null; + } + } + + /* + * Clears the open requests channel map. + */ + clear() { + this.#openRequests.clear(); + } + + /** + * Suspend observer activity. This is called when the Network monitor actor stops + * listening. + */ + destroy() { + if (Services.appinfo.processType != Ci.nsIXULRuntime.PROCESS_TYPE_CONTENT) { + gActivityDistributor.removeObserver(this); + Services.obs.removeObserver( + this.#httpResponseExaminer, + "http-on-examine-response" + ); + Services.obs.removeObserver( + this.#httpResponseExaminer, + "http-on-examine-cached-response" + ); + Services.obs.removeObserver( + this.#httpModifyExaminer, + "http-on-modify-request" + ); + Services.obs.removeObserver( + this.#httpStopRequest, + "http-on-stop-request" + ); + } else { + Services.obs.removeObserver( + this.#httpFailedOpening, + "http-on-failed-opening-request" + ); + } + + Services.obs.removeObserver( + this.#serviceWorkerRequest, + "service-worker-synthesized-response" + ); + + this.#ignoreChannelFunction = null; + this.#onNetworkEvent = null; + this.#throttler = null; + this.#decodedCertificateCache.clear(); + this.clear(); + + this.#isDestroyed = true; + } +} diff --git a/devtools/shared/network-observer/NetworkOverride.sys.mjs b/devtools/shared/network-observer/NetworkOverride.sys.mjs new file mode 100644 index 0000000000..a7857bcd76 --- /dev/null +++ b/devtools/shared/network-observer/NetworkOverride.sys.mjs @@ -0,0 +1,70 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** + * This modules focuses on redirecting requests to a particular local file. + */ + +import { XPCOMUtils } from "resource://gre/modules/XPCOMUtils.sys.mjs"; + +const { NetUtil } = ChromeUtils.import("resource://gre/modules/NetUtil.jsm"); + +const lazy = {}; + +ChromeUtils.defineESModuleGetters(lazy, { + FileUtils: "resource://gre/modules/FileUtils.sys.mjs", +}); + +XPCOMUtils.defineLazyServiceGetter( + lazy, + "mimeService", + "@mozilla.org/mime;1", + "nsIMIMEService" +); + +function readFile(file) { + const fstream = Cc["@mozilla.org/network/file-input-stream;1"].createInstance( + Ci.nsIFileInputStream + ); + fstream.init(file, -1, 0, 0); + const data = NetUtil.readInputStreamToString(fstream, fstream.available()); + fstream.close(); + return data; +} + +/** + * Given an in-flight channel, we will force to replace the content of this request + * with the content of a local file. + * + * @param {nsIHttpChannel} channel + * The request to replace content for. + * @param {String} path + * The absolute path to the local file to read content from. + */ +function overrideChannelWithFilePath(channel, path) { + // For JS it isn't important, but for HTML we ought to set the right content type on the data URI. + let mimeType = ""; + try { + // getTypeFromURI will throw if there is no extension at the end of the URI + mimeType = lazy.mimeService.getTypeFromURI(channel.URI); + } catch (e) {} + + // Redirect to a data: URI as we can't redirect to file:// URI + // without many security issues. We are leveraging the `allowInsecureRedirectToDataURI` + // attribute used by WebExtension. + const file = lazy.FileUtils.File(path); + const data = readFile(file); + const redirectURI = Services.io.newURI( + `data:${mimeType};base64,${btoa(data)}` + ); + + channel.redirectTo(redirectURI); + + // Prevents having CORS exception and various issues because of redirecting to data: URI. + channel.loadInfo.allowInsecureRedirectToDataURI = true; +} + +export const NetworkOverride = { + overrideChannelWithFilePath, +}; diff --git a/devtools/shared/network-observer/NetworkResponseListener.sys.mjs b/devtools/shared/network-observer/NetworkResponseListener.sys.mjs new file mode 100644 index 0000000000..23246c73de --- /dev/null +++ b/devtools/shared/network-observer/NetworkResponseListener.sys.mjs @@ -0,0 +1,609 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import { XPCOMUtils } from "resource://gre/modules/XPCOMUtils.sys.mjs"; + +const lazy = {}; + +ChromeUtils.defineESModuleGetters(lazy, { + getResponseCacheObject: + "resource://devtools/shared/platform/CacheEntry.sys.mjs", + NetworkHelper: + "resource://devtools/shared/network-observer/NetworkHelper.sys.mjs", + NetworkUtils: + "resource://devtools/shared/network-observer/NetworkUtils.sys.mjs", +}); + +XPCOMUtils.defineLazyModuleGetters(lazy, { + NetUtil: "resource://gre/modules/NetUtil.jsm", +}); + +// Network logging + +/** + * The network response listener implements the nsIStreamListener and + * nsIRequestObserver interfaces. This is used within the NetworkObserver feature + * to get the response body of the request. + * + * The code is mostly based on code listings from: + * + * http://www.softwareishard.com/blog/firebug/ + * nsitraceablechannel-intercept-http-traffic/ + * + * @constructor + * @param {Object} httpActivity + * HttpActivity object associated with this request. See NetworkObserver + * more information. + * @param {Map} decodedCertificateCache + * A Map of certificate fingerprints to decoded certificates, to avoid + * repeatedly decoding previously-seen certificates. + */ +export class NetworkResponseListener { + /** + * The compressed and encoded response body size. Will progressively increase + * until the full response is received. + * + * @type {Number} + */ + #bodySize = 0; + /** + * The uncompressed, decoded response body size. + * + * @type {Number} + */ + #decodedBodySize = 0; + /** + * nsIStreamListener created by nsIStreamConverterService.asyncConvertData + * + * @type {nsIStreamListener} + */ + #converter = null; + /** + * See constructor argument of the same name. + * + * @type {Map} + */ + #decodedCertificateCache; + /** + * Is the channel from a service worker + * + * @type {boolean} + */ + #fromServiceWorker; + /** + * See constructor argument of the same name. + * + * @type {Object} + */ + #httpActivity; + /** + * Set from sink.inputStream, mainly to prevent GC. + * + * @type {nsIInputStream} + */ + #inputStream = null; + /** + * Explicit flag to check if this listener was already destroyed. + * + * @type {boolean} + */ + #isDestroyed = false; + /** + * Internal promise used to hold the completion of #getSecurityInfo. + * + * @type {Promise} + */ + #onSecurityInfo = null; + /** + * Offset for the onDataAvailable calls where we pass the data from our pipe + * to the converter. + * + * @type {Number} + */ + #offset = 0; + /** + * Stores the received data as a string. + * + * @type {string} + */ + #receivedData = ""; + /** + * The nsIRequest we are started for. + * + * @type {nsIRequest} + */ + #request = null; + /** + * The response will be written into the outputStream of this nsIPipe. + * Both ends of the pipe must be blocking. + * + * @type {nsIPipe} + */ + #sink = null; + /** + * Indicates if the response had a size greater than response body limit. + * + * @type {boolean} + */ + #truncated = false; + /** + * Backup for existing notificationCallbacks set on the monitored channel. + * Initialized in the constructor. + * + * @type {Object} + */ + #wrappedNotificationCallbacks; + + constructor(httpActivity, decodedCertificateCache, fromServiceWorker) { + this.#httpActivity = httpActivity; + this.#decodedCertificateCache = decodedCertificateCache; + this.#fromServiceWorker = fromServiceWorker; + + // Note that this is really only needed for the non-e10s case. + // See bug 1309523. + const channel = this.#httpActivity.channel; + // If the channel already had notificationCallbacks, hold them here + // internally so that we can forward getInterface requests to that object. + this.#wrappedNotificationCallbacks = channel.notificationCallbacks; + channel.notificationCallbacks = this; + } + + set inputStream(inputStream) { + this.#inputStream = inputStream; + } + + set sink(sink) { + this.#sink = sink; + } + + // nsIInterfaceRequestor implementation + + /** + * This object implements nsIProgressEventSink, but also needs to forward + * interface requests to the notification callbacks of other objects. + */ + getInterface(iid) { + if (iid.equals(Ci.nsIProgressEventSink)) { + return this; + } + if (this.#wrappedNotificationCallbacks) { + return this.#wrappedNotificationCallbacks.getInterface(iid); + } + throw Components.Exception("", Cr.NS_ERROR_NO_INTERFACE); + } + + /** + * Forward notifications for interfaces this object implements, in case other + * objects also implemented them. + */ + #forwardNotification(iid, method, args) { + if (!this.#wrappedNotificationCallbacks) { + return; + } + try { + const impl = this.#wrappedNotificationCallbacks.getInterface(iid); + impl[method].apply(impl, args); + } catch (e) { + if (e.result != Cr.NS_ERROR_NO_INTERFACE) { + throw e; + } + } + } + + /** + * Set the async listener for the given nsIAsyncInputStream. This allows us to + * wait asynchronously for any data coming from the stream. + * + * @param nsIAsyncInputStream stream + * The input stream from where we are waiting for data to come in. + * @param nsIInputStreamCallback listener + * The input stream callback you want. This is an object that must have + * the onInputStreamReady() method. If the argument is null, then the + * current callback is removed. + * @return void + */ + setAsyncListener(stream, listener) { + // Asynchronously wait for the stream to be readable or closed. + stream.asyncWait(listener, 0, 0, Services.tm.mainThread); + } + + /** + * Stores the received data, if request/response body logging is enabled. It + * also does limit the number of stored bytes, based on the + * `devtools.netmonitor.responseBodyLimit` pref. + * + * Learn more about nsIStreamListener at: + * https://developer.mozilla.org/en/XPCOM_Interface_Reference/nsIStreamListener + * + * @param nsIRequest request + * @param nsISupports context + * @param nsIInputStream inputStream + * @param unsigned long offset + * @param unsigned long count + */ + onDataAvailable(request, inputStream, offset, count) { + const data = lazy.NetUtil.readInputStreamToString(inputStream, count); + + this.#decodedBodySize += count; + + if (!this.#httpActivity.discardResponseBody) { + const limit = Services.prefs.getIntPref( + "devtools.netmonitor.responseBodyLimit" + ); + if (this.#receivedData.length <= limit || limit == 0) { + this.#receivedData += lazy.NetworkHelper.convertToUnicode( + data, + request.contentCharset + ); + } + if (this.#receivedData.length > limit && limit > 0) { + this.#receivedData = this.#receivedData.substr(0, limit); + this.#truncated = true; + } + } + } + + /** + * See documentation at + * https://developer.mozilla.org/En/NsIRequestObserver + * + * @param nsIRequest request + * @param nsISupports context + */ + onStartRequest(request) { + request = request.QueryInterface(Ci.nsIChannel); + // Converter will call this again, we should just ignore that. + if (this.#request) { + return; + } + + this.#request = request; + this.#onSecurityInfo = this.#getSecurityInfo(); + // We need to track the offset for the onDataAvailable calls where + // we pass the data from our pipe to the converter. + this.#offset = 0; + + const channel = this.#request; + + // Bug 1372115 - We should load bytecode cached requests from cache as the actual + // channel content is going to be optimized data that reflects platform internals + // instead of the content user expects (i.e. content served by HTTP server) + // Note that bytecode cached is one example, there may be wasm or other usecase in + // future. + let isOptimizedContent = false; + try { + if (channel instanceof Ci.nsICacheInfoChannel) { + isOptimizedContent = channel.alternativeDataType; + } + } catch (e) { + // Accessing `alternativeDataType` for some SW requests throws. + } + if (isOptimizedContent) { + let charset; + try { + charset = this.#request.contentCharset; + } catch (e) { + // Accessing the charset sometimes throws NS_ERROR_NOT_AVAILABLE when + // reloading the page + } + if (!charset) { + charset = this.#httpActivity.charset; + } + lazy.NetworkHelper.loadFromCache( + this.#httpActivity.url, + charset, + this.#onComplete.bind(this) + ); + return; + } + + // In the multi-process mode, the conversion happens on the child + // side while we can only monitor the channel on the parent + // side. If the content is gzipped, we have to unzip it + // ourself. For that we use the stream converter services. Do not + // do that for Service workers as they are run in the child + // process. + if ( + !this.#fromServiceWorker && + channel instanceof Ci.nsIEncodedChannel && + channel.contentEncodings && + !channel.applyConversion + ) { + const encodingHeader = channel.getResponseHeader("Content-Encoding"); + const scs = Cc["@mozilla.org/streamConverters;1"].getService( + Ci.nsIStreamConverterService + ); + const encodings = encodingHeader.split(/\s*\t*,\s*\t*/); + let nextListener = this; + const acceptedEncodings = [ + "gzip", + "deflate", + "br", + "x-gzip", + "x-deflate", + ]; + for (const i in encodings) { + // There can be multiple conversions applied + const enc = encodings[i].toLowerCase(); + if (acceptedEncodings.indexOf(enc) > -1) { + this.#converter = scs.asyncConvertData( + enc, + "uncompressed", + nextListener, + null + ); + nextListener = this.#converter; + } + } + if (this.#converter) { + this.#converter.onStartRequest(this.#request, null); + } + } + // Asynchronously wait for the data coming from the request. + this.setAsyncListener(this.#sink.inputStream, this); + } + + /** + * Parse security state of this request and report it to the client. + */ + async #getSecurityInfo() { + // Many properties of the securityInfo (e.g., the server certificate or HPKP + // status) are not available in the content process and can't be even touched safely, + // because their C++ getters trigger assertions. This function is called in content + // process for synthesized responses from service workers, in the parent otherwise. + if (Services.appinfo.processType == Ci.nsIXULRuntime.PROCESS_TYPE_CONTENT) { + return; + } + + // Take the security information from the original nsIHTTPChannel instead of + // the nsIRequest received in onStartRequest. If response to this request + // was a redirect from http to https, the request object seems to contain + // security info for the https request after redirect. + const secinfo = this.#httpActivity.channel.securityInfo; + const info = await lazy.NetworkHelper.parseSecurityInfo( + secinfo, + this.#request.loadInfo.originAttributes, + this.#httpActivity, + this.#decodedCertificateCache + ); + let isRacing = false; + try { + const channel = this.#httpActivity.channel; + if (channel instanceof Ci.nsICacheInfoChannel) { + isRacing = channel.isRacing(); + } + } catch (err) { + // See the following bug for more details: + // https://bugzilla.mozilla.org/show_bug.cgi?id=1582589 + } + + this.#httpActivity.owner.addSecurityInfo(info, isRacing); + } + + /** + * Fetches cache information from CacheEntry + * @private + */ + async #fetchCacheInformation() { + // TODO: This method is async and #httpActivity is nullified in the #destroy + // method of this class. Backup httpActivity to avoid errors here. + const httpActivity = this.#httpActivity; + const cacheEntry = await lazy.getResponseCacheObject(this.#request); + httpActivity.owner.addResponseCache({ + responseCache: cacheEntry, + }); + } + + /** + * Handle the onStopRequest by closing the sink output stream. + * + * For more documentation about nsIRequestObserver go to: + * https://developer.mozilla.org/En/NsIRequestObserver + */ + onStopRequest() { + // Bug 1429365: onStopRequest may be called after onComplete for resources loaded + // from bytecode cache. + if (!this.#httpActivity) { + return; + } + this.#sink.outputStream.close(); + } + + // nsIProgressEventSink implementation + + /** + * Handle progress event as data is transferred. This is used to record the + * size on the wire, which may be compressed / encoded. + */ + onProgress(request, progress, progressMax) { + this.#bodySize = progress; + + // Need to forward as well to keep things like Download Manager's progress + // bar working properly. + this.#forwardNotification(Ci.nsIProgressEventSink, "onProgress", arguments); + } + + onStatus() { + this.#forwardNotification(Ci.nsIProgressEventSink, "onStatus", arguments); + } + + /** + * Clean up the response listener once the response input stream is closed. + * This is called from onStopRequest() or from onInputStreamReady() when the + * stream is closed. + * @return void + */ + onStreamClose() { + if (!this.#httpActivity) { + return; + } + // Remove our listener from the request input stream. + this.setAsyncListener(this.#sink.inputStream, null); + + let responseStatus; + try { + responseStatus = this.#httpActivity.channel.responseStatus; + } catch (e) { + // Will throw NS_ERROR_NOT_AVAILABLE if the response has not been received + // yet. + } + if (this.#request.fromCache || responseStatus == 304) { + this.#fetchCacheInformation(); + } + + if (!this.#httpActivity.discardResponseBody && this.#receivedData.length) { + this.#onComplete(this.#receivedData); + } else if ( + !this.#httpActivity.discardResponseBody && + responseStatus == 304 + ) { + // Response is cached, so we load it from cache. + let charset; + try { + charset = this.#request.contentCharset; + } catch (e) { + // Accessing the charset sometimes throws NS_ERROR_NOT_AVAILABLE when + // reloading the page + } + if (!charset) { + charset = this.#httpActivity.charset; + } + lazy.NetworkHelper.loadFromCache( + this.#httpActivity.url, + charset, + this.#onComplete.bind(this) + ); + } else { + this.#onComplete(); + } + } + + /** + * Handler for when the response completes. This function cleans up the + * response listener. + * + * @param string [data] + * Optional, the received data coming from the response listener or + * from the cache. + */ + #onComplete(data) { + // Make sure all the security and response content info are sent + this.#getResponseContent(data); + this.#onSecurityInfo.then(() => this.#destroy()); + } + + /** + * Create the response object and send it to the client. + */ + #getResponseContent(data) { + const response = { + mimeType: "", + text: data || "", + }; + + response.bodySize = this.#bodySize; + response.decodedBodySize = this.#decodedBodySize; + // TODO: Stop exposing the decodedBodySize as `size` which is ambiguous. + // Consumers should use `decodedBodySize` instead. See Bug 1808560. + response.size = this.#decodedBodySize; + response.headersSize = this.#httpActivity.headersSize; + response.transferredSize = this.#bodySize + this.#httpActivity.headersSize; + + try { + response.mimeType = this.#request.contentType; + } catch (ex) { + // Ignore. + } + + if ( + !response.mimeType || + !lazy.NetworkHelper.isTextMimeType(response.mimeType) + ) { + response.encoding = "base64"; + try { + response.text = btoa(response.text); + } catch (err) { + // Ignore. + } + } + + if (response.mimeType && this.#request.contentCharset) { + response.mimeType += "; charset=" + this.#request.contentCharset; + } + + this.#receivedData = ""; + + // Check any errors or blocking scenarios which happen late in the cycle + // e.g If a host is not found (NS_ERROR_UNKNOWN_HOST) or CORS blocking. + const { blockingExtension, blockedReason } = + lazy.NetworkUtils.getBlockedReason(this.#httpActivity.channel); + + this.#httpActivity.owner.addResponseContent(response, { + discardResponseBody: this.#httpActivity.discardResponseBody, + truncated: this.#truncated, + blockedReason, + blockingExtension, + }); + } + + #destroy() { + this.#wrappedNotificationCallbacks = null; + this.#httpActivity = null; + this.#sink = null; + this.#inputStream = null; + this.#converter = null; + this.#request = null; + + this.#isDestroyed = true; + } + + /** + * The nsIInputStreamCallback for when the request input stream is ready - + * either it has more data or it is closed. + * + * @param nsIAsyncInputStream stream + * The sink input stream from which data is coming. + * @returns void + */ + onInputStreamReady(stream) { + if (!(stream instanceof Ci.nsIAsyncInputStream) || !this.#httpActivity) { + return; + } + + let available = -1; + try { + // This may throw if the stream is closed normally or due to an error. + available = stream.available(); + } catch (ex) { + // Ignore. + } + + if (available != -1) { + if (available != 0) { + if (this.#converter) { + this.#converter.onDataAvailable( + this.#request, + stream, + this.#offset, + available + ); + } else { + this.onDataAvailable(this.#request, stream, this.#offset, available); + } + } + this.#offset += available; + this.setAsyncListener(stream, this); + } else { + this.onStreamClose(); + this.#offset = 0; + } + } + + QueryInterface = ChromeUtils.generateQI([ + "nsIStreamListener", + "nsIInputStreamCallback", + "nsIRequestObserver", + "nsIInterfaceRequestor", + ]); +} diff --git a/devtools/shared/network-observer/NetworkThrottleManager.sys.mjs b/devtools/shared/network-observer/NetworkThrottleManager.sys.mjs new file mode 100644 index 0000000000..a643861004 --- /dev/null +++ b/devtools/shared/network-observer/NetworkThrottleManager.sys.mjs @@ -0,0 +1,495 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +const ArrayBufferInputStream = Components.Constructor( + "@mozilla.org/io/arraybuffer-input-stream;1", + "nsIArrayBufferInputStream" +); +const BinaryInputStream = Components.Constructor( + "@mozilla.org/binaryinputstream;1", + "nsIBinaryInputStream", + "setInputStream" +); + +import { XPCOMUtils } from "resource://gre/modules/XPCOMUtils.sys.mjs"; + +const lazy = {}; + +XPCOMUtils.defineLazyServiceGetter( + lazy, + "gActivityDistributor", + "@mozilla.org/network/http-activity-distributor;1", + "nsIHttpActivityDistributor" +); + +ChromeUtils.defineESModuleGetters(lazy, { + setTimeout: "resource://gre/modules/Timer.sys.mjs", +}); + +class NetworkThrottleListener { + #activities; + #offset; + #originalListener; + #pendingData; + #pendingException; + #queue; + #responseStarted; + + /** + * Construct a new nsIStreamListener that buffers data and provides a + * method to notify another listener when data is available. This is + * used to throttle network data on a per-channel basis. + * + * After construction, @see setOriginalListener must be called on the + * new object. + * + * @param {NetworkThrottleQueue} queue the NetworkThrottleQueue to + * which status changes should be reported + */ + constructor(queue) { + this.#activities = {}; + this.#offset = 0; + this.#pendingData = []; + this.#pendingException = null; + this.#queue = queue; + this.#responseStarted = false; + } + + /** + * Set the original listener for this object. The original listener + * will receive requests from this object when the queue allows data + * through. + * + * @param {nsIStreamListener} originalListener the original listener + * for the channel, to which all requests will be sent + */ + setOriginalListener(originalListener) { + this.#originalListener = originalListener; + } + + /** + * @see nsIStreamListener.onStartRequest. + */ + onStartRequest(request) { + this.#originalListener.onStartRequest(request); + this.#queue.start(this); + } + + /** + * @see nsIStreamListener.onStopRequest. + */ + onStopRequest(request, statusCode) { + this.#pendingData.push({ request, statusCode }); + this.#queue.dataAvailable(this); + } + + /** + * @see nsIStreamListener.onDataAvailable. + */ + onDataAvailable(request, inputStream, offset, count) { + if (this.#pendingException) { + throw this.#pendingException; + } + + const bin = new BinaryInputStream(inputStream); + const bytes = new ArrayBuffer(count); + bin.readArrayBuffer(count, bytes); + + const stream = new ArrayBufferInputStream(); + stream.setData(bytes, 0, count); + + this.#pendingData.push({ request, stream, count }); + this.#queue.dataAvailable(this); + } + + /** + * Allow some buffered data from this object to be forwarded to this + * object's originalListener. + * + * @param {Number} bytesPermitted The maximum number of bytes + * permitted to be sent. + * @return {Object} an object of the form {length, done}, where + * |length| is the number of bytes actually forwarded, and + * |done| is a boolean indicating whether this particular + * request has been completed. (A NetworkThrottleListener + * may be queued multiple times, so this does not mean that + * all available data has been sent.) + */ + sendSomeData(bytesPermitted) { + if (this.#pendingData.length === 0) { + // Shouldn't happen. + return { length: 0, done: true }; + } + + const { request, stream, count, statusCode } = this.#pendingData[0]; + + if (statusCode !== undefined) { + this.#pendingData.shift(); + this.#originalListener.onStopRequest(request, statusCode); + return { length: 0, done: true }; + } + + if (bytesPermitted > count) { + bytesPermitted = count; + } + + try { + this.#originalListener.onDataAvailable( + request, + stream, + this.#offset, + bytesPermitted + ); + } catch (e) { + this.#pendingException = e; + } + + let done = false; + if (bytesPermitted === count) { + this.#pendingData.shift(); + done = true; + } else { + this.#pendingData[0].count -= bytesPermitted; + } + + this.#offset += bytesPermitted; + // Maybe our state has changed enough to emit an event. + this.#maybeEmitEvents(); + + return { length: bytesPermitted, done }; + } + + /** + * Return the number of pending data requests available for this + * listener. + */ + pendingCount() { + return this.#pendingData.length; + } + + /** + * This is called when an http activity event is delivered. This + * object delays the event until the appropriate moment. + */ + addActivityCallback( + callback, + httpActivity, + channel, + activityType, + activitySubtype, + timestamp, + extraSizeData, + extraStringData + ) { + const datum = { + callback, + httpActivity, + channel, + activityType, + activitySubtype, + extraSizeData, + extraStringData, + }; + this.#activities[activitySubtype] = datum; + + if ( + activitySubtype === + lazy.gActivityDistributor.ACTIVITY_SUBTYPE_RESPONSE_COMPLETE + ) { + this.totalSize = extraSizeData; + } + + this.#maybeEmitEvents(); + } + + /** + * This is called for a download throttler when the latency timeout + * has ended. + */ + responseStart() { + this.#responseStarted = true; + this.#maybeEmitEvents(); + } + + /** + * Check our internal state and emit any http activity events as + * needed. Note that we wait until both our internal state has + * changed and we've received the real http activity event from + * platform. This approach ensures we can both pass on the correct + * data from the original event, and update the reported time to be + * consistent with the delay we're introducing. + */ + #maybeEmitEvents() { + if (this.#responseStarted) { + this.#maybeEmit( + lazy.gActivityDistributor.ACTIVITY_SUBTYPE_RESPONSE_START + ); + this.#maybeEmit( + lazy.gActivityDistributor.ACTIVITY_SUBTYPE_RESPONSE_HEADER + ); + } + + if (this.totalSize !== undefined && this.#offset >= this.totalSize) { + this.#maybeEmit( + lazy.gActivityDistributor.ACTIVITY_SUBTYPE_RESPONSE_COMPLETE + ); + this.#maybeEmit( + lazy.gActivityDistributor.ACTIVITY_SUBTYPE_TRANSACTION_CLOSE + ); + } + } + + /** + * Emit an event for |code|, if the appropriate entry in + * |activities| is defined. + */ + #maybeEmit(code) { + if (this.#activities[code] !== undefined) { + const { + callback, + httpActivity, + channel, + activityType, + activitySubtype, + extraSizeData, + extraStringData, + } = this.#activities[code]; + const now = Date.now() * 1000; + callback( + httpActivity, + channel, + activityType, + activitySubtype, + now, + extraSizeData, + extraStringData + ); + this.#activities[code] = undefined; + } + } + + QueryInterface = ChromeUtils.generateQI([ + "nsIStreamListener", + "nsIInterfaceRequestor", + ]); +} + +class NetworkThrottleQueue { + #downloadQueue; + #latencyMax; + #latencyMean; + #maxBPS; + #meanBPS; + #pendingRequests; + #previousReads; + #pumping; + + /** + * Construct a new queue that can be used to throttle the network for + * a group of related network requests. + * + * meanBPS {Number} Mean bytes per second. + * maxBPS {Number} Maximum bytes per second. + * latencyMean {Number} Mean latency in milliseconds. + * latencyMax {Number} Maximum latency in milliseconds. + */ + constructor(meanBPS, maxBPS, latencyMean, latencyMax) { + this.#meanBPS = meanBPS; + this.#maxBPS = maxBPS; + this.#latencyMean = latencyMean; + this.#latencyMax = latencyMax; + + this.#pendingRequests = new Set(); + this.#downloadQueue = []; + this.#previousReads = []; + + this.#pumping = false; + } + + /** + * A helper function that lets the indicating listener start sending + * data. This is called after the initial round trip time for the + * listener has elapsed. + */ + #allowDataFrom(throttleListener) { + throttleListener.responseStart(); + this.#pendingRequests.delete(throttleListener); + const count = throttleListener.pendingCount(); + for (let i = 0; i < count; ++i) { + this.#downloadQueue.push(throttleListener); + } + this.#pump(); + } + + /** + * An internal function that permits individual listeners to send + * data. + */ + #pump() { + // A redirect will cause two NetworkThrottleListeners to be on a + // listener chain. In this case, we might recursively call into + // this method. Avoid infinite recursion here. + if (this.#pumping) { + return; + } + this.#pumping = true; + + const now = Date.now(); + const oneSecondAgo = now - 1000; + + while ( + this.#previousReads.length && + this.#previousReads[0].when < oneSecondAgo + ) { + this.#previousReads.shift(); + } + + const totalBytes = this.#previousReads.reduce((sum, elt) => { + return sum + elt.numBytes; + }, 0); + + let thisSliceBytes = this.#random(this.#meanBPS, this.#maxBPS); + if (totalBytes < thisSliceBytes) { + thisSliceBytes -= totalBytes; + let readThisTime = 0; + while (thisSliceBytes > 0 && this.#downloadQueue.length) { + const { length, done } = + this.#downloadQueue[0].sendSomeData(thisSliceBytes); + thisSliceBytes -= length; + readThisTime += length; + if (done) { + this.#downloadQueue.shift(); + } + } + this.#previousReads.push({ when: now, numBytes: readThisTime }); + } + + // If there is more data to download, then schedule ourselves for + // one second after the oldest previous read. + if (this.#downloadQueue.length) { + const when = this.#previousReads[0].when + 1000; + lazy.setTimeout(this.#pump.bind(this), when - now); + } + + this.#pumping = false; + } + + /** + * A helper function that, given a mean and a maximum, returns a + * random integer between (mean - (max - mean)) and max. + */ + #random(mean, max) { + return mean - (max - mean) + Math.floor(2 * (max - mean) * Math.random()); + } + + /** + * Notice a new listener object. This is called by the + * NetworkThrottleListener when the request has started. Initially + * a new listener object is put into a "pending" state, until the + * round-trip time has elapsed. This is used to simulate latency. + * + * @param {NetworkThrottleListener} throttleListener the new listener + */ + start(throttleListener) { + this.#pendingRequests.add(throttleListener); + const delay = this.#random(this.#latencyMean, this.#latencyMax); + if (delay > 0) { + lazy.setTimeout(() => this.#allowDataFrom(throttleListener), delay); + } else { + this.#allowDataFrom(throttleListener); + } + } + + /** + * Note that new data is available for a given listener. Each time + * data is available, the listener will be re-queued. + * + * @param {NetworkThrottleListener} throttleListener the listener + * which has data available. + */ + dataAvailable(throttleListener) { + if (!this.#pendingRequests.has(throttleListener)) { + this.#downloadQueue.push(throttleListener); + this.#pump(); + } + } +} + +/** + * Construct a new object that can be used to throttle the network for + * a group of related network requests. + * + * @param {Object} An object with the following attributes: + * latencyMean {Number} Mean latency in milliseconds. + * latencyMax {Number} Maximum latency in milliseconds. + * downloadBPSMean {Number} Mean bytes per second for downloads. + * downloadBPSMax {Number} Maximum bytes per second for downloads. + * uploadBPSMean {Number} Mean bytes per second for uploads. + * uploadBPSMax {Number} Maximum bytes per second for uploads. + * + * Download throttling will not be done if downloadBPSMean and + * downloadBPSMax are <= 0. Upload throttling will not be done if + * uploadBPSMean and uploadBPSMax are <= 0. + */ +export class NetworkThrottleManager { + #downloadQueue; + + constructor({ + latencyMean, + latencyMax, + downloadBPSMean, + downloadBPSMax, + uploadBPSMean, + uploadBPSMax, + }) { + if (downloadBPSMax <= 0 && downloadBPSMean <= 0) { + this.#downloadQueue = null; + } else { + this.#downloadQueue = new NetworkThrottleQueue( + downloadBPSMean, + downloadBPSMax, + latencyMean, + latencyMax + ); + } + if (uploadBPSMax <= 0 && uploadBPSMean <= 0) { + this.uploadQueue = null; + } else { + this.uploadQueue = Cc[ + "@mozilla.org/network/throttlequeue;1" + ].createInstance(Ci.nsIInputChannelThrottleQueue); + this.uploadQueue.init(uploadBPSMean, uploadBPSMax); + } + } + + /** + * Create a new NetworkThrottleListener for a given channel and + * install it using |setNewListener|. + * + * @param {nsITraceableChannel} channel the channel to manage + * @return {NetworkThrottleListener} the new listener, or null if + * download throttling is not being done. + */ + manage(channel) { + if (this.#downloadQueue) { + const listener = new NetworkThrottleListener(this.#downloadQueue); + const originalListener = channel.setNewListener(listener); + listener.setOriginalListener(originalListener); + return listener; + } + return null; + } + + /** + * Throttle uploads taking place on the given channel. + * + * @param {nsITraceableChannel} channel the channel to manage + */ + manageUpload(channel) { + if (this.uploadQueue) { + channel = channel.QueryInterface(Ci.nsIThrottledInputChannel); + channel.throttleQueue = this.uploadQueue; + } + } +} diff --git a/devtools/shared/network-observer/NetworkUtils.sys.mjs b/devtools/shared/network-observer/NetworkUtils.sys.mjs new file mode 100644 index 0000000000..44de9fa353 --- /dev/null +++ b/devtools/shared/network-observer/NetworkUtils.sys.mjs @@ -0,0 +1,670 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import { XPCOMUtils } from "resource://gre/modules/XPCOMUtils.sys.mjs"; + +const lazy = {}; + +ChromeUtils.defineESModuleGetters(lazy, { + NetworkHelper: + "resource://devtools/shared/network-observer/NetworkHelper.sys.mjs", +}); + +XPCOMUtils.defineLazyGetter(lazy, "tpFlagsMask", () => { + const trackingProtectionLevel2Enabled = Services.prefs + .getStringPref("urlclassifier.trackingTable") + .includes("content-track-digest256"); + + return trackingProtectionLevel2Enabled + ? ~Ci.nsIClassifiedChannel.CLASSIFIED_ANY_BASIC_TRACKING & + ~Ci.nsIClassifiedChannel.CLASSIFIED_ANY_STRICT_TRACKING + : ~Ci.nsIClassifiedChannel.CLASSIFIED_ANY_BASIC_TRACKING & + Ci.nsIClassifiedChannel.CLASSIFIED_ANY_STRICT_TRACKING; +}); + +/** + * Convert a nsIContentPolicy constant to a display string + */ +const LOAD_CAUSE_STRINGS = { + [Ci.nsIContentPolicy.TYPE_INVALID]: "invalid", + [Ci.nsIContentPolicy.TYPE_OTHER]: "other", + [Ci.nsIContentPolicy.TYPE_SCRIPT]: "script", + [Ci.nsIContentPolicy.TYPE_IMAGE]: "img", + [Ci.nsIContentPolicy.TYPE_STYLESHEET]: "stylesheet", + [Ci.nsIContentPolicy.TYPE_OBJECT]: "object", + [Ci.nsIContentPolicy.TYPE_DOCUMENT]: "document", + [Ci.nsIContentPolicy.TYPE_SUBDOCUMENT]: "subdocument", + [Ci.nsIContentPolicy.TYPE_PING]: "ping", + [Ci.nsIContentPolicy.TYPE_XMLHTTPREQUEST]: "xhr", + [Ci.nsIContentPolicy.TYPE_OBJECT_SUBREQUEST]: "objectSubdoc", + [Ci.nsIContentPolicy.TYPE_DTD]: "dtd", + [Ci.nsIContentPolicy.TYPE_FONT]: "font", + [Ci.nsIContentPolicy.TYPE_MEDIA]: "media", + [Ci.nsIContentPolicy.TYPE_WEBSOCKET]: "websocket", + [Ci.nsIContentPolicy.TYPE_CSP_REPORT]: "csp", + [Ci.nsIContentPolicy.TYPE_XSLT]: "xslt", + [Ci.nsIContentPolicy.TYPE_BEACON]: "beacon", + [Ci.nsIContentPolicy.TYPE_FETCH]: "fetch", + [Ci.nsIContentPolicy.TYPE_IMAGESET]: "imageset", + [Ci.nsIContentPolicy.TYPE_WEB_MANIFEST]: "webManifest", + [Ci.nsIContentPolicy.TYPE_WEB_IDENTITY]: "webidentity", +}; + +function causeTypeToString(causeType, loadFlags, internalContentPolicyType) { + let prefix = ""; + if ( + (causeType == Ci.nsIContentPolicy.TYPE_IMAGESET || + internalContentPolicyType == Ci.nsIContentPolicy.TYPE_INTERNAL_IMAGE) && + loadFlags & Ci.nsIRequest.LOAD_BACKGROUND + ) { + prefix = "lazy-"; + } + + return prefix + LOAD_CAUSE_STRINGS[causeType] || "unknown"; +} + +function stringToCauseType(value) { + return Object.keys(LOAD_CAUSE_STRINGS).find( + key => LOAD_CAUSE_STRINGS[key] === value + ); +} + +function isChannelFromSystemPrincipal(channel) { + let principal = null; + let browsingContext = channel.loadInfo.browsingContext; + if (!browsingContext) { + const topFrame = lazy.NetworkHelper.getTopFrameForRequest(channel); + if (topFrame) { + browsingContext = topFrame.browsingContext; + } else { + // Fallback to the triggering principal when browsingContext and topFrame is null + // e.g some chrome requests + principal = channel.loadInfo.triggeringPrincipal; + } + } + + // When in the parent process, we can get the documentPrincipal from the + // WindowGlobal which is available on the BrowsingContext + if (!principal) { + principal = CanonicalBrowsingContext.isInstance(browsingContext) + ? browsingContext.currentWindowGlobal.documentPrincipal + : browsingContext.window.document.nodePrincipal; + } + return principal.isSystemPrincipal; +} + +/** + * Get the browsing context id for the channel. + * + * @param {*} channel + * @returns {number} + */ +function getChannelBrowsingContextID(channel) { + if (channel.loadInfo.browsingContextID) { + return channel.loadInfo.browsingContextID; + } + // At least WebSocket channel aren't having a browsingContextID set on their loadInfo + // We fallback on top frame element, which works, but will be wrong for WebSocket + // in same-process iframes... + const topFrame = lazy.NetworkHelper.getTopFrameForRequest(channel); + // topFrame is typically null for some chrome requests like favicons + if (topFrame && topFrame.browsingContext) { + return topFrame.browsingContext.id; + } + return null; +} + +/** + * Get the innerWindowId for the channel. + * + * @param {*} channel + * @returns {number} + */ +function getChannelInnerWindowId(channel) { + if (channel.loadInfo.innerWindowID) { + return channel.loadInfo.innerWindowID; + } + // At least WebSocket channel aren't having a browsingContextID set on their loadInfo + // We fallback on top frame element, which works, but will be wrong for WebSocket + // in same-process iframes... + const topFrame = lazy.NetworkHelper.getTopFrameForRequest(channel); + // topFrame is typically null for some chrome requests like favicons + if (topFrame?.browsingContext?.currentWindowGlobal) { + return topFrame.browsingContext.currentWindowGlobal.innerWindowId; + } + return null; +} + +/** + * Does this channel represent a Preload request. + * + * @param {*} channel + * @returns {boolean} + */ +function isPreloadRequest(channel) { + const type = channel.loadInfo.internalContentPolicyType; + return ( + type == Ci.nsIContentPolicy.TYPE_INTERNAL_SCRIPT_PRELOAD || + type == Ci.nsIContentPolicy.TYPE_INTERNAL_MODULE_PRELOAD || + type == Ci.nsIContentPolicy.TYPE_INTERNAL_IMAGE_PRELOAD || + type == Ci.nsIContentPolicy.TYPE_INTERNAL_STYLESHEET_PRELOAD || + type == Ci.nsIContentPolicy.TYPE_INTERNAL_FONT_PRELOAD + ); +} + +/** + * Get the channel cause details. + * + * @param {nsIChannel} channel + * @returns {Object} + * - loadingDocumentUri {string} uri of the document which created the + * channel + * - type {string} cause type as string + */ +function getCauseDetails(channel) { + // Determine the cause and if this is an XHR request. + let causeType = Ci.nsIContentPolicy.TYPE_OTHER; + let causeUri = null; + + if (channel.loadInfo) { + causeType = channel.loadInfo.externalContentPolicyType; + const { loadingPrincipal } = channel.loadInfo; + if (loadingPrincipal) { + causeUri = loadingPrincipal.spec; + } + } + + return { + loadingDocumentUri: causeUri, + type: causeTypeToString( + causeType, + channel.loadFlags, + channel.loadInfo.internalContentPolicyType + ), + }; +} + +/** + * Get the channel priority. Priority is a number which typically ranges from + * -20 (lowest priority) to 20 (highest priority). Can be null if the channel + * does not implement nsISupportsPriority. + * + * @param {nsIChannel} channel + * @returns {number|undefined} + */ +function getChannelPriority(channel) { + if (channel instanceof Ci.nsISupportsPriority) { + return channel.priority; + } + + return null; +} + +/** + * Get the channel HTTP version as an uppercase string starting with "HTTP/" + * (eg "HTTP/2"). + * + * @param {nsIChannel} channel + * @returns {string} + */ +function getHttpVersion(channel) { + // Determine the HTTP version. + const httpVersionMaj = {}; + const httpVersionMin = {}; + + channel.QueryInterface(Ci.nsIHttpChannelInternal); + channel.getResponseVersion(httpVersionMaj, httpVersionMin); + + // The official name HTTP version 2.0 and 3.0 are HTTP/2 and HTTP/3, omit the + // trailing `.0`. + if (httpVersionMin.value == 0) { + return "HTTP/" + httpVersionMaj.value; + } + + return "HTTP/" + httpVersionMaj.value + "." + httpVersionMin.value; +} + +const UNKNOWN_PROTOCOL_STRINGS = ["", "unknown"]; +const HTTP_PROTOCOL_STRINGS = ["http", "https"]; +/** + * Get the protocol for the provided httpActivity. Either the ALPN negotiated + * protocol or as a fallback a protocol computed from the scheme and the + * response status. + * + * TODO: The `protocol` is similar to another response property called + * `httpVersion`. `httpVersion` is uppercase and purely computed from the + * response status, whereas `protocol` uses nsIHttpChannel.protocolVersion by + * default and otherwise falls back on `httpVersion`. Ideally we should merge + * the two properties. + * + * @param {Object} httpActivity + * The httpActivity object for which we need to get the protocol. + * + * @returns {string} + * The protocol as a string. + */ +function getProtocol(channel) { + let protocol = ""; + try { + const httpChannel = channel.QueryInterface(Ci.nsIHttpChannel); + // protocolVersion corresponds to ALPN negotiated protocol. + protocol = httpChannel.protocolVersion; + } catch (e) { + // Ignore errors reading protocolVersion. + } + + if (UNKNOWN_PROTOCOL_STRINGS.includes(protocol)) { + protocol = channel.URI.scheme; + const httpVersion = getHttpVersion(channel); + if ( + typeof httpVersion == "string" && + HTTP_PROTOCOL_STRINGS.includes(protocol) + ) { + protocol = httpVersion.toLowerCase(); + } + } + + return protocol; +} + +/** + * Get the channel referrer policy as a string + * (eg "strict-origin-when-cross-origin"). + * + * @param {nsIChannel} channel + * @returns {string} + */ +function getReferrerPolicy(channel) { + return channel.referrerInfo + ? channel.referrerInfo.getReferrerPolicyString() + : ""; +} + +/** + * Check if the channel is private. + * + * @param {nsIChannel} channel + * @returns {boolean} + */ +function isChannelPrivate(channel) { + channel.QueryInterface(Ci.nsIPrivateBrowsingChannel); + return channel.isChannelPrivate; +} + +/** + * Check if the channel data is loaded from the cache or not. + * + * @param {nsIChannel} channel + * The channel for which we need to check the cache status. + * + * @returns {boolean} + * True if the channel data is loaded from the cache, false otherwise. + */ +function isFromCache(channel) { + if (channel instanceof Ci.nsICacheInfoChannel) { + return channel.isFromCache(); + } + + return false; +} + +const REDIRECT_STATES = [ + 301, // HTTP Moved Permanently + 302, // HTTP Found + 303, // HTTP See Other + 307, // HTTP Temporary Redirect +]; +/** + * Check if the channel's status corresponds to a known redirect status. + * + * @param {nsIChannel} channel + * The channel for which we need to check the redirect status. + * + * @returns {boolean} + * True if the channel data is a redirect, false otherwise. + */ +function isRedirectedChannel(channel) { + try { + return REDIRECT_STATES.includes(channel.responseStatus); + } catch (e) { + // Throws NS_ERROR_NOT_AVAILABLE if the request was not sent yet. + } + return false; +} + +/** + * isNavigationRequest is true for the one request used to load a new top level + * document of a given tab, or top level window. It will typically be false for + * navigation requests of iframes, i.e. the request loading another document in + * an iframe. + * + * @param {nsIChannel} channel + * @return {boolean} + */ +function isNavigationRequest(channel) { + return channel.isMainDocumentChannel && channel.loadInfo.isTopLevelLoad; +} + +/** + * Returns true if the channel has been processed by URL-Classifier features + * and is considered third-party with the top window URI, and if it has loaded + * a resource that is classified as a tracker. + * + * @param {nsIChannel} channel + * @return {boolean} + */ +function isThirdPartyTrackingResource(channel) { + // Only consider channels classified as level-1 to be trackers if our preferences + // would not cause such channels to be blocked in strict content blocking mode. + // Make sure the value produced here is a boolean. + return !!( + channel instanceof Ci.nsIClassifiedChannel && + channel.isThirdPartyTrackingResource() && + (channel.thirdPartyClassificationFlags & lazy.tpFlagsMask) == 0 + ); +} + +/** + * Retrieve the websocket channel for the provided channel, if available. + * Returns null otherwise. + * + * @param {nsIChannel} channel + * @returns {nsIWebSocketChannel|null} + */ +function getWebSocketChannel(channel) { + let wsChannel = null; + if (channel.notificationCallbacks) { + try { + wsChannel = channel.notificationCallbacks.QueryInterface( + Ci.nsIWebSocketChannel + ); + } catch (e) { + // Not all channels implement nsIWebSocketChannel. + } + } + return wsChannel; +} + +/** + * For a given channel, fetch the request's headers and cookies. + * + * @param {nsIChannel} channel + * @return {Object} + * An object with two properties: + * @property {Array<Object>} cookies + * Array of { name, value } objects. + * @property {Array<Object>} headers + * Array of { name, value } objects. + */ +function fetchRequestHeadersAndCookies(channel) { + const headers = []; + let cookies = []; + let cookieHeader = null; + + // Copy the request header data. + channel.visitRequestHeaders({ + visitHeader(name, value) { + if (name == "Cookie") { + cookieHeader = value; + } + headers.push({ name, value }); + }, + }); + + if (cookieHeader) { + cookies = lazy.NetworkHelper.parseCookieHeader(cookieHeader); + } + + return { cookies, headers }; +} + +/** + * For a given channel, fetch the response's headers and cookies. + * + * @param {nsIChannel} channel + * @return {Object} + * An object with two properties: + * @property {Array<Object>} cookies + * Array of { name, value } objects. + * @property {Array<Object>} headers + * Array of { name, value } objects. + */ +function fetchResponseHeadersAndCookies(channel) { + // Read response headers and cookies. + const headers = []; + const setCookieHeaders = []; + + const SET_COOKIE_REGEXP = /set-cookie/i; + channel.visitOriginalResponseHeaders({ + visitHeader(name, value) { + if (SET_COOKIE_REGEXP.test(name)) { + setCookieHeaders.push(value); + } + headers.push({ name, value }); + }, + }); + + return { + cookies: lazy.NetworkHelper.parseSetCookieHeaders(setCookieHeaders), + headers, + }; +} + +/** + * Check if a given network request should be logged by a network monitor + * based on the specified filters. + * + * @param nsIHttpChannel channel + * Request to check. + * @param filters + * NetworkObserver filters to match against. An object with one of the following attributes: + * - sessionContext: When inspecting requests from the parent process, pass the WatcherActor's session context. + * This helps know what is the overall debugged scope. + * See watcher actor constructor for more info. + * - targetActor: When inspecting requests from the content process, pass the WindowGlobalTargetActor. + * This helps know what exact subset of request we should accept. + * This is especially useful to behave correctly regarding EFT, where we should include or not + * iframes requests. + * - browserId, addonId, window: All these attributes are legacy. + * Only browserId attribute is still used by the legacy WebConsoleActor startListener API. + * @return boolean + * True if the network request should be logged, false otherwise. + */ +function matchRequest(channel, filters) { + // NetworkEventWatcher should now pass a session context for the parent process codepath + if (filters.sessionContext) { + const { type } = filters.sessionContext; + if (type == "all") { + return true; + } + + // Ignore requests from chrome or add-on code when we don't monitor the whole browser + if ( + channel.loadInfo?.loadingDocument === null && + (channel.loadInfo.loadingPrincipal === + Services.scriptSecurityManager.getSystemPrincipal() || + channel.loadInfo.isInDevToolsContext) + ) { + return false; + } + + if (type == "browser-element") { + if (!channel.loadInfo.browsingContext) { + const topFrame = lazy.NetworkHelper.getTopFrameForRequest(channel); + // `topFrame` is typically null for some chrome requests like favicons + // And its `browsingContext` attribute might be null if the request happened + // while the tab is being closed. + return ( + topFrame?.browsingContext?.browserId == + filters.sessionContext.browserId + ); + } + return ( + channel.loadInfo.browsingContext.browserId == + filters.sessionContext.browserId + ); + } + if (type == "webextension") { + return ( + channel.loadInfo?.loadingPrincipal?.addonId === + filters.sessionContext.addonId + ); + } + throw new Error("Unsupported session context type: " + type); + } + + // NetworkEventContentWatcher and NetworkEventStackTraces pass a target actor instead, from the content processes + // Because of EFT, we can't use session context as we have to know what exact windows the target actor covers. + if (filters.targetActor) { + // Bug 1769982 the target actor might be destroying and accessing windows will throw. + // Ignore all further request when this happens. + let windows; + try { + windows = filters.targetActor.windows; + } catch (e) { + return false; + } + const win = lazy.NetworkHelper.getWindowForRequest(channel); + return windows.includes(win); + } + + // This is fallback code for the legacy WebConsole.startListeners codepath, + // which may still pass individual browserId/window/addonId attributes. + // This should be removable once we drop the WebConsole codepath for network events + // (bug 1721592 and followups) + return legacyMatchRequest(channel, filters); +} + +function legacyMatchRequest(channel, filters) { + // Log everything if no filter is specified + if (!filters.browserId && !filters.window && !filters.addonId) { + return true; + } + + // Ignore requests from chrome or add-on code when we are monitoring + // content. + if ( + channel.loadInfo?.loadingDocument === null && + (channel.loadInfo.loadingPrincipal === + Services.scriptSecurityManager.getSystemPrincipal() || + channel.loadInfo.isInDevToolsContext) + ) { + return false; + } + + if (filters.window) { + let win = lazy.NetworkHelper.getWindowForRequest(channel); + if (filters.matchExactWindow) { + return win == filters.window; + } + + // Since frames support, this.window may not be the top level content + // frame, so that we can't only compare with win.top. + while (win) { + if (win == filters.window) { + return true; + } + if (win.parent == win) { + break; + } + win = win.parent; + } + return false; + } + + if (filters.browserId) { + const topFrame = lazy.NetworkHelper.getTopFrameForRequest(channel); + // `topFrame` is typically null for some chrome requests like favicons + // And its `browsingContext` attribute might be null if the request happened + // while the tab is being closed. + if (topFrame?.browsingContext?.browserId == filters.browserId) { + return true; + } + + // If we couldn't get the top frame BrowsingContext from the loadContext, + // look for it on channel.loadInfo instead. + if (channel.loadInfo?.browsingContext?.browserId == filters.browserId) { + return true; + } + } + + if ( + filters.addonId && + channel.loadInfo?.loadingPrincipal?.addonId === filters.addonId + ) { + return true; + } + + return false; +} + +function getBlockedReason(channel) { + let blockingExtension, blockedReason; + const { status } = channel; + + try { + const request = channel.QueryInterface(Ci.nsIHttpChannel); + const properties = request.QueryInterface(Ci.nsIPropertyBag); + + blockedReason = request.loadInfo.requestBlockingReason; + blockingExtension = properties.getProperty("cancelledByExtension"); + + // WebExtensionPolicy is not available for workers + if (typeof WebExtensionPolicy !== "undefined") { + blockingExtension = WebExtensionPolicy.getByID(blockingExtension).name; + } + } catch (err) { + // "cancelledByExtension" doesn't have to be available. + } + // These are platform errors which are not exposed to the users, + // usually the requests (with these errors) might be displayed with various + // other status codes. + const ignoreList = [ + // This is emited when the request is already in the cache. + "NS_ERROR_PARSED_DATA_CACHED", + // This is emited when there is some issues around images e.g When the img.src + // links to a non existent url. This is typically shown as a 404 request. + "NS_IMAGELIB_ERROR_FAILURE", + // This is emited when there is a redirect. They are shown as 301 requests. + "NS_BINDING_REDIRECTED", + // E.g Emited by send beacon requests. + "NS_ERROR_ABORT", + ]; + + // If the request has not failed or is not blocked by a web extension, check for + // any errors not on the ignore list. e.g When a host is not found (NS_ERROR_UNKNOWN_HOST). + if ( + blockedReason == 0 && + !Components.isSuccessCode(status) && + !ignoreList.includes(ChromeUtils.getXPCOMErrorName(status)) + ) { + blockedReason = ChromeUtils.getXPCOMErrorName(status); + } + + return { blockingExtension, blockedReason }; +} + +export const NetworkUtils = { + causeTypeToString, + fetchRequestHeadersAndCookies, + fetchResponseHeadersAndCookies, + getCauseDetails, + getChannelBrowsingContextID, + getChannelInnerWindowId, + getChannelPriority, + getHttpVersion, + getProtocol, + getReferrerPolicy, + getWebSocketChannel, + isChannelFromSystemPrincipal, + isChannelPrivate, + isFromCache, + isNavigationRequest, + isPreloadRequest, + isRedirectedChannel, + isThirdPartyTrackingResource, + matchRequest, + stringToCauseType, + getBlockedReason, +}; diff --git a/devtools/shared/network-observer/README.md b/devtools/shared/network-observer/README.md new file mode 100644 index 0000000000..7c2d41a959 --- /dev/null +++ b/devtools/shared/network-observer/README.md @@ -0,0 +1,9 @@ +# Network Observer modules + +The NetworkObserver module and associated helpers allow to: +- monitor network events (requests and responses) +- block requests +- throttle responses + +The DevTools network-observer modules are used both by DevTools and by WebDriver BiDi, found under /remote. +Breaking changes should be discussed and reviewed both by devtools and webdriver peers. diff --git a/devtools/shared/network-observer/WildcardToRegexp.sys.mjs b/devtools/shared/network-observer/WildcardToRegexp.sys.mjs new file mode 100644 index 0000000000..4c495fdd78 --- /dev/null +++ b/devtools/shared/network-observer/WildcardToRegexp.sys.mjs @@ -0,0 +1,28 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** + * Converts a URL-like string which might include the `*` character as a wildcard + * to a regular expression. They are used to match against actual URLs for the + * request blocking feature from DevTools. + * + * The returned regular expression is case insensitive. + * + * @param {string} url + * A URL-like string which can contain one or several `*` as wildcard + * characters. + * @return {RegExp} + * A regular expression which can be used to match URLs compatible with the + * provided url "template". + */ +export function wildcardToRegExp(url) { + return new RegExp(url.split("*").map(regExpEscape).join(".*"), "i"); +} + +/** + * Escapes all special RegExp characters in the given string. + */ +const regExpEscape = s => { + return s.replace(/[|\\{}()[\]^$+*?.]/g, "\\$&"); +}; diff --git a/devtools/shared/network-observer/moz.build b/devtools/shared/network-observer/moz.build new file mode 100644 index 0000000000..71ba8bf3db --- /dev/null +++ b/devtools/shared/network-observer/moz.build @@ -0,0 +1,20 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +BROWSER_CHROME_MANIFESTS += ["test/browser/browser.ini"] + +XPCSHELL_TESTS_MANIFESTS += ["test/xpcshell/xpcshell.ini"] + +DevToolsModules( + "ChannelMap.sys.mjs", + "NetworkHelper.sys.mjs", + "NetworkObserver.sys.mjs", + "NetworkOverride.sys.mjs", + "NetworkResponseListener.sys.mjs", + "NetworkThrottleManager.sys.mjs", + "NetworkUtils.sys.mjs", + "WildcardToRegexp.sys.mjs", +) diff --git a/devtools/shared/network-observer/test/browser/browser.ini b/devtools/shared/network-observer/test/browser/browser.ini new file mode 100644 index 0000000000..f1d09d2c1b --- /dev/null +++ b/devtools/shared/network-observer/test/browser/browser.ini @@ -0,0 +1,15 @@ +[DEFAULT] +tags = devtools +subsuite = devtools +support-files = + head.js + doc_network-observer.html + gzipped.sjs + override.html + override.js + sjs_network-observer-test-server.sjs + +[browser_networkobserver.js] +skip-if = http3 # Bug 1829298 +[browser_networkobserver_invalid_constructor.js] +[browser_networkobserver_override.js] diff --git a/devtools/shared/network-observer/test/browser/browser_networkobserver.js b/devtools/shared/network-observer/test/browser/browser_networkobserver.js new file mode 100644 index 0000000000..73e9f8510a --- /dev/null +++ b/devtools/shared/network-observer/test/browser/browser_networkobserver.js @@ -0,0 +1,72 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +const TEST_URL = URL_ROOT + "doc_network-observer.html"; +const REQUEST_URL = + URL_ROOT + `sjs_network-observer-test-server.sjs?sts=200&fmt=html`; + +// Check that the NetworkObserver can detect basic requests and calls the +// onNetworkEvent callback when expected. +add_task(async function testSingleRequest() { + await addTab(TEST_URL); + + const onNetworkEvents = waitForNetworkEvents(REQUEST_URL, 1); + await SpecialPowers.spawn(gBrowser.selectedBrowser, [REQUEST_URL], _url => { + content.wrappedJSObject.sendRequest(_url); + }); + + const eventsCount = await onNetworkEvents; + is(eventsCount, 1, "Received the expected number of network events"); +}); + +add_task(async function testMultipleRequests() { + await addTab(TEST_URL); + const EXPECTED_REQUESTS_COUNT = 5; + + const onNetworkEvents = waitForNetworkEvents( + REQUEST_URL, + EXPECTED_REQUESTS_COUNT + ); + await SpecialPowers.spawn( + gBrowser.selectedBrowser, + [REQUEST_URL, EXPECTED_REQUESTS_COUNT], + (_url, _count) => { + for (let i = 0; i < _count; i++) { + content.wrappedJSObject.sendRequest(_url); + } + } + ); + + const eventsCount = await onNetworkEvents; + is( + eventsCount, + EXPECTED_REQUESTS_COUNT, + "Received the expected number of network events" + ); +}); + +add_task(async function testOnNetworkEventArguments() { + await addTab(TEST_URL); + + const onNetworkEvent = new Promise(resolve => { + const networkObserver = new NetworkObserver({ + ignoreChannelFunction: () => false, + onNetworkEvent: (...args) => { + resolve(args); + return createNetworkEventOwner(); + }, + }); + registerCleanupFunction(() => networkObserver.destroy()); + }); + + await SpecialPowers.spawn(gBrowser.selectedBrowser, [REQUEST_URL], _url => { + content.wrappedJSObject.sendRequest(_url); + }); + + const args = await onNetworkEvent; + is(args.length, 2, "Received two arguments"); + is(typeof args[0], "object", "First argument is an object"); + ok(args[1] instanceof Ci.nsIChannel, "Second argument is a channel"); +}); diff --git a/devtools/shared/network-observer/test/browser/browser_networkobserver_invalid_constructor.js b/devtools/shared/network-observer/test/browser/browser_networkobserver_invalid_constructor.js new file mode 100644 index 0000000000..76a93d938a --- /dev/null +++ b/devtools/shared/network-observer/test/browser/browser_networkobserver_invalid_constructor.js @@ -0,0 +1,49 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +// Check that the NetworkObserver constructor validates its arguments. +add_task(async function testInvalidConstructorArguments() { + Assert.throws( + () => new NetworkObserver(), + /Expected "ignoreChannelFunction" to be a function, got undefined/, + "NetworkObserver constructor should throw if no argument was provided" + ); + + Assert.throws( + () => new NetworkObserver({}), + /Expected "ignoreChannelFunction" to be a function, got undefined/, + "NetworkObserver constructor should throw if ignoreChannelFunction was not provided" + ); + + const invalidValues = [null, true, false, 12, "str", ["arr"], { obj: "obj" }]; + for (const invalidValue of invalidValues) { + Assert.throws( + () => new NetworkObserver({ ignoreChannelFunction: invalidValue }), + /Expected "ignoreChannelFunction" to be a function, got/, + `NetworkObserver constructor should throw if a(n) ${typeof invalidValue} was provided for ignoreChannelFunction` + ); + } + + const EMPTY_FN = () => {}; + Assert.throws( + () => new NetworkObserver({ ignoreChannelFunction: EMPTY_FN }), + /Expected "onNetworkEvent" to be a function, got undefined/, + "NetworkObserver constructor should throw if onNetworkEvent was not provided" + ); + + // Now we will pass a function for `ignoreChannelFunction`, and will do the + // same tests for onNetworkEvent + for (const invalidValue of invalidValues) { + Assert.throws( + () => + new NetworkObserver({ + ignoreChannelFunction: EMPTY_FN, + onNetworkEvent: invalidValue, + }), + /Expected "onNetworkEvent" to be a function, got/, + `NetworkObserver constructor should throw if a(n) ${typeof invalidValue} was provided for onNetworkEvent` + ); + } +}); diff --git a/devtools/shared/network-observer/test/browser/browser_networkobserver_override.js b/devtools/shared/network-observer/test/browser/browser_networkobserver_override.js new file mode 100644 index 0000000000..3b00c4b2e9 --- /dev/null +++ b/devtools/shared/network-observer/test/browser/browser_networkobserver_override.js @@ -0,0 +1,179 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +const TEST_URL = URL_ROOT + "doc_network-observer.html"; +const REQUEST_URL = + URL_ROOT + `sjs_network-observer-test-server.sjs?sts=200&fmt=html`; +const GZIPPED_REQUEST_URL = URL_ROOT + `gzipped.sjs`; +const OVERRIDE_FILENAME = "override.js"; +const OVERRIDE_HTML_FILENAME = "override.html"; + +add_task(async function testLocalOverride() { + await addTab(TEST_URL); + + let eventsCount = 0; + const networkObserver = new NetworkObserver({ + ignoreChannelFunction: channel => channel.URI.spec !== REQUEST_URL, + onNetworkEvent: event => { + info("received a network event"); + eventsCount++; + return createNetworkEventOwner(event); + }, + }); + + const overrideFile = getChromeDir(getResolvedURI(gTestPath)); + overrideFile.append(OVERRIDE_FILENAME); + info(" override " + REQUEST_URL + " to " + overrideFile.path + "\n"); + networkObserver.override(REQUEST_URL, overrideFile.path); + + info("Assert that request and cached request are overriden"); + await SpecialPowers.spawn( + gBrowser.selectedBrowser, + [REQUEST_URL], + async _url => { + const request = await content.wrappedJSObject.fetch(_url); + const requestcontent = await request.text(); + is( + requestcontent, + `"use strict";\ndocument.title = "evaluated";\n`, + "the request content has been overriden" + ); + const secondRequest = await content.wrappedJSObject.fetch(_url); + const secondRequestcontent = await secondRequest.text(); + is( + secondRequestcontent, + `"use strict";\ndocument.title = "evaluated";\n`, + "the cached request content has been overriden" + ); + } + ); + + info("Assert that JS scripts can be overriden"); + await SpecialPowers.spawn( + gBrowser.selectedBrowser, + [REQUEST_URL], + async _url => { + const script = await content.document.createElement("script"); + const onLoad = new Promise(resolve => + script.addEventListener("load", resolve, { once: true }) + ); + script.src = _url; + content.document.body.appendChild(script); + await onLoad; + is( + content.document.title, + "evaluated", + "The <script> tag content has been overriden and correctly evaluated" + ); + } + ); + + await BrowserTestUtils.waitForCondition(() => eventsCount >= 1); + + networkObserver.destroy(); +}); + +add_task(async function testHtmlFileOverride() { + let eventsCount = 0; + const networkObserver = new NetworkObserver({ + ignoreChannelFunction: channel => channel.URI.spec !== TEST_URL, + onNetworkEvent: event => { + info("received a network event"); + eventsCount++; + return createNetworkEventOwner(event); + }, + }); + + const overrideFile = getChromeDir(getResolvedURI(gTestPath)); + overrideFile.append(OVERRIDE_HTML_FILENAME); + info(" override " + TEST_URL + " to " + overrideFile.path + "\n"); + networkObserver.override(TEST_URL, overrideFile.path); + + await addTab(TEST_URL); + await SpecialPowers.spawn( + gBrowser.selectedBrowser, + [TEST_URL], + async pageUrl => { + is( + content.document.documentElement.outerHTML, + "<html><head></head><body>Overriden!\n</body></html>", + "The content of the HTML has been overriden" + ); + // For now, all overriden request have their location changed to an internal data: URI + // Bug xxx aims at keeping the original URI. + todo_is( + content.location.href, + pageUrl, + "The location of the page is still the original one" + ); + } + ); + await BrowserTestUtils.waitForCondition(() => eventsCount >= 1); + networkObserver.destroy(); +}); + +// Exact same test, but with a gzipped request, which requires very special treatment +add_task(async function testLocalOverrideGzipped() { + await addTab(TEST_URL); + + let eventsCount = 0; + const networkObserver = new NetworkObserver({ + ignoreChannelFunction: channel => channel.URI.spec !== GZIPPED_REQUEST_URL, + onNetworkEvent: event => { + info("received a network event"); + eventsCount++; + return createNetworkEventOwner(event); + }, + }); + + const overrideFile = getChromeDir(getResolvedURI(gTestPath)); + overrideFile.append(OVERRIDE_FILENAME); + info(" override " + GZIPPED_REQUEST_URL + " to " + overrideFile.path + "\n"); + networkObserver.override(GZIPPED_REQUEST_URL, overrideFile.path); + + await SpecialPowers.spawn( + gBrowser.selectedBrowser, + [GZIPPED_REQUEST_URL], + async _url => { + const request = await content.wrappedJSObject.fetch(_url); + const requestcontent = await request.text(); + is( + requestcontent, + `"use strict";\ndocument.title = "evaluated";\n`, + "the request content has been overriden" + ); + const secondRequest = await content.wrappedJSObject.fetch(_url); + const secondRequestcontent = await secondRequest.text(); + is( + secondRequestcontent, + `"use strict";\ndocument.title = "evaluated";\n`, + "the cached request content has been overriden" + ); + } + ); + + await SpecialPowers.spawn( + gBrowser.selectedBrowser, + [GZIPPED_REQUEST_URL], + async _url => { + const script = await content.document.createElement("script"); + const onLoad = new Promise(resolve => + script.addEventListener("load", resolve, { once: true }) + ); + script.src = _url; + content.document.body.appendChild(script); + await onLoad; + is( + content.document.title, + "evaluated", + "The <script> tag content has been overriden and correctly evaluated" + ); + } + ); + + await BrowserTestUtils.waitForCondition(() => eventsCount >= 1); + + networkObserver.destroy(); +}); diff --git a/devtools/shared/network-observer/test/browser/doc_network-observer.html b/devtools/shared/network-observer/test/browser/doc_network-observer.html new file mode 100644 index 0000000000..90cd9be69b --- /dev/null +++ b/devtools/shared/network-observer/test/browser/doc_network-observer.html @@ -0,0 +1,25 @@ +<!-- Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ --> +<!doctype html> + +<html> + <head> + <meta charset="utf-8"/> + <meta http-equiv="Cache-Control" content="no-cache, no-store, must-revalidate" /> + <meta http-equiv="Pragma" content="no-cache" /> + <meta http-equiv="Expires" content="0" /> + <title>Network Observer test page</title> + </head> + <body> + <p>Network Observer test page</p> + <script type="text/javascript"> + "use strict"; + + window.sendRequest = url => { + const xhr = new XMLHttpRequest(); + xhr.open("GET", url, true); + xhr.send(null); + } + </script> + </body> +</html> diff --git a/devtools/shared/network-observer/test/browser/gzipped.sjs b/devtools/shared/network-observer/test/browser/gzipped.sjs new file mode 100644 index 0000000000..09d0b249b1 --- /dev/null +++ b/devtools/shared/network-observer/test/browser/gzipped.sjs @@ -0,0 +1,44 @@ +"use strict"; + +function gzipCompressString(string, obs) { + const scs = Cc["@mozilla.org/streamConverters;1"].getService( + Ci.nsIStreamConverterService + ); + const listener = Cc["@mozilla.org/network/stream-loader;1"].createInstance( + Ci.nsIStreamLoader + ); + listener.init(obs); + const converter = scs.asyncConvertData( + "uncompressed", + "gzip", + listener, + null + ); + const stringStream = Cc[ + "@mozilla.org/io/string-input-stream;1" + ].createInstance(Ci.nsIStringInputStream); + stringStream.data = string; + converter.onStartRequest(null, null); + converter.onDataAvailable(null, stringStream, 0, string.length); + converter.onStopRequest(null, null, null); +} + +const ORIGINAL_JS_CONTENT = `console.log("original javascript content");`; + +function handleRequest(request, response) { + response.processAsync(); + + // Generate data + response.setHeader("Content-Type", "application/javascript", false); + response.setHeader("Content-Encoding", "gzip", false); + + const observer = { + onStreamComplete(loader, context, status, length, result) { + const buffer = String.fromCharCode.apply(this, result); + response.setHeader("Content-Length", "" + buffer.length, false); + response.write(buffer); + response.finish(); + }, + }; + gzipCompressString(ORIGINAL_JS_CONTENT, observer); +} diff --git a/devtools/shared/network-observer/test/browser/head.js b/devtools/shared/network-observer/test/browser/head.js new file mode 100644 index 0000000000..ab03163d9d --- /dev/null +++ b/devtools/shared/network-observer/test/browser/head.js @@ -0,0 +1,92 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +ChromeUtils.defineESModuleGetters(this, { + NetworkObserver: + "resource://devtools/shared/network-observer/NetworkObserver.sys.mjs", +}); + +const TEST_DIR = gTestPath.substr(0, gTestPath.lastIndexOf("/")); +const CHROME_URL_ROOT = TEST_DIR + "/"; +const URL_ROOT = CHROME_URL_ROOT.replace( + "chrome://mochitests/content/", + "https://example.com/" +); + +/** + * Load the provided url in an existing browser. + * Returns a promise which will resolve when the page is loaded. + * + * @param {Browser} browser + * The browser element where the URL should be loaded. + * @param {String} url + * The URL to load in the new tab + */ +async function loadURL(browser, url) { + const loaded = BrowserTestUtils.browserLoaded(browser); + BrowserTestUtils.loadURIString(browser, url); + return loaded; +} + +/** + * Create a new foreground tab loading the provided url. + * Returns a promise which will resolve when the page is loaded. + * + * @param {String} url + * The URL to load in the new tab + */ +async function addTab(url) { + const tab = await BrowserTestUtils.openNewForegroundTab(gBrowser, url); + registerCleanupFunction(() => { + gBrowser.removeTab(tab); + }); + return tab; +} + +/** + * Create a simple network event owner, with empty implementations of all + * the expected APIs for a NetworkEventOwner. + */ +function createNetworkEventOwner(event) { + return { + addEventTimings: () => {}, + addResponseCache: () => {}, + addResponseContent: () => {}, + addResponseStart: () => {}, + addSecurityInfo: () => {}, + addServerTimings: () => {}, + }; +} + +/** + * Wait for network events matching the provided URL, until the count reaches + * the provided expected count. + * + * @param {string} expectedUrl + * The URL which should be monitored by the NetworkObserver. + * @param {number} expectedRequestsCount + * How many different events (requests) are expected. + * @returns {Promise} + * A promise which will resolve with the current count, when the expected + * count is reached. + */ +async function waitForNetworkEvents(expectedUrl, expectedRequestsCount) { + let eventsCount = 0; + const networkObserver = new NetworkObserver({ + ignoreChannelFunction: channel => channel.URI.spec !== expectedUrl, + onNetworkEvent: event => { + info("waitForNetworkEvents received a new event"); + eventsCount++; + return createNetworkEventOwner(event); + }, + }); + registerCleanupFunction(() => networkObserver.destroy()); + + info("Wait until the events count reaches " + expectedRequestsCount); + await BrowserTestUtils.waitForCondition( + () => eventsCount >= expectedRequestsCount + ); + return eventsCount; +} diff --git a/devtools/shared/network-observer/test/browser/override.html b/devtools/shared/network-observer/test/browser/override.html new file mode 100644 index 0000000000..0e3878e313 --- /dev/null +++ b/devtools/shared/network-observer/test/browser/override.html @@ -0,0 +1 @@ +<html><head></head><body>Overriden!</body></html> diff --git a/devtools/shared/network-observer/test/browser/override.js b/devtools/shared/network-observer/test/browser/override.js new file mode 100644 index 0000000000..7b000fcd0f --- /dev/null +++ b/devtools/shared/network-observer/test/browser/override.js @@ -0,0 +1,2 @@ +"use strict"; +document.title = "evaluated"; diff --git a/devtools/shared/network-observer/test/browser/sjs_network-observer-test-server.sjs b/devtools/shared/network-observer/test/browser/sjs_network-observer-test-server.sjs new file mode 100644 index 0000000000..918d6e8447 --- /dev/null +++ b/devtools/shared/network-observer/test/browser/sjs_network-observer-test-server.sjs @@ -0,0 +1,198 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +Cu.importGlobalProperties(["TextEncoder"]); + +// Simple server which can handle several response types and states. +// Trimmed down from devtools/client/netmonitor/test/sjs_content-type-test-server.sjs +// Additional features can be ported if needed. +function handleRequest(request, response) { + response.processAsync(); + + const params = request.queryString.split("&"); + const format = (params.filter(s => s.includes("fmt="))[0] || "").split( + "=" + )[1]; + const status = + (params.filter(s => s.includes("sts="))[0] || "").split("=")[1] || 200; + + const cacheExpire = 60; // seconds + + function setCacheHeaders() { + if (status != 304) { + response.setHeader( + "Cache-Control", + "no-cache, no-store, must-revalidate" + ); + response.setHeader("Pragma", "no-cache"); + response.setHeader("Expires", "0"); + return; + } + + response.setHeader("Expires", Date(Date.now() + cacheExpire * 1000), false); + } + + let timer = Cc["@mozilla.org/timer;1"].createInstance(Ci.nsITimer); + + timer.initWithCallback( + // eslint-disable-next-line complexity + () => { + // to avoid garbage collection + timer = null; + switch (format) { + case "txt": { + response.setStatusLine(request.httpVersion, status, "DA DA DA"); + response.setHeader("Content-Type", "text/plain", false); + setCacheHeaders(); + + function convertToUtf8(str) { + return String.fromCharCode(...new TextEncoder().encode(str)); + } + + // This script must be evaluated as UTF-8 for this to write out the + // bytes of the string in UTF-8. If it's evaluated as Latin-1, the + // written bytes will be the result of UTF-8-encoding this string + // *twice*. + const data = "Братан, ты вообще качаешься?"; + const stringOfUtf8Bytes = convertToUtf8(data); + response.write(stringOfUtf8Bytes); + + response.finish(); + break; + } + case "xml": { + response.setStatusLine(request.httpVersion, status, "OK"); + response.setHeader("Content-Type", "text/xml; charset=utf-8", false); + setCacheHeaders(); + response.write("<label value='greeting'>Hello XML!</label>"); + response.finish(); + break; + } + case "html": { + const content = ( + params.filter(s => s.includes("res="))[0] || "" + ).split("=")[1]; + response.setStatusLine(request.httpVersion, status, "OK"); + response.setHeader("Content-Type", "text/html; charset=utf-8", false); + setCacheHeaders(); + response.write(content || "<p>Hello HTML!</p>"); + response.finish(); + break; + } + case "xhtml": { + response.setStatusLine(request.httpVersion, status, "OK"); + response.setHeader( + "Content-Type", + "application/xhtml+xml; charset=utf-8", + false + ); + setCacheHeaders(); + response.write("<p>Hello XHTML!</p>"); + response.finish(); + break; + } + case "html-long": { + const str = new Array(102400 /* 100 KB in bytes */).join("."); + response.setStatusLine(request.httpVersion, status, "OK"); + response.setHeader("Content-Type", "text/html; charset=utf-8", false); + setCacheHeaders(); + response.write("<p>" + str + "</p>"); + response.finish(); + break; + } + case "css": { + response.setStatusLine(request.httpVersion, status, "OK"); + response.setHeader("Content-Type", "text/css; charset=utf-8", false); + setCacheHeaders(); + response.write("body:pre { content: 'Hello CSS!' }"); + response.finish(); + break; + } + case "js": { + response.setStatusLine(request.httpVersion, status, "OK"); + response.setHeader( + "Content-Type", + "application/javascript; charset=utf-8", + false + ); + setCacheHeaders(); + response.write("function() { return 'Hello JS!'; }"); + response.finish(); + break; + } + case "json": { + response.setStatusLine(request.httpVersion, status, "OK"); + response.setHeader( + "Content-Type", + "application/json; charset=utf-8", + false + ); + setCacheHeaders(); + response.write('{ "greeting": "Hello JSON!" }'); + response.finish(); + break; + } + + case "font": { + response.setStatusLine(request.httpVersion, status, "OK"); + response.setHeader("Content-Type", "font/woff", false); + setCacheHeaders(); + response.finish(); + break; + } + case "image": { + response.setStatusLine(request.httpVersion, status, "OK"); + response.setHeader("Content-Type", "image/png", false); + setCacheHeaders(); + response.finish(); + break; + } + case "application-ogg": { + response.setStatusLine(request.httpVersion, status, "OK"); + response.setHeader("Content-Type", "application/ogg", false); + setCacheHeaders(); + response.finish(); + break; + } + case "audio": { + response.setStatusLine(request.httpVersion, status, "OK"); + response.setHeader("Content-Type", "audio/ogg", false); + setCacheHeaders(); + response.finish(); + break; + } + case "video": { + response.setStatusLine(request.httpVersion, status, "OK"); + response.setHeader("Content-Type", "video/webm", false); + setCacheHeaders(); + response.finish(); + break; + } + case "ws": { + response.setStatusLine( + request.httpVersion, + 101, + "Switching Protocols" + ); + response.setHeader("Connection", "upgrade", false); + response.setHeader("Upgrade", "websocket", false); + setCacheHeaders(); + response.finish(); + break; + } + default: { + response.setStatusLine(request.httpVersion, 404, "Not Found"); + response.setHeader("Content-Type", "text/html; charset=utf-8", false); + setCacheHeaders(); + response.write("<blink>Not Found</blink>"); + response.finish(); + break; + } + } + }, + 10, + Ci.nsITimer.TYPE_ONE_SHOT + ); // Make sure this request takes a few ms. +} diff --git a/devtools/shared/network-observer/test/xpcshell/head.js b/devtools/shared/network-observer/test/xpcshell/head.js new file mode 100644 index 0000000000..93b66e4632 --- /dev/null +++ b/devtools/shared/network-observer/test/xpcshell/head.js @@ -0,0 +1,9 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +ChromeUtils.defineESModuleGetters(this, { + NetworkHelper: + "resource://devtools/shared/network-observer/NetworkHelper.sys.mjs", +}); diff --git a/devtools/shared/network-observer/test/xpcshell/test_network_helper.js b/devtools/shared/network-observer/test/xpcshell/test_network_helper.js new file mode 100644 index 0000000000..ff514ab98e --- /dev/null +++ b/devtools/shared/network-observer/test/xpcshell/test_network_helper.js @@ -0,0 +1,90 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +function run_test() { + test_isTextMimeType(); + test_parseCookieHeader(); +} + +function test_isTextMimeType() { + Assert.equal(NetworkHelper.isTextMimeType("text/plain"), true); + Assert.equal(NetworkHelper.isTextMimeType("application/javascript"), true); + Assert.equal(NetworkHelper.isTextMimeType("application/json"), true); + Assert.equal(NetworkHelper.isTextMimeType("text/css"), true); + Assert.equal(NetworkHelper.isTextMimeType("text/html"), true); + Assert.equal(NetworkHelper.isTextMimeType("image/svg+xml"), true); + Assert.equal(NetworkHelper.isTextMimeType("application/xml"), true); + + // Test custom JSON subtype + Assert.equal( + NetworkHelper.isTextMimeType("application/vnd.tent.posts-feed.v0+json"), + true + ); + Assert.equal( + NetworkHelper.isTextMimeType("application/vnd.tent.posts-feed.v0-json"), + true + ); + // Test custom XML subtype + Assert.equal( + NetworkHelper.isTextMimeType("application/vnd.tent.posts-feed.v0+xml"), + true + ); + Assert.equal( + NetworkHelper.isTextMimeType("application/vnd.tent.posts-feed.v0-xml"), + false + ); + // Test case-insensitive + Assert.equal( + NetworkHelper.isTextMimeType("application/vnd.BIG-CORP+json"), + true + ); + // Test non-text type + Assert.equal(NetworkHelper.isTextMimeType("image/png"), false); + // Test invalid types + Assert.equal(NetworkHelper.isTextMimeType("application/foo-+json"), false); + Assert.equal(NetworkHelper.isTextMimeType("application/-foo+json"), false); + Assert.equal( + NetworkHelper.isTextMimeType("application/foo--bar+json"), + false + ); + + // Test we do not cause internal errors with unoptimized regex. Bug 961097 + Assert.equal( + NetworkHelper.isTextMimeType("application/vnd.google.safebrowsing-chunk"), + false + ); +} + +function test_parseCookieHeader() { + let result = NetworkHelper.parseSetCookieHeaders(["Test=1; SameSite=Strict"]); + Assert.deepEqual(result, [{ name: "Test", value: "1", samesite: "Strict" }]); + + result = NetworkHelper.parseSetCookieHeaders(["Test=1; SameSite=strict"]); + Assert.deepEqual(result, [{ name: "Test", value: "1", samesite: "Strict" }]); + + result = NetworkHelper.parseSetCookieHeaders(["Test=1; SameSite=STRICT"]); + Assert.deepEqual(result, [{ name: "Test", value: "1", samesite: "Strict" }]); + + result = NetworkHelper.parseSetCookieHeaders(["Test=1; SameSite=None"]); + Assert.deepEqual(result, [{ name: "Test", value: "1", samesite: "None" }]); + + result = NetworkHelper.parseSetCookieHeaders(["Test=1; SameSite=NONE"]); + Assert.deepEqual(result, [{ name: "Test", value: "1", samesite: "None" }]); + + result = NetworkHelper.parseSetCookieHeaders(["Test=1; SameSite=lax"]); + Assert.deepEqual(result, [{ name: "Test", value: "1", samesite: "Lax" }]); + + result = NetworkHelper.parseSetCookieHeaders(["Test=1; SameSite=Lax"]); + Assert.deepEqual(result, [{ name: "Test", value: "1", samesite: "Lax" }]); + + result = NetworkHelper.parseSetCookieHeaders([ + "Test=1; SameSite=Lax", + "Foo=2; SameSite=None", + ]); + Assert.deepEqual(result, [ + { name: "Test", value: "1", samesite: "Lax" }, + { name: "Foo", value: "2", samesite: "None" }, + ]); +} diff --git a/devtools/shared/network-observer/test/xpcshell/test_security-info-certificate.js b/devtools/shared/network-observer/test/xpcshell/test_security-info-certificate.js new file mode 100644 index 0000000000..00f482b8ca --- /dev/null +++ b/devtools/shared/network-observer/test/xpcshell/test_security-info-certificate.js @@ -0,0 +1,84 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ +"use strict"; + +// Tests that NetworkHelper.parseCertificateInfo parses certificate information +// correctly. + +const DUMMY_CERT = { + getBase64DERString() { + // This is the base64-encoded contents of the "DigiCert ECC Secure Server CA" + // intermediate certificate as issued by "DigiCert Global Root CA". It was + // chosen as a test certificate because it has an issuer common name, + // organization, and organizational unit that are somewhat distinct from + // its subject common name and organization name. + return "MIIDrDCCApSgAwIBAgIQCssoukZe5TkIdnRw883GEjANBgkqhkiG9w0BAQwFADBhMQswCQYDVQQGEwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3d3cuZGlnaWNlcnQuY29tMSAwHgYDVQQDExdEaWdpQ2VydCBHbG9iYWwgUm9vdCBDQTAeFw0xMzAzMDgxMjAwMDBaFw0yMzAzMDgxMjAwMDBaMEwxCzAJBgNVBAYTAlVTMRUwEwYDVQQKEwxEaWdpQ2VydCBJbmMxJjAkBgNVBAMTHURpZ2lDZXJ0IEVDQyBTZWN1cmUgU2VydmVyIENBMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAE4ghC6nfYJN6gLGSkE85AnCNyqQIKDjc/ITa4jVMU9tWRlUvzlgKNcR7E2Munn17voOZ/WpIRllNv68DLP679Wz9HJOeaBy6Wvqgvu1cYr3GkvXg6HuhbPGtkESvMNCuMo4IBITCCAR0wEgYDVR0TAQH/BAgwBgEB/wIBADAOBgNVHQ8BAf8EBAMCAYYwNAYIKwYBBQUHAQEEKDAmMCQGCCsGAQUFBzABhhhodHRwOi8vb2NzcC5kaWdpY2VydC5jb20wQgYDVR0fBDswOTA3oDWgM4YxaHR0cDovL2NybDMuZGlnaWNlcnQuY29tL0RpZ2lDZXJ0R2xvYmFsUm9vdENBLmNybDA9BgNVHSAENjA0MDIGBFUdIAAwKjAoBggrBgEFBQcCARYcaHR0cHM6Ly93d3cuZGlnaWNlcnQuY29tL0NQUzAdBgNVHQ4EFgQUo53mH/naOU/AbuiRy5Wl2jHiCp8wHwYDVR0jBBgwFoAUA95QNVbRTLtm8KPiGxvDl7I90VUwDQYJKoZIhvcNAQEMBQADggEBAMeKoENL7HTJxavVHzA1Nm6YVntIrAVjrnuaVyRXzG/63qttnMe2uuzO58pzZNvfBDcKAEmzP58mrZGMIOgfiA4q+2Y3yDDo0sIkp0VILeoBUEoxlBPfjV/aKrtJPGHzecicZpIalir0ezZYoyxBEHQa0+1IttK7igZFcTMQMHp6mCHdJLnsnLWSB62DxsRq+HfmNb4TDydkskO/g+l3VtsIh5RHFPVfKK+jaEyDj2D3loB5hWp2Jp2VDCADjT7ueihlZGak2YPqmXTNbk19HOuNssWvFhtOyPNV6og4ETQdEa8/B6hPatJ0ES8q/HO3X8IVQwVs1n3aAr0im0/T+Xc="; + }, +}; + +add_task(async function run_test() { + info("Testing NetworkHelper.parseCertificateInfo."); + + const result = await NetworkHelper.parseCertificateInfo( + DUMMY_CERT, + new Map() + ); + + // Subject + equal( + result.subject.commonName, + "DigiCert ECC Secure Server CA", + "Common name is correct." + ); + equal( + result.subject.organization, + "DigiCert Inc", + "Organization is correct." + ); + equal( + result.subject.organizationUnit, + undefined, + "Organizational unit is correct." + ); + + // Issuer + equal( + result.issuer.commonName, + "DigiCert Global Root CA", + "Common name of the issuer is correct." + ); + equal( + result.issuer.organization, + "DigiCert Inc", + "Organization of the issuer is correct." + ); + equal( + result.issuer.organizationUnit, + "www.digicert.com", + "Organizational unit of the issuer is correct." + ); + + // Validity + equal( + result.validity.start, + "Fri, 08 Mar 2013 12:00:00 GMT", + "Start of the validity period is correct." + ); + equal( + result.validity.end, + "Wed, 08 Mar 2023 12:00:00 GMT", + "End of the validity period is correct." + ); + + // Fingerprints + equal( + result.fingerprint.sha1, + "56:EE:7C:27:06:83:16:2D:83:BA:EA:CC:79:0E:22:47:1A:DA:AB:E8", + "Certificate SHA1 fingerprint is correct." + ); + equal( + result.fingerprint.sha256, + "45:84:46:BA:75:D9:32:E9:14:F2:3C:2B:57:B7:D1:92:ED:DB:C2:18:1D:95:8E:11:81:AD:52:51:74:7A:1E:E8", + "Certificate SHA256 fingerprint is correct." + ); +}); diff --git a/devtools/shared/network-observer/test/xpcshell/test_security-info-parser.js b/devtools/shared/network-observer/test/xpcshell/test_security-info-parser.js new file mode 100644 index 0000000000..9515851a8b --- /dev/null +++ b/devtools/shared/network-observer/test/xpcshell/test_security-info-parser.js @@ -0,0 +1,54 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ +"use strict"; + +// Test that NetworkHelper.parseSecurityInfo returns correctly formatted object. + +const wpl = Ci.nsIWebProgressListener; +const MockCertificate = { + getBase64DERString() { + // This is the same test certificate as in + // test_security-info-certificate.js for consistency. + return "MIIDrDCCApSgAwIBAgIQCssoukZe5TkIdnRw883GEjANBgkqhkiG9w0BAQwFADBhMQswCQYDVQQGEwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3d3cuZGlnaWNlcnQuY29tMSAwHgYDVQQDExdEaWdpQ2VydCBHbG9iYWwgUm9vdCBDQTAeFw0xMzAzMDgxMjAwMDBaFw0yMzAzMDgxMjAwMDBaMEwxCzAJBgNVBAYTAlVTMRUwEwYDVQQKEwxEaWdpQ2VydCBJbmMxJjAkBgNVBAMTHURpZ2lDZXJ0IEVDQyBTZWN1cmUgU2VydmVyIENBMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAE4ghC6nfYJN6gLGSkE85AnCNyqQIKDjc/ITa4jVMU9tWRlUvzlgKNcR7E2Munn17voOZ/WpIRllNv68DLP679Wz9HJOeaBy6Wvqgvu1cYr3GkvXg6HuhbPGtkESvMNCuMo4IBITCCAR0wEgYDVR0TAQH/BAgwBgEB/wIBADAOBgNVHQ8BAf8EBAMCAYYwNAYIKwYBBQUHAQEEKDAmMCQGCCsGAQUFBzABhhhodHRwOi8vb2NzcC5kaWdpY2VydC5jb20wQgYDVR0fBDswOTA3oDWgM4YxaHR0cDovL2NybDMuZGlnaWNlcnQuY29tL0RpZ2lDZXJ0R2xvYmFsUm9vdENBLmNybDA9BgNVHSAENjA0MDIGBFUdIAAwKjAoBggrBgEFBQcCARYcaHR0cHM6Ly93d3cuZGlnaWNlcnQuY29tL0NQUzAdBgNVHQ4EFgQUo53mH/naOU/AbuiRy5Wl2jHiCp8wHwYDVR0jBBgwFoAUA95QNVbRTLtm8KPiGxvDl7I90VUwDQYJKoZIhvcNAQEMBQADggEBAMeKoENL7HTJxavVHzA1Nm6YVntIrAVjrnuaVyRXzG/63qttnMe2uuzO58pzZNvfBDcKAEmzP58mrZGMIOgfiA4q+2Y3yDDo0sIkp0VILeoBUEoxlBPfjV/aKrtJPGHzecicZpIalir0ezZYoyxBEHQa0+1IttK7igZFcTMQMHp6mCHdJLnsnLWSB62DxsRq+HfmNb4TDydkskO/g+l3VtsIh5RHFPVfKK+jaEyDj2D3loB5hWp2Jp2VDCADjT7ueihlZGak2YPqmXTNbk19HOuNssWvFhtOyPNV6og4ETQdEa8/B6hPatJ0ES8q/HO3X8IVQwVs1n3aAr0im0/T+Xc="; + }, +}; + +// This *cannot* be used as an nsITransportSecurityInfo (since that interface is +// builtinclass) but the methods being tested aren't defined by XPCOM and aren't +// calling QueryInterface, so this usage is fine. +const MockSecurityInfo = { + securityState: wpl.STATE_IS_SECURE, + errorCode: 0, + cipherName: "TLS_ECDH_ECDSA_WITH_AES_128_GCM_SHA256", + // TLS_VERSION_1_2 + protocolVersion: 3, + serverCert: MockCertificate, +}; + +add_task(async function run_test() { + const result = await NetworkHelper.parseSecurityInfo( + MockSecurityInfo, + {}, + {}, + new Map() + ); + + equal(result.state, "secure", "State is correct."); + + equal( + result.cipherSuite, + MockSecurityInfo.cipherName, + "Cipher suite is correct." + ); + + equal(result.protocolVersion, "TLSv1.2", "Protocol version is correct."); + + deepEqual( + result.cert, + await NetworkHelper.parseCertificateInfo(MockCertificate, new Map()), + "Certificate information is correct." + ); + + equal(result.hpkp, false, "HPKP is false when URI is not available."); + equal(result.hsts, false, "HSTS is false when URI is not available."); +}); diff --git a/devtools/shared/network-observer/test/xpcshell/test_security-info-protocol-version.js b/devtools/shared/network-observer/test/xpcshell/test_security-info-protocol-version.js new file mode 100644 index 0000000000..a81f7ce73c --- /dev/null +++ b/devtools/shared/network-observer/test/xpcshell/test_security-info-protocol-version.js @@ -0,0 +1,48 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ +"use strict"; + +// Tests that NetworkHelper.formatSecurityProtocol returns correct +// protocol version strings. + +const TEST_CASES = [ + { + description: "TLS_VERSION_1", + input: 1, + expected: "TLSv1", + }, + { + description: "TLS_VERSION_1.1", + input: 2, + expected: "TLSv1.1", + }, + { + description: "TLS_VERSION_1.2", + input: 3, + expected: "TLSv1.2", + }, + { + description: "TLS_VERSION_1.3", + input: 4, + expected: "TLSv1.3", + }, + { + description: "invalid version", + input: -1, + expected: "Unknown", + }, +]; + +function run_test() { + info("Testing NetworkHelper.formatSecurityProtocol."); + + for (const { description, input, expected } of TEST_CASES) { + info("Testing " + description); + + equal( + NetworkHelper.formatSecurityProtocol(input), + expected, + "Got the expected protocol string." + ); + } +} diff --git a/devtools/shared/network-observer/test/xpcshell/test_security-info-state.js b/devtools/shared/network-observer/test/xpcshell/test_security-info-state.js new file mode 100644 index 0000000000..be622b2019 --- /dev/null +++ b/devtools/shared/network-observer/test/xpcshell/test_security-info-state.js @@ -0,0 +1,122 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ +"use strict"; + +// Tests that security info parser gives correct general security state for +// different cases. + +const wpl = Ci.nsIWebProgressListener; + +// This *cannot* be used as an nsITransportSecurityInfo (since that interface is +// builtinclass) but the methods being tested aren't defined by XPCOM and aren't +// calling QueryInterface, so this usage is fine. +const MockSecurityInfo = { + securityState: wpl.STATE_IS_BROKEN, + errorCode: 0, + // nsISSLStatus.TLS_VERSION_1_2 + protocolVersion: 3, + cipherName: "TLS_ECDH_ECDSA_WITH_AES_128_GCM_SHA256", +}; + +add_task(async function run_test() { + await test_nullSecurityInfo(); + await test_insecureSecurityInfoWithNSSError(); + await test_insecureSecurityInfoWithoutNSSError(); + await test_brokenSecurityInfo(); + await test_secureSecurityInfo(); +}); + +/** + * Test that undefined security information is returns "insecure". + */ +async function test_nullSecurityInfo() { + const result = await NetworkHelper.parseSecurityInfo(null, {}, {}, new Map()); + equal( + result.state, + "insecure", + "state == 'insecure' when securityInfo was undefined" + ); +} + +/** + * Test that STATE_IS_INSECURE with NSSError returns "broken" + */ +async function test_insecureSecurityInfoWithNSSError() { + MockSecurityInfo.securityState = wpl.STATE_IS_INSECURE; + + // Taken from security/manager/ssl/tests/unit/head_psm.js. + MockSecurityInfo.errorCode = -8180; + + const result = await NetworkHelper.parseSecurityInfo( + MockSecurityInfo, + {}, + {}, + new Map() + ); + equal( + result.state, + "broken", + "state == 'broken' if securityState contains STATE_IS_INSECURE flag AND " + + "errorCode is NSS error." + ); + + MockSecurityInfo.errorCode = 0; +} + +/** + * Test that STATE_IS_INSECURE without NSSError returns "insecure" + */ +async function test_insecureSecurityInfoWithoutNSSError() { + MockSecurityInfo.securityState = wpl.STATE_IS_INSECURE; + + const result = await NetworkHelper.parseSecurityInfo( + MockSecurityInfo, + {}, + {}, + new Map() + ); + equal( + result.state, + "insecure", + "state == 'insecure' if securityState contains STATE_IS_INSECURE flag BUT " + + "errorCode is not NSS error." + ); +} + +/** + * Test that STATE_IS_SECURE returns "secure" + */ +async function test_secureSecurityInfo() { + MockSecurityInfo.securityState = wpl.STATE_IS_SECURE; + + const result = await NetworkHelper.parseSecurityInfo( + MockSecurityInfo, + {}, + {}, + new Map() + ); + equal( + result.state, + "secure", + "state == 'secure' if securityState contains STATE_IS_SECURE flag" + ); +} + +/** + * Test that STATE_IS_BROKEN returns "weak" + */ +async function test_brokenSecurityInfo() { + MockSecurityInfo.securityState = wpl.STATE_IS_BROKEN; + + const result = await NetworkHelper.parseSecurityInfo( + MockSecurityInfo, + {}, + {}, + new Map() + ); + equal( + result.state, + "weak", + "state == 'weak' if securityState contains STATE_IS_BROKEN flag" + ); +} diff --git a/devtools/shared/network-observer/test/xpcshell/test_security-info-static-hpkp.js b/devtools/shared/network-observer/test/xpcshell/test_security-info-static-hpkp.js new file mode 100644 index 0000000000..f1aa883b93 --- /dev/null +++ b/devtools/shared/network-observer/test/xpcshell/test_security-info-static-hpkp.js @@ -0,0 +1,41 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ +"use strict"; + +// Test that NetworkHelper.parseSecurityInfo correctly detects static hpkp pins + +const wpl = Ci.nsIWebProgressListener; + +// This *cannot* be used as an nsITransportSecurityInfo (since that interface is +// builtinclass) but the methods being tested aren't defined by XPCOM and aren't +// calling QueryInterface, so this usage is fine. +const MockSecurityInfo = { + securityState: wpl.STATE_IS_SECURE, + errorCode: 0, + cipherName: "TLS_ECDH_ECDSA_WITH_AES_128_GCM_SHA256", + // TLS_VERSION_1_2 + protocolVersion: 3, + serverCert: { + getBase64DERString() { + // This is the same test certificate as in + // test_security-info-certificate.js for consistency. + return "MIIDrDCCApSgAwIBAgIQCssoukZe5TkIdnRw883GEjANBgkqhkiG9w0BAQwFADBhMQswCQYDVQQGEwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3d3cuZGlnaWNlcnQuY29tMSAwHgYDVQQDExdEaWdpQ2VydCBHbG9iYWwgUm9vdCBDQTAeFw0xMzAzMDgxMjAwMDBaFw0yMzAzMDgxMjAwMDBaMEwxCzAJBgNVBAYTAlVTMRUwEwYDVQQKEwxEaWdpQ2VydCBJbmMxJjAkBgNVBAMTHURpZ2lDZXJ0IEVDQyBTZWN1cmUgU2VydmVyIENBMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAE4ghC6nfYJN6gLGSkE85AnCNyqQIKDjc/ITa4jVMU9tWRlUvzlgKNcR7E2Munn17voOZ/WpIRllNv68DLP679Wz9HJOeaBy6Wvqgvu1cYr3GkvXg6HuhbPGtkESvMNCuMo4IBITCCAR0wEgYDVR0TAQH/BAgwBgEB/wIBADAOBgNVHQ8BAf8EBAMCAYYwNAYIKwYBBQUHAQEEKDAmMCQGCCsGAQUFBzABhhhodHRwOi8vb2NzcC5kaWdpY2VydC5jb20wQgYDVR0fBDswOTA3oDWgM4YxaHR0cDovL2NybDMuZGlnaWNlcnQuY29tL0RpZ2lDZXJ0R2xvYmFsUm9vdENBLmNybDA9BgNVHSAENjA0MDIGBFUdIAAwKjAoBggrBgEFBQcCARYcaHR0cHM6Ly93d3cuZGlnaWNlcnQuY29tL0NQUzAdBgNVHQ4EFgQUo53mH/naOU/AbuiRy5Wl2jHiCp8wHwYDVR0jBBgwFoAUA95QNVbRTLtm8KPiGxvDl7I90VUwDQYJKoZIhvcNAQEMBQADggEBAMeKoENL7HTJxavVHzA1Nm6YVntIrAVjrnuaVyRXzG/63qttnMe2uuzO58pzZNvfBDcKAEmzP58mrZGMIOgfiA4q+2Y3yDDo0sIkp0VILeoBUEoxlBPfjV/aKrtJPGHzecicZpIalir0ezZYoyxBEHQa0+1IttK7igZFcTMQMHp6mCHdJLnsnLWSB62DxsRq+HfmNb4TDydkskO/g+l3VtsIh5RHFPVfKK+jaEyDj2D3loB5hWp2Jp2VDCADjT7ueihlZGak2YPqmXTNbk19HOuNssWvFhtOyPNV6og4ETQdEa8/B6hPatJ0ES8q/HO3X8IVQwVs1n3aAr0im0/T+Xc="; + }, + }, +}; + +const MockHttpInfo = { + hostname: "include-subdomains.pinning.example.com", + private: false, +}; + +add_task(async function run_test() { + Services.prefs.setIntPref("security.cert_pinning.enforcement_level", 1); + const result = await NetworkHelper.parseSecurityInfo( + MockSecurityInfo, + {}, + MockHttpInfo, + new Map() + ); + equal(result.hpkp, true, "Static HPKP detected."); +}); diff --git a/devtools/shared/network-observer/test/xpcshell/test_security-info-weakness-reasons.js b/devtools/shared/network-observer/test/xpcshell/test_security-info-weakness-reasons.js new file mode 100644 index 0000000000..71d7675284 --- /dev/null +++ b/devtools/shared/network-observer/test/xpcshell/test_security-info-weakness-reasons.js @@ -0,0 +1,39 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ +"use strict"; + +// Tests that NetworkHelper.getReasonsForWeakness returns correct reasons for +// weak requests. + +const wpl = Ci.nsIWebProgressListener; +const TEST_CASES = [ + { + description: "weak cipher", + input: wpl.STATE_IS_BROKEN | wpl.STATE_USES_WEAK_CRYPTO, + expected: ["cipher"], + }, + { + description: "only STATE_IS_BROKEN flag", + input: wpl.STATE_IS_BROKEN, + expected: [], + }, + { + description: "only STATE_IS_SECURE flag", + input: wpl.STATE_IS_SECURE, + expected: [], + }, +]; + +function run_test() { + info("Testing NetworkHelper.getReasonsForWeakness."); + + for (const { description, input, expected } of TEST_CASES) { + info("Testing " + description); + + deepEqual( + NetworkHelper.getReasonsForWeakness(input), + expected, + "Got the expected reasons for weakness." + ); + } +} diff --git a/devtools/shared/network-observer/test/xpcshell/test_throttle.js b/devtools/shared/network-observer/test/xpcshell/test_throttle.js new file mode 100644 index 0000000000..5f8ef589fb --- /dev/null +++ b/devtools/shared/network-observer/test/xpcshell/test_throttle.js @@ -0,0 +1,162 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +/* eslint-disable mozilla/use-chromeutils-generateqi */ + +const { NetworkThrottleManager } = ChromeUtils.importESModule( + "resource://devtools/shared/network-observer/NetworkThrottleManager.sys.mjs" +); +const nsIScriptableInputStream = Ci.nsIScriptableInputStream; + +function TestStreamListener() { + this.state = "initial"; +} +TestStreamListener.prototype = { + onStartRequest() { + this.setState("start"); + }, + + onStopRequest() { + this.setState("stop"); + }, + + onDataAvailable(request, inputStream, offset, count) { + const sin = Cc["@mozilla.org/scriptableinputstream;1"].createInstance( + nsIScriptableInputStream + ); + sin.init(inputStream); + this.data = sin.read(count); + this.setState("data"); + }, + + setState(state) { + this.state = state; + if (this._deferred) { + this._deferred.resolve(state); + this._deferred = null; + } + }, + + onStateChanged() { + if (!this._deferred) { + let resolve, reject; + const promise = new Promise(function (res, rej) { + resolve = res; + reject = rej; + }); + this._deferred = { resolve, reject, promise }; + } + return this._deferred.promise; + }, +}; + +function TestChannel() { + this.state = "initial"; + this.testListener = new TestStreamListener(); + this._throttleQueue = null; +} +TestChannel.prototype = { + QueryInterface() { + return this; + }, + + get throttleQueue() { + return this._throttleQueue; + }, + + set throttleQueue(q) { + this._throttleQueue = q; + this.state = "throttled"; + }, + + setNewListener(listener) { + this.listener = listener; + this.state = "listener"; + return this.testListener; + }, +}; + +add_task(async function () { + const throttler = new NetworkThrottleManager({ + latencyMean: 1, + latencyMax: 1, + downloadBPSMean: 500, + downloadBPSMax: 500, + uploadBPSMean: 500, + uploadBPSMax: 500, + }); + + const uploadChannel = new TestChannel(); + throttler.manageUpload(uploadChannel); + equal( + uploadChannel.state, + "throttled", + "NetworkThrottleManager set throttleQueue" + ); + + const downloadChannel = new TestChannel(); + const testListener = downloadChannel.testListener; + + const listener = throttler.manage(downloadChannel); + equal( + downloadChannel.state, + "listener", + "NetworkThrottleManager called setNewListener" + ); + + equal(testListener.state, "initial", "test listener in initial state"); + + // This method must be passed through immediately. + listener.onStartRequest(null); + equal(testListener.state, "start", "test listener started"); + + const TEST_INPUT = "hi bob"; + + const testStream = Cc["@mozilla.org/storagestream;1"].createInstance( + Ci.nsIStorageStream + ); + testStream.init(512, 512); + const out = testStream.getOutputStream(0); + out.write(TEST_INPUT, TEST_INPUT.length); + out.close(); + const testInputStream = testStream.newInputStream(0); + + const activityDistributor = Cc[ + "@mozilla.org/network/http-activity-distributor;1" + ].getService(Ci.nsIHttpActivityDistributor); + let activitySeen = false; + listener.addActivityCallback( + () => { + activitySeen = true; + }, + null, + null, + null, + activityDistributor.ACTIVITY_SUBTYPE_RESPONSE_COMPLETE, + null, + TEST_INPUT.length, + null + ); + + // onDataAvailable is required to immediately read the data. + listener.onDataAvailable(null, testInputStream, 0, 6); + equal(testInputStream.available(), 0, "no more data should be available"); + equal( + testListener.state, + "start", + "test listener should not have received data" + ); + equal(activitySeen, false, "activity not distributed yet"); + + let newState = await testListener.onStateChanged(); + equal(newState, "data", "test listener received data"); + equal(testListener.data, TEST_INPUT, "test listener received all the data"); + equal(activitySeen, true, "activity has been distributed"); + + const onChange = testListener.onStateChanged(); + listener.onStopRequest(null, null); + newState = await onChange; + equal(newState, "stop", "onStateChanged reported"); +}); diff --git a/devtools/shared/network-observer/test/xpcshell/xpcshell.ini b/devtools/shared/network-observer/test/xpcshell/xpcshell.ini new file mode 100644 index 0000000000..3f5c1b674a --- /dev/null +++ b/devtools/shared/network-observer/test/xpcshell/xpcshell.ini @@ -0,0 +1,15 @@ +[DEFAULT] +tags = devtools +head = head.js +firefox-appdir = browser +skip-if = toolkit == 'android' +support-files = + +[test_network_helper.js] +[test_security-info-certificate.js] +[test_security-info-parser.js] +[test_security-info-protocol-version.js] +[test_security-info-state.js] +[test_security-info-static-hpkp.js] +[test_security-info-weakness-reasons.js] +[test_throttle.js] |