diff options
Diffstat (limited to 'devtools/client/netmonitor/src/utils/request-utils.js')
| -rw-r--r-- | devtools/client/netmonitor/src/utils/request-utils.js | 769 |
1 files changed, 769 insertions, 0 deletions
diff --git a/devtools/client/netmonitor/src/utils/request-utils.js b/devtools/client/netmonitor/src/utils/request-utils.js new file mode 100644 index 0000000000..11273016be --- /dev/null +++ b/devtools/client/netmonitor/src/utils/request-utils.js @@ -0,0 +1,769 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +"use strict"; + +const { + getUnicodeUrl, + getUnicodeUrlPath, + getUnicodeHostname, +} = require("resource://devtools/client/shared/unicode-url.js"); + +const { + UPDATE_PROPS, +} = require("resource://devtools/client/netmonitor/src/constants.js"); + +const CONTENT_MIME_TYPE_ABBREVIATIONS = { + ecmascript: "js", + javascript: "js", + "x-javascript": "js", +}; + +/** + * Extracts any urlencoded form data sections (e.g. "?foo=bar&baz=42") from a + * POST request. + * + * @param {object} headers - the "requestHeaders". + * @param {object} uploadHeaders - the "requestHeadersFromUploadStream". + * @param {object} postData - the "requestPostData". + * @return {array} a promise list that is resolved with the extracted form data. + */ +async function getFormDataSections( + headers, + uploadHeaders, + postData, + getLongString +) { + const formDataSections = []; + + const requestHeaders = headers.headers; + const payloadHeaders = uploadHeaders ? uploadHeaders.headers : []; + const allHeaders = [...payloadHeaders, ...requestHeaders]; + + const contentTypeHeader = allHeaders.find(e => { + return e.name.toLowerCase() == "content-type"; + }); + + const contentTypeLongString = contentTypeHeader + ? contentTypeHeader.value + : ""; + + const contentType = await getLongString(contentTypeLongString); + + if (contentType && contentType.includes("x-www-form-urlencoded")) { + const postDataLongString = postData.postData.text; + const text = await getLongString(postDataLongString); + + for (const section of text.trim().split(/\r\n|\r|\n/)) { + // Before displaying it, make sure this section of the POST data + // isn't a line containing upload stream headers. + if (payloadHeaders.every(header => !section.startsWith(header.name))) { + formDataSections.push(section); + } + } + } + + return formDataSections; +} + +/** + * Fetch headers full content from actor server + * + * @param {object} headers - a object presents headers data + * @return {object} a headers object with updated content payload + */ +async function fetchHeaders(headers, getLongString) { + for (const { value } of headers.headers) { + headers.headers.value = await getLongString(value); + } + return headers; +} + +/** + * Fetch network event update packets from actor server + * Expect to fetch a couple of network update packets from a given request. + * + * @param {function} requestData - requestData function for lazily fetch data + * @param {object} request - request object + * @param {array} updateTypes - a list of network event update types + */ +function fetchNetworkUpdatePacket(requestData, request, updateTypes) { + const promises = []; + if (request) { + updateTypes.forEach(updateType => { + // Only stackTrace will be handled differently + if (updateType === "stackTrace") { + if (request.cause.stacktraceAvailable && !request.stacktrace) { + promises.push(requestData(request.id, updateType)); + } + return; + } + + if (request[`${updateType}Available`] && !request[updateType]) { + promises.push(requestData(request.id, updateType)); + } + }); + } + + return Promise.all(promises); +} + +/** + * Form a data: URI given a mime type, encoding, and some text. + * + * @param {string} mimeType - mime type + * @param {string} encoding - encoding to use; if not set, the + * text will be base64-encoded. + * @param {string} text - text of the URI. + * @return {string} a data URI + */ +function formDataURI(mimeType, encoding, text) { + if (!encoding) { + encoding = "base64"; + text = btoa(unescape(encodeURIComponent(text))); + } + return "data:" + mimeType + ";" + encoding + "," + text; +} + +/** + * Write out a list of headers into a chunk of text + * + * @param {array} headers - array of headers info { name, value } + * @param {string} preHeaderText - first line of the headers request/response + * @return {string} list of headers in text format + */ +function writeHeaderText(headers, preHeaderText) { + let result = ""; + if (preHeaderText) { + result += preHeaderText + "\r\n"; + } + result += headers.map(({ name, value }) => name + ": " + value).join("\r\n"); + result += "\r\n\r\n"; + return result; +} + +/** + * Decode base64 string. + * + * @param {string} url - a string + * @return {string} decoded string + */ +function decodeUnicodeBase64(string) { + try { + return decodeURIComponent(atob(string)); + } catch (err) { + // Ignore error and return input string directly. + } + return string; +} + +/** + * Helper for getting an abbreviated string for a mime type. + * + * @param {string} mimeType - mime type + * @return {string} abbreviated mime type + */ +function getAbbreviatedMimeType(mimeType) { + if (!mimeType) { + return ""; + } + const abbrevType = (mimeType.split(";")[0].split("/")[1] || "").split("+")[0]; + return CONTENT_MIME_TYPE_ABBREVIATIONS[abbrevType] || abbrevType; +} + +/** + * Helpers for getting a filename from a mime type. + * + * @param {string} baseNameWithQuery - unicode basename and query of a url + * @return {string} unicode filename portion of a url + */ +function getFileName(baseNameWithQuery) { + const basename = baseNameWithQuery && baseNameWithQuery.split("?")[0]; + return basename && basename.includes(".") ? basename : null; +} + +/** + * Helpers for retrieving a URL object from a string + * + * @param {string} url - unvalidated url string + * @return {URL} The URL object + */ +function getUrl(url) { + try { + return new URL(url); + } catch (err) { + return null; + } +} + +/** + * Helpers for retrieving the value of a URL object property + * + * @param {string} input - unvalidated url string + * @param {string} string - desired property in the URL object + * @return {string} unicode query of a url + */ +function getUrlProperty(input, property) { + const url = getUrl(input); + return url?.[property] ? url[property] : ""; +} + +/** + * Helpers for getting the last portion of a url. + * For example helper returns "basename" from http://domain.com/path/basename + * If basename portion is empty, it returns the url pathname. + * + * @param {string} input - unvalidated url string + * @return {string} unicode basename of a url + */ +function getUrlBaseName(url) { + const pathname = getUrlProperty(url, "pathname"); + return getUnicodeUrlPath(pathname.replace(/\S*\//, "") || pathname || "/"); +} + +/** + * Helpers for getting the query portion of a url. + * + * @param {string} url - unvalidated url string + * @return {string} unicode query of a url + */ +function getUrlQuery(url) { + return getUrlProperty(url, "search").replace(/^\?/, ""); +} + +/** + * Helpers for getting unicode name and query portions of a url. + * + * @param {string} url - unvalidated url string + * @return {string} unicode basename and query portions of a url + */ +function getUrlBaseNameWithQuery(url) { + const basename = getUrlBaseName(url); + const search = getUrlProperty(url, "search"); + return basename + getUnicodeUrlPath(search); +} + +/** + * Helpers for getting hostname portion of an URL. + * + * @param {string} url - unvalidated url string + * @return {string} unicode hostname of a url + */ +function getUrlHostName(url) { + return getUrlProperty(url, "hostname"); +} + +/** + * Helpers for getting host portion of an URL. + * + * @param {string} url - unvalidated url string + * @return {string} unicode host of a url + */ +function getUrlHost(url) { + return getUrlProperty(url, "host"); +} + +/** + * Helpers for getting the shceme portion of a url. + * For example helper returns "http" from http://domain.com/path/basename + * + * @param {string} url - unvalidated url string + * @return {string} string scheme of a url + */ +function getUrlScheme(url) { + const protocol = getUrlProperty(url, "protocol"); + return protocol.replace(":", "").toLowerCase(); +} + +/** + * Extract several details fields from a URL at once. + */ +function getUrlDetails(url) { + const baseNameWithQuery = getUrlBaseNameWithQuery(url); + let host = getUrlHost(url); + const hostname = getUrlHostName(url); + const unicodeUrl = getUnicodeUrl(url); + const scheme = getUrlScheme(url); + + // If the hostname contains unreadable ASCII characters, we need to do the + // following two steps: + // 1. Converting the unreadable hostname to a readable Unicode domain name. + // For example, converting xn--g6w.xn--8pv into a Unicode domain name. + // 2. Replacing the unreadable hostname portion in the `host` with the + // readable hostname. + // For example, replacing xn--g6w.xn--8pv:8000 with [Unicode domain]:8000 + // After finishing the two steps, we get a readable `host`. + const unicodeHostname = getUnicodeHostname(hostname); + if (unicodeHostname !== hostname) { + host = host.replace(hostname, unicodeHostname); + } + + // Mark local hosts specially, where "local" is as defined in the W3C + // spec for secure contexts. + // http://www.w3.org/TR/powerful-features/ + // + // * If the name falls under 'localhost' + // * If the name is an IPv4 address within 127.0.0.0/8 + // * If the name is an IPv6 address within ::1/128 + // + // IPv6 parsing is a little sloppy; it assumes that the address has + // been validated before it gets here. + const isLocal = + hostname.match(/(.+\.)?localhost$/) || + hostname.match(/^127\.\d{1,3}\.\d{1,3}\.\d{1,3}/) || + hostname.match(/\[[0:]+1\]/); + + return { + baseNameWithQuery, + host, + scheme, + unicodeUrl, + isLocal, + url, + }; +} + +/** + * Parse a url's query string into its components + * + * @param {string} query - query string of a url portion + * @return {array} array of query params { name, value } + */ +function parseQueryString(query) { + if (!query) { + return null; + } + return query + .replace(/^[?&]/, "") + .split("&") + .map(e => { + const param = e.split("="); + return { + name: param[0] ? getUnicodeUrlPath(param[0].replace(/\+/g, " ")) : "", + value: param[1] + ? getUnicodeUrlPath(param.slice(1).join("=").replace(/\+/g, " ")) + : "", + }; + }); +} + +/** + * Parse a string of formdata sections into its components + * + * @param {string} sections - sections of formdata joined by & + * @return {array} array of formdata params { name, value } + */ +function parseFormData(sections) { + if (!sections) { + return []; + } + + return sections + .replace(/^&/, "") + .split("&") + .map(e => { + const param = e.split("="); + return { + name: param[0] ? getUnicodeUrlPath(param[0]) : "", + value: param[1] ? getUnicodeUrlPath(param[1]) : "", + }; + }); +} + +/** + * Reduces an IP address into a number for easier sorting + * + * @param {string} ip - IP address to reduce + * @return {number} the number representing the IP address + */ +function ipToLong(ip) { + if (!ip) { + // Invalid IP + return -1; + } + + let base; + let octets = ip.split("."); + + if (octets.length === 4) { + // IPv4 + base = 10; + } else if (ip.includes(":")) { + // IPv6 + const numberOfZeroSections = + 8 - ip.replace(/^:+|:+$/g, "").split(/:+/g).length; + octets = ip + .replace("::", `:${"0:".repeat(numberOfZeroSections)}`) + .replace(/^:|:$/g, "") + .split(":"); + base = 16; + } else { + // Invalid IP + return -1; + } + return octets + .map((val, ix, arr) => { + return parseInt(val, base) * Math.pow(256, arr.length - 1 - ix); + }) + .reduce((sum, val) => { + return sum + val; + }, 0); +} + +/** + * Compare two objects on a subset of their properties + */ +function propertiesEqual(props, item1, item2) { + return item1 === item2 || props.every(p => item1[p] === item2[p]); +} + +/** + * Calculate the start time of a request, which is the time from start + * of 1st request until the start of this request. + * + * Without a firstRequestStartedMs argument the wrong time will be returned. + * However, it can be omitted when comparing two start times and neither supplies + * a firstRequestStartedMs. + */ +function getStartTime(item, firstRequestStartedMs = 0) { + return item.startedMs - firstRequestStartedMs; +} + +/** + * Calculate the end time of a request, which is the time from start + * of 1st request until the end of this response. + * + * Without a firstRequestStartedMs argument the wrong time will be returned. + * However, it can be omitted when comparing two end times and neither supplies + * a firstRequestStartedMs. + */ +function getEndTime(item, firstRequestStartedMs = 0) { + const { startedMs, totalTime } = item; + return startedMs + totalTime - firstRequestStartedMs; +} + +/** + * Calculate the response time of a request, which is the time from start + * of 1st request until the beginning of download of this response. + * + * Without a firstRequestStartedMs argument the wrong time will be returned. + * However, it can be omitted when comparing two response times and neither supplies + * a firstRequestStartedMs. + */ +function getResponseTime(item, firstRequestStartedMs = 0) { + const { startedMs, totalTime, eventTimings = { timings: {} } } = item; + return ( + startedMs + totalTime - firstRequestStartedMs - eventTimings.timings.receive + ); +} + +/** + * Format the protocols used by the request. + */ +function getFormattedProtocol(item) { + const { httpVersion = "", responseHeaders = { headers: [] } } = item; + const protocol = [httpVersion]; + responseHeaders.headers.some(h => { + if (h.hasOwnProperty("name") && h.name.toLowerCase() === "x-firefox-spdy") { + /** + * First we make sure h.value is defined and not an empty string. + * Then check that HTTP version and x-firefox-spdy == "http/1.1". + * If not, check that HTTP version and x-firefox-spdy have the same + * numeric value when of the forms "http/<x>" and "h<x>" respectively. + * If not, will push to protocol the non-standard x-firefox-spdy value. + * + * @see https://bugzilla.mozilla.org/show_bug.cgi?id=1501357 + */ + if (h.value !== undefined && h.value.length) { + if ( + h.value.toLowerCase() !== "http/1.1" || + protocol[0].toLowerCase() !== "http/1.1" + ) { + if ( + parseFloat(h.value.toLowerCase().split("")[1]) !== + parseFloat(protocol[0].toLowerCase().split("/")[1]) + ) { + protocol.push(h.value); + return true; + } + } + } + } + return false; + }); + return protocol.join("+"); +} + +/** + * Get the value of a particular response header, or null if not + * present. + */ +function getResponseHeader(item, header) { + const { responseHeaders } = item; + if (!responseHeaders || !responseHeaders.headers.length) { + return null; + } + header = header.toLowerCase(); + for (const responseHeader of responseHeaders.headers) { + if (responseHeader.name.toLowerCase() == header) { + return responseHeader.value; + } + } + return null; +} + +/** + * Get the value of a particular request header, or null if not + * present. + */ +function getRequestHeader(item, header) { + const { requestHeaders } = item; + if (!requestHeaders || !requestHeaders.headers.length) { + return null; + } + header = header.toLowerCase(); + for (const requestHeader of requestHeaders.headers) { + if (requestHeader.name.toLowerCase() == header) { + return requestHeader.value; + } + } + return null; +} + +/** + * Extracts any urlencoded form data sections from a POST request. + */ +async function updateFormDataSections(props) { + const { connector, request = {}, updateRequest } = props; + let { + id, + formDataSections, + requestHeaders, + requestHeadersAvailable, + requestHeadersFromUploadStream, + requestPostData, + requestPostDataAvailable, + } = request; + + if (requestHeadersAvailable && !requestHeaders) { + requestHeaders = await connector.requestData(id, "requestHeaders"); + } + + if (requestPostDataAvailable && !requestPostData) { + requestPostData = await connector.requestData(id, "requestPostData"); + } + + if ( + !formDataSections && + requestHeaders && + requestPostData && + requestHeadersFromUploadStream + ) { + formDataSections = await getFormDataSections( + requestHeaders, + requestHeadersFromUploadStream, + requestPostData, + connector.getLongString + ); + + updateRequest(request.id, { formDataSections }, true); + } +} + +/** + * This helper function helps to resolve the full payload of a message + * that is wrapped in a LongStringActor object. + */ +async function getMessagePayload(payload, getLongString) { + const result = await getLongString(payload); + return result; +} + +/** + * This helper function is used for additional processing of + * incoming network update packets. It makes sure the only valid + * update properties and the values are correct. + * It's used by Network and Console panel reducers. + * @param {object} update + * The new update payload + * @param {object} request + * The current request in the state + */ +function processNetworkUpdates(update) { + const newRequest = {}; + for (const [key, value] of Object.entries(update)) { + if (UPDATE_PROPS.includes(key)) { + newRequest[key] = value; + if (key == "requestPostData") { + newRequest.requestHeadersFromUploadStream = value.uploadHeaders; + } + } + } + return newRequest; +} + +/** + * This method checks that the response is base64 encoded by + * comparing these 2 values: + * 1. The original response + * 2. The value of doing a base64 decode on the + * response and then base64 encoding the result. + * If the values are different or an error is thrown, + * the method will return false. + */ +function isBase64(payload) { + try { + return btoa(atob(payload)) == payload; + } catch (err) { + return false; + } +} + +/** + * Checks if the payload is of JSON type. + * This function also handles JSON with XSSI-escaping characters by stripping them + * and returning the stripped chars in the strippedChars property + * This function also handles Base64 encoded JSON. + * @returns {Object} shape: + * {Object} json: parsed JSON object + * {Error} error: JSON parsing error + * {string} strippedChars: XSSI stripped chars removed from JSON payload + */ +function parseJSON(payloadUnclean) { + let json; + const jsonpRegex = /^\s*([\w$]+)\s*\(\s*([^]*)\s*\)\s*;?\s*$/; + const [, jsonpCallback, jsonp] = payloadUnclean.match(jsonpRegex) || []; + if (jsonpCallback && jsonp) { + let error; + try { + json = parseJSON(jsonp).json; + } catch (err) { + error = err; + } + return { json, error, jsonpCallback }; + } + + let { payload, strippedChars, error } = removeXSSIString(payloadUnclean); + + try { + json = JSON.parse(payload); + } catch (err) { + if (isBase64(payload)) { + try { + json = JSON.parse(atob(payload)); + } catch (err64) { + error = err64; + } + } else { + error = err; + } + } + + // Do not present JSON primitives (e.g. boolean, strings in quotes, numbers) + // as JSON expandable tree. + if (!error) { + if (typeof json !== "object") { + return {}; + } + } + return { + json, + error, + strippedChars, + }; +} + +/** + * Removes XSSI prevention sequences from JSON payloads + * @param {string} payloadUnclean: JSON payload that may or may have a + * XSSI prevention sequence + * @returns {Object} Shape: + * {string} payload: the JSON witht the XSSI prevention sequence removed + * {string} strippedChars: XSSI string that was removed, null if no XSSI + * prevention sequence was found + * {Error} error: error attempting to strip XSSI prevention sequence + */ +function removeXSSIString(payloadUnclean) { + // Regex that finds the XSSI protection sequences )]}'\n for(;;); and while(1); + const xssiRegex = /(^\)\]\}',?\n)|(^for ?\(;;\);?)|(^while ?\(1\);?)/; + let payload, strippedChars, error; + const xssiRegexMatch = payloadUnclean.match(xssiRegex); + + // Remove XSSI string if there was one found + if (xssiRegexMatch?.length > 0) { + const xssiLen = xssiRegexMatch[0].length; + try { + // substring the payload by the length of the XSSI match to remove it + // and save the match to report + payload = payloadUnclean.substring(xssiLen); + strippedChars = xssiRegexMatch[0]; + } catch (err) { + error = err; + payload = payloadUnclean; + } + } else { + // if there was no XSSI match just return the raw payload + payload = payloadUnclean; + } + return { + payload, + strippedChars, + error, + }; +} + +/** + * Computes the request headers of an HTTP request + * + * @param {string} method: request method + * @param {string} httpVersion: request http version + * @param {object} requestHeaders: request headers + * @param {object} urlDetails: request url details + * + * @return {string} the request headers + */ +function getRequestHeadersRawText( + method, + httpVersion, + requestHeaders, + urlDetails +) { + const url = new URL(urlDetails.url); + const path = url ? `${url.pathname}${url.search}` : "<unknown>"; + const preHeaderText = `${method} ${path} ${httpVersion}`; + return writeHeaderText(requestHeaders.headers, preHeaderText).trim(); +} + +module.exports = { + decodeUnicodeBase64, + getFormDataSections, + fetchHeaders, + fetchNetworkUpdatePacket, + formDataURI, + writeHeaderText, + getAbbreviatedMimeType, + getFileName, + getEndTime, + getFormattedProtocol, + getMessagePayload, + getRequestHeader, + getResponseHeader, + getResponseTime, + getStartTime, + getUrlBaseName, + getUrlBaseNameWithQuery, + getUrlDetails, + getUrlHost, + getUrlHostName, + getUrlQuery, + getUrlScheme, + parseQueryString, + parseFormData, + updateFormDataSections, + processNetworkUpdates, + propertiesEqual, + ipToLong, + parseJSON, + getRequestHeadersRawText, +}; |