summaryrefslogtreecommitdiffstats
path: root/taskcluster/docker/periodic-updates/scripts/getHSTSPreloadList.js
diff options
context:
space:
mode:
authorDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-07 17:32:43 +0000
committerDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-07 17:32:43 +0000
commit6bf0a5cb5034a7e684dcc3500e841785237ce2dd (patch)
treea68f146d7fa01f0134297619fbe7e33db084e0aa /taskcluster/docker/periodic-updates/scripts/getHSTSPreloadList.js
parentInitial commit. (diff)
downloadthunderbird-6bf0a5cb5034a7e684dcc3500e841785237ce2dd.tar.xz
thunderbird-6bf0a5cb5034a7e684dcc3500e841785237ce2dd.zip
Adding upstream version 1:115.7.0.upstream/1%115.7.0upstream
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'taskcluster/docker/periodic-updates/scripts/getHSTSPreloadList.js')
-rw-r--r--taskcluster/docker/periodic-updates/scripts/getHSTSPreloadList.js557
1 files changed, 557 insertions, 0 deletions
diff --git a/taskcluster/docker/periodic-updates/scripts/getHSTSPreloadList.js b/taskcluster/docker/periodic-updates/scripts/getHSTSPreloadList.js
new file mode 100644
index 0000000000..aeaa29bc2d
--- /dev/null
+++ b/taskcluster/docker/periodic-updates/scripts/getHSTSPreloadList.js
@@ -0,0 +1,557 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed with this file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+"use strict";
+
+// How to run this file:
+// 1. [obtain firefox source code]
+// 2. [build/obtain firefox binaries]
+// 3. run `[path to]/run-mozilla.sh [path to]/xpcshell [path to]/getHSTSPreloadlist.js [absolute path to]/nsSTSPreloadlist.inc'
+// Note: Running this file outputs a new nsSTSPreloadlist.inc in the current
+// working directory.
+
+var gSSService = Cc["@mozilla.org/ssservice;1"].getService(
+ Ci.nsISiteSecurityService
+);
+
+const { FileUtils } = ChromeUtils.importESModule(
+ "resource://gre/modules/FileUtils.sys.mjs"
+);
+
+const SOURCE =
+ "https://chromium.googlesource.com/chromium/src/+/refs/heads/main/net/http/transport_security_state_static.json?format=TEXT";
+const TOOL_SOURCE =
+ "https://hg.mozilla.org/mozilla-central/file/default/taskcluster/docker/periodic-updates/scripts/getHSTSPreloadList.js";
+const OUTPUT = "nsSTSPreloadList.inc";
+const MINIMUM_REQUIRED_MAX_AGE = 60 * 60 * 24 * 7 * 18;
+const MAX_CONCURRENT_REQUESTS = 500;
+const MAX_RETRIES = 1;
+const REQUEST_TIMEOUT = 30 * 1000;
+const ERROR_NONE = "no error";
+const ERROR_CONNECTING_TO_HOST = "could not connect to host";
+const ERROR_NO_HSTS_HEADER = "did not receive HSTS header";
+const ERROR_MAX_AGE_TOO_LOW = "max-age too low: ";
+const HEADER = `/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/*****************************************************************************/
+/* This is an automatically generated file. If you're not */
+/* nsSiteSecurityService.cpp, you shouldn't be #including it. */
+/*****************************************************************************/
+
+#include <stdint.h>
+`;
+
+const GPERF_DELIM = "%%\n";
+
+function download() {
+ let req = new XMLHttpRequest();
+ req.open("GET", SOURCE, false); // doing the request synchronously
+ try {
+ req.send();
+ } catch (e) {
+ throw new Error(`ERROR: problem downloading '${SOURCE}': ${e}`);
+ }
+
+ if (req.status != 200) {
+ throw new Error(
+ "ERROR: problem downloading '" + SOURCE + "': status " + req.status
+ );
+ }
+
+ let resultDecoded;
+ try {
+ resultDecoded = atob(req.responseText);
+ } catch (e) {
+ throw new Error(
+ "ERROR: could not decode data as base64 from '" + SOURCE + "': " + e
+ );
+ }
+
+ // we have to filter out '//' comments, while not mangling the json
+ let result = resultDecoded.replace(/^(\s*)?\/\/[^\n]*\n/gm, "");
+ let data = null;
+ try {
+ data = JSON.parse(result);
+ } catch (e) {
+ throw new Error(`ERROR: could not parse data from '${SOURCE}': ${e}`);
+ }
+ return data;
+}
+
+function getHosts(rawdata) {
+ let hosts = [];
+
+ if (!rawdata || !rawdata.entries) {
+ throw new Error(
+ "ERROR: source data not formatted correctly: 'entries' not found"
+ );
+ }
+
+ for (let entry of rawdata.entries) {
+ if (entry.mode && entry.mode == "force-https") {
+ if (entry.name) {
+ // We trim the entry name here to avoid malformed URI exceptions when we
+ // later try to connect to the domain.
+ entry.name = entry.name.trim();
+ entry.retries = MAX_RETRIES;
+ // We prefer the camelCase variable to the JSON's snake case version
+ entry.includeSubdomains = entry.include_subdomains;
+ hosts.push(entry);
+ } else {
+ throw new Error("ERROR: entry not formatted correctly: no name found");
+ }
+ }
+ }
+
+ return hosts;
+}
+
+function processStsHeader(host, header, status, securityInfo) {
+ let maxAge = {
+ value: 0,
+ };
+ let includeSubdomains = {
+ value: false,
+ };
+ let error = ERROR_NONE;
+ if (
+ header != null &&
+ securityInfo != null &&
+ securityInfo.overridableErrorCategory ==
+ Ci.nsITransportSecurityInfo.ERROR_UNSET
+ ) {
+ try {
+ let uri = Services.io.newURI("https://" + host.name);
+ gSSService.processHeader(uri, header, {}, maxAge, includeSubdomains);
+ } catch (e) {
+ dump(
+ "ERROR: could not process header '" +
+ header +
+ "' from " +
+ host.name +
+ ": " +
+ e +
+ "\n"
+ );
+ error = e;
+ }
+ } else if (status == 0) {
+ error = ERROR_CONNECTING_TO_HOST;
+ } else {
+ error = ERROR_NO_HSTS_HEADER;
+ }
+
+ if (error == ERROR_NONE && maxAge.value < MINIMUM_REQUIRED_MAX_AGE) {
+ error = ERROR_MAX_AGE_TOO_LOW;
+ }
+
+ return {
+ name: host.name,
+ maxAge: maxAge.value,
+ includeSubdomains: includeSubdomains.value,
+ error,
+ retries: host.retries - 1,
+ forceInclude: host.forceInclude,
+ };
+}
+
+// RedirectAndAuthStopper prevents redirects and HTTP authentication
+function RedirectAndAuthStopper() {}
+
+RedirectAndAuthStopper.prototype = {
+ // nsIChannelEventSink
+ asyncOnChannelRedirect(oldChannel, newChannel, flags, callback) {
+ throw Components.Exception("", Cr.NS_ERROR_ENTITY_CHANGED);
+ },
+
+ // nsIAuthPrompt2
+ promptAuth(channel, level, authInfo) {
+ return false;
+ },
+
+ asyncPromptAuth(channel, callback, context, level, authInfo) {
+ throw Components.Exception("", Cr.NS_ERROR_NOT_IMPLEMENTED);
+ },
+
+ getInterface(iid) {
+ return this.QueryInterface(iid);
+ },
+
+ QueryInterface: ChromeUtils.generateQI([
+ "nsIChannelEventSink",
+ "nsIAuthPrompt2",
+ ]),
+};
+
+function fetchstatus(host) {
+ return new Promise((resolve, reject) => {
+ let xhr = new XMLHttpRequest();
+ let uri = "https://" + host.name + "/";
+
+ xhr.open("head", uri, true);
+ xhr.setRequestHeader("X-Automated-Tool", TOOL_SOURCE);
+ xhr.timeout = REQUEST_TIMEOUT;
+
+ let errorHandler = () => {
+ dump("ERROR: exception making request to " + host.name + "\n");
+ resolve(
+ processStsHeader(
+ host,
+ null,
+ xhr.status,
+ xhr.channel && xhr.channel.securityInfo
+ )
+ );
+ };
+
+ xhr.onerror = errorHandler;
+ xhr.ontimeout = errorHandler;
+ xhr.onabort = errorHandler;
+
+ xhr.onload = () => {
+ let header = xhr.getResponseHeader("strict-transport-security");
+ resolve(
+ processStsHeader(host, header, xhr.status, xhr.channel.securityInfo)
+ );
+ };
+
+ xhr.channel.notificationCallbacks = new RedirectAndAuthStopper();
+ xhr.send();
+ });
+}
+
+async function getHSTSStatus(host) {
+ do {
+ host = await fetchstatus(host);
+ } while (shouldRetry(host));
+ return host;
+}
+
+function compareHSTSStatus(a, b) {
+ if (a.name > b.name) {
+ return 1;
+ }
+ if (a.name < b.name) {
+ return -1;
+ }
+ return 0;
+}
+
+function writeTo(string, fos) {
+ fos.write(string, string.length);
+}
+
+// Determines and returns a string representing a declaration of when this
+// preload list should no longer be used.
+// This is the current time plus MINIMUM_REQUIRED_MAX_AGE.
+function getExpirationTimeString() {
+ let now = new Date();
+ let nowMillis = now.getTime();
+ // MINIMUM_REQUIRED_MAX_AGE is in seconds, so convert to milliseconds
+ let expirationMillis = nowMillis + MINIMUM_REQUIRED_MAX_AGE * 1000;
+ let expirationMicros = expirationMillis * 1000;
+ return (
+ "const PRTime gPreloadListExpirationTime = INT64_C(" +
+ expirationMicros +
+ ");\n"
+ );
+}
+
+function shouldRetry(response) {
+ return (
+ response.error != ERROR_NO_HSTS_HEADER &&
+ response.error != ERROR_MAX_AGE_TOO_LOW &&
+ response.error != ERROR_NONE &&
+ response.retries > 0
+ );
+}
+
+// Copied from browser/components/migration/MigrationUtils.sys.mjs
+function spinResolve(promise) {
+ if (!(promise instanceof Promise)) {
+ return promise;
+ }
+ let done = false;
+ let result = null;
+ let error = null;
+ promise
+ .catch(e => {
+ error = e;
+ })
+ .then(r => {
+ result = r;
+ done = true;
+ });
+
+ Services.tm.spinEventLoopUntil(
+ "getHSTSPreloadList.js:spinResolve",
+ () => done
+ );
+ if (error) {
+ throw error;
+ } else {
+ return result;
+ }
+}
+
+async function probeHSTSStatuses(inHosts) {
+ let totalLength = inHosts.length;
+ dump("Examining " + totalLength + " hosts.\n");
+
+ // Make requests in batches of MAX_CONCURRENT_REQUESTS. Otherwise, we have
+ // too many in-flight requests and the time it takes to process them causes
+ // them all to time out.
+ let allResults = [];
+ while (inHosts.length) {
+ let promises = [];
+ for (let i = 0; i < MAX_CONCURRENT_REQUESTS && inHosts.length; i++) {
+ let host = inHosts.shift();
+ promises.push(getHSTSStatus(host));
+ }
+ let results = await Promise.all(promises);
+ let progress = (
+ (100 * (totalLength - inHosts.length)) /
+ totalLength
+ ).toFixed(2);
+ dump(progress + "% done\n");
+ allResults = allResults.concat(results);
+ }
+
+ dump("HSTS Probe received " + allResults.length + " statuses.\n");
+ return allResults;
+}
+
+function readCurrentList(filename) {
+ var currentHosts = {};
+ var file = Cc["@mozilla.org/file/local;1"].createInstance(Ci.nsIFile);
+ file.initWithPath(filename);
+ var fis = Cc["@mozilla.org/network/file-input-stream;1"].createInstance(
+ Ci.nsILineInputStream
+ );
+ fis.init(file, -1, -1, Ci.nsIFileInputStream.CLOSE_ON_EOF);
+ var line = {};
+
+ // While we generate entries matching the latest version format,
+ // we still need to be able to read entries in the previous version formats
+ // for bootstrapping a latest version preload list from a previous version
+ // preload list. Hence these regexes.
+ const entryRegexes = [
+ /([^,]+), (0|1)/, // v3
+ / {2}\/\* "([^"]*)", (true|false) \*\//, // v2
+ / {2}{ "([^"]*)", (true|false) },/, // v1
+ ];
+
+ while (fis.readLine(line)) {
+ let match;
+ entryRegexes.find(r => {
+ match = r.exec(line.value);
+ return match;
+ });
+ if (match) {
+ currentHosts[match[1]] = match[2] == "1" || match[2] == "true";
+ }
+ }
+ return currentHosts;
+}
+
+function combineLists(newHosts, currentHosts) {
+ let newHostsSet = new Set();
+
+ for (let newHost of newHosts) {
+ newHostsSet.add(newHost.name);
+ }
+
+ for (let currentHost in currentHosts) {
+ if (!newHostsSet.has(currentHost)) {
+ newHosts.push({ name: currentHost, retries: MAX_RETRIES });
+ }
+ }
+}
+
+const TEST_ENTRIES = [
+ {
+ name: "includesubdomains.preloaded.test",
+ includeSubdomains: true,
+ },
+ {
+ name: "includesubdomains2.preloaded.test",
+ includeSubdomains: true,
+ },
+ {
+ name: "noincludesubdomains.preloaded.test",
+ includeSubdomains: false,
+ },
+];
+
+function deleteTestHosts(currentHosts) {
+ for (let testEntry of TEST_ENTRIES) {
+ delete currentHosts[testEntry.name];
+ }
+}
+
+function getTestHosts() {
+ let hosts = [];
+ for (let testEntry of TEST_ENTRIES) {
+ hosts.push({
+ name: testEntry.name,
+ maxAge: MINIMUM_REQUIRED_MAX_AGE,
+ includeSubdomains: testEntry.includeSubdomains,
+ error: ERROR_NONE,
+ // This deliberately doesn't have a value for `retries` (because we should
+ // never attempt to connect to this host).
+ forceInclude: true,
+ });
+ }
+ return hosts;
+}
+
+async function insertHosts(inoutHostList, inAddedHosts) {
+ for (let host of inAddedHosts) {
+ inoutHostList.push(host);
+ }
+}
+
+function filterForcedInclusions(inHosts, outNotForced, outForced) {
+ // Apply our filters (based on policy today) to determine which entries
+ // will be included without being checked (forced); the others will be
+ // checked using active probing.
+ for (let host of inHosts) {
+ if (
+ host.policy == "google" ||
+ host.policy == "public-suffix" ||
+ host.policy == "public-suffix-requested"
+ ) {
+ host.forceInclude = true;
+ host.error = ERROR_NONE;
+ outForced.push(host);
+ } else {
+ outNotForced.push(host);
+ }
+ }
+}
+
+function output(statuses) {
+ dump("INFO: Writing output to " + OUTPUT + "\n");
+ try {
+ let file = new FileUtils.File(
+ PathUtils.join(Services.dirsvc.get("CurWorkD", Ci.nsIFile).path, OUTPUT)
+ );
+ let fos = FileUtils.openSafeFileOutputStream(file);
+ writeTo(HEADER, fos);
+ writeTo(getExpirationTimeString(), fos);
+
+ writeTo(GPERF_DELIM, fos);
+
+ for (let status of statuses) {
+ let includeSubdomains = status.includeSubdomains ? 1 : 0;
+ writeTo(status.name + ", " + includeSubdomains + "\n", fos);
+ }
+
+ writeTo(GPERF_DELIM, fos);
+ FileUtils.closeSafeFileOutputStream(fos);
+ dump("finished writing output file\n");
+ } catch (e) {
+ dump("ERROR: problem writing output to '" + OUTPUT + "': " + e + "\n");
+ throw e;
+ }
+}
+
+function errorToString(status) {
+ return status.error == ERROR_MAX_AGE_TOO_LOW
+ ? status.error + status.maxAge
+ : status.error;
+}
+
+async function main(args) {
+ if (args.length != 1) {
+ throw new Error(
+ "Usage: getHSTSPreloadList.js <absolute path to current nsSTSPreloadList.inc>"
+ );
+ }
+
+ // get the current preload list
+ let currentHosts = readCurrentList(args[0]);
+ // delete any hosts we use in tests so we don't actually connect to them
+ deleteTestHosts(currentHosts);
+ // disable the current preload list so it won't interfere with requests we make
+ Services.prefs.setBoolPref(
+ "network.stricttransportsecurity.preloadlist",
+ false
+ );
+ // download and parse the raw json file from the Chromium source
+ let rawdata = download();
+ // get just the hosts with mode: "force-https"
+ let hosts = getHosts(rawdata);
+ // add hosts in the current list to the new list (avoiding duplicates)
+ combineLists(hosts, currentHosts);
+
+ // Don't contact hosts that are forced to be included anyway
+ let hostsToContact = [];
+ let forcedHosts = [];
+ filterForcedInclusions(hosts, hostsToContact, forcedHosts);
+
+ // Initialize the final status list
+ let hstsStatuses = [];
+ // Add the hosts we use in tests
+ dump("Adding test hosts\n");
+ insertHosts(hstsStatuses, getTestHosts());
+ // Add in the hosts that are forced
+ dump("Adding forced hosts\n");
+ insertHosts(hstsStatuses, forcedHosts);
+
+ let total = await probeHSTSStatuses(hostsToContact)
+ .then(function (probedStatuses) {
+ return hstsStatuses.concat(probedStatuses);
+ })
+ .then(function (statuses) {
+ return statuses.sort(compareHSTSStatus);
+ })
+ .then(function (statuses) {
+ for (let status of statuses) {
+ // If we've encountered an error for this entry (other than the site not
+ // sending an HSTS header), be safe and don't remove it from the list
+ // (given that it was already on the list).
+ if (
+ !status.forceInclude &&
+ status.error != ERROR_NONE &&
+ status.error != ERROR_NO_HSTS_HEADER &&
+ status.error != ERROR_MAX_AGE_TOO_LOW &&
+ status.name in currentHosts
+ ) {
+ // dump("INFO: error connecting to or processing " + status.name + " - using previous status on list\n");
+ status.maxAge = MINIMUM_REQUIRED_MAX_AGE;
+ status.includeSubdomains = currentHosts[status.name];
+ }
+ }
+ return statuses;
+ })
+ .then(function (statuses) {
+ // Filter out entries we aren't including.
+ var includedStatuses = statuses.filter(function (status) {
+ if (status.maxAge < MINIMUM_REQUIRED_MAX_AGE && !status.forceInclude) {
+ // dump("INFO: " + status.name + " NOT ON the preload list\n");
+ return false;
+ }
+
+ // dump("INFO: " + status.name + " ON the preload list (includeSubdomains: " + status.includeSubdomains + ")\n");
+ if (status.forceInclude && status.error != ERROR_NONE) {
+ dump(
+ status.name +
+ ": " +
+ errorToString(status) +
+ " (error ignored - included regardless)\n"
+ );
+ }
+ return true;
+ });
+ return includedStatuses;
+ });
+
+ // Write the output file
+ output(total);
+
+ dump("HSTS probing all done\n");
+}
+
+// arguments is a global within xpcshell
+spinResolve(main(arguments));