Adding upstream version 1:115.7.0.upstream/1%115.7.0upstream

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

1 files changed, 557 insertions, 0 deletions

diff --git a/taskcluster/docker/periodic-updates/scripts/getHSTSPreloadList.js b/taskcluster/docker/periodic-updates/scripts/getHSTSPreloadList.js
new file mode 100644
index 0000000000..aeaa29bc2d
--- /dev/null
+++ b/taskcluster/docker/periodic-updates/scripts/getHSTSPreloadList.js
@@ -0,0 +1,557 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed with this file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+"use strict";
+
+// How to run this file:
+// 1. [obtain firefox source code]
+// 2. [build/obtain firefox binaries]
+// 3. run `[path to]/run-mozilla.sh [path to]/xpcshell [path to]/getHSTSPreloadlist.js [absolute path to]/nsSTSPreloadlist.inc'
+// Note: Running this file outputs a new nsSTSPreloadlist.inc in the current
+//       working directory.
+
+var gSSService = Cc["@mozilla.org/ssservice;1"].getService(
+  Ci.nsISiteSecurityService
+);
+
+const { FileUtils } = ChromeUtils.importESModule(
+  "resource://gre/modules/FileUtils.sys.mjs"
+);
+
+const SOURCE =
+  "https://chromium.googlesource.com/chromium/src/+/refs/heads/main/net/http/transport_security_state_static.json?format=TEXT";
+const TOOL_SOURCE =
+  "https://hg.mozilla.org/mozilla-central/file/default/taskcluster/docker/periodic-updates/scripts/getHSTSPreloadList.js";
+const OUTPUT = "nsSTSPreloadList.inc";
+const MINIMUM_REQUIRED_MAX_AGE = 60 * 60 * 24 * 7 * 18;
+const MAX_CONCURRENT_REQUESTS = 500;
+const MAX_RETRIES = 1;
+const REQUEST_TIMEOUT = 30 * 1000;
+const ERROR_NONE = "no error";
+const ERROR_CONNECTING_TO_HOST = "could not connect to host";
+const ERROR_NO_HSTS_HEADER = "did not receive HSTS header";
+const ERROR_MAX_AGE_TOO_LOW = "max-age too low: ";
+const HEADER = `/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/*****************************************************************************/
+/* This is an automatically generated file. If you're not                    */
+/* nsSiteSecurityService.cpp, you shouldn't be #including it.                */
+/*****************************************************************************/
+
+#include <stdint.h>
+`;
+
+const GPERF_DELIM = "%%\n";
+
+function download() {
+  let req = new XMLHttpRequest();
+  req.open("GET", SOURCE, false); // doing the request synchronously
+  try {
+    req.send();
+  } catch (e) {
+    throw new Error(`ERROR: problem downloading '${SOURCE}': ${e}`);
+  }
+
+  if (req.status != 200) {
+    throw new Error(
+      "ERROR: problem downloading '" + SOURCE + "': status " + req.status
+    );
+  }
+
+  let resultDecoded;
+  try {
+    resultDecoded = atob(req.responseText);
+  } catch (e) {
+    throw new Error(
+      "ERROR: could not decode data as base64 from '" + SOURCE + "': " + e
+    );
+  }
+
+  // we have to filter out '//' comments, while not mangling the json
+  let result = resultDecoded.replace(/^(\s*)?\/\/[^\n]*\n/gm, "");
+  let data = null;
+  try {
+    data = JSON.parse(result);
+  } catch (e) {
+    throw new Error(`ERROR: could not parse data from '${SOURCE}': ${e}`);
+  }
+  return data;
+}
+
+function getHosts(rawdata) {
+  let hosts = [];
+
+  if (!rawdata || !rawdata.entries) {
+    throw new Error(
+      "ERROR: source data not formatted correctly: 'entries' not found"
+    );
+  }
+
+  for (let entry of rawdata.entries) {
+    if (entry.mode && entry.mode == "force-https") {
+      if (entry.name) {
+        // We trim the entry name here to avoid malformed URI exceptions when we
+        // later try to connect to the domain.
+        entry.name = entry.name.trim();
+        entry.retries = MAX_RETRIES;
+        // We prefer the camelCase variable to the JSON's snake case version
+        entry.includeSubdomains = entry.include_subdomains;
+        hosts.push(entry);
+      } else {
+        throw new Error("ERROR: entry not formatted correctly: no name found");
+      }
+    }
+  }
+
+  return hosts;
+}
+
+function processStsHeader(host, header, status, securityInfo) {
+  let maxAge = {
+    value: 0,
+  };
+  let includeSubdomains = {
+    value: false,
+  };
+  let error = ERROR_NONE;
+  if (
+    header != null &&
+    securityInfo != null &&
+    securityInfo.overridableErrorCategory ==
+      Ci.nsITransportSecurityInfo.ERROR_UNSET
+  ) {
+    try {
+      let uri = Services.io.newURI("https://" + host.name);
+      gSSService.processHeader(uri, header, {}, maxAge, includeSubdomains);
+    } catch (e) {
+      dump(
+        "ERROR: could not process header '" +
+          header +
+          "' from " +
+          host.name +
+          ": " +
+          e +
+          "\n"
+      );
+      error = e;
+    }
+  } else if (status == 0) {
+    error = ERROR_CONNECTING_TO_HOST;
+  } else {
+    error = ERROR_NO_HSTS_HEADER;
+  }
+
+  if (error == ERROR_NONE && maxAge.value < MINIMUM_REQUIRED_MAX_AGE) {
+    error = ERROR_MAX_AGE_TOO_LOW;
+  }
+
+  return {
+    name: host.name,
+    maxAge: maxAge.value,
+    includeSubdomains: includeSubdomains.value,
+    error,
+    retries: host.retries - 1,
+    forceInclude: host.forceInclude,
+  };
+}
+
+// RedirectAndAuthStopper prevents redirects and HTTP authentication
+function RedirectAndAuthStopper() {}
+
+RedirectAndAuthStopper.prototype = {
+  // nsIChannelEventSink
+  asyncOnChannelRedirect(oldChannel, newChannel, flags, callback) {
+    throw Components.Exception("", Cr.NS_ERROR_ENTITY_CHANGED);
+  },
+
+  // nsIAuthPrompt2
+  promptAuth(channel, level, authInfo) {
+    return false;
+  },
+
+  asyncPromptAuth(channel, callback, context, level, authInfo) {
+    throw Components.Exception("", Cr.NS_ERROR_NOT_IMPLEMENTED);
+  },
+
+  getInterface(iid) {
+    return this.QueryInterface(iid);
+  },
+
+  QueryInterface: ChromeUtils.generateQI([
+    "nsIChannelEventSink",
+    "nsIAuthPrompt2",
+  ]),
+};
+
+function fetchstatus(host) {
+  return new Promise((resolve, reject) => {
+    let xhr = new XMLHttpRequest();
+    let uri = "https://" + host.name + "/";
+
+    xhr.open("head", uri, true);
+    xhr.setRequestHeader("X-Automated-Tool", TOOL_SOURCE);
+    xhr.timeout = REQUEST_TIMEOUT;
+
+    let errorHandler = () => {
+      dump("ERROR: exception making request to " + host.name + "\n");
+      resolve(
+        processStsHeader(
+          host,
+          null,
+          xhr.status,
+          xhr.channel && xhr.channel.securityInfo
+        )
+      );
+    };
+
+    xhr.onerror = errorHandler;
+    xhr.ontimeout = errorHandler;
+    xhr.onabort = errorHandler;
+
+    xhr.onload = () => {
+      let header = xhr.getResponseHeader("strict-transport-security");
+      resolve(
+        processStsHeader(host, header, xhr.status, xhr.channel.securityInfo)
+      );
+    };
+
+    xhr.channel.notificationCallbacks = new RedirectAndAuthStopper();
+    xhr.send();
+  });
+}
+
+async function getHSTSStatus(host) {
+  do {
+    host = await fetchstatus(host);
+  } while (shouldRetry(host));
+  return host;
+}
+
+function compareHSTSStatus(a, b) {
+  if (a.name > b.name) {
+    return 1;
+  }
+  if (a.name < b.name) {
+    return -1;
+  }
+  return 0;
+}
+
+function writeTo(string, fos) {
+  fos.write(string, string.length);
+}
+
+// Determines and returns a string representing a declaration of when this
+// preload list should no longer be used.
+// This is the current time plus MINIMUM_REQUIRED_MAX_AGE.
+function getExpirationTimeString() {
+  let now = new Date();
+  let nowMillis = now.getTime();
+  // MINIMUM_REQUIRED_MAX_AGE is in seconds, so convert to milliseconds
+  let expirationMillis = nowMillis + MINIMUM_REQUIRED_MAX_AGE * 1000;
+  let expirationMicros = expirationMillis * 1000;
+  return (
+    "const PRTime gPreloadListExpirationTime = INT64_C(" +
+    expirationMicros +
+    ");\n"
+  );
+}
+
+function shouldRetry(response) {
+  return (
+    response.error != ERROR_NO_HSTS_HEADER &&
+    response.error != ERROR_MAX_AGE_TOO_LOW &&
+    response.error != ERROR_NONE &&
+    response.retries > 0
+  );
+}
+
+// Copied from browser/components/migration/MigrationUtils.sys.mjs
+function spinResolve(promise) {
+  if (!(promise instanceof Promise)) {
+    return promise;
+  }
+  let done = false;
+  let result = null;
+  let error = null;
+  promise
+    .catch(e => {
+      error = e;
+    })
+    .then(r => {
+      result = r;
+      done = true;
+    });
+
+  Services.tm.spinEventLoopUntil(
+    "getHSTSPreloadList.js:spinResolve",
+    () => done
+  );
+  if (error) {
+    throw error;
+  } else {
+    return result;
+  }
+}
+
+async function probeHSTSStatuses(inHosts) {
+  let totalLength = inHosts.length;
+  dump("Examining " + totalLength + " hosts.\n");
+
+  // Make requests in batches of MAX_CONCURRENT_REQUESTS. Otherwise, we have
+  // too many in-flight requests and the time it takes to process them causes
+  // them all to time out.
+  let allResults = [];
+  while (inHosts.length) {
+    let promises = [];
+    for (let i = 0; i < MAX_CONCURRENT_REQUESTS && inHosts.length; i++) {
+      let host = inHosts.shift();
+      promises.push(getHSTSStatus(host));
+    }
+    let results = await Promise.all(promises);
+    let progress = (
+      (100 * (totalLength - inHosts.length)) /
+      totalLength
+    ).toFixed(2);
+    dump(progress + "% done\n");
+    allResults = allResults.concat(results);
+  }
+
+  dump("HSTS Probe received " + allResults.length + " statuses.\n");
+  return allResults;
+}
+
+function readCurrentList(filename) {
+  var currentHosts = {};
+  var file = Cc["@mozilla.org/file/local;1"].createInstance(Ci.nsIFile);
+  file.initWithPath(filename);
+  var fis = Cc["@mozilla.org/network/file-input-stream;1"].createInstance(
+    Ci.nsILineInputStream
+  );
+  fis.init(file, -1, -1, Ci.nsIFileInputStream.CLOSE_ON_EOF);
+  var line = {};
+
+  // While we generate entries matching the latest version format,
+  // we still need to be able to read entries in the previous version formats
+  // for bootstrapping a latest version preload list from a previous version
+  // preload list. Hence these regexes.
+  const entryRegexes = [
+    /([^,]+), (0|1)/, // v3
+    / {2}\/\* "([^"]*)", (true|false) \*\//, // v2
+    / {2}{ "([^"]*)", (true|false) },/, // v1
+  ];
+
+  while (fis.readLine(line)) {
+    let match;
+    entryRegexes.find(r => {
+      match = r.exec(line.value);
+      return match;
+    });
+    if (match) {
+      currentHosts[match[1]] = match[2] == "1" || match[2] == "true";
+    }
+  }
+  return currentHosts;
+}
+
+function combineLists(newHosts, currentHosts) {
+  let newHostsSet = new Set();
+
+  for (let newHost of newHosts) {
+    newHostsSet.add(newHost.name);
+  }
+
+  for (let currentHost in currentHosts) {
+    if (!newHostsSet.has(currentHost)) {
+      newHosts.push({ name: currentHost, retries: MAX_RETRIES });
+    }
+  }
+}
+
+const TEST_ENTRIES = [
+  {
+    name: "includesubdomains.preloaded.test",
+    includeSubdomains: true,
+  },
+  {
+    name: "includesubdomains2.preloaded.test",
+    includeSubdomains: true,
+  },
+  {
+    name: "noincludesubdomains.preloaded.test",
+    includeSubdomains: false,
+  },
+];
+
+function deleteTestHosts(currentHosts) {
+  for (let testEntry of TEST_ENTRIES) {
+    delete currentHosts[testEntry.name];
+  }
+}
+
+function getTestHosts() {
+  let hosts = [];
+  for (let testEntry of TEST_ENTRIES) {
+    hosts.push({
+      name: testEntry.name,
+      maxAge: MINIMUM_REQUIRED_MAX_AGE,
+      includeSubdomains: testEntry.includeSubdomains,
+      error: ERROR_NONE,
+      // This deliberately doesn't have a value for `retries` (because we should
+      // never attempt to connect to this host).
+      forceInclude: true,
+    });
+  }
+  return hosts;
+}
+
+async function insertHosts(inoutHostList, inAddedHosts) {
+  for (let host of inAddedHosts) {
+    inoutHostList.push(host);
+  }
+}
+
+function filterForcedInclusions(inHosts, outNotForced, outForced) {
+  // Apply our filters (based on policy today) to determine which entries
+  // will be included without being checked (forced); the others will be
+  // checked using active probing.
+  for (let host of inHosts) {
+    if (
+      host.policy == "google" ||
+      host.policy == "public-suffix" ||
+      host.policy == "public-suffix-requested"
+    ) {
+      host.forceInclude = true;
+      host.error = ERROR_NONE;
+      outForced.push(host);
+    } else {
+      outNotForced.push(host);
+    }
+  }
+}
+
+function output(statuses) {
+  dump("INFO: Writing output to " + OUTPUT + "\n");
+  try {
+    let file = new FileUtils.File(
+      PathUtils.join(Services.dirsvc.get("CurWorkD", Ci.nsIFile).path, OUTPUT)
+    );
+    let fos = FileUtils.openSafeFileOutputStream(file);
+    writeTo(HEADER, fos);
+    writeTo(getExpirationTimeString(), fos);
+
+    writeTo(GPERF_DELIM, fos);
+
+    for (let status of statuses) {
+      let includeSubdomains = status.includeSubdomains ? 1 : 0;
+      writeTo(status.name + ", " + includeSubdomains + "\n", fos);
+    }
+
+    writeTo(GPERF_DELIM, fos);
+    FileUtils.closeSafeFileOutputStream(fos);
+    dump("finished writing output file\n");
+  } catch (e) {
+    dump("ERROR: problem writing output to '" + OUTPUT + "': " + e + "\n");
+    throw e;
+  }
+}
+
+function errorToString(status) {
+  return status.error == ERROR_MAX_AGE_TOO_LOW
+    ? status.error + status.maxAge
+    : status.error;
+}
+
+async function main(args) {
+  if (args.length != 1) {
+    throw new Error(
+      "Usage: getHSTSPreloadList.js <absolute path to current nsSTSPreloadList.inc>"
+    );
+  }
+
+  // get the current preload list
+  let currentHosts = readCurrentList(args[0]);
+  // delete any hosts we use in tests so we don't actually connect to them
+  deleteTestHosts(currentHosts);
+  // disable the current preload list so it won't interfere with requests we make
+  Services.prefs.setBoolPref(
+    "network.stricttransportsecurity.preloadlist",
+    false
+  );
+  // download and parse the raw json file from the Chromium source
+  let rawdata = download();
+  // get just the hosts with mode: "force-https"
+  let hosts = getHosts(rawdata);
+  // add hosts in the current list to the new list (avoiding duplicates)
+  combineLists(hosts, currentHosts);
+
+  // Don't contact hosts that are forced to be included anyway
+  let hostsToContact = [];
+  let forcedHosts = [];
+  filterForcedInclusions(hosts, hostsToContact, forcedHosts);
+
+  // Initialize the final status list
+  let hstsStatuses = [];
+  // Add the hosts we use in tests
+  dump("Adding test hosts\n");
+  insertHosts(hstsStatuses, getTestHosts());
+  // Add in the hosts that are forced
+  dump("Adding forced hosts\n");
+  insertHosts(hstsStatuses, forcedHosts);
+
+  let total = await probeHSTSStatuses(hostsToContact)
+    .then(function (probedStatuses) {
+      return hstsStatuses.concat(probedStatuses);
+    })
+    .then(function (statuses) {
+      return statuses.sort(compareHSTSStatus);
+    })
+    .then(function (statuses) {
+      for (let status of statuses) {
+        // If we've encountered an error for this entry (other than the site not
+        // sending an HSTS header), be safe and don't remove it from the list
+        // (given that it was already on the list).
+        if (
+          !status.forceInclude &&
+          status.error != ERROR_NONE &&
+          status.error != ERROR_NO_HSTS_HEADER &&
+          status.error != ERROR_MAX_AGE_TOO_LOW &&
+          status.name in currentHosts
+        ) {
+          // dump("INFO: error connecting to or processing " + status.name + " - using previous status on list\n");
+          status.maxAge = MINIMUM_REQUIRED_MAX_AGE;
+          status.includeSubdomains = currentHosts[status.name];
+        }
+      }
+      return statuses;
+    })
+    .then(function (statuses) {
+      // Filter out entries we aren't including.
+      var includedStatuses = statuses.filter(function (status) {
+        if (status.maxAge < MINIMUM_REQUIRED_MAX_AGE && !status.forceInclude) {
+          // dump("INFO: " + status.name + " NOT ON the preload list\n");
+          return false;
+        }
+
+        // dump("INFO: " + status.name + " ON the preload list (includeSubdomains: " + status.includeSubdomains + ")\n");
+        if (status.forceInclude && status.error != ERROR_NONE) {
+          dump(
+            status.name +
+              ": " +
+              errorToString(status) +
+              " (error ignored - included regardless)\n"
+          );
+        }
+        return true;
+      });
+      return includedStatuses;
+    });
+
+  // Write the output file
+  output(total);
+
+  dump("HSTS probing all done\n");
+}
+
+// arguments is a global within xpcshell
+spinResolve(main(arguments));