firefox/taskcluster/docker/periodic-updates/scripts/getHSTSPreloadList.js

/* This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed with this file, You can obtain one at http://mozilla.org/MPL/2.0/. */
"use strict";

// How to run this file:
// 1. [obtain firefox source code]
// 2. [build/obtain firefox binaries]
// 3. run `[path to]/firefox -xpcshell [path to]/getHSTSPreloadlist.js [absolute path to]/nsSTSPreloadlist.inc'
// Note: Running this file outputs a new nsSTSPreloadlist.inc in the current
//       working directory.

var gSSService = Cc["@mozilla.org/ssservice;1"].getService(
  Ci.nsISiteSecurityService
);

const { FileUtils } = ChromeUtils.importESModule(
  "resource://gre/modules/FileUtils.sys.mjs"
);

const SOURCE =
  "https://chromium.googlesource.com/chromium/src/+/refs/heads/main/net/http/transport_security_state_static.json?format=TEXT";
const TOOL_SOURCE =
  "https://hg.mozilla.org/mozilla-central/file/default/taskcluster/docker/periodic-updates/scripts/getHSTSPreloadList.js";
const OUTPUT = "nsSTSPreloadList.inc";
const MINIMUM_REQUIRED_MAX_AGE = 60 * 60 * 24 * 7 * 18;
const MAX_CONCURRENT_REQUESTS = 500;
const MAX_RETRIES = 1;
const REQUEST_TIMEOUT = 30 * 1000;
const ERROR_NONE = "no error";
const ERROR_CONNECTING_TO_HOST = "could not connect to host";
const ERROR_NO_HSTS_HEADER = "did not receive HSTS header";
const ERROR_MAX_AGE_TOO_LOW = "max-age too low: ";
const HEADER = `/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

/*****************************************************************************/
/* This is an automatically generated file. If you're not                    */
/* nsSiteSecurityService.cpp, you shouldn't be #including it.                */
/*****************************************************************************/

#include <stdint.h>
`;

const GPERF_DELIM = "%%\n";

async function download() {
  var resp = await fetch(SOURCE);
  if (resp.status != 200) {
    throw new Error(
      "ERROR: problem downloading '" + SOURCE + "': status " + resp.status
    );
  }

  let text = await resp.text();
  let resultDecoded;
  try {
    resultDecoded = atob(text);
  } catch (e) {
    throw new Error(
      "ERROR: could not decode data as base64 from '" + SOURCE + "': " + e
    );
  }

  // we have to filter out '//' comments, while not mangling the json
  let result = resultDecoded.replace(/^(\s*)?\/\/[^\n]*\n/gm, "");
  let data = null;
  try {
    data = JSON.parse(result);
  } catch (e) {
    throw new Error(`ERROR: could not parse data from '${SOURCE}': ${e}`);
  }
  return data;
}

function getHosts(rawdata) {
  let hosts = [];

  if (!rawdata || !rawdata.entries) {
    throw new Error(
      "ERROR: source data not formatted correctly: 'entries' not found"
    );
  }

  for (let entry of rawdata.entries) {
    if (entry.mode && entry.mode == "force-https") {
      if (entry.name) {
        // We trim the entry name here to avoid malformed URI exceptions when we
        // later try to connect to the domain.
        entry.name = entry.name.trim();
        entry.retries = MAX_RETRIES;
        // We prefer the camelCase variable to the JSON's snake case version
        entry.includeSubdomains = entry.include_subdomains;
        hosts.push(entry);
      } else {
        throw new Error("ERROR: entry not formatted correctly: no name found");
      }
    }
  }

  return hosts;
}

function processStsHeader(host, header, status, securityInfo) {
  let maxAge = {
    value: 0,
  };
  let includeSubdomains = {
    value: false,
  };
  let error = ERROR_NONE;
  if (
    header != null &&
    securityInfo != null &&
    securityInfo.overridableErrorCategory ==
      Ci.nsITransportSecurityInfo.ERROR_UNSET
  ) {
    try {
      let uri = Services.io.newURI("https://" + host.name);
      gSSService.processHeader(uri, header, {}, maxAge, includeSubdomains);
    } catch (e) {
      dump(
        "ERROR: could not process header '" +
          header +
          "' from " +
          host.name +
          ": " +
          e +
          "\n"
      );
      error = e;
    }
  } else if (status == 0) {
    error = ERROR_CONNECTING_TO_HOST;
  } else {
    error = ERROR_NO_HSTS_HEADER;
  }

  if (error == ERROR_NONE && maxAge.value < MINIMUM_REQUIRED_MAX_AGE) {
    error = ERROR_MAX_AGE_TOO_LOW;
  }

  return {
    name: host.name,
    maxAge: maxAge.value,
    includeSubdomains: includeSubdomains.value,
    error,
    retries: host.retries - 1,
    forceInclude: host.forceInclude,
  };
}

// RedirectAndAuthStopper prevents redirects and HTTP authentication
function RedirectAndAuthStopper() {}

RedirectAndAuthStopper.prototype = {
  // nsIChannelEventSink
  asyncOnChannelRedirect() {
    throw Components.Exception("", Cr.NS_ERROR_ENTITY_CHANGED);
  },

  // nsIAuthPrompt2
  promptAuth() {
    return false;
  },

  asyncPromptAuth() {
    throw Components.Exception("", Cr.NS_ERROR_NOT_IMPLEMENTED);
  },

  getInterface(iid) {
    return this.QueryInterface(iid);
  },

  QueryInterface: ChromeUtils.generateQI([
    "nsIChannelEventSink",
    "nsIAuthPrompt2",
  ]),
};

function fetchstatus(host) {
  return new Promise(resolve => {
    let xhr = new XMLHttpRequest();
    let uri = "https://" + host.name + "/";

    xhr.open("head", uri, true);
    xhr.setRequestHeader("X-Automated-Tool", TOOL_SOURCE);
    xhr.timeout = REQUEST_TIMEOUT;

    let errorHandler = () => {
      dump("ERROR: exception making request to " + host.name + "\n");
      resolve(
        processStsHeader(
          host,
          null,
          xhr.status,
          xhr.channel && xhr.channel.securityInfo
        )
      );
    };

    xhr.onerror = errorHandler;
    xhr.ontimeout = errorHandler;
    xhr.onabort = errorHandler;

    xhr.onload = () => {
      let header = xhr.getResponseHeader("strict-transport-security");
      resolve(
        processStsHeader(host, header, xhr.status, xhr.channel.securityInfo)
      );
    };

    xhr.channel.notificationCallbacks = new RedirectAndAuthStopper();
    xhr.send();
  });
}

async function getHSTSStatus(host) {
  do {
    host = await fetchstatus(host);
  } while (shouldRetry(host));
  return host;
}

function compareHSTSStatus(a, b) {
  if (a.name > b.name) {
    return 1;
  }
  if (a.name < b.name) {
    return -1;
  }
  return 0;
}

function writeTo(string, fos) {
  fos.write(string, string.length);
}

// Determines and returns a string representing a declaration of when this
// preload list should no longer be used.
// This is the current time plus MINIMUM_REQUIRED_MAX_AGE.
function getExpirationTimeString() {
  let now = new Date();
  let nowMillis = now.getTime();
  // MINIMUM_REQUIRED_MAX_AGE is in seconds, so convert to milliseconds
  let expirationMillis = nowMillis + MINIMUM_REQUIRED_MAX_AGE * 1000;
  let expirationMicros = expirationMillis * 1000;
  return (
    "const PRTime gPreloadListExpirationTime = INT64_C(" +
    expirationMicros +
    ");\n"
  );
}

function shouldRetry(response) {
  return (
    response.error != ERROR_NO_HSTS_HEADER &&
    response.error != ERROR_MAX_AGE_TOO_LOW &&
    response.error != ERROR_NONE &&
    response.retries > 0
  );
}

// Copied from browser/components/migration/MigrationUtils.sys.mjs
function spinResolve(promise) {
  if (!(promise instanceof Promise)) {
    return promise;
  }
  let done = false;
  let result = null;
  let error = null;
  promise
    .catch(e => {
      error = e;
    })
    .then(r => {
      result = r;
      done = true;
    });

  Services.tm.spinEventLoopUntil(
    "getHSTSPreloadList.js:spinResolve",
    () => done
  );
  if (error) {
    throw error;
  } else {
    return result;
  }
}

async function probeHSTSStatuses(inHosts) {
  let totalLength = inHosts.length;
  dump("Examining " + totalLength + " hosts.\n");

  // Make requests in batches of MAX_CONCURRENT_REQUESTS. Otherwise, we have
  // too many in-flight requests and the time it takes to process them causes
  // them all to time out.
  let allResults = [];
  while (inHosts.length) {
    let promises = [];
    for (let i = 0; i < MAX_CONCURRENT_REQUESTS && inHosts.length; i++) {
      let host = inHosts.shift();
      promises.push(getHSTSStatus(host));
    }
    let results = await Promise.all(promises);
    let progress = (
      (100 * (totalLength - inHosts.length)) /
      totalLength
    ).toFixed(2);
    dump(progress + "% done\n");
    allResults = allResults.concat(results);
  }

  dump("HSTS Probe received " + allResults.length + " statuses.\n");
  return allResults;
}

function readCurrentList(filename) {
  var currentHosts = {};
  var file = Cc["@mozilla.org/file/local;1"].createInstance(Ci.nsIFile);
  file.initWithPath(filename);
  var fis = Cc["@mozilla.org/network/file-input-stream;1"].createInstance(
    Ci.nsILineInputStream
  );
  fis.init(file, -1, -1, Ci.nsIFileInputStream.CLOSE_ON_EOF);
  var line = {};

  // While we generate entries matching the latest version format,
  // we still need to be able to read entries in the previous version formats
  // for bootstrapping a latest version preload list from a previous version
  // preload list. Hence these regexes.
  const entryRegexes = [
    /([^,]+), (0|1)/, // v3
    / {2}\/\* "([^"]*)", (true|false) \*\//, // v2
    / {2}{ "([^"]*)", (true|false) },/, // v1
  ];

  while (fis.readLine(line)) {
    let match;
    entryRegexes.find(r => {
      match = r.exec(line.value);
      return match;
    });
    if (match) {
      currentHosts[match[1]] = match[2] == "1" || match[2] == "true";
    }
  }
  return currentHosts;
}

function combineLists(newHosts, currentHosts) {
  let newHostsSet = new Set();

  for (let newHost of newHosts) {
    newHostsSet.add(newHost.name);
  }

  for (let currentHost in currentHosts) {
    if (!newHostsSet.has(currentHost)) {
      newHosts.push({ name: currentHost, retries: MAX_RETRIES });
    }
  }
}

const TEST_ENTRIES = [
  {
    name: "includesubdomains.preloaded.test",
    includeSubdomains: true,
  },
  {
    name: "includesubdomains2.preloaded.test",
    includeSubdomains: true,
  },
  {
    name: "noincludesubdomains.preloaded.test",
    includeSubdomains: false,
  },
];

function deleteTestHosts(currentHosts) {
  for (let testEntry of TEST_ENTRIES) {
    delete currentHosts[testEntry.name];
  }
}

function getTestHosts() {
  let hosts = [];
  for (let testEntry of TEST_ENTRIES) {
    hosts.push({
      name: testEntry.name,
      maxAge: MINIMUM_REQUIRED_MAX_AGE,
      includeSubdomains: testEntry.includeSubdomains,
      error: ERROR_NONE,
      // This deliberately doesn't have a value for `retries` (because we should
      // never attempt to connect to this host).
      forceInclude: true,
    });
  }
  return hosts;
}

async function insertHosts(inoutHostList, inAddedHosts) {
  for (let host of inAddedHosts) {
    inoutHostList.push(host);
  }
}

function filterForcedInclusions(inHosts, outNotForced, outForced) {
  // Apply our filters (based on policy today) to determine which entries
  // will be included without being checked (forced); the others will be
  // checked using active probing.
  for (let host of inHosts) {
    if (
      host.policy == "google" ||
      host.policy == "public-suffix" ||
      host.policy == "public-suffix-requested"
    ) {
      host.forceInclude = true;
      host.error = ERROR_NONE;
      outForced.push(host);
    } else {
      outNotForced.push(host);
    }
  }
}

function output(statuses) {
  dump("INFO: Writing output to " + OUTPUT + "\n");
  try {
    let file = new FileUtils.File(
      PathUtils.join(Services.dirsvc.get("CurWorkD", Ci.nsIFile).path, OUTPUT)
    );
    let fos = FileUtils.openSafeFileOutputStream(file);
    writeTo(HEADER, fos);
    writeTo(getExpirationTimeString(), fos);

    writeTo(GPERF_DELIM, fos);

    for (let status of statuses) {
      let includeSubdomains = status.includeSubdomains ? 1 : 0;
      writeTo(status.name + ", " + includeSubdomains + "\n", fos);
    }

    writeTo(GPERF_DELIM, fos);
    FileUtils.closeSafeFileOutputStream(fos);
    dump("finished writing output file\n");
  } catch (e) {
    dump("ERROR: problem writing output to '" + OUTPUT + "': " + e + "\n");
    throw e;
  }
}

function errorToString(status) {
  return status.error == ERROR_MAX_AGE_TOO_LOW
    ? status.error + status.maxAge
    : status.error;
}

async function main(args) {
  if (args.length != 1) {
    throw new Error(
      "Usage: getHSTSPreloadList.js <absolute path to current nsSTSPreloadList.inc>"
    );
  }

  // get the current preload list
  let currentHosts = readCurrentList(args[0]);
  // delete any hosts we use in tests so we don't actually connect to them
  deleteTestHosts(currentHosts);
  // disable the current preload list so it won't interfere with requests we make
  Services.prefs.setBoolPref(
    "network.stricttransportsecurity.preloadlist",
    false
  );
  // download and parse the raw json file from the Chromium source
  let rawdata = await download();
  // get just the hosts with mode: "force-https"
  let hosts = getHosts(rawdata);
  // add hosts in the current list to the new list (avoiding duplicates)
  combineLists(hosts, currentHosts);

  // Don't contact hosts that are forced to be included anyway
  let hostsToContact = [];
  let forcedHosts = [];
  filterForcedInclusions(hosts, hostsToContact, forcedHosts);

  // Initialize the final status list
  let hstsStatuses = [];
  // Add the hosts we use in tests
  dump("Adding test hosts\n");
  insertHosts(hstsStatuses, getTestHosts());
  // Add in the hosts that are forced
  dump("Adding forced hosts\n");
  insertHosts(hstsStatuses, forcedHosts);

  let total = await probeHSTSStatuses(hostsToContact)
    .then(function (probedStatuses) {
      return hstsStatuses.concat(probedStatuses);
    })
    .then(function (statuses) {
      return statuses.sort(compareHSTSStatus);
    })
    .then(function (statuses) {
      for (let status of statuses) {
        // If we've encountered an error for this entry (other than the site not
        // sending an HSTS header), be safe and don't remove it from the list
        // (given that it was already on the list).
        if (
          !status.forceInclude &&
          status.error != ERROR_NONE &&
          status.error != ERROR_NO_HSTS_HEADER &&
          status.error != ERROR_MAX_AGE_TOO_LOW &&
          status.name in currentHosts
        ) {
          // dump("INFO: error connecting to or processing " + status.name + " - using previous status on list\n");
          status.maxAge = MINIMUM_REQUIRED_MAX_AGE;
          status.includeSubdomains = currentHosts[status.name];
        }
      }
      return statuses;
    })
    .then(function (statuses) {
      // Filter out entries we aren't including.
      var includedStatuses = statuses.filter(function (status) {
        if (status.maxAge < MINIMUM_REQUIRED_MAX_AGE && !status.forceInclude) {
          // dump("INFO: " + status.name + " NOT ON the preload list\n");
          return false;
        }

        // dump("INFO: " + status.name + " ON the preload list (includeSubdomains: " + status.includeSubdomains + ")\n");
        if (status.forceInclude && status.error != ERROR_NONE) {
          dump(
            status.name +
              ": " +
              errorToString(status) +
              " (error ignored - included regardless)\n"
          );
        }
        return true;
      });
      return includedStatuses;
    });

  // Write the output file
  output(total);

  dump("HSTS probing all done\n");
}

// arguments is a global within xpcshell
spinResolve(main(arguments));