diff options
Diffstat (limited to 'browser/components/attribution')
18 files changed, 2777 insertions, 0 deletions
diff --git a/browser/components/attribution/AttributionCode.sys.mjs b/browser/components/attribution/AttributionCode.sys.mjs new file mode 100644 index 0000000000..354270c13a --- /dev/null +++ b/browser/components/attribution/AttributionCode.sys.mjs @@ -0,0 +1,451 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** + * This is a policy object used to override behavior for testing. + */ +export const AttributionIOUtils = { + write: async (path, bytes) => IOUtils.write(path, bytes), + read: async path => IOUtils.read(path), + readUTF8: async path => IOUtils.readUTF8(path), + exists: async path => IOUtils.exists(path), +}; + +import { XPCOMUtils } from "resource://gre/modules/XPCOMUtils.sys.mjs"; +import { AppConstants } from "resource://gre/modules/AppConstants.sys.mjs"; + +const lazy = {}; +ChromeUtils.defineESModuleGetters(lazy, { + MacAttribution: "resource:///modules/MacAttribution.sys.mjs", +}); +XPCOMUtils.defineLazyGetter(lazy, "log", () => { + let { ConsoleAPI } = ChromeUtils.importESModule( + "resource://gre/modules/Console.sys.mjs" + ); + let consoleOptions = { + // tip: set maxLogLevel to "debug" and use lazy.log.debug() to create + // detailed messages during development. See LOG_LEVELS in Console.sys.mjs + // for details. + maxLogLevel: "error", + maxLogLevelPref: "browser.attribution.loglevel", + prefix: "AttributionCode", + }; + return new ConsoleAPI(consoleOptions); +}); + +// This maximum length was originally based on how much space we have in the PE +// file header that we store attribution codes in for full and stub installers. +// Windows Store builds instead use a "Campaign ID" passed through URLs to send +// attribution information, which Microsoft's documentation claims must be no +// longer than 100 characters. In our own testing, we've been able to retrieve +// the first 208 characters of the Campaign ID. Either way, the "max" length +// for Microsoft Store builds is much lower than this limit implies. +const ATTR_CODE_MAX_LENGTH = 1010; +const ATTR_CODE_VALUE_REGEX = /[a-zA-Z0-9_%\\-\\.\\(\\)]*/; +const ATTR_CODE_FIELD_SEPARATOR = "%26"; // URL-encoded & +const ATTR_CODE_KEY_VALUE_SEPARATOR = "%3D"; // URL-encoded = +const ATTR_CODE_KEYS = [ + "source", + "medium", + "campaign", + "content", + "experiment", + "variation", + "ua", + "dltoken", + "msstoresignedin", + "dlsource", +]; + +let gCachedAttrData = null; + +export var AttributionCode = { + /** + * Wrapper to pull campaign IDs from MSIX builds. + * This function solely exists to make it easy to mock out for tests. + */ + get msixCampaignId() { + return Cc["@mozilla.org/windows-package-manager;1"] + .createInstance(Ci.nsIWindowsPackageManager) + .getCampaignId(); + }, + + /** + * Returns a platform-specific nsIFile for the file containing the attribution + * data, or null if the current platform does not support (caching) + * attribution data. + */ + get attributionFile() { + if (AppConstants.platform == "win") { + let file = Services.dirsvc.get("GreD", Ci.nsIFile); + file.append("postSigningData"); + return file; + } else if (AppConstants.platform == "macosx") { + // There's no `UpdRootD` in xpcshell tests. Some existing tests override + // it, which is onerous and difficult to share across tests. When testing, + // if it's not defined, fallback to a nested subdirectory of the xpcshell + // temp directory. Nesting more closely replicates the situation where the + // update directory does not (yet) exist, testing a scenario witnessed in + // development. + let file; + try { + file = Services.dirsvc.get("UpdRootD", Ci.nsIFile); + } catch (ex) { + // It's most common to test for the profile dir, even though we actually + // are using the temp dir. + if ( + ex instanceof Ci.nsIException && + ex.result == Cr.NS_ERROR_FAILURE && + Services.env.exists("XPCSHELL_TEST_PROFILE_DIR") + ) { + let path = Services.env.get("XPCSHELL_TEST_TEMP_DIR"); + file = Cc["@mozilla.org/file/local;1"].createInstance(Ci.nsIFile); + file.initWithPath(path); + file.append("nested_UpdRootD_1"); + file.append("nested_UpdRootD_2"); + } else { + throw ex; + } + } + file.append("macAttributionData"); + return file; + } + + return null; + }, + + /** + * Write the given attribution code to the attribution file. + * @param {String} code to write. + */ + async writeAttributionFile(code) { + // Writing attribution files is only used as part of test code, and Mac + // attribution, so bailing here for MSIX builds is no big deal. + if ( + AppConstants.platform === "win" && + Services.sysinfo.getProperty("hasWinPackageId") + ) { + Services.console.logStringMessage( + "Attribution code cannot be written for MSIX builds, aborting." + ); + return; + } + let file = AttributionCode.attributionFile; + await IOUtils.makeDirectory(file.parent.path); + let bytes = new TextEncoder().encode(code); + await AttributionIOUtils.write(file.path, bytes); + }, + + /** + * Returns an array of allowed attribution code keys. + */ + get allowedCodeKeys() { + return [...ATTR_CODE_KEYS]; + }, + + /** + * Returns an object containing a key-value pair for each piece of attribution + * data included in the passed-in attribution code string. + * If the string isn't a valid attribution code, returns an empty object. + */ + parseAttributionCode(code) { + if (code.length > ATTR_CODE_MAX_LENGTH) { + return {}; + } + + let isValid = true; + let parsed = {}; + for (let param of code.split(ATTR_CODE_FIELD_SEPARATOR)) { + let [key, value] = param.split(ATTR_CODE_KEY_VALUE_SEPARATOR, 2); + if (key && ATTR_CODE_KEYS.includes(key)) { + if (value && ATTR_CODE_VALUE_REGEX.test(value)) { + if (key === "msstoresignedin") { + if (value === "true") { + parsed[key] = true; + } else if (value === "false") { + parsed[key] = false; + } else { + throw new Error("Couldn't parse msstoresignedin"); + } + } else { + parsed[key] = value; + } + } + } else { + lazy.log.debug( + `parseAttributionCode: "${code}" => isValid = false: "${key}", "${value}"` + ); + isValid = false; + break; + } + } + + if (isValid) { + return parsed; + } + + Services.telemetry + .getHistogramById("BROWSER_ATTRIBUTION_ERRORS") + .add("decode_error"); + + return {}; + }, + + /** + * Returns an object containing a key-value pair for each piece of attribution + * data included in the passed-in URL containing a query string encoding an + * attribution code. + * + * We have less control of the attribution codes on macOS so we accept more + * URLs than we accept attribution codes on Windows. + * + * If the URL is empty, returns an empty object. + * + * If the URL doesn't parse, throws. + */ + parseAttributionCodeFromUrl(url) { + if (!url) { + return {}; + } + + let parsed = {}; + + let params = new URL(url).searchParams; + for (let key of ATTR_CODE_KEYS) { + // We support the key prefixed with utm_ or not, but intentionally + // choose non-utm params over utm params. + for (let paramKey of [`utm_${key}`, `funnel_${key}`, key]) { + if (params.has(paramKey)) { + // We expect URI-encoded components in our attribution codes. + let value = encodeURIComponent(params.get(paramKey)); + if (value && ATTR_CODE_VALUE_REGEX.test(value)) { + parsed[key] = value; + } + } + } + } + + return parsed; + }, + + /** + * Returns a string serializing the given attribution data. + * + * It is expected that the given values are already URL-encoded. + */ + serializeAttributionData(data) { + // Iterating in this way makes the order deterministic. + let s = ""; + for (let key of ATTR_CODE_KEYS) { + if (key in data) { + let value = data[key]; + if (s) { + s += ATTR_CODE_FIELD_SEPARATOR; // URL-encoded & + } + s += `${key}${ATTR_CODE_KEY_VALUE_SEPARATOR}${value}`; // URL-encoded = + } + } + return s; + }, + + /** + * Reads the attribution code, either from disk or a cached version. + * Returns a promise that fulfills with an object containing the parsed + * attribution data if the code could be read and is valid, + * or an empty object otherwise. + * + * On windows the attribution service converts utm_* keys, removing "utm_". + * On OSX the attributions are set directly on download and retain "utm_". We + * strip "utm_" while retrieving the params. + */ + async getAttrDataAsync() { + if (gCachedAttrData != null) { + lazy.log.debug( + `getAttrDataAsync: attribution is cached: ${JSON.stringify( + gCachedAttrData + )}` + ); + return gCachedAttrData; + } + + gCachedAttrData = {}; + let attributionFile = this.attributionFile; + if (!attributionFile) { + // This platform doesn't support attribution. + lazy.log.debug( + `getAttrDataAsync: no attribution (attributionFile is null)` + ); + return gCachedAttrData; + } + + if ( + AppConstants.platform == "macosx" && + !(await AttributionIOUtils.exists(attributionFile.path)) + ) { + lazy.log.debug( + `getAttrDataAsync: macOS && !exists("${attributionFile.path}")` + ); + + // On macOS, we fish the attribution data from the system quarantine DB. + try { + let referrer = await lazy.MacAttribution.getReferrerUrl(); + lazy.log.debug( + `getAttrDataAsync: macOS attribution getReferrerUrl: "${referrer}"` + ); + + gCachedAttrData = this.parseAttributionCodeFromUrl(referrer); + } catch (ex) { + // Avoid partial attribution data. + gCachedAttrData = {}; + + // No attributions. Just `warn` 'cuz this isn't necessarily an error. + lazy.log.warn("Caught exception fetching macOS attribution codes!", ex); + + if ( + ex instanceof Ci.nsIException && + ex.result == Cr.NS_ERROR_UNEXPECTED + ) { + // Bad quarantine data. + Services.telemetry + .getHistogramById("BROWSER_ATTRIBUTION_ERRORS") + .add("quarantine_error"); + } + } + + lazy.log.debug( + `macOS attribution data is ${JSON.stringify(gCachedAttrData)}` + ); + + // We only want to try to fetch the referrer from the quarantine + // database once on macOS. + try { + let code = this.serializeAttributionData(gCachedAttrData); + lazy.log.debug(`macOS attribution data serializes as "${code}"`); + await this.writeAttributionFile(code); + } catch (ex) { + lazy.log.debug( + `Caught exception writing "${attributionFile.path}"`, + ex + ); + Services.telemetry + .getHistogramById("BROWSER_ATTRIBUTION_ERRORS") + .add("write_error"); + return gCachedAttrData; + } + + lazy.log.debug( + `Returning after successfully writing "${attributionFile.path}"` + ); + return gCachedAttrData; + } + + lazy.log.debug( + `getAttrDataAsync: !macOS || !exists("${attributionFile.path}")` + ); + + let bytes; + try { + if ( + AppConstants.platform === "win" && + Services.sysinfo.getProperty("hasWinPackageId") + ) { + // This comes out of windows-package-manager _not_ URL encoded or in an ArrayBuffer, + // but the parsing code wants it that way. It's easier to just provide that + // than have the parsing code support both. + lazy.log.debug( + `winPackageFamilyName is: ${Services.sysinfo.getProperty( + "winPackageFamilyName" + )}` + ); + let encoder = new TextEncoder(); + bytes = encoder.encode(encodeURIComponent(this.msixCampaignId)); + } else { + bytes = await AttributionIOUtils.read(attributionFile.path); + } + } catch (ex) { + if (DOMException.isInstance(ex) && ex.name == "NotFoundError") { + lazy.log.debug( + `getAttrDataAsync: !exists("${ + attributionFile.path + }"), returning ${JSON.stringify(gCachedAttrData)}` + ); + return gCachedAttrData; + } + lazy.log.debug( + `other error trying to read attribution data: + attributionFile.path is: ${attributionFile.path}` + ); + lazy.log.debug("Full exception is:"); + lazy.log.debug(ex); + + Services.telemetry + .getHistogramById("BROWSER_ATTRIBUTION_ERRORS") + .add("read_error"); + } + if (bytes) { + try { + let decoder = new TextDecoder(); + let code = decoder.decode(bytes); + lazy.log.debug( + `getAttrDataAsync: attribution bytes deserializes to ${code}` + ); + if (AppConstants.platform == "macosx" && !code) { + // On macOS, an empty attribution code is fine. (On Windows, that + // means the stub/full installer has been incorrectly attributed, + // which is an error.) + return gCachedAttrData; + } + + gCachedAttrData = this.parseAttributionCode(code); + lazy.log.debug( + `getAttrDataAsync: ${code} parses to ${JSON.stringify( + gCachedAttrData + )}` + ); + } catch (ex) { + // TextDecoder can throw an error + Services.telemetry + .getHistogramById("BROWSER_ATTRIBUTION_ERRORS") + .add("decode_error"); + } + } + + return gCachedAttrData; + }, + + /** + * Return the cached attribution data synchronously without hitting + * the disk. + * @returns A dictionary with the attribution data if it's available, + * null otherwise. + */ + getCachedAttributionData() { + return gCachedAttrData; + }, + + /** + * Deletes the attribution data file. + * Returns a promise that resolves when the file is deleted, + * or if the file couldn't be deleted (the promise is never rejected). + */ + async deleteFileAsync() { + try { + await IOUtils.remove(this.attributionFile.path); + } catch (ex) { + // The attribution file may already have been deleted, + // or it may have never been installed at all; + // failure to delete it isn't an error. + } + }, + + /** + * Clears the cached attribution code value, if any. + * Does nothing if called from outside of an xpcshell test. + */ + _clearCache() { + if (Services.env.exists("XPCSHELL_TEST_PROFILE_DIR")) { + gCachedAttrData = null; + } + }, +}; diff --git a/browser/components/attribution/MacAttribution.sys.mjs b/browser/components/attribution/MacAttribution.sys.mjs new file mode 100644 index 0000000000..cb4a8373c8 --- /dev/null +++ b/browser/components/attribution/MacAttribution.sys.mjs @@ -0,0 +1,177 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import { XPCOMUtils } from "resource://gre/modules/XPCOMUtils.sys.mjs"; + +const lazy = {}; +XPCOMUtils.defineLazyGetter(lazy, "log", () => { + let { ConsoleAPI } = ChromeUtils.importESModule( + "resource://gre/modules/Console.sys.mjs" + ); + let consoleOptions = { + // tip: set maxLogLevel to "debug" and use lazy.log.debug() to create + // detailed messages during development. See LOG_LEVELS in Console.sys.mjs + // for details. + maxLogLevel: "error", + maxLogLevelPref: "browser.attribution.mac.loglevel", + prefix: "MacAttribution", + }; + return new ConsoleAPI(consoleOptions); +}); + +ChromeUtils.defineESModuleGetters(lazy, { + Subprocess: "resource://gre/modules/Subprocess.sys.mjs", +}); + +/** + * Get the location of the user's macOS quarantine database. + * @return {String} path. + */ +function getQuarantineDatabasePath() { + let file = Services.dirsvc.get("Home", Ci.nsIFile); + file.append("Library"); + file.append("Preferences"); + file.append("com.apple.LaunchServices.QuarantineEventsV2"); + return file.path; +} + +/** + * Query given path for quarantine extended attributes. + * @param {String} path of the file to query. + * @return {[String, String]} pair of the quarantine data GUID and remaining + * quarantine data (usually, Gatekeeper flags). + * @throws NS_ERROR_NOT_AVAILABLE if there is no quarantine GUID for the given path. + * @throws NS_ERROR_UNEXPECTED if there is a quarantine GUID, but it is malformed. + */ +async function getQuarantineAttributes(path) { + let bytes = await IOUtils.getMacXAttr(path, "com.apple.quarantine"); + if (!bytes) { + throw new Components.Exception( + `No macOS quarantine xattrs found for ${path}`, + Cr.NS_ERROR_NOT_AVAILABLE + ); + } + + let string = new TextDecoder("utf-8").decode(bytes); + let parts = string.split(";"); + if (!parts.length) { + throw new Components.Exception( + `macOS quarantine data is not ; separated`, + Cr.NS_ERROR_UNEXPECTED + ); + } + let guid = parts[parts.length - 1]; + if (guid.length != 36) { + // Like "12345678-90AB-CDEF-1234-567890ABCDEF". + throw new Components.Exception( + `macOS quarantine data guid is not length 36: ${guid.length}`, + Cr.NS_ERROR_UNEXPECTED + ); + } + + return { guid, parts }; +} + +/** + * Invoke system SQLite binary to extract the referrer URL corresponding to + * the given GUID from the given macOS quarantine database. + * @param {String} path of the user's macOS quarantine database. + * @param {String} guid to query. + * @return {String} referrer URL. + */ +async function queryQuarantineDatabase( + guid, + path = getQuarantineDatabasePath() +) { + let query = `SELECT COUNT(*), LSQuarantineOriginURLString + FROM LSQuarantineEvent + WHERE LSQuarantineEventIdentifier = '${guid}' + ORDER BY LSQuarantineTimeStamp DESC LIMIT 1`; + + let proc = await lazy.Subprocess.call({ + command: "/usr/bin/sqlite3", + arguments: [path, query], + environment: {}, + stderr: "stdout", + }); + + let stdout = await proc.stdout.readString(); + + let { exitCode } = await proc.wait(); + if (exitCode != 0) { + throw new Components.Exception( + "Failed to run sqlite3", + Cr.NS_ERROR_UNEXPECTED + ); + } + + // Output is like "integer|url". + let parts = stdout.split("|", 2); + if (parts.length != 2) { + throw new Components.Exception( + "Failed to parse sqlite3 output", + Cr.NS_ERROR_UNEXPECTED + ); + } + + if (parts[0].trim() == "0") { + throw new Components.Exception( + `Quarantine database does not contain URL for guid ${guid}`, + Cr.NS_ERROR_UNEXPECTED + ); + } + + return parts[1].trim(); +} + +export var MacAttribution = { + /** + * The file path to the `.app` directory. + */ + get applicationPath() { + // On macOS, `GreD` is like "App.app/Contents/macOS". Return "App.app". + return Services.dirsvc.get("GreD", Ci.nsIFile).parent.parent.path; + }, + + /** + * Used by the Attributions system to get the download referrer. + * + * @param {String} path to get the quarantine data from. + * Usually this is a `.app` directory but can be any + * (existing) file or directory. Default: `this.applicationPath`. + * @return {String} referrer URL. + * @throws NS_ERROR_NOT_AVAILABLE if there is no quarantine GUID for the given path. + * @throws NS_ERROR_UNEXPECTED if there is a quarantine GUID, but no corresponding referrer URL is known. + */ + async getReferrerUrl(path = this.applicationPath) { + lazy.log.debug(`getReferrerUrl(${JSON.stringify(path)})`); + + // First, determine the quarantine GUID assigned by macOS to the given path. + let guid; + try { + guid = (await getQuarantineAttributes(path)).guid; + } catch (ex) { + throw new Components.Exception( + `No macOS quarantine GUID found for ${path}`, + Cr.NS_ERROR_NOT_AVAILABLE + ); + } + lazy.log.debug(`getReferrerUrl: guid: ${guid}`); + + // Second, fish the relevant record from the quarantine database. + let url = ""; + try { + url = await queryQuarantineDatabase(guid); + lazy.log.debug(`getReferrerUrl: url: ${url}`); + } catch (ex) { + // This path is known to macOS but we failed to extract a referrer -- be noisy. + throw new Components.Exception( + `No macOS quarantine referrer URL found for ${path} with GUID ${guid}`, + Cr.NS_ERROR_UNEXPECTED + ); + } + + return url; + }, +}; diff --git a/browser/components/attribution/ProvenanceData.sys.mjs b/browser/components/attribution/ProvenanceData.sys.mjs new file mode 100644 index 0000000000..128b0214aa --- /dev/null +++ b/browser/components/attribution/ProvenanceData.sys.mjs @@ -0,0 +1,544 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import { AppConstants } from "resource://gre/modules/AppConstants.sys.mjs"; + +const lazy = {}; +ChromeUtils.defineESModuleGetters(lazy, { + AttributionIOUtils: "resource:///modules/AttributionCode.sys.mjs", +}); + +let gReadZoneIdPromise = null; +let gTelemetryPromise = null; + +export var ProvenanceData = { + /** + * Clears cached code/Promises. For testing only. + */ + _clearCache() { + if (Services.env.exists("XPCSHELL_TEST_PROFILE_DIR")) { + gReadZoneIdPromise = null; + gTelemetryPromise = null; + } + }, + + /** + * Returns an nsIFile for the file containing the zone identifier-based + * provenance data. This currently only exists on Windows. On other platforms, + * this will return null. + */ + get zoneIdProvenanceFile() { + if (AppConstants.platform == "win") { + let file = Services.dirsvc.get("GreD", Ci.nsIFile); + file.append("zoneIdProvenanceData"); + return file; + } + return null; + }, + + /** + * Loads the provenance data that the installer copied (along with some + * metadata) from the Firefox installer used to create the current + * installation. + * + * If it doesn't already exist, creates a global Promise that loads the data + * from the file and caches it. Subsequent calls get the same Promise. There + * is no support for re-reading the file from the disk because there is no + * good reason that the contents of the file should change. + * + * Only expected contents will be pulled out of the file. This does not + * extract arbitrary, unexpected data. Data will be validated, to the extent + * possible. Most possible data returned has a potential key indicating that + * we read an unexpected value out of the file. + * + * @returns `null` on unsupported OSs. Otherwise, an object with these + * possible keys: + * `readProvenanceError` + * Will be present if there was a problem when Firefox tried to read the + * file that ought to have been written by the installer. Possible values + * are: + * `noSuchFile`, `readError`, `parseError` + * If this key is present, no other keys will be present. + * `fileSystem` + * What filesystem the installer was on. If available, this will be a + * string returned from `GetVolumeInformationByHandleW` via its + * `lpFileSystemNameBuffer` parameter. Possible values are: + * `NTFS`, `FAT32`, `other`, `missing`, `readIniError` + * `other` will be used if the value read does not match one of the + * expected file systems. + * `missing` will be used if the file didn't contain `fileSystem` or + * `readFsError` information. + * `readIniError` will be used if an error is encountered when reading + * the key from the file in the installation directory. + * `readFsError` + * The reason why the installer file system could not be determined. Will + * be present if `readProvenanceError` and `fileSystem` are not. Possible + * values are: + * `openFile`, `getVolInfo`, `fsUnterminated`, `getBufferSize`, + * `convertString`, `unexpected`, `readIniError` + * `unexpected` will be used if the value read from the file didn't match + * any of the expected values, for some reason. + * `missing` will be used if the file didn't contain `fileSystem` or + * `readFsError` information. + * `readIniError` will be used if an error is encountered when reading + * the key from the file in the installation directory. + * `readFsErrorCode` + * An integer returned by `GetLastError()` indicating, in more detail, + * why we failed to obtain the file system. This key may exist if + * `readFsError` exists. + * `readZoneIdError` + * The reason why the installer was unable to read its zone identifier + * ADS. Possible values are: + * `openFile`, `readFile`, `unexpected`, `readIniError` + * `unexpected` will be used if the value read from the file didn't match + * any of the expected values, for some reason. + * `readIniError` will be used if an error is encountered when reading + * the key from the file in the installation directory. + * `readZoneIdErrorCode` + * An integer returned by `GetLastError()` indicating, in more detail, + * why we failed to read the zone identifier ADS. This key may exist if + * `readZoneIdError` exists. + * `zoneIdFileSize` + * This key should exist if Firefox successfully read the file in the + * installation directory and the installer successfully opened the ADS. + * If the installer failed to get the size of the ADS prior to reading + * it, this will be `unknown`. If the installer was able to get the ADS + * size, this will be an integer describing how many bytes long it was. + * If this value in installation directory's file isn't `unknown` or an + * integer, this will be `unexpected`. If an error is encountered when + * reading the key from the file in the installation directory, this will + * be `readIniError`. + * `zoneIdBufferLargeEnough` + * This key should exist if Firefox successfully read the file in the + * installation directory and the installer successfully opened the ADS. + * Indicates whether the zone identifier ADS size was bigger than the + * maximum size that the installer will read from it. If we failed to + * determine the ADS size, this will be `unknown`. If the installation + * directory's file contains an invalid value, this will be `unexpected`. + * If an error is encountered when reading the key from the file in the + * installation directory, this will be `readIniError`. + * Otherwise, this will be a boolean indicating whether or not the buffer + * was large enough to fit the ADS data into. + * `zoneIdTruncated` + * This key should exist if Firefox successfully read the file in the + * installation directory and the installer successfully read the ADS. + * Indicates whether or not we read through the end of the ADS data when + * we copied it. If the installer failed to determine this, this value + * will be `unknown`. If the installation directory's file contains an + * invalid value, this will be `unexpected`. If an error is encountered + * when reading the key from the file in the installation directory, this + * will be `readIniError`. Otherwise, this will be a boolean value + * indicating whether or not the data that we copied was truncated. + * `zoneId` + * The Security Zone that the Zone Identifier data indicates that + * installer was downloaded from. See this documentation: + * https://learn.microsoft.com/en-us/previous-versions/windows/internet-explorer/ie-developer/platform-apis/ms537183(v=vs.85) + * This key will be present if `readProvenanceError` and + * `readZoneIdError` are not. It will either be a valid zone ID (an + * integer between 0 and 4, inclusive), or else it will be `unexpected`, + * `missing`, or `readIniError`. + * `referrerUrl` + * The URL of the download referrer. This key will be present if + * `readProvenanceError` and `readZoneIdError` are not. It will either be + * a `URL` object, or else it will be `unexpected`, `missing`, or + * `readIniError`. + * `referrerUrlIsMozilla` + * This key will be present if `ReferrerUrl` is a `URL` object. It will + * be `true`` if the download referrer appears to be a Mozilla URL. + * Otherwise it will be `false`. + * `hostUrl` + * The URL of the download source. This key will be present if + * `readProvenanceError` and `readZoneIdError` are not. It will either be + * a `URL` object, or else it will be `unexpected`, `missing`, or + * `readIniError` + * `hostUrlIsMozilla` + * This key will be present if `HostUrl` is a `URL` object. It will + * be `true`` if the download source appears to be a Mozilla URL. + * Otherwise it will be `false`. + */ + async readZoneIdProvenanceFile() { + if (gReadZoneIdPromise) { + return gReadZoneIdPromise; + } + gReadZoneIdPromise = (async () => { + let file = this.zoneIdProvenanceFile; + if (!file) { + return null; + } + let iniData; + try { + iniData = await lazy.AttributionIOUtils.readUTF8(file.path); + } catch (ex) { + if (DOMException.isInstance(ex) && ex.name == "NotFoundError") { + return { readProvenanceError: "noSuchFile" }; + } + return { readProvenanceError: "readError" }; + } + + let ini; + try { + // We would rather use asynchronous I/O, so we are going to read the + // file with IOUtils and then pass the result into the INI parser + // rather than just giving the INI parser factory the file. + ini = Cc["@mozilla.org/xpcom/ini-parser-factory;1"] + .getService(Ci.nsIINIParserFactory) + .createINIParser(null); + ini.initFromString(iniData); + } catch (ex) { + return { readProvenanceError: "parseError" }; + } + + const unexpectedValueError = "unexpected"; + const missingKeyError = "missing"; + const readIniError = "readIniError"; + const possibleIniErrors = [missingKeyError, readIniError]; + + // Format is {"IniSectionName": {"IniKeyName": "IniValue"}} + let iniValues = { + Mozilla: { + fileSystem: null, + readFsError: null, + readFsErrorCode: null, + readZoneIdError: null, + readZoneIdErrorCode: null, + zoneIdFileSize: null, + zoneIdBufferLargeEnough: null, + zoneIdTruncated: null, + }, + ZoneTransfer: { + ZoneId: null, + ReferrerUrl: null, + HostUrl: null, + }, + }; + + // The ini reader interface is a little weird in that if we just try to + // read a value from a known section/key and the section/key doesn't + // exist, we just get a generic error rather than an indication that + // the section/key doesn't exist. To distinguish missing keys from any + // other potential errors, we are going to enumerate the sections and + // keys use that to determine if we should try to read from them. + let existingSections; + try { + existingSections = Array.from(ini.getSections()); + } catch (ex) { + return { readProvenanceError: "parseError" }; + } + for (const section in iniValues) { + if (!existingSections.includes(section)) { + for (const key in iniValues[section]) { + iniValues[section][key] = missingKeyError; + } + continue; + } + + let existingKeys; + try { + existingKeys = Array.from(ini.getKeys(section)); + } catch (ex) { + for (const key in iniValues[section]) { + iniValues[section][key] = readIniError; + } + continue; + } + + for (const key in iniValues[section]) { + if (!existingKeys.includes(key)) { + iniValues[section][key] = missingKeyError; + continue; + } + + let value; + try { + value = ini.getString(section, key).trim(); + } catch (ex) { + value = readIniError; + } + iniValues[section][key] = value; + } + } + + // This helps with how verbose the validation gets. + const fileSystem = iniValues.Mozilla.fileSystem; + const readFsError = iniValues.Mozilla.readFsError; + const readFsErrorCode = iniValues.Mozilla.readFsErrorCode; + const readZoneIdError = iniValues.Mozilla.readZoneIdError; + const readZoneIdErrorCode = iniValues.Mozilla.readZoneIdErrorCode; + const zoneIdFileSize = iniValues.Mozilla.zoneIdFileSize; + const zoneIdBufferLargeEnough = iniValues.Mozilla.zoneIdBufferLargeEnough; + const zoneIdTruncated = iniValues.Mozilla.zoneIdTruncated; + const zoneId = iniValues.ZoneTransfer.ZoneId; + const referrerUrl = iniValues.ZoneTransfer.ReferrerUrl; + const hostUrl = iniValues.ZoneTransfer.HostUrl; + + let returnObject = {}; + + // readFsError, readFsErrorCode, fileSystem + const validReadFsErrors = [ + "openFile", + "getVolInfo", + "fsUnterminated", + "getBufferSize", + "convertString", + ]; + // These must be upper case + const validFileSystemValues = ["NTFS", "FAT32"]; + if (fileSystem == missingKeyError && readFsError != missingKeyError) { + if ( + possibleIniErrors.includes(readFsError) || + validReadFsErrors.includes(readFsError) + ) { + returnObject.readFsError = readFsError; + } else { + returnObject.readFsError = unexpectedValueError; + } + if (readFsErrorCode != missingKeyError) { + let code = parseInt(readFsErrorCode, 10); + if (!isNaN(code)) { + returnObject.readFsErrorCode = code; + } + } + } else if (possibleIniErrors.includes(fileSystem)) { + returnObject.fileSystem = fileSystem; + } else if (validFileSystemValues.includes(fileSystem.toUpperCase())) { + returnObject.fileSystem = fileSystem.toUpperCase(); + } else { + returnObject.fileSystem = "other"; + } + + // zoneIdFileSize + if (zoneIdFileSize == missingKeyError) { + // We don't include this one if it's missing. + } else if ( + zoneIdFileSize == readIniError || + zoneIdFileSize == "unknown" + ) { + returnObject.zoneIdFileSize = zoneIdFileSize; + } else { + let size = parseInt(zoneIdFileSize, 10); + if (isNaN(size)) { + returnObject.zoneIdFileSize = unexpectedValueError; + } else { + returnObject.zoneIdFileSize = size; + } + } + + // zoneIdBufferLargeEnough + if (zoneIdBufferLargeEnough == missingKeyError) { + // We don't include this one if it's missing. + } else if ( + zoneIdBufferLargeEnough == readIniError || + zoneIdBufferLargeEnough == "unknown" + ) { + returnObject.zoneIdBufferLargeEnough = zoneIdBufferLargeEnough; + } else if (zoneIdBufferLargeEnough.toLowerCase() == "true") { + returnObject.zoneIdBufferLargeEnough = true; + } else if (zoneIdBufferLargeEnough.toLowerCase() == "false") { + returnObject.zoneIdBufferLargeEnough = false; + } else { + returnObject.zoneIdBufferLargeEnough = unexpectedValueError; + } + + // zoneIdTruncated + if (zoneIdTruncated == missingKeyError) { + // We don't include this one if it's missing. + } else if ( + zoneIdTruncated == readIniError || + zoneIdTruncated == "unknown" + ) { + returnObject.zoneIdTruncated = zoneIdTruncated; + } else if (zoneIdTruncated.toLowerCase() == "true") { + returnObject.zoneIdTruncated = true; + } else if (zoneIdTruncated.toLowerCase() == "false") { + returnObject.zoneIdTruncated = false; + } else { + returnObject.zoneIdTruncated = unexpectedValueError; + } + + // readZoneIdError, readZoneIdErrorCode, zoneId, referrerUrl, hostUrl, + // referrerUrlIsMozilla, hostUrlIsMozilla + const validReadZoneIdErrors = ["openFile", "readFile"]; + if ( + readZoneIdError != missingKeyError && + zoneId == missingKeyError && + referrerUrl == missingKeyError && + hostUrl == missingKeyError + ) { + if ( + possibleIniErrors.includes(readZoneIdError) || + validReadZoneIdErrors.includes(readZoneIdError) + ) { + returnObject.readZoneIdError = readZoneIdError; + } else { + returnObject.readZoneIdError = unexpectedValueError; + } + if (readZoneIdErrorCode != missingKeyError) { + let code = parseInt(readZoneIdErrorCode, 10); + if (!isNaN(code)) { + returnObject.readZoneIdErrorCode = code; + } + } + } else { + if (possibleIniErrors.includes(zoneId)) { + returnObject.zoneId = zoneId; + } else { + let id = parseInt(zoneId, 10); + if (isNaN(id) || id < 0 || id > 4) { + returnObject.zoneId = unexpectedValueError; + } else { + returnObject.zoneId = id; + } + } + + let isMozillaURL = url => { + const mozillaDomains = ["mozilla.com", "mozilla.net", "mozilla.org"]; + for (const domain of mozillaDomains) { + if (url.hostname == domain) { + return true; + } + if (url.hostname.endsWith("." + domain)) { + return true; + } + } + return false; + }; + + if (possibleIniErrors.includes(referrerUrl)) { + returnObject.referrerUrl = referrerUrl; + } else { + try { + returnObject.referrerUrl = new URL(referrerUrl); + } catch (ex) { + returnObject.referrerUrl = unexpectedValueError; + } + if (URL.isInstance(returnObject.referrerUrl)) { + returnObject.referrerUrlIsMozilla = isMozillaURL( + returnObject.referrerUrl + ); + } + } + + if (possibleIniErrors.includes(hostUrl)) { + returnObject.hostUrl = hostUrl; + } else { + try { + returnObject.hostUrl = new URL(hostUrl); + } catch (ex) { + returnObject.hostUrl = unexpectedValueError; + } + if (URL.isInstance(returnObject.hostUrl)) { + returnObject.hostUrlIsMozilla = isMozillaURL(returnObject.hostUrl); + } + } + } + + return returnObject; + })(); + return gReadZoneIdPromise; + }, + + /** + * Only submits telemetry once, no matter how many times it is called. + * Has no effect on OSs where provenance data is not supported. + * + * @returns An object indicating the values submitted. Keys may not match the + * Scalar names since the returned object is intended to be suitable + * for use as a Telemetry Event's `extra` object, which has shorter + * limits for extra key names than the limits for Scalar names. + * Values will be converted to strings since Telemetry Event's + * `extra` objects must have string values. + * On platforms that do not support provenance data, this will always + * return an empty object. + */ + async submitProvenanceTelemetry() { + if (gTelemetryPromise) { + return gTelemetryPromise; + } + gTelemetryPromise = (async () => { + const errorValue = "error"; + + let extra = {}; + + let provenance = await this.readZoneIdProvenanceFile(); + if (!provenance) { + return extra; + } + + let setTelemetry = (scalarName, extraKey, value) => { + Services.telemetry.scalarSet(scalarName, value); + extra[extraKey] = value.toString(); + }; + + setTelemetry( + "attribution.provenance.data_exists", + "data_exists", + !provenance.readProvenanceError + ); + if (provenance.readProvenanceError) { + return extra; + } + + setTelemetry( + "attribution.provenance.file_system", + "file_system", + provenance.fileSystem ?? errorValue + ); + + // https://learn.microsoft.com/en-us/windows/win32/debug/system-error-codes--0-499-#ERROR_FILE_NOT_FOUND + const ERROR_FILE_NOT_FOUND = 2; + + let ads_exists = + !provenance.readProvenanceError && + !( + provenance.readZoneIdError == "openFile" && + provenance.readZoneIdErrorCode == ERROR_FILE_NOT_FOUND + ); + setTelemetry( + "attribution.provenance.ads_exists", + "ads_exists", + ads_exists + ); + if (!ads_exists) { + return extra; + } + + setTelemetry( + "attribution.provenance.security_zone", + "security_zone", + "zoneId" in provenance ? provenance.zoneId.toString() : errorValue + ); + + let haveReferrerUrl = URL.isInstance(provenance.referrerUrl); + setTelemetry( + "attribution.provenance.referrer_url_exists", + "refer_url_exist", + haveReferrerUrl + ); + if (haveReferrerUrl) { + setTelemetry( + "attribution.provenance.referrer_url_is_mozilla", + "refer_url_moz", + provenance.referrerUrlIsMozilla + ); + } + + let haveHostUrl = URL.isInstance(provenance.hostUrl); + setTelemetry( + "attribution.provenance.host_url_exists", + "host_url_exist", + haveHostUrl + ); + if (haveHostUrl) { + setTelemetry( + "attribution.provenance.host_url_is_mozilla", + "host_url_moz", + provenance.hostUrlIsMozilla + ); + } + + return extra; + })(); + return gTelemetryPromise; + }, +}; diff --git a/browser/components/attribution/docs/index.rst b/browser/components/attribution/docs/index.rst new file mode 100644 index 0000000000..573afa70e2 --- /dev/null +++ b/browser/components/attribution/docs/index.rst @@ -0,0 +1,131 @@ +======================== +Installation Attribution +======================== + +*Installation Attribution* is a system that allows us to do a few things: + +- Gauge the success of marketing campaigns +- Determine (roughly) the origin of an installer that a user ran +- Support the Return to AMO workflow. + +We accomplish these things by adding an *attribution code* to Firefox installers which generally contains information supplied by www.mozilla.org (Bedrock and related services). This information is read by Firefox during and after installation, and sent back to Mozilla through Firefox Telemetry. + +The following information is supported by this system: + +* Traditional `UTM parameters <https://en.wikipedia.org/wiki/UTM_parameters>`_ (*source*, *medium*, *campaign*, and *content*) +* *experiment* +* *variation* +* *ua* +* *dltoken* +* *dlsource* +* *msstoresignedin* + +Descriptions of each of these can be found in :ref:`the Telemetry Environment documentation <environment>`. + +-------------------------- +Firefox Windows Installers +-------------------------- + +Installs done with either the stub or full NSIS installer are capable of being attributed. When these installers are created, they are given initial attribution data of *dlsource=mozillaci*. Users who download their installer via www.mozilla.org will typically have this attribution data overwritten (unless they have Do-not-track (DNT) enabled), with *dlsource=mozillaci*, a *dltoken*, and whatever UTM parameters Bedrock deems appropriate. + +An additional complication here is that the attribution system is used (or abused, depending on your view) to support the Return to AMO workflow -- forcing the *campaign* and *content* UTM parameters to specific values. + +The below diagram illustrates the flow of the cases above: + +.. mermaid:: + + flowchart TD + subgraph Legend + direction LR + start1[ ] --->|"(1) Bedrock, Do-Not-Track enabled"| stop1[ ] + start2[ ] --->|"(2) Bedrock, Do-Not-Track disabled"| stop2[ ] + start3[ ] --->|"(3) Bedrock via Search Engine"| stop3[ ] + start4[ ] --->|"(4) Bedrock via Return to AMO flow"| stop4[ ] + start5[ ] --->|"(5) Direct download from CDN Origin"| stop5[ ] + start6[ ] --->|"Common Paths"| stop6[ ] + + %% Legend colours + linkStyle 0 stroke-width:2px,fill:none,stroke:blue; + linkStyle 1 stroke-width:2px,fill:none,stroke:green; + linkStyle 2 stroke-width:2px,fill:none,stroke:red; + linkStyle 3 stroke-width:2px,fill:none,stroke:purple; + linkStyle 4 stroke-width:2px,fill:none,stroke:pink; + end + + subgraph Download Flow + User([User]) + Search[Search Engine] + AMO[AMO<br>addons.mozilla.org] + Bedrock[Bedrock<br>mozilla.org] + Bouncer[Bouncer<br>download.mozilla.org] + CDNOrigin[CDN Origin<br>archive.mozilla.org] + CDN[CDN<br>download-installer.cdn.mozilla.net] + Attr[Attribution Service<br>stubdownloader.services.mozilla.net] + AttrCDN[Attribution CDN<br>cdn.stubdownloader.services.mozilla.net] + Inst([Installer Downloaded]) + + %% Case 1: Bedrock, DNT enabled + User ----> Bedrock + Bedrock ---->|"No attribution data set"| Bouncer + Bouncer ----> CDN + CDN ---->|"Contains static attribution data:<br><i>dlsource</i>: mozillaci"| Inst + + linkStyle 6,7,8,9 stroke-width:2px,fill:none,stroke:blue; + + %% Case 2: Bedrock, DNT disabled + User ---> Bedrock + + linkStyle 10 stroke-width:2px,fill:none,stroke:green; + + %% Case 3: Bedrock via Search Engine + User ----> Search + Search ---->|"Sets UTM parameters"| Bedrock + + linkStyle 11,12 stroke-width:2px,fill:none,stroke:red; + + %% Case 4: Bedrock via Return to AMO flow + User ---->|"Initiates Return-to-AMO request"| AMO + AMO ---->|"Sets <i>campaign</i> and<br><i>content</i> UTM parameters"| Bedrock + + linkStyle 13,14 stroke-width:2px,fill:none,stroke:purple; + + %% Case 5: Direct download from CDN + User --> CDNOrigin + CDNOrigin -->|"Contains static attribution data:<br><i>dlsource</i>: mozillaci"| Inst + + linkStyle 15,16 stroke-width:2px,fill:none,stroke:pink; + + %% Common links for cases 2, 3, and 4 + Bedrock ---->|"Attribution data forwarded:<br><i>dlsource</i>: mozorg<br><i>dltoken</i>: present<br>any UTM parameters set"| Bouncer + Bouncer ---->|"Forwards attribution data"| Attr + Attr <---->|"Fetches installer"| CDN + Attr ---->|"Places modified installer<br>on Attribution CDN"| AttrCDN + AttrCDN ---->|"Contains dynamic attribution data:<br><i>dlsource</i>: mozorg<br><i>dltoken</i>: present<br>any UTM parameters set"| Inst + + %% Common links for everything + CDN <---->|"Fetches installer"| CDNOrigin + end + + +--------------- +Microsoft Store +--------------- + +Firefox installs done through the Microsoft Store support extracting campaign IDs that may be embedded into them. This allows us to attribute installs through different channels by providing particular links to the Microsoft Store with attribution data included. For example: + +`ms-windows-store://pdp/?productid=9NZVDKPMR9RD&cid=source%3Dgoogle.com%26medium%3Dorganic%26campaign%3D(not%20set)%26content%3D(not%20set) <ms-windows-store://pdp/?productid=9NZVDKPMR9RD&cid=source%3Dgoogle.com%26medium%3Dorganic%26campaign%3D(not%20set)%26content%3D(not%20set)>`_ + + +`https://www.microsoft.com/store/apps/9NZVDKPMR9RD?cid=source%3Dgoogle.com%26medium%3Dorganic%26campaign%3D(not%20set)%26content%3D(not%20set) <https://www.microsoft.com/store/apps/9NZVDKPMR9RD?cid=source%3Dgoogle.com%26medium%3Dorganic%26campaign%3D(not%20set)%26content%3D(not%20set)>`_ + + +For more on how custom campaign IDs work in general in the Microsoft Store environment, `see Microsoft's documentation <https://docs.microsoft.com/en-us/windows/uwp/publish/create-a-custom-app-promotion-campaign>`_. + +The Microsoft Store provides a single `cid` (Campaign ID). Their documentation claims it is limited to 100 characters, although in our own testing we've been able to retrieve the first 208 characters of Campaign IDs. Firefox expects this Campaign ID to follow the same format as stub and full installer attribution codes, which have a maximum of length of 1010 characters. Since Campaign IDs are more limited than what Firefox allows, we need to be a bit more thoughtful about what we include in them vs. stub and full installer attribution. At the time of writing, we've yet to be able to test whether we can reliably pull more than the advertised 100 characters of a Campaign ID in the real world -- something that we should do before we send any crucial information past the first 100 characters. + +In addition to the attribution data retrieved through the campaign ID, we also add an extra key to it to indicate whether or not the user was signed into the Microsoft Store when they installed. This `msstoresignedin` key can have a value of `true` or `false`. + +There are a couple of other caveats to keep in mind: + +* A campaign ID is only set the *first* time a user installs Firefox through the Store. Subsequent installs will inherit the original campaign ID (even if it was an empty string). This means that only brand new installs will be attributed -- not reinstalls. +* At the time of writing, it is not clear whether or not installs done without being signed into the Microsoft Store will be able to find their campaign ID. Microsoft's documentation claims they can, but our own testing has not been able to verify this. diff --git a/browser/components/attribution/moz.build b/browser/components/attribution/moz.build new file mode 100644 index 0000000000..cbbd53768b --- /dev/null +++ b/browser/components/attribution/moz.build @@ -0,0 +1,40 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +with Files("**"): + BUG_COMPONENT = ("Toolkit", "Telemetry") + +XPCSHELL_TESTS_MANIFESTS += ["test/xpcshell/xpcshell.ini"] + +BROWSER_CHROME_MANIFESTS += ["test/browser/browser.ini"] + +EXTRA_JS_MODULES += [ + "AttributionCode.sys.mjs", + "ProvenanceData.sys.mjs", +] + +SPHINX_TREES["docs"] = "docs" + +if CONFIG["MOZ_WIDGET_TOOLKIT"] == "cocoa": + XPIDL_SOURCES += [ + "nsIMacAttribution.idl", + ] + + XPIDL_MODULE = "attribution" + + EXPORTS += [ + "nsMacAttribution.h", + ] + + SOURCES += [ + "nsMacAttribution.cpp", + ] + + FINAL_LIBRARY = "browsercomps" + + EXTRA_JS_MODULES += [ + "MacAttribution.sys.mjs", + ] diff --git a/browser/components/attribution/nsIMacAttribution.idl b/browser/components/attribution/nsIMacAttribution.idl new file mode 100644 index 0000000000..5bd1818817 --- /dev/null +++ b/browser/components/attribution/nsIMacAttribution.idl @@ -0,0 +1,23 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "nsISupports.idl" + +[scriptable, uuid(6FC66A78-6CBC-4B3F-B7BA-379289B29276)] +interface nsIMacAttributionService : nsISupports +{ + /** + * Set the referrer URL on a given path. + * + * @param aFilePath A path to the file to set the quarantine data on. + * @param aReferrer A url to set as the referrer for the download. + * @param aCreate If true, creates new quarantine properties, overwriting + * any existing properties. If false, the referrer is only + * set if quarantine properties already exist on the file. + */ + void setReferrerUrl(in AUTF8String aFilePath, + in AUTF8String aReferrer, + in boolean aCreate); +}; diff --git a/browser/components/attribution/nsMacAttribution.cpp b/browser/components/attribution/nsMacAttribution.cpp new file mode 100644 index 0000000000..7397285794 --- /dev/null +++ b/browser/components/attribution/nsMacAttribution.cpp @@ -0,0 +1,51 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "nsMacAttribution.h" + +#include <CoreFoundation/CoreFoundation.h> +#include <ApplicationServices/ApplicationServices.h> + +#include "../../../xpcom/io/CocoaFileUtils.h" +#include "nsCocoaFeatures.h" +#include "nsString.h" + +using namespace mozilla; + +NS_IMPL_ISUPPORTS(nsMacAttributionService, nsIMacAttributionService) + +NS_IMETHODIMP +nsMacAttributionService::SetReferrerUrl(const nsACString& aFilePath, + const nsACString& aReferrerUrl, + const bool aCreate) { + const nsCString& flat = PromiseFlatCString(aFilePath); + CFStringRef filePath = ::CFStringCreateWithCString( + kCFAllocatorDefault, flat.get(), kCFStringEncodingUTF8); + + if (!filePath) { + return NS_ERROR_UNEXPECTED; + } + + const nsCString& flatReferrer = PromiseFlatCString(aReferrerUrl); + CFStringRef referrer = ::CFStringCreateWithCString( + kCFAllocatorDefault, flatReferrer.get(), kCFStringEncodingUTF8); + if (!referrer) { + ::CFRelease(filePath); + return NS_ERROR_UNEXPECTED; + } + CFURLRef referrerURL = + ::CFURLCreateWithString(kCFAllocatorDefault, referrer, nullptr); + + CocoaFileUtils::AddQuarantineMetadataToFile(filePath, NULL, referrerURL, true, + aCreate); + + ::CFRelease(filePath); + ::CFRelease(referrer); + if (referrerURL) { + ::CFRelease(referrerURL); + } + + return NS_OK; +} diff --git a/browser/components/attribution/nsMacAttribution.h b/browser/components/attribution/nsMacAttribution.h new file mode 100644 index 0000000000..9ccbee4ea4 --- /dev/null +++ b/browser/components/attribution/nsMacAttribution.h @@ -0,0 +1,22 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef nsmacattribution_h____ +#define nsmacattribution_h____ + +#include "nsIMacAttribution.h" + +class nsMacAttributionService : public nsIMacAttributionService { + public: + nsMacAttributionService(){}; + + NS_DECL_ISUPPORTS + NS_DECL_NSIMACATTRIBUTIONSERVICE + + protected: + virtual ~nsMacAttributionService(){}; +}; + +#endif // nsmacattribution_h____ diff --git a/browser/components/attribution/test/browser/browser.ini b/browser/components/attribution/test/browser/browser.ini new file mode 100644 index 0000000000..080df0afef --- /dev/null +++ b/browser/components/attribution/test/browser/browser.ini @@ -0,0 +1,9 @@ +[DEFAULT] +support-files = + head.js + +[browser_AttributionCode_Mac_telemetry.js] +skip-if = toolkit != "cocoa" # macOS only telemetry. +[browser_AttributionCode_telemetry.js] +skip-if = (os != "win" && toolkit != "cocoa") # Windows and macOS only telemetry. + diff --git a/browser/components/attribution/test/browser/browser_AttributionCode_Mac_telemetry.js b/browser/components/attribution/test/browser/browser_AttributionCode_Mac_telemetry.js new file mode 100644 index 0000000000..7523f0f930 --- /dev/null +++ b/browser/components/attribution/test/browser/browser_AttributionCode_Mac_telemetry.js @@ -0,0 +1,248 @@ +ChromeUtils.defineESModuleGetters(this, { + TelemetryTestUtils: "resource://testing-common/TelemetryTestUtils.sys.mjs", +}); +const { MacAttribution } = ChromeUtils.importESModule( + "resource:///modules/MacAttribution.sys.mjs" +); +const { AttributionIOUtils } = ChromeUtils.importESModule( + "resource:///modules/AttributionCode.sys.mjs" +); +const { sinon } = ChromeUtils.importESModule( + "resource://testing-common/Sinon.sys.mjs" +); + +async function assertCacheExistsAndIsEmpty() { + // We should have written to the cache, and be able to read back + // with no errors. + const histogram = Services.telemetry.getHistogramById( + "BROWSER_ATTRIBUTION_ERRORS" + ); + histogram.clear(); + + ok(await AttributionIOUtils.exists(AttributionCode.attributionFile.path)); + Assert.deepEqual( + "", + new TextDecoder().decode( + await AttributionIOUtils.read(AttributionCode.attributionFile.path) + ) + ); + + AttributionCode._clearCache(); + let result = await AttributionCode.getAttrDataAsync(); + Assert.deepEqual(result, {}, "Should be able to get cached result"); + + Assert.deepEqual({}, histogram.snapshot().values || {}); +} + +add_task(async function test_write_error() { + const sandbox = sinon.createSandbox(); + let attributionSvc = Cc["@mozilla.org/mac-attribution;1"].getService( + Ci.nsIMacAttributionService + ); + attributionSvc.setReferrerUrl( + MacAttribution.applicationPath, + "https://example.com?content=content", + true + ); + + await AttributionCode.deleteFileAsync(); + AttributionCode._clearCache(); + + const histogram = Services.telemetry.getHistogramById( + "BROWSER_ATTRIBUTION_ERRORS" + ); + + let oldExists = AttributionIOUtils.exists; + let oldWrite = AttributionIOUtils.write; + try { + // Clear any existing telemetry + histogram.clear(); + + // Force the file to not exist and then cause a write error. This is delicate + // because various background tasks may invoke `IOUtils.writeAtomic` while + // this test is running. Be careful to only stub the one call. + AttributionIOUtils.exists = () => false; + AttributionIOUtils.write = () => { + throw new Error("write_error"); + }; + + // Try to read the attribution code. + let result = await AttributionCode.getAttrDataAsync(); + Assert.deepEqual( + result, + { content: "content" }, + "Should be able to get a result even if the file doesn't write" + ); + + TelemetryTestUtils.assertHistogram(histogram, INDEX_WRITE_ERROR, 1); + } finally { + AttributionIOUtils.exists = oldExists; + AttributionIOUtils.write = oldWrite; + await AttributionCode.deleteFileAsync(); + AttributionCode._clearCache(); + histogram.clear(); + sandbox.restore(); + } +}); + +add_task(async function test_unusual_referrer() { + // This referrer URL looks malformed, but the malformed bits are dropped, so + // it's actually ok. This is what allows extraneous bits like `fbclid` tags + // to be ignored. + let attributionSvc = Cc["@mozilla.org/mac-attribution;1"].getService( + Ci.nsIMacAttributionService + ); + attributionSvc.setReferrerUrl( + MacAttribution.applicationPath, + "https://example.com?content=&=campaign", + true + ); + + await AttributionCode.deleteFileAsync(); + AttributionCode._clearCache(); + + const histogram = Services.telemetry.getHistogramById( + "BROWSER_ATTRIBUTION_ERRORS" + ); + try { + // Clear any existing telemetry + histogram.clear(); + + // Try to read the attribution code + await AttributionCode.getAttrDataAsync(); + + let result = await AttributionCode.getAttrDataAsync(); + Assert.deepEqual(result, {}, "Should be able to get empty result"); + + Assert.deepEqual({}, histogram.snapshot().values || {}); + + await assertCacheExistsAndIsEmpty(); + } finally { + await AttributionCode.deleteFileAsync(); + AttributionCode._clearCache(); + histogram.clear(); + } +}); + +add_task(async function test_blank_referrer() { + let attributionSvc = Cc["@mozilla.org/mac-attribution;1"].getService( + Ci.nsIMacAttributionService + ); + attributionSvc.setReferrerUrl(MacAttribution.applicationPath, "", true); + + await AttributionCode.deleteFileAsync(); + AttributionCode._clearCache(); + + const histogram = Services.telemetry.getHistogramById( + "BROWSER_ATTRIBUTION_ERRORS" + ); + + try { + // Clear any existing telemetry + histogram.clear(); + + // Try to read the attribution code + let result = await AttributionCode.getAttrDataAsync(); + Assert.deepEqual(result, {}, "Should be able to get empty result"); + + Assert.deepEqual({}, histogram.snapshot().values || {}); + + await assertCacheExistsAndIsEmpty(); + } finally { + await AttributionCode.deleteFileAsync(); + AttributionCode._clearCache(); + histogram.clear(); + } +}); + +add_task(async function test_no_referrer() { + const sandbox = sinon.createSandbox(); + let newApplicationPath = MacAttribution.applicationPath + ".test"; + sandbox.stub(MacAttribution, "applicationPath").get(() => newApplicationPath); + + await AttributionCode.deleteFileAsync(); + AttributionCode._clearCache(); + + const histogram = Services.telemetry.getHistogramById( + "BROWSER_ATTRIBUTION_ERRORS" + ); + try { + // Clear any existing telemetry + histogram.clear(); + + // Try to read the attribution code + await AttributionCode.getAttrDataAsync(); + + let result = await AttributionCode.getAttrDataAsync(); + Assert.deepEqual(result, {}, "Should be able to get empty result"); + + Assert.deepEqual({}, histogram.snapshot().values || {}); + + await assertCacheExistsAndIsEmpty(); + } finally { + await AttributionCode.deleteFileAsync(); + AttributionCode._clearCache(); + histogram.clear(); + sandbox.restore(); + } +}); + +add_task(async function test_broken_referrer() { + let attributionSvc = Cc["@mozilla.org/mac-attribution;1"].getService( + Ci.nsIMacAttributionService + ); + attributionSvc.setReferrerUrl( + MacAttribution.applicationPath, + "https://example.com?content=content", + true + ); + + // This uses macOS internals to change the GUID so that it will look like the + // application has quarantine data but nothing will be pressent in the + // quarantine database. This shouldn't happen in the wild. + function generateQuarantineGUID() { + let str = Services.uuid.generateUUID().toString().toUpperCase(); + // Strip {}. + return str.substring(1, str.length - 1); + } + + // These magic constants are macOS GateKeeper flags. + let string = [ + "01c1", + "5991b778", + "Safari.app", + generateQuarantineGUID(), + ].join(";"); + let bytes = new TextEncoder().encode(string); + await IOUtils.setMacXAttr( + MacAttribution.applicationPath, + "com.apple.quarantine", + bytes + ); + + await AttributionCode.deleteFileAsync(); + AttributionCode._clearCache(); + + const histogram = Services.telemetry.getHistogramById( + "BROWSER_ATTRIBUTION_ERRORS" + ); + try { + // Clear any existing telemetry + histogram.clear(); + + // Try to read the attribution code + await AttributionCode.getAttrDataAsync(); + + let result = await AttributionCode.getAttrDataAsync(); + Assert.deepEqual(result, {}, "Should be able to get empty result"); + + TelemetryTestUtils.assertHistogram(histogram, INDEX_QUARANTINE_ERROR, 1); + histogram.clear(); + + await assertCacheExistsAndIsEmpty(); + } finally { + await AttributionCode.deleteFileAsync(); + AttributionCode._clearCache(); + histogram.clear(); + } +}); diff --git a/browser/components/attribution/test/browser/browser_AttributionCode_telemetry.js b/browser/components/attribution/test/browser/browser_AttributionCode_telemetry.js new file mode 100644 index 0000000000..43888a1e74 --- /dev/null +++ b/browser/components/attribution/test/browser/browser_AttributionCode_telemetry.js @@ -0,0 +1,101 @@ +ChromeUtils.defineESModuleGetters(this, { + TelemetryTestUtils: "resource://testing-common/TelemetryTestUtils.sys.mjs", +}); +const { AttributionIOUtils } = ChromeUtils.importESModule( + "resource:///modules/AttributionCode.sys.mjs" +); + +add_task(async function test_parse_error() { + if (AppConstants.platform == "macosx") { + // On macOS, the underlying data is the OS-level quarantine + // database. We need to start from nothing to isolate the cache. + const { MacAttribution } = ChromeUtils.importESModule( + "resource:///modules/MacAttribution.sys.mjs" + ); + let attributionSvc = Cc["@mozilla.org/mac-attribution;1"].getService( + Ci.nsIMacAttributionService + ); + attributionSvc.setReferrerUrl(MacAttribution.applicationPath, "", true); + } + + registerCleanupFunction(async () => { + await AttributionCode.deleteFileAsync(); + AttributionCode._clearCache(); + }); + const histogram = Services.telemetry.getHistogramById( + "BROWSER_ATTRIBUTION_ERRORS" + ); + // Delete the file to trigger a read error + await AttributionCode.deleteFileAsync(); + AttributionCode._clearCache(); + // Clear any existing telemetry + histogram.clear(); + let result = await AttributionCode.getAttrDataAsync(); + Assert.deepEqual( + result, + {}, + "Shouldn't be able to get a result if the file doesn't exist" + ); + + // Write an invalid file to trigger a decode error + // Skip this for MSIX packages though - we can't write or delete + // the attribution file there, everything happens in memory instead. + if ( + AppConstants.platform === "win" && + !Services.sysinfo.getProperty("hasWinPackageId") + ) { + await AttributionCode.deleteFileAsync(); + AttributionCode._clearCache(); + // Empty string is valid on macOS. + await AttributionCode.writeAttributionFile( + AppConstants.platform == "macosx" ? "invalid" : "" + ); + result = await AttributionCode.getAttrDataAsync(); + Assert.deepEqual(result, {}, "Should have failed to parse"); + + // `assertHistogram` also ensures that `read_error` index 0 is 0 + // as we should not have recorded telemetry from the previous `getAttrDataAsync` call + TelemetryTestUtils.assertHistogram(histogram, INDEX_DECODE_ERROR, 1); + // Reset + histogram.clear(); + } +}); + +add_task(async function test_read_error() { + registerCleanupFunction(async () => { + await AttributionCode.deleteFileAsync(); + AttributionCode._clearCache(); + }); + const histogram = Services.telemetry.getHistogramById( + "BROWSER_ATTRIBUTION_ERRORS" + ); + + // Delete the file to trigger a read error + await AttributionCode.deleteFileAsync(); + AttributionCode._clearCache(); + // Clear any existing telemetry + histogram.clear(); + + // Force the file to exist but then cause a read error + let oldExists = AttributionIOUtils.exists; + AttributionIOUtils.exists = () => true; + + let oldRead = AttributionIOUtils.read; + AttributionIOUtils.read = () => { + throw new Error("read_error"); + }; + + registerCleanupFunction(() => { + AttributionIOUtils.exists = oldExists; + AttributionIOUtils.read = oldRead; + }); + + // Try to read the file + await AttributionCode.getAttrDataAsync(); + + // It should record the read error + TelemetryTestUtils.assertHistogram(histogram, INDEX_READ_ERROR, 1); + + // Clear any existing telemetry + histogram.clear(); +}); diff --git a/browser/components/attribution/test/browser/head.js b/browser/components/attribution/test/browser/head.js new file mode 100644 index 0000000000..61b24284ee --- /dev/null +++ b/browser/components/attribution/test/browser/head.js @@ -0,0 +1,23 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ + */ +"use strict"; + +const { AttributionCode } = ChromeUtils.importESModule( + "resource:///modules/AttributionCode.sys.mjs" +); + +// Keep in sync with `BROWSER_ATTRIBUTION_ERRORS` in Histograms.json. +const INDEX_READ_ERROR = 0; +const INDEX_DECODE_ERROR = 1; +const INDEX_WRITE_ERROR = 2; +const INDEX_QUARANTINE_ERROR = 3; + +add_setup(function () { + // AttributionCode._clearCache is only possible in a testing environment + Services.env.set("XPCSHELL_TEST_PROFILE_DIR", "testing"); + + registerCleanupFunction(() => { + Services.env.set("XPCSHELL_TEST_PROFILE_DIR", null); + }); +}); diff --git a/browser/components/attribution/test/xpcshell/head.js b/browser/components/attribution/test/xpcshell/head.js new file mode 100644 index 0000000000..67a93563d4 --- /dev/null +++ b/browser/components/attribution/test/xpcshell/head.js @@ -0,0 +1,136 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ + */ +"use strict"; + +const { AttributionCode } = ChromeUtils.importESModule( + "resource:///modules/AttributionCode.sys.mjs" +); + +let validAttrCodes = [ + { + code: "source%3Dgoogle.com%26medium%3Dorganic%26campaign%3D(not%20set)%26content%3D(not%20set)", + parsed: { + source: "google.com", + medium: "organic", + campaign: "(not%20set)", + content: "(not%20set)", + }, + }, + { + code: "source%3Dgoogle.com%26medium%3Dorganic%26campaign%3D(not%20set)%26content%3D(not%20set)%26msstoresignedin%3Dtrue", + parsed: { + source: "google.com", + medium: "organic", + campaign: "(not%20set)", + content: "(not%20set)", + msstoresignedin: true, + }, + platforms: ["win"], + }, + { + code: "source%3Dgoogle.com%26medium%3Dorganic%26campaign%3D%26content%3D", + parsed: { source: "google.com", medium: "organic" }, + doesNotRoundtrip: true, // `campaign=` and `=content` are dropped. + }, + { + code: "source%3Dgoogle.com%26medium%3Dorganic%26campaign%3D(not%20set)", + parsed: { + source: "google.com", + medium: "organic", + campaign: "(not%20set)", + }, + }, + { + code: "source%3Dgoogle.com%26medium%3Dorganic", + parsed: { source: "google.com", medium: "organic" }, + }, + { code: "source%3Dgoogle.com", parsed: { source: "google.com" } }, + { code: "medium%3Dgoogle.com", parsed: { medium: "google.com" } }, + { code: "campaign%3Dgoogle.com", parsed: { campaign: "google.com" } }, + { code: "content%3Dgoogle.com", parsed: { content: "google.com" } }, + { + code: "experiment%3Dexperimental", + parsed: { experiment: "experimental" }, + }, + { code: "variation%3Dvaried", parsed: { variation: "varied" } }, + { + code: "ua%3DGoogle%20Chrome%20123", + parsed: { ua: "Google%20Chrome%20123" }, + }, + { + code: "dltoken%3Dc18f86a3-f228-4d98-91bb-f90135c0aa9c", + parsed: { dltoken: "c18f86a3-f228-4d98-91bb-f90135c0aa9c" }, + }, + { + code: "dlsource%3Dsome-dl-source", + parsed: { + dlsource: "some-dl-source", + }, + }, +]; + +let invalidAttrCodes = [ + // Empty string + "", + // Not escaped + "source=google.com&medium=organic&campaign=(not set)&content=(not set)", + // Too long + "campaign%3D" + "a".repeat(1000), + // Unknown key name + "source%3Dgoogle.com%26medium%3Dorganic%26large%3Dgeneticallymodified", + // Empty key name + "source%3Dgoogle.com%26medium%3Dorganic%26%3Dgeneticallymodified", +]; + +/** + * Arrange for each test to have a unique application path for storing + * quarantine data. + * + * The quarantine data is necessarily a shared system resource, managed by the + * OS, so we need to avoid polluting it during tests. + * + * There are at least two ways to achieve this. Here we use Sinon to stub the + * relevant accessors: this has the advantage of being local and relatively easy + * to follow. In the App Update Service tests, an `nsIDirectoryServiceProvider` + * is installed, which is global and much harder to extract for re-use. + */ +async function setupStubs() { + // Local imports to avoid polluting the global namespace. + const { AppConstants } = ChromeUtils.importESModule( + "resource://gre/modules/AppConstants.sys.mjs" + ); + const { sinon } = ChromeUtils.importESModule( + "resource://testing-common/Sinon.sys.mjs" + ); + + // This depends on the caller to invoke it by name. We do try to + // prevent the most obvious incorrect invocation, namely + // `add_task(setupStubs)`. + let caller = Components.stack.caller; + const testID = caller.filename.toString().split("/").pop().split(".")[0]; + notEqual(testID, "head"); + + let applicationFile = do_get_tempdir(); + applicationFile.append(testID); + applicationFile.append("App.app"); + + if (AppConstants.platform == "macosx") { + // We're implicitly using the fact that modules are shared between importers here. + const { MacAttribution } = ChromeUtils.importESModule( + "resource:///modules/MacAttribution.sys.mjs" + ); + sinon + .stub(MacAttribution, "applicationPath") + .get(() => applicationFile.path); + } + + // The macOS quarantine database applies to existing paths only, so make + // sure our mock application path exists. This also creates the parent + // directory for the attribution file, needed on both macOS and Windows. We + // don't ignore existing paths because we're inside a temporary directory: + // this should never be invoked twice for the same test. + await IOUtils.makeDirectory(applicationFile.path, { + from: do_get_tempdir().path, + }); +} diff --git a/browser/components/attribution/test/xpcshell/test_AttributionCode.js b/browser/components/attribution/test/xpcshell/test_AttributionCode.js new file mode 100644 index 0000000000..1296420cd7 --- /dev/null +++ b/browser/components/attribution/test/xpcshell/test_AttributionCode.js @@ -0,0 +1,131 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ + */ +"use strict"; + +const { AppConstants } = ChromeUtils.importESModule( + "resource://gre/modules/AppConstants.sys.mjs" +); + +const { sinon } = ChromeUtils.importESModule( + "resource://testing-common/Sinon.sys.mjs" +); + +add_task(async () => { + await setupStubs(); +}); + +/** + * Test validation of attribution codes, + * to make sure we reject bad ones and accept good ones. + */ +add_task(async function testValidAttrCodes() { + let currentCode = null; + for (let entry of validAttrCodes) { + currentCode = entry.code; + // Attribution for MSIX builds works quite differently than regular Windows + // builds: the attribution codes come from a Windows API that we've wrapped + // with an XPCOM class. We don't have a way to inject codes into the build, + // so instead we mock out our XPCOM class and return the desired values from + // there. (A potential alternative is to have MSIX tests rely on attribution + // files, but that more or less invalidates the tests.) + if ( + AppConstants.platform === "win" && + Services.sysinfo.getProperty("hasWinPackageId") + ) { + // In real life, the attribution codes returned from Microsoft APIs + // are not URI encoded, and the AttributionCode code that deals with + // them expects that - so we have to simulate that as well. + sinon + .stub(AttributionCode, "msixCampaignId") + .get(() => decodeURIComponent(currentCode)); + } else { + await AttributionCode.writeAttributionFile(currentCode); + } + AttributionCode._clearCache(); + let result = await AttributionCode.getAttrDataAsync(); + Assert.deepEqual( + result, + entry.parsed, + "Parsed code should match expected value, code was: " + currentCode + ); + } + AttributionCode._clearCache(); +}); + +/** + * Make sure codes with various formatting errors are not seen as valid. + */ +add_task(async function testInvalidAttrCodes() { + let currentCode = null; + for (let code of invalidAttrCodes) { + currentCode = code; + + if ( + AppConstants.platform === "win" && + Services.sysinfo.getProperty("hasWinPackageId") + ) { + if (code.includes("not set")) { + // One of the "invalid" codes that we test is an unescaped one. + // This is valid for most platforms, but we actually _expect_ + // unescaped codes for MSIX builds, so that particular test is not + // valid for this case. + continue; + } + + sinon + .stub(AttributionCode, "msixCampaignId") + .get(() => decodeURIComponent(currentCode)); + } else { + await AttributionCode.writeAttributionFile(currentCode); + } + AttributionCode._clearCache(); + let result = await AttributionCode.getAttrDataAsync(); + Assert.deepEqual( + result, + {}, + "Code should have failed to parse: " + currentCode + ); + } + AttributionCode._clearCache(); +}); + +/** + * Test the cache by deleting the attribution data file + * and making sure we still get the expected code. + */ +let condition = { + // MSIX attribution codes are not cached by us, thus this test is + // unnecessary for those builds. + skip_if: () => + AppConstants.platform === "win" && + Services.sysinfo.getProperty("hasWinPackageId"), +}; +add_task(condition, async function testDeletedFile() { + // Set up the test by clearing the cache and writing a valid file. + await AttributionCode.writeAttributionFile(validAttrCodes[0].code); + let result = await AttributionCode.getAttrDataAsync(); + Assert.deepEqual( + result, + validAttrCodes[0].parsed, + "The code should be readable directly from the file" + ); + + // Delete the file and make sure we can still read the value back from cache. + await AttributionCode.deleteFileAsync(); + result = await AttributionCode.getAttrDataAsync(); + Assert.deepEqual( + result, + validAttrCodes[0].parsed, + "The code should be readable from the cache" + ); + + // Clear the cache and check we can't read anything. + AttributionCode._clearCache(); + result = await AttributionCode.getAttrDataAsync(); + Assert.deepEqual( + result, + {}, + "Shouldn't be able to get a code after file is deleted and cache is cleared" + ); +}); diff --git a/browser/components/attribution/test/xpcshell/test_MacAttribution.js b/browser/components/attribution/test/xpcshell/test_MacAttribution.js new file mode 100644 index 0000000000..078b056dac --- /dev/null +++ b/browser/components/attribution/test/xpcshell/test_MacAttribution.js @@ -0,0 +1,97 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ + */ + +"use strict"; + +const { MacAttribution } = ChromeUtils.importESModule( + "resource:///modules/MacAttribution.sys.mjs" +); + +add_task(async () => { + await setupStubs(); +}); + +add_task(async function testValidAttrCodes() { + let appPath = MacAttribution.applicationPath; + let attributionSvc = Cc["@mozilla.org/mac-attribution;1"].getService( + Ci.nsIMacAttributionService + ); + + for (let entry of validAttrCodes) { + if (entry.platforms && !entry.platforms.includes("mac")) { + continue; + } + + // Set a url referrer. In the macOS quarantine database, the + // referrer URL has components that areURI-encoded. Our test data + // URI-encodes the components and also the separators (?, &, =). + // So we decode it and re-encode it to leave just the components + // URI-encoded. + let url = `http://example.com?${encodeURI(decodeURIComponent(entry.code))}`; + attributionSvc.setReferrerUrl(appPath, url, true); + let referrer = await MacAttribution.getReferrerUrl(appPath); + equal(referrer, url, "overwrite referrer url"); + + // Read attribution code from referrer. + await AttributionCode.deleteFileAsync(); + AttributionCode._clearCache(); + let result = await AttributionCode.getAttrDataAsync(); + Assert.deepEqual( + result, + entry.parsed, + "Parsed code should match expected value, code was: " + entry.code + ); + + // Read attribution code from file. + AttributionCode._clearCache(); + result = await AttributionCode.getAttrDataAsync(); + Assert.deepEqual( + result, + entry.parsed, + "Parsed code should match expected value, code was: " + entry.code + ); + + // Does not overwrite cached existing attribution code. + attributionSvc.setReferrerUrl(appPath, "http://test.com", false); + referrer = await MacAttribution.getReferrerUrl(appPath); + equal(referrer, url, "update referrer url"); + + result = await AttributionCode.getAttrDataAsync(); + Assert.deepEqual( + result, + entry.parsed, + "Parsed code should match expected value, code was: " + entry.code + ); + } +}); + +add_task(async function testInvalidAttrCodes() { + let appPath = MacAttribution.applicationPath; + let attributionSvc = Cc["@mozilla.org/mac-attribution;1"].getService( + Ci.nsIMacAttributionService + ); + + for (let code of invalidAttrCodes) { + // Set a url referrer. Not all of these invalid codes can be represented + // in the quarantine database; skip those ones. + let url = `http://example.com?${code}`; + let referrer; + try { + attributionSvc.setReferrerUrl(appPath, url, true); + referrer = await MacAttribution.getReferrerUrl(appPath); + } catch (ex) { + continue; + } + if (!referrer) { + continue; + } + equal(referrer, url, "overwrite referrer url"); + + // Read attribution code from referrer. + await AttributionCode.deleteFileAsync(); + AttributionCode._clearCache(); + let result = await AttributionCode.getAttrDataAsync(); + Assert.deepEqual(result, {}, "Code should have failed to parse: " + code); + } +}); diff --git a/browser/components/attribution/test/xpcshell/test_attribution_parsing.js b/browser/components/attribution/test/xpcshell/test_attribution_parsing.js new file mode 100644 index 0000000000..abf2456ce4 --- /dev/null +++ b/browser/components/attribution/test/xpcshell/test_attribution_parsing.js @@ -0,0 +1,44 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ + */ +"use strict"; + +/** + * This test file exists to be run on any platform during development, + * whereas the test_AttributionCode.js will test the attribution file + * in the app local data dir on Windows. It will only run under + * Windows on try. + */ + +/** + * Test validation of attribution codes. + */ +add_task(async function testValidAttrCodes() { + for (let entry of validAttrCodes) { + let result = AttributionCode.parseAttributionCode(entry.code); + Assert.deepEqual( + result, + entry.parsed, + "Parsed code should match expected value, code was: " + entry.code + ); + + result = AttributionCode.serializeAttributionData(entry.parsed); + if (!entry.doesNotRoundtrip) { + Assert.deepEqual( + result, + entry.code, + "Serialized data should match expected value, code was: " + entry.code + ); + } + } +}); + +/** + * Make sure codes with various formatting errors are not seen as valid. + */ +add_task(async function testInvalidAttrCodes() { + for (let code of invalidAttrCodes) { + let result = AttributionCode.parseAttributionCode(code); + Assert.deepEqual(result, {}, "Code should have failed to parse: " + code); + } +}); diff --git a/browser/components/attribution/test/xpcshell/test_zoneId_parsing.js b/browser/components/attribution/test/xpcshell/test_zoneId_parsing.js new file mode 100644 index 0000000000..2cab8678fd --- /dev/null +++ b/browser/components/attribution/test/xpcshell/test_zoneId_parsing.js @@ -0,0 +1,538 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ + */ +"use strict"; + +/** + * Test that we can properly parse and validate `zoneIdProvenanceData`. + */ + +const { AttributionIOUtils } = ChromeUtils.importESModule( + "resource:///modules/AttributionCode.sys.mjs" +); +const { ProvenanceData } = ChromeUtils.importESModule( + "resource:///modules/ProvenanceData.sys.mjs" +); +const { TelemetryTestUtils } = ChromeUtils.importESModule( + "resource://testing-common/TelemetryTestUtils.sys.mjs" +); + +add_setup(function setup() { + let origReadUTF8 = AttributionIOUtils.readUTF8; + registerCleanupFunction(() => { + AttributionIOUtils.readUTF8 = origReadUTF8; + }); +}); + +// If `iniFileContents` is passed as `null`, we will simulate an error reading +// the INI. +async function testProvenance(iniFileContents, testFn, telemetryTestFn) { + // The extra data returned by `ProvenanceData.submitProvenanceTelemetry` uses + // names that don't actually match the scalar names due to name length + // restrictions. + let scalarToExtraKeyMap = { + "attribution.provenance.data_exists": "data_exists", + "attribution.provenance.file_system": "file_system", + "attribution.provenance.ads_exists": "ads_exists", + "attribution.provenance.security_zone": "security_zone", + "attribution.provenance.referrer_url_exists": "refer_url_exist", + "attribution.provenance.referrer_url_is_mozilla": "refer_url_moz", + "attribution.provenance.host_url_exists": "host_url_exist", + "attribution.provenance.host_url_is_mozilla": "host_url_moz", + }; + + if (iniFileContents == null) { + AttributionIOUtils.readUTF8 = async path => { + throw new Error("test error: simulating provenance file read error"); + }; + } else { + AttributionIOUtils.readUTF8 = async path => iniFileContents; + } + let provenance = await ProvenanceData.readZoneIdProvenanceFile(); + if (testFn) { + await testFn(provenance); + } + if (telemetryTestFn) { + Services.telemetry.clearScalars(); + let extras = await ProvenanceData.submitProvenanceTelemetry(); + let scalars = Services.telemetry.getSnapshotForScalars( + "new-profile", + false /* aClear */ + ).parent; + let checkScalar = (scalarName, expectedValue) => { + TelemetryTestUtils.assertScalar(scalars, scalarName, expectedValue); + let extraKey = scalarToExtraKeyMap[scalarName]; + Assert.equal(extras[extraKey], expectedValue.toString()); + }; + telemetryTestFn(checkScalar); + } + ProvenanceData._clearCache(); +} + +add_task(async function unableToReadFile() { + await testProvenance( + null, + provenance => { + Assert.ok("readProvenanceError" in provenance); + }, + checkScalar => { + checkScalar("attribution.provenance.data_exists", false); + } + ); +}); + +add_task(async function expectedMozilla() { + await testProvenance( + ` +[Mozilla] +fileSystem=NTFS +zoneIdFileSize=194 +zoneIdBufferLargeEnough=true +zoneIdTruncated=false + +[MozillaZoneIdentifierStartSentinel] +[ZoneTransfer] +ZoneId=3 +ReferrerUrl=https://mozilla.org/ +HostUrl=https://download-installer.cdn.mozilla.net/pub/firefox/nightly/latest-mozilla-central-l10n/Firefox%20Installer.en-US.exe +`, + provenance => { + Assert.deepEqual(provenance, { + fileSystem: "NTFS", + zoneIdFileSize: 194, + zoneIdBufferLargeEnough: true, + zoneIdTruncated: false, + zoneId: 3, + referrerUrl: "https://mozilla.org/", + hostUrl: + "https://download-installer.cdn.mozilla.net/pub/firefox/nightly/latest-mozilla-central-l10n/Firefox%20Installer.en-US.exe", + referrerUrlIsMozilla: true, + hostUrlIsMozilla: true, + }); + }, + checkScalar => { + checkScalar("attribution.provenance.data_exists", true); + checkScalar("attribution.provenance.file_system", "NTFS"); + checkScalar("attribution.provenance.ads_exists", true); + checkScalar("attribution.provenance.security_zone", "3"); + checkScalar("attribution.provenance.referrer_url_exists", true); + checkScalar("attribution.provenance.referrer_url_is_mozilla", true); + checkScalar("attribution.provenance.host_url_exists", true); + checkScalar("attribution.provenance.host_url_is_mozilla", true); + } + ); +}); + +add_task(async function expectedNonMozilla() { + await testProvenance( + ` +[ZoneTransfer] +ReferrerUrl=https://mozilla.foobar.org/ +HostUrl=https://download-installer.cdn.mozilla.foobar.net/pub/firefox/nightly/latest-mozilla-central-l10n/Firefox%20Installer.en-US.exe +`, + provenance => { + Assert.equal(provenance.referrerUrlIsMozilla, false); + Assert.equal(provenance.hostUrlIsMozilla, false); + }, + checkScalar => { + checkScalar("attribution.provenance.referrer_url_exists", true); + checkScalar("attribution.provenance.referrer_url_is_mozilla", false); + checkScalar("attribution.provenance.host_url_exists", true); + checkScalar("attribution.provenance.host_url_is_mozilla", false); + } + ); +}); + +add_task(async function readFsError() { + await testProvenance( + ` +[Mozilla] +readFsError=openFile +readFsErrorCode=1234 +`, + provenance => { + Assert.equal(provenance.readFsError, "openFile"); + Assert.equal(provenance.readFsErrorCode, 1234); + }, + checkScalar => { + checkScalar("attribution.provenance.file_system", "error"); + } + ); +}); + +add_task(async function unexpectedReadFsError() { + await testProvenance( + ` +[Mozilla] +readFsError=openFile +readFsErrorCode=bazqux +`, + provenance => { + Assert.equal(provenance.readFsError, "openFile"); + Assert.ok(!("readFsErrorCode" in provenance)); + }, + checkScalar => { + checkScalar("attribution.provenance.file_system", "error"); + } + ); +}); + +add_task(async function unexpectedReadFsError() { + await testProvenance( + ` +[Mozilla] +readFsError=foobar +`, + provenance => { + Assert.equal(provenance.readFsError, "unexpected"); + }, + checkScalar => { + checkScalar("attribution.provenance.file_system", "error"); + } + ); +}); + +add_task(async function missingFileSystem() { + await testProvenance( + ``, + provenance => { + Assert.equal(provenance.fileSystem, "missing"); + }, + checkScalar => { + checkScalar("attribution.provenance.file_system", "missing"); + } + ); +}); + +add_task(async function fileSystem() { + await testProvenance( + ` +[Mozilla] +fileSystem=ntfs +`, + provenance => { + Assert.equal(provenance.fileSystem, "NTFS"); + }, + checkScalar => { + checkScalar("attribution.provenance.file_system", "NTFS"); + } + ); +}); + +add_task(async function unexpectedFileSystem() { + await testProvenance( + ` +[Mozilla] +fileSystem=foobar +`, + provenance => { + Assert.equal(provenance.fileSystem, "other"); + }, + checkScalar => { + checkScalar("attribution.provenance.file_system", "other"); + } + ); +}); + +add_task(async function zoneIdFileSize() { + await testProvenance( + ` +[Mozilla] +zoneIdFileSize=1234 +`, + provenance => { + Assert.equal(provenance.zoneIdFileSize, 1234); + }, + null + ); +}); + +add_task(async function unknownZoneIdFileSize() { + await testProvenance( + ` +[Mozilla] +zoneIdFileSize=unknown +`, + provenance => { + Assert.equal(provenance.zoneIdFileSize, "unknown"); + }, + null + ); +}); + +add_task(async function unexpectedZoneIdFileSize() { + await testProvenance( + ` +[Mozilla] +zoneIdFileSize=foobar +`, + provenance => { + Assert.equal(provenance.zoneIdFileSize, "unexpected"); + }, + null + ); +}); + +add_task(async function missingZoneIdFileSize() { + await testProvenance( + ``, + provenance => { + Assert.ok(!("zoneIdFileSize" in provenance)); + }, + null + ); +}); + +add_task(async function zoneIdBufferLargeEnoughTrue() { + await testProvenance( + ` +[Mozilla] +zoneIdBufferLargeEnough=TrUe +`, + provenance => { + Assert.equal(provenance.zoneIdBufferLargeEnough, true); + }, + null + ); +}); + +add_task(async function zoneIdBufferLargeEnoughFalse() { + await testProvenance( + ` +[Mozilla] +zoneIdBufferLargeEnough=FaLsE +`, + provenance => { + Assert.equal(provenance.zoneIdBufferLargeEnough, false); + }, + null + ); +}); + +add_task(async function unknownZoneIdBufferLargeEnough() { + await testProvenance( + ` +[Mozilla] +zoneIdBufferLargeEnough=unknown +`, + provenance => { + Assert.equal(provenance.zoneIdBufferLargeEnough, "unknown"); + }, + null + ); +}); + +add_task(async function unknownZoneIdBufferLargeEnough() { + await testProvenance( + ` +[Mozilla] +zoneIdBufferLargeEnough=foobar +`, + provenance => { + Assert.equal(provenance.zoneIdBufferLargeEnough, "unexpected"); + }, + null + ); +}); + +add_task(async function missingZoneIdBufferLargeEnough() { + await testProvenance(``, provenance => { + Assert.ok(!("zoneIdBufferLargeEnough" in provenance)); + }); +}); + +add_task(async function zoneIdTruncatedTrue() { + await testProvenance( + ` +[Mozilla] +zoneIdTruncated=TrUe +`, + provenance => { + Assert.equal(provenance.zoneIdTruncated, true); + }, + null + ); +}); + +add_task(async function zoneIdTruncatedFalse() { + await testProvenance( + ` +[Mozilla] +zoneIdTruncated=FaLsE +`, + provenance => { + Assert.equal(provenance.zoneIdTruncated, false); + }, + null + ); +}); + +add_task(async function unknownZoneIdTruncated() { + await testProvenance( + ` +[Mozilla] +zoneIdTruncated=unknown +`, + provenance => { + Assert.equal(provenance.zoneIdTruncated, "unknown"); + }, + null + ); +}); + +add_task(async function unexpectedZoneIdTruncated() { + await testProvenance( + ` +[Mozilla] +zoneIdTruncated=foobar +`, + provenance => { + Assert.equal(provenance.zoneIdTruncated, "unexpected"); + }, + null + ); +}); + +add_task(async function missingZoneIdTruncated() { + await testProvenance(``, provenance => { + Assert.ok(!("zoneIdTruncated" in provenance)); + }); +}); + +add_task(async function readZoneIdError() { + await testProvenance( + ` +[Mozilla] +readZoneIdError=openFile +readZoneIdErrorCode=1234 +`, + provenance => { + Assert.equal(provenance.readZoneIdError, "openFile"); + Assert.equal(provenance.readZoneIdErrorCode, 1234); + }, + null + ); +}); + +add_task(async function unexpectedReadZoneIdErrorCode() { + await testProvenance( + ` +[Mozilla] +readZoneIdError=openFile +readZoneIdErrorCode=bazqux +`, + provenance => { + Assert.equal(provenance.readZoneIdError, "openFile"); + Assert.ok(!("readZoneIdErrorCode" in provenance)); + }, + null + ); +}); + +add_task(async function noAdsOnInstaller() { + await testProvenance( + ` +[Mozilla] +readZoneIdError=openFile +readZoneIdErrorCode=2 +`, + provenance => { + Assert.equal(provenance.readZoneIdError, "openFile"); + Assert.equal(provenance.readZoneIdErrorCode, 2); + }, + checkScalar => { + checkScalar("attribution.provenance.ads_exists", false); + } + ); +}); + +add_task(async function unexpectedReadZoneIdError() { + await testProvenance( + ` +[Mozilla] +readZoneIdError=foobar +`, + provenance => { + Assert.equal(provenance.readZoneIdError, "unexpected"); + }, + null + ); +}); + +add_task(async function missingZoneId() { + await testProvenance( + ``, + provenance => { + Assert.equal(provenance.zoneId, "missing"); + }, + null + ); +}); + +add_task(async function unexpectedZoneId() { + await testProvenance( + ` +[ZoneTransfer] +ZoneId=9999999999 +`, + provenance => { + Assert.equal(provenance.zoneId, "unexpected"); + }, + checkScalar => { + checkScalar("attribution.provenance.ads_exists", true); + checkScalar("attribution.provenance.security_zone", "unexpected"); + } + ); +}); + +add_task(async function missingReferrerUrl() { + await testProvenance( + ``, + provenance => { + Assert.equal(provenance.referrerUrl, "missing"); + }, + checkScalar => { + checkScalar("attribution.provenance.ads_exists", true); + checkScalar("attribution.provenance.referrer_url_exists", false); + } + ); +}); + +add_task(async function unexpectedReferrerUrl() { + await testProvenance( + ` +[ZoneTransfer] +ReferrerUrl=foobar +`, + provenance => { + Assert.equal(provenance.referrerUrl, "unexpected"); + }, + null + ); +}); + +add_task(async function missingHostUrl() { + await testProvenance( + ``, + provenance => { + Assert.equal(provenance.hostUrl, "missing"); + }, + checkScalar => { + checkScalar("attribution.provenance.ads_exists", true); + checkScalar("attribution.provenance.referrer_url_exists", false); + } + ); +}); + +add_task(async function unexpectedHostUrl() { + await testProvenance( + ` +[ZoneTransfer] +HostUrl=foobar +`, + provenance => { + Assert.equal(provenance.hostUrl, "unexpected"); + }, + null + ); +}); diff --git a/browser/components/attribution/test/xpcshell/xpcshell.ini b/browser/components/attribution/test/xpcshell/xpcshell.ini new file mode 100644 index 0000000000..0ab5f6b0cb --- /dev/null +++ b/browser/components/attribution/test/xpcshell/xpcshell.ini @@ -0,0 +1,11 @@ +[DEFAULT] +firefox-appdir = browser +skip-if = (os != "win" && toolkit != "cocoa") # Only available on Windows and macOS +head = head.js + +[test_AttributionCode.js] +[test_MacAttribution.js] +skip-if = toolkit != "cocoa" # osx specific tests +[test_attribution_parsing.js] +[test_zoneId_parsing.js] +run-if = os == 'win' # Feature only available on Windows |