diff options
Diffstat (limited to 'toolkit/components/telemetry/docs/data')
27 files changed, 3417 insertions, 0 deletions
diff --git a/toolkit/components/telemetry/docs/data/addons-malware-ping.rst b/toolkit/components/telemetry/docs/data/addons-malware-ping.rst new file mode 100644 index 0000000000..18502d7489 --- /dev/null +++ b/toolkit/components/telemetry/docs/data/addons-malware-ping.rst @@ -0,0 +1,42 @@ + +Add-ons malware ping +==================== + +This ping is generated by an add-on created by Mozilla and shipped to users on older versions of Firefox (44-46). The ping contains information about the profile that might have been altered by a third party malicious add-on. + +Structure: + +.. code-block:: js + + { + type: "malware-addon-states", + ... + clientId: <UUID>, + environment: { ... }, + // Common ping data. + payload: { + // True if the blocklist was disabled at startup time. + blocklistDisabled: <bool>, + // True if the malicious add-on exists and is enabled. False if it + // exists and is disabled or null if the add-on was not found. + mainAddonActive: <bool | null>, + // A value of the malicious add-on block list state, or null if the + // add-on was not found. + mainAddonBlocked: <int | null>, + // True if a malicious user.js file was found in the profile. + foundUserJS: <bool>, + // If a malicious secmodd.db file was found the extension ID that the // file contained.. + secmoddAddon: <string | null>, . + // A list of IDs for extensions which were hidden by malicious CSS. + hiddenAddons: [ + <string>, + ... + ], + // A mapping of installed add-on IDs with known malicious + // update URL patterns to their exact update URLs. + updateURLs: { + <extensionID>: <updateURL>, + ... + } + } + } diff --git a/toolkit/components/telemetry/docs/data/anonymous-ping.rst b/toolkit/components/telemetry/docs/data/anonymous-ping.rst new file mode 100644 index 0000000000..53e217fc9d --- /dev/null +++ b/toolkit/components/telemetry/docs/data/anonymous-ping.rst @@ -0,0 +1,68 @@ + +"anonymous" ping +================ + +.. note:: + + This ping is no longer sent by Firefox or Fennec. + +This ping is only for product survey purpose and will not track/associate client ID. It's used +to evaluate custom tab usage and see which app is using our custom tab. + +Submission interval & triggers +Since this ping is used to measure the feature usage, it should be sent each time the client app uses our custom tab. + +Dataset: +Only opt-in users will send out this ping. +Since all other pings will collect client ID. We need this custom ping to not do that. + +Size and volume: +The size of submitted payload is small. And this custom ping should be deprecated after it's released for 6 months. + +Privacy: +We won't collect customer information so there'll be no PI leak. + +Data contents: +The content of this ping will let us know which app is using our custom tab. +Just like other feature usage measurement, we only need it for opt-in users (which consider as heavy users). + +Structure: + +.. code-block:: js + + { + "payload": { + "client": <string> // The package name of the caller app. + } + type: <string>, // "anonymous", "activation", "deletion", "saved-session", ... + id: <UUID>, // a UUID that identifies this ping + creationDate: <ISO date>, // the date the ping was generated + version: <number>, // the version of the ping format, currently 4 + + application: { + architecture: <string>, // build architecture, e.g. x86 + buildId: <string>, // "20141126041045" + name: <string>, // "Firefox" + version: <string>, // "35.0" + displayVersion: <string>, // "35.0b3" + vendor: <string>, // "Mozilla" + platformVersion: <string>, // "35.0" + xpcomAbi: <string>, // e.g. "x86-msvc" + channel: <string>, // "beta" + }, + } + +Field details +------------- + +client +~~~~~~ +It could be ``com.example.app``, which is the identifier of the app. + +Version history +--------------- +* v1: initial version - Will be shipped in `Fennec 55 <https://bugzilla.mozilla.org/show_bug.cgi?id=1329157>`_. + +Notes +~~~~~ +There's no option in this custom ping since we don't collect clientId nor environment data. diff --git a/toolkit/components/telemetry/docs/data/backgroundhangmonitor-ping.rst b/toolkit/components/telemetry/docs/data/backgroundhangmonitor-ping.rst new file mode 100644 index 0000000000..db4a28dd8b --- /dev/null +++ b/toolkit/components/telemetry/docs/data/backgroundhangmonitor-ping.rst @@ -0,0 +1,162 @@ + +"backgroundhangmonitor" ping +============================ + +Whenever a thread monitored by the Background Hang Monitor hangs, a stack and +some non-identifying information about the hang is captured. When 50 of these +hangs are collected across all processes, or the browser exits, this ping is +transmitted with the collected hang information. + +This ping is only collected in nightly builds, to avoid the high volume of pings +which would be produced in Beta. + +Structure: + +.. code-block:: js + + { + "type": "backgroundhangmonitor", + ... // common ping data + "environment": { ... }, + "payload": { + "timeSinceLastPing": <number>, // uptime since last backgroundhangmonitor ping (ms). + "modules": [ + [ + <string>, // Name of the file holding the debug information. + <string> // Breakpad ID of this module. + ], + ... + ], + "hangs": [ + { + "duration": <number>, // duration of the hang in milliseconds. + "thread": <string>, // name of the hanging thread. + "runnableName": <string>, // name of the runnable executing during the hang. + // Runnable names are only collected for the XPCOM main thread. + "process": <string>, // Type of process that hung, see below for a list of types. + "remoteType": <string>, // Remote type of process which hung, see below. + "annotations": [ ... ], // A list of annotations on the hang, see below. + "pseudoStack": [ ... ], // List of label stack frames and js frames. + "stack": [ ... ], // interleaved hang stack, see below. + }, + ... + ] + } + } + +.. note:: + + Hangs are collected whenever the current runnable takes over 128ms. + +Process Types +------------- + +The ``process`` field is a string denoting the kind of process that hung. Hangs +are currently sent only for the processes below: + ++---------------+---------------------------------------------------+ +| Kind | Description | ++===============+===================================================+ +| default | Main process, also known as the browser process | ++---------------+---------------------------------------------------+ +| tab | Content process | ++---------------+---------------------------------------------------+ +| gpu | GPU process | ++---------------+---------------------------------------------------+ + +Remote Type +----------- + +The ``remoteType`` field is a string denoting the type of content process that +hung. As such it is only non-null if ``processType`` contains the ``tab`` value. + +The supported ``remoteType`` values are documented in the crash ping +documentation: :ref:`remote-process-types`. + +Stack Traces +------------ + +Each hang object contains a ``stack`` field which has been populated with an +interleaved stack trace of the hung thread. An interleaved stack consists of a +native backtrace with additional frames interleaved, representing chrome JS and +label stack entries. + +The structure that manages the label stack and the JS stack was called +"PseudoStack" in the past and is now called "ProfilingStack". + +Note that this field only contains native stack frames, label stack and chrome +JS script frames. If native stacks can not be collected on the target platform, +or stackwalking was not initialized, there will be no native frames present, and +the stack will consist only of label stack and chrome JS script frames. + +A single frame in the stack is either a raw string, representing a label stack +or chrome JS script frame, or a native stack frame. + +Label stack frames contain the static string from all insances of the +AUTO_PROFILER_LABEL* macros. Additionally, dynamic strings are collected from +all usages of the AUTO_PROFILER_LABEL_DYNAMIC*_NONSENSITIVE macros. The dynamic +strings are simply appended to the static strings after a space character. + +Current dynamic string collections are as follows: + ++--------------------------------------------------+-----------------------------------------+ +| Static string | Dynamic | ++==================================================+=========================================+ +| ChromeUtils::Import | Associated chrome:// or resource:// URI | ++--------------------------------------------------+-----------------------------------------+ +| nsJSContext::GarbageCollectNow | GC reason string | ++--------------------------------------------------+-----------------------------------------+ +| mozJSSubScriptLoader::DoLoadSubScriptWithOptions | Associated chrome:// or resource:// URI | ++--------------------------------------------------+-----------------------------------------+ +| PresShell::DoFlushPendingNotifications | Flush type | ++--------------------------------------------------+-----------------------------------------+ +| nsObserverService::NotifyObservers | Associated observer topic | ++--------------------------------------------------+-----------------------------------------+ + +Native stack frames are as such: + +.. code-block:: js + + [ + <number>, // Index in the payload.modules list of the module description. + // -1 if this frame was not in a valid module. + <string> // Hex string (e.g. "FF0F") of the frame offset in the module. + ] + +Annotations +----------- + +The annotations field is an array of key-value pairs, for example if the user +was interacting during a hang the annotations field would look something like +this: + +.. code-block:: js + + [ + [ + "UserInteracting", + "true" + ] + ] + +The following annotations are currently present in tree: + ++-----------------+-------------------------------------------------+ +| Name | Description | ++=================+=================================================+ +| UserInteracting | "true" if the user was actively interacting | ++-----------------+-------------------------------------------------+ +| pluginName | Name of the currently running plugin | ++-----------------+-------------------------------------------------+ +| pluginVersion | Version of the currently running plugin | ++-----------------+-------------------------------------------------+ +| HangUIShown | "true" if the hang UI was shown | ++-----------------+-------------------------------------------------+ +| HangUIContinued | "true" if continue was selected in the hang UI | ++-----------------+-------------------------------------------------+ +| HangUIDontShow | "true" if the hang UI was not shown | ++-----------------+-------------------------------------------------+ +| Unrecovered | "true" if the hang persisted until process exit | ++-----------------+-------------------------------------------------+ + +Additional annotations can be added at run-time via :doc:`../collection/user-interactions`. diff --git a/toolkit/components/telemetry/docs/data/common-ping.rst b/toolkit/components/telemetry/docs/data/common-ping.rst new file mode 100644 index 0000000000..1587131808 --- /dev/null +++ b/toolkit/components/telemetry/docs/data/common-ping.rst @@ -0,0 +1,42 @@ + +Common ping format +================== + +This defines the top-level structure of a Telemetry ping. +It contains basic information shared between different ping types, which enables proper storage and processing of the raw pings server-side. + +It also contains optional further information: + +* the :doc:`environment data <../data/environment>`, which contains important info to correlate the measurements against +* the ``clientId``, a UUID identifying a profile and allowing user-oriented correlation of data + +*Note:* Both are not submitted with all ping types due to privacy concerns. This and the data it that can be correlated against is inspected under the `data collection policy <https://wiki.mozilla.org/Firefox/Data_Collection>`_. + +Finally, the structure also contains the `payload`, which is the specific data submitted for the respective *ping type*. + +Structure: + +.. code-block:: js + + { + type: <string>, // "main", "activation", "optout", "saved-session", ... + id: <UUID>, // a UUID that identifies this ping + creationDate: <ISO date>, // the date the ping was generated + version: <number>, // the version of the ping format, currently 4 + + application: { + architecture: <string>, // build architecture, e.g. x86 + buildId: <string>, // "20141126041045" + name: <string>, // "Firefox" + version: <string>, // "35.0" + displayVersion: <string>, // "35.0b3" + vendor: <string>, // "Mozilla" + platformVersion: <string>, // "35.0" + xpcomAbi: <string>, // e.g. "x86-msvc" + channel: <string>, // "beta" + }, + + clientId: <UUID>, // optional + environment: { ... }, // optional, not all pings contain the environment + payload: { ... }, // the actual payload data for this ping type + } diff --git a/toolkit/components/telemetry/docs/data/coverage-ping.rst b/toolkit/components/telemetry/docs/data/coverage-ping.rst new file mode 100644 index 0000000000..3db93d9074 --- /dev/null +++ b/toolkit/components/telemetry/docs/data/coverage-ping.rst @@ -0,0 +1,40 @@ + +"coverage" ping +=============== + +This ping is not enabled by default. When enabled, a ping is generated a total of once per profile, as a diagnostic tool +to determine whether Telemetry is working for users. + +This ping contains no client id and no environment data. + +You can find more background information in `this blog post <https://blog.mozilla.org/data/2018/08/20/effectively-measuring-search-in-firefox/>`_. + +Structure: + +.. code-block:: js + + { + "appVersion": "63.0a1", + "appUpdateChannel": "nightly", + "osName": "Darwin", + "osVersion": "17.7.0", + "telemetryEnabled": 1 + } + +Expected behaviours +------------------- +The following is a list of expected behaviours for the ``coverage`` ping: + +- The ping will only be sent once per ping version, per profile. +- If sending the ping fails, it will be retried on startup. +- A totally arbitrary UUID is generated on first run on a new profile, to use for filtering duplicates. +- The ping is sent to a different endpoint not using existing Telemetry. +- The ping does not honor the Telemetry enabled preference, but provides its own opt-out preference: `toolkit.coverage.opt-out`. +- The ping is disabled by default. It is intended to be enabled for users on an experimental basis using the preference `toolkit.coverage.enabled`. + +Version History +--------------- + +- Firefox 64: + + - "coverage" ping shipped (`bug 1492656 <https://bugzilla.mozilla.org/show_bug.cgi?id=1492656>`_). diff --git a/toolkit/components/telemetry/docs/data/crash-ping.rst b/toolkit/components/telemetry/docs/data/crash-ping.rst new file mode 100644 index 0000000000..7d25f97319 --- /dev/null +++ b/toolkit/components/telemetry/docs/data/crash-ping.rst @@ -0,0 +1,262 @@ +Crash ping +========== + +This ping is captured after the main Firefox process crashes or after a child process +process crashes, whether or not the crash report is submitted to +crash-stats.mozilla.org. It includes non-identifying metadata about the crash. + +This ping is sent either by the ``CrashManager`` or by the crash reporter +client. The ``CrashManager`` is responsible for sending crash pings for the +child processes crashes, which are sent right after the crash is detected, +as well as for main process crashes, which are sent after Firefox restarts +successfully. The crash reporter client sends crash pings only for main process +crashes whether or not the user also reports the crash. The crash reporter +client will not send the crash ping if telemetry has been disabled in Firefox. + +The environment block that is sent with this ping varies: if Firefox was running +long enough to record the environment block before the crash, then the environment +at the time of the crash will be recorded and ``hasCrashEnvironment`` will be true. +If Firefox crashed before the environment was recorded, ``hasCrashEnvironment`` will +be false and the recorded environment will be the environment at time of submission. + +The client ID is submitted with this ping. + +The metadata field holds a subset of the crash annotations, all field values +are stored as strings but some may be interpreted either as numbers or +boolean values. Numbers are integral unless stated otherwise in the +description. Boolean values are set to "1" when true, "0" when false. If +they're absent then they should be interpreted as false. + +Structure: + +.. code-block:: js + + { + type: "crash", + ... common ping data + clientId: <UUID>, + environment: { ... }, + payload: { + crashDate: "YYYY-MM-DD", + crashTime: <ISO Date>, // per-hour resolution + version: 1, + sessionId: <UUID>, // Telemetry ID of crashing session. May be missing for crashes that happen early in startup + crashId: <UUID>, // Optional, ID of the associated crash + minidumpSha256Hash: <hash>, // SHA256 hash of the minidump file + processType: <type>, // Type of process that crashed, see below for a list of types + stackTraces: { ... }, // Optional, see below + metadata: { // Annotations saved while Firefox was running. See CrashAnnotations.yaml for more information + ProductID: "{ec8030f7-c20a-464f-9b0e-13a3a9e97384}", + ProductName: "Firefox", + ReleaseChannel: <channel>, + Version: <version number>, + BuildID: "YYYYMMDDHHMMSS", + AsyncShutdownTimeout: <json>, // Optional, present when a shutdown blocker failed to respond within a reasonable amount of time + AvailablePageFile: <size>, // Windows-only, available paging file in bytes + AvailablePhysicalMemory: <size>, // Windows-only, available physical memory in bytes + AvailableSwapMemory: <size>, // macOS- and Linux-only, available swap space + AvailableVirtualMemory: <size>, // Windows-only, available virtual memory in bytes + BackgroundTaskName: "task_name", // Optional, if the app was invoked in background task mode via `--backgroundtask task_name` + BlockedDllList: <list>, // Windows-only, see WindowsDllBlocklist.cpp for details + BlocklistInitFailed: "1", // Windows-only, present only if the DLL blocklist initialization failed + CrashTime: <time>, // Seconds since the Epoch + DOMFissionEnabled: "1", // Optional, if set indicates that a Fission window had been opened + EventLoopNestingLevel: <levels>, // Optional, present only if >0, indicates the nesting level of the event-loop + ExperimentalFeatures: <features>, // Optional, a comma-separated string that specifies the enabled experimental features from about:preferences#experimental + FontName: <name>, // Optional, the font family name that is being loaded when the crash occurred + GPUProcessLaunchCount: <num>, // Number of times the GPU process was launched + HeadlessMode: "1", // Optional, "1" if the app was invoked in headless mode via `--headless ...` or `--backgroundtask ...` + ipc_channel_error: <error string>, // Optional, contains the string processing error reason for an ipc-based content crash + IsGarbageCollecting: "1", // Optional, if set indicates that the crash occurred while the garbage collector was running + LowCommitSpaceEvents: <num>, // Windows-only, present only if >0, number of low commit space events detected by the available memory tracker + MainThreadRunnableName: <name>, // Optional, Nightly-only, name of the currently executing nsIRunnable on the main thread + MozCrashReason: <reason>, // Optional, contains the string passed to MOZ_CRASH() + OOMAllocationSize: <size>, // Size of the allocation that caused an OOM + ProfilerChildShutdownPhase: <string>, // Profiler shutdown phase + PurgeablePhysicalMemory: <size>, // macOS-only, amount of memory that can be deallocated by the OS in case of memory pressure + QuotaManagerShutdownTimeout: <log-string>, // Optional, contains a list of shutdown steps and status of the quota manager clients + RemoteType: <type>, // Optional, type of content process, see below for a list of types + SecondsSinceLastCrash: <duration>, // Seconds elapsed since the last crash occurred + ShutdownProgress: <phase>, // Optional, contains a string describing the shutdown phase in which the crash occurred + SystemMemoryUsePercentage: <percentage>, // Windows-only, percent of memory in use + StartupCrash: "1", // Optional, if set indicates that Firefox crashed during startup + TextureUsage: <usage>, // Optional, usage of texture memory in bytes + TotalPageFile: <size>, // Windows-only, paging file in use expressed in bytes + TotalPhysicalMemory: <size>, // Windows-only, physical memory in use expressed in bytes + TotalVirtualMemory: <size>, // Windows-only, virtual memory in use expressed in bytes + UptimeTS: <duration>, // Seconds since Firefox was started, this can have a fractional component + User32BeforeBlocklist: "1", // Windows-only, present only if user32.dll was loaded before the DLL blocklist has been initialized + WindowsErrorReporting: "1", // Windows-only, present only if the crash was intercepted by the WER runtime exception module + WindowsPackageFamilyName: <string>, // Windows-only, a string containing the "Package Family Name" of Firefox, if installed through an MSIX package + }, + hasCrashEnvironment: bool + } + } + +.. note:: + + For "crash" pings generated by the crashreporter we are deliberately truncating the ``creationTime`` + field to hours. See `bug 1345108 <https://bugzilla.mozilla.org/show_bug.cgi?id=1345108>`_ for context. + +Process Types +------------- + +The ``processType`` field contains the type of process that crashed. There are +currently multiple process types defined in ``nsICrashService`` but crash pings +are sent only for the ones below: + ++---------------+-------------------------------------------------------------------------------+ +| Type | Description | ++===============+===============================================================================+ +| main | :ref:`Main process <parent-process>`, also known as parent or browser process | ++---------------+-------------------------------------------------------------------------------+ +| content | :ref:`Content process <content-process>` | ++---------------+-------------------------------------------------------------------------------+ +| gmplugin | :ref:`Gecko media plugin <gecko-media-plugins-process>` | ++---------------+-------------------------------------------------------------------------------+ +| gpu | :ref:`GPU process <gpu-process>` | ++---------------+-------------------------------------------------------------------------------+ +| vr | :ref:`VR process <vr-process>` | ++---------------+-------------------------------------------------------------------------------+ +| rdd | :ref:`Data decoder process <data-decoder-process>` | ++---------------+-------------------------------------------------------------------------------+ +| socket | :ref:`Network socket process <network-socket-process>` | ++---------------+-------------------------------------------------------------------------------+ +| sandboxbroker | :ref:`Remote sandbox broker <remote-sandbox-process>` | ++---------------+-------------------------------------------------------------------------------+ +| forkserver | :ref:`Fork server <fork-server>` | ++---------------+-------------------------------------------------------------------------------+ +| utility | :ref:`Utility process <utility-process>` | ++---------------+-------------------------------------------------------------------------------+ + +.. _remote-process-types: + +Remote Process Types +-------------------- + +The optional ``remoteType`` field contains the type of the content process that +crashed. As such it is present only if ``processType`` contains the ``content`` +value. The following content process types are currently defined: + ++-----------+--------------------------------------------------------+ +| Type | Description | ++===========+========================================================+ +| web | The content process was running code from a web page | ++-----------+--------------------------------------------------------+ +| file | The content process was running code from a local file | ++-----------+--------------------------------------------------------+ +| extension | The content process was running code from an extension | ++-----------+--------------------------------------------------------+ + +Stack Traces +------------ + +The crash ping may contain a ``stackTraces`` field which has been populated +with stack traces for all threads in the crashed process. The format of this +field is similar to the one used by Socorro for representing a crash. The main +differences are that redundant fields are not stored and that the module a +frame belongs to is referenced by index in the module array rather than by its +file name. + +Note that this field does not contain data from the application; only bare +stack traces and module lists are stored. + +.. code-block:: js + + { + status: <string>, // Status of the analysis, "OK" or an error message + crash_info: { // Basic crash information + type: <string>, // Type of crash, SIGSEGV, assertion, etc... + address: <addr>, // Crash address crash, hex format, see the notes below + crashing_thread: <index> // Index in the thread array below + }, + main_module: <index>, // Index of Firefox' executable in the module list + modules: [{ + base_addr: <addr>, // Base address of the module, hex format + end_addr: <addr>, // End address of the module, hex format + code_id: <string>, // Unique ID of this module, see the notes below + debug_file: <string>, // Name of the file holding the debug information + debug_id: <string>, // ID or hash of the debug information file + filename: <string>, // File name + version: <string>, // Library/executable version + }, + ... // List of modules ordered by base memory address + ], + threads: [{ // Stack traces for every thread + frames: [{ + module_index: <index>, // Index of the module this frame belongs to + ip: <ip>, // Program counter, hex format + trust: <string> // Trust of this frame, see the notes below + }, + ... // List of frames, the first frame is the topmost + ] + }] + } + +Notes +~~~~~ + +Memory addresses and instruction pointers are always stored as strings in +hexadecimal format (e.g. "0x4000"). They can be made of up to 16 characters for +64-bit addresses. + +The crash type is both OS and CPU dependent and can be either a descriptive +string (e.g. SIGSEGV, EXCEPTION_ACCESS_VIOLATION) or a raw numeric value. The +crash address meaning depends on the type of crash. In a segmentation fault the +crash address will be the memory address whose access caused the fault; in a +crash triggered by an illegal instruction exception the address will be the +instruction pointer where the invalid instruction resides. +See `breakpad <https://chromium.googlesource.com/breakpad/breakpad/+/c99d374dde62654a024840accfb357b2851daea0/src/processor/minidump_processor.cc#675>`__'s +relevant code for further information. + +Since it's not always possible to establish with certainty the address of the +previous frame while walking the stack, every frame has a trust value that +represents how it was found and thus how certain we are that it's a real frame. +The trust levels are (from least trusted to most trusted): + ++---------------+---------------------------------------------------+ +| Trust | Description | ++===============+===================================================+ +| context | Given as instruction pointer in a context | ++---------------+---------------------------------------------------+ +| prewalked | Explicitly provided by some external stack walker | ++---------------+---------------------------------------------------+ +| cfi | Derived from call frame info | ++---------------+---------------------------------------------------+ +| frame_pointer | Derived from frame pointer | ++---------------+---------------------------------------------------+ +| cfi_scan | Found while scanning stack using call frame info | ++---------------+---------------------------------------------------+ +| scan | Scanned the stack, found this | ++---------------+---------------------------------------------------+ +| none | Unknown, this is most likely not a valid frame | ++---------------+---------------------------------------------------+ + +The ``code_id`` field holds a unique ID used to distinguish between different +versions and builds of the same module. See `breakpad <https://chromium.googlesource.com/breakpad/breakpad/+/24f5931c5e0120982c0cbf1896641e3ef2bdd52f/src/google_breakpad/processor/code_module.h#60>`__'s +description for further information. This field is populated only on Windows. + +Version History +--------------- + +- Firefox 58: Added ipc_channel_error (`bug 1410143 <https://bugzilla.mozilla.org/show_bug.cgi?id=1410143>`_). +- Firefox 62: Added LowCommitSpaceEvents (`bug 1464773 <https://bugzilla.mozilla.org/show_bug.cgi?id=1464773>`_). +- Firefox 63: Added RecordReplayError (`bug 1481009 <https://bugzilla.mozilla.org/show_bug.cgi?id=1481009>`_). +- Firefox 64: Added MemoryErrorCorrection (`bug 1498609 <https://bugzilla.mozilla.org/show_bug.cgi?id=1498609>`_). +- Firefox 68: Added IndexedDBShutdownTimeout and LocalStorageShutdownTimeout + (`bug 1539750 <https://bugzilla.mozilla.org/show_bug.cgi?id=1539750>`_). +- Firefox 74: Added AvailableSwapMemory and PurgeablePhysicalMemory + (`bug 1587721 <https://bugzilla.mozilla.org/show_bug.cgi?id=1587721>`_). +- Firefox 74: Added MainThreadRunnableName (`bug 1608158 <https://bugzilla.mozilla.org/show_bug.cgi?id=1608158>`_). +- Firefox 76: Added DOMFissionEnabled (`bug 1602918 <https://bugzilla.mozilla.org/show_bug.cgi?id=1602918>`_). +- Firefox 79: Added ExperimentalFeatures (`bug 1644544 <https://bugzilla.mozilla.org/show_bug.cgi?id=1644544>`_). +- Firefox 85: Added QuotaManagerShutdownTimeout, removed IndexedDBShutdownTimeout and LocalStorageShutdownTimeout + (`bug 1672369 <https://bugzilla.mozilla.org/show_bug.cgi?id=1672369>`_). +- Firefox 89: Added GPUProcessLaunchCount (`bug 1710448 <https://bugzilla.mozilla.org/show_bug.cgi?id=1710448>`_) + and ProfilerChildShutdownPhase (`bug 1704680 <https://bugzilla.mozilla.org/show_bug.cgi?id=1704680>`_). +- Firefox 90: Removed MemoryErrorCorrection (`bug 1710152 <https://bugzilla.mozilla.org/show_bug.cgi?id=1710152>`_) + and added WindowsErrorReporting (`bug 1703761 <https://bugzilla.mozilla.org/show_bug.cgi?id=1703761>`_). +- Firefox 95: Added HeadlessMode and BackgroundTaskName (`bug 1697875 <https://bugzilla.mozilla.org/show_bug.cgi?id=1697875>`_). +- Firefox 96: Added WindowsPackageFamilyName (`bug 1738375 <https://bugzilla.mozilla.org/show_bug.cgi?id=1738375>`_). +- Firefox 103: Removed ContainsMemoryReport (`bug 1776279 <https://bugzilla.mozilla.org/show_bug.cgi?id=1776279>`_). +- Firefox 107: Added UtilityActorsName (`bug 1788596 <https://bugzilla.mozilla.org/show_bug.cgi?id=1788596>`_). diff --git a/toolkit/components/telemetry/docs/data/default-browser-ping.rst b/toolkit/components/telemetry/docs/data/default-browser-ping.rst new file mode 100644 index 0000000000..17eacee960 --- /dev/null +++ b/toolkit/components/telemetry/docs/data/default-browser-ping.rst @@ -0,0 +1,90 @@ +====================== +"default-browser" ping +====================== + +This opt-out ping is sent from the Default Browser Agent, which is a Windows-only program that registers itself during Firefox installation with the Windows scheduled tasks system so that it runs automatically every 24 hours, whether Firefox is running or not. The scheduled task gathers the data for this ping and then sends it by handing it off to :doc:`../internals/pingsender`. + +Even though this ping is generated by a binary separate from Firefox itself, opting out of telemetry does disable it; the pref value is copied to the registry so that the default browser agent can read it without needing to work with profiles. Relevant policies are consulted as well. The agent also has its own pref, ``default-agent.enabled``, which if set to false disables all agent functionality, including generating this ping. + +Each installation of Firefox has its own copy of the agent and its own scheduled task, so one ping will be sent every day for each installation on a given machine. This is needed because the default browser setting is per-user, and different installations may have been created by different users. If multiple operating system-level users are all using one copy of Firefox, only one scheduled task will have been created and only one ping will be sent, even though the users might have different default browser settings. + +The namespace this ping is in is called ``default-browser-agent``. + +For more information about the default browser agent itself, see :doc:`its documentation </toolkit/mozapps/defaultagent/default-browser-agent/index>`. + +Structure +========= + +Since this ping is sent from an external binary, it's structured as its own ping document type and not in the standard Firefox telemetry format. It's also missing lots of data that would normally be present; for instance, there is no ``clientId``, because the agent does not load any profile and so has no way to find any, and no environment block because the agent doesn't contain the telemetry library code to build it. + +Here's the format of the ping data, with example values for each property: + +.. code-block:: js + + { + build_channel: <string>, // ex. "nightly", or "beta", or "release" + version: <string>, // ex. "72.0.2" + os_version: <string>, // ex. 10.0.18363.592 + os_locale: <string>, // ex. en-US + default_browser: <string>, // ex. "firefox" + previous_default_browser: <string>, // ex. "edge" + default_pdf_viewer_raw: <string>, // ex. "firefox" + notification_type: <string>, // ex. "initial" or "followup" + notification_shown: <string>, // ex. "shown", or "not-shown", or "error" + notification_action: <string>, // ex. "no-action" or "make-firefox-default-button" + previous_notification_action: <string>, // Same possible values as notification_action + } + +``build_channel`` +----------------- +The Firefox channel. + +``version`` +----------- +The Firefox version. + +``os_version`` +-------------- +The Windows version number. Below Windows 10, this is in the format ``[major].[minor].[build]``; for Windows 10, the format is ``10.0.[build].[UBR]``. + +``os_locale`` +------------- +The locale that the user has selected for the operating system (NOT for Firefox). + +``default_browser`` +------------------- +Which browser is currently set as the system default web browser. This is simply a string with the name of the browser; the possible values include "firefox", "chrome", "edge", "edge-chrome", "ie", "opera", and "brave". + +``previous_default_browser`` +---------------------------- +Which browser was set as the system default before it was changed to the current setting. The possible values are the same as for ``default_browser``. + +The OS does not keep track of previous default settings, so the agent records this information itself. That means that it will be inaccurate until the first time the default is changed after the agent task begins running. Before then, the value of ``previous_default_browser`` will be the same as ``default_browser``. + +This value is updated every time the Default Browser Agent runs, so when the default browser is first changed the values for ``default_browser`` and ``previous_default_browser`` will be different. But on subsequent executions of the Default Browser Agent, the two values will be the same. + +``default_pdf_viewer_raw`` +-------------------------- +Which pdf viewer is currently set as the system default. This is simply a string with the name of the pdf viewer. + +``notification_type`` +--------------------- +Which notification type was shown. There are currently two types of notifications, "initial" and "followup". The initial notification is shown first and has a "Remind me later" button. The followup notification is only shown if the "Remind me later" button is clicked and has a "Never ask again" button instead of the "Remind me later" button. Note that the value of ``notification_shown`` should be consulted to determine whether the notification type specified was actually shown. + +``notification_shown`` +---------------------- +Whether a notification was shown or not. Possible value include "shown", "not-shown", and "error". + +``notification_action`` +----------------------- +The action that the user took in response to the notification. Possible values currently include "dismissed-by-timeout", "dismissed-to-action-center", "dismissed-by-button", "dismissed-by-application-hidden", "remind-me-later", "make-firefox-default-button", "toast-clicked", "no-action". + +Many of the values correspond to buttons on the notification and should be pretty self explanatory, but a few are less so. The action "no-action" will be used if and only if the value of ``notification_shown`` is not "shown" to indicate that no action was taken because no notification was displayed. The action "dismissed-to-action-center" will be used if the user clicks the arrow in the top right corner of the notification to dismiss it to the action center. The action "dismissed-by-application-hidden" is provided because that is a method of dismissal that the notification API could give but, in practice, should never be seen. The action "dismissed-by-timeout" indicates that the user did not interact with the notification and it timed out. + +``previous_notification_action`` +-------------------------------- +The action that the user took in response to the previous notification. Possible values are the same as those of ``notification_action``. + +If no notification has ever been shown, this will be "no-action". If ``notification_shown`` is "shown", this will be the action that was taken on the notification before the one that was just shown (or "no-action" if there was no previous notification). Otherwise, this will be the action that the user took the last time a notification was shown. + +Note that because this feature was added later, there may be people in configurations that might seem impossible, like having the combination of ``notification_type`` being "followup" with a ``previous_notification_action`` of "no-action", because the first notification action was taken before we started storing that value. diff --git a/toolkit/components/telemetry/docs/data/deletion-request-ping.rst b/toolkit/components/telemetry/docs/data/deletion-request-ping.rst new file mode 100644 index 0000000000..0c1c00c301 --- /dev/null +++ b/toolkit/components/telemetry/docs/data/deletion-request-ping.rst @@ -0,0 +1,68 @@ +"deletion-request" ping +======================= + +This ping is submitted when a user opts out of sending technical and interaction data to Mozilla. +(In other words, when the +``datareporting.healthreport.uploadEnabled`` +:doc:`preference <../internals/preferences>` is set to ``false``.) + +This ping contains the client id. +This ping does not contain any environment data. + +This ping is intended to communicate to the Data Pipeline that the user wishes +to have their reported Telemetry data deleted. +As such it attempts to send itself at the moment the user opts out of data collection, +and continues to try and send itself. + +This ping contains scalars present in the "deletion-request" store. + +Structure: + +.. code-block:: js + + { + version: 4, + type: "deletion-request", + ... common ping data (including clientId) + payload: { + scalars: { + <process-type>: { // like "parent" or "content" + <id name>: <id>, // like "deletion.request.impression_id": "<RFC 4122 GUID>" + }, + }, + } + } + +Expected behaviours +------------------- +The following is a list of expected behaviours for the ``deletion-request`` ping: + +- Telemetry will try to send the ping even if upload is disabled. +- Telemetry may persist this ping if it can't be immediately sent, and may try to resend it later. + +Analysis Gotchas +---------------- +It is `known <https://bugzilla.mozilla.org/show_bug.cgi?id=1741252>`_ that, +on a release week, "deletion-request" pings from the previous release version will spike in volume. + +There is a strong geo component to these spikes (China and Russia mostly). +The pings behave like they're coming from real Firefox instances (one per ``client_id``). +However, we've received no "main" pings from these clients from the previous 28 days +(per ``clients_last_seen``) which makes them rather unusual. + +We suspect this is some sort of automation or pseudo-automation. + +Including these pings in our self-service deletion processes doesn't make them any slower, +and ingesting them isn't breaking the bank, so at time of writing we're inclined to take a +"document and then ignore" approach. + +Version History +--------------- + +- Firefox 72: + + - "deletion-request" ping replaces the "optout" ping (`bug 1585410 <https://bugzilla.mozilla.org/show_bug.cgi?id=1585410>`_). + +- Firefox 73: + + - Added support for subordinate ids in the "deletion-request" store (`bug 1604312 <https://bugzilla.mozilla.org/show_bug.cgi?id=1604312>`_). diff --git a/toolkit/components/telemetry/docs/data/downgrade-ping.rst b/toolkit/components/telemetry/docs/data/downgrade-ping.rst new file mode 100644 index 0000000000..5366255834 --- /dev/null +++ b/toolkit/components/telemetry/docs/data/downgrade-ping.rst @@ -0,0 +1,30 @@ + +"downgrade" ping +================ + +This ping is captured when attempting to use a profile that was previously used +with a newer version of the application. + +This ping is submitted directly through the ```pingsender```. The common ping +data relates to the profile and application that the user attempted to use. + +The client ID is submitted with this ping. No environment block is included with +this ping. + +Structure: + +.. code-block:: js + + { + type: "downgrade", + ... common ping data + clientId: <UUID>, + payload: { + lastVersion: "", // The last version of the application that ran this profile + hasSync: <bool>, // Whether the profile is signed in to sync + hasBinary: <bool>, // Whether the last version of the application is available to run + button: <int> // The button the user chose to click from the UI: + // 0 - Quit + // 1 - Create new profile + } + } diff --git a/toolkit/components/telemetry/docs/data/environment.rst b/toolkit/components/telemetry/docs/data/environment.rst new file mode 100644 index 0000000000..b61c2a35e4 --- /dev/null +++ b/toolkit/components/telemetry/docs/data/environment.rst @@ -0,0 +1,615 @@ + +Environment +=========== + +The environment consists of data that is expected to be characteristic for performance and other behavior and not expected to change too often. + +Changes to most of these data points are detected (where possible and sensible) and will lead to a session split in the :doc:`main-ping`. +The environment data may also be submitted by other ping types. + +*Note:* This is not submitted with all ping types due to privacy concerns. This and other data is inspected under the `data collection policy <https://wiki.mozilla.org/Firefox/Data_Collection>`_. + +Some parts of the environment must be fetched asynchronously at startup. We don't want other Telemetry components to block on waiting for the environment, so some items may be missing from it until the async fetching finished. +This currently affects the following sections: + +- profile +- add-ons +- services + + +Structure: + +.. code-block:: js + + { + build: { + applicationId: <string>, // nsIXULAppInfo.ID + applicationName: <string>, // "Firefox" + architecture: <string>, // e.g. "x86", build architecture for the active build + buildId: <string>, // e.g. "20141126041045" + version: <string>, // e.g. "35.0" + vendor: <string>, // e.g. "Mozilla" + displayVersion: <string>, // e.g. "35.0b1" + platformVersion: <string>, // e.g. "35.0" + xpcomAbi: <string>, // e.g. "x86-msvc" + updaterAvailable: <bool>, // Whether the app was built with app update available (MOZ_UPDATER) + }, + settings: { + addonCompatibilityCheckEnabled: <bool>, // Whether application compatibility is respected for add-ons + blocklistEnabled: <bool>, // true on failure + isDefaultBrowser: <bool>, // whether Firefox is the default browser. On Windows, this is operationalized as whether Firefox is the default HTTP protocol handler and the default HTML file handler. + defaultSearchEngine: <string>, // e.g. "yahoo" + defaultSearchEngineData: {, // data about the current default engine + name: <string>, // engine name, e.g. "Yahoo"; or "NONE" if no default + loadPath: <string>, // where the engine line is located; missing if no default + origin: <string>, // 'default', 'verified', 'unverified', or 'invalid'; based on the presence and validity of the engine's loadPath verification hash. + submissionURL: <string> // set for default engines or well known search domains + }, + defaultPrivateSearchEngine: <string>, // e.g. "duckduckgo" + defaultPrivateSearchEngine: {, + // data about the current default engine for private browsing mode. Same as defaultSearchEngineData. + }, + launcherProcessState: <integer>, // optional, values correspond to values of mozilla::LauncherRegistryInfo::EnabledState enum + e10sEnabled: <bool>, // whether e10s is on, i.e. browser tabs open by default in a different process + e10sMultiProcesses: <integer>, // Maximum number of processes that will be launched for regular web content + fissionEnabled: <bool>, // whether fission is enabled this session, and subframes can load in a different process + telemetryEnabled: <bool>, // false on failure + locale: <string>, // e.g. "it", null on failure + intl: { + requestedLocales: [ <string>, ... ], // The locales that are being requested. + availableLocales: [ <string>, ... ], // The locales that are available for use. + appLocales: [ <string>, ... ], // The negotiated locales that are being used. + systemLocales: [ <string>, ... ], // The locales for the OS. + regionalPrefsLocales: [ <string>, ... ], // The regional preferences for the OS. + acceptLanguages: [ <string>, ... ], // The languages for the Accept-Languages header. + }, + update: { + channel: <string>, // e.g. "release", null on failure + enabled: <bool>, // true on failure + autoDownload: <bool>, // true on failure + background: <bool>, // Indicates whether updates may be installed when Firefox is not running. + }, + userPrefs: { + // Only prefs which are changed are listed in this block + "pref.name.value": value // some prefs send the value + "pref.name.url": "<user-set>" // For some privacy-sensitive prefs + // only the fact that the value has been changed is recorded + }, + attribution: { // optional, only present if the installation has attribution data + // all of these values are optional. + source: <string>, // referring partner domain, when install happens via a known partner + medium: <string>, // category of the source, such as "organic" for a search engine + campaign: <string>, // identifier of the particular campaign that led to the download of the product + content: <string>, // identifier to indicate the particular link within a campaign + variation: <string>, // name/id of the variation cohort used in the enrolled funnel experiment + experiment: <string>, // name/id of the enrolled funnel experiment + ua: <string>, // identifier derived from the user agent downloading the installer, e.g., chrome, Google Chrome 123 + dltoken: <string>, // Unique token created at Firefox download time. ex: c18f86a3-f228-4d98-91bb-f90135c0aa9c + msstoresignedin: <boolean>, // optional, only present if the installation was done through the Microsoft Store, and was able to retrieve the "campaign ID" it was first installed with. this value is "true" if the user was signed into the Microsoft Store when they first installed, and false otherwise + dlsource: <string>, // identifier that indicate where installations of Firefox originate + }, + sandbox: { + effectiveContentProcessLevel: <integer>, + contentWin32kLockdownState: <integer>, + } + }, + // Optional, missing if fetching the information failed or had not yet completed. + services: { + // True if the user has a firefox account + accountEnabled: <bool>, + // True if the user has sync enabled. + syncEnabled: <bool> + }, + profile: { + creationDate: <integer>, // integer days since UNIX epoch, e.g. 16446 + resetDate: <integer>, // integer days since UNIX epoch, e.g. 16446 - optional + firstUseDate: <integer>, // integer days since UNIX epoch, e.g. 16446 - optional + }, + partner: { // This section may not be immediately available on startup + distributionId: <string>, // pref "distribution.id", null on failure + distributionVersion: <string>, // pref "distribution.version", null on failure + partnerId: <string>, // pref mozilla.partner.id, null on failure + distributor: <string>, // pref app.distributor, null on failure + distributorChannel: <string>, // pref app.distributor.channel, null on failure + partnerNames: [ + // list from prefs app.partner.<name>=<name> + ], + }, + system: { + memoryMB: <number>, + virtualMaxMB: <number>, // windows-only + isWow64: <bool>, // windows-only + isWowARM64: <bool>, // windows-only + hasWinPackageId: <bool>, // windows-only + winPackageFamilyName: <string>, // windows-only + cpu: { + count: <number>, // desktop only, e.g. 8, or null on failure - logical cpus + cores: <number>, // desktop only, e.g., 4, or null on failure - physical cores + vendor: <string>, // desktop only, e.g. "GenuineIntel", or null on failure + name: <string>, // desktop only, e.g. "Intel(R) Core(TM) i9-8950HK CPU @ 2.90GHz", + // or null on failure + family: <number>, // desktop only, null on failure + model: <number, // desktop only, null on failure + stepping: <number>, // desktop only, null on failure + l2cacheKB: <number>, // L2 cache size in KB, only on windows & mac + l3cacheKB: <number>, // desktop only, L3 cache size in KB + speedMHz: <number>, // desktop only, cpu clock speed in MHz + extensions: [ + <string>, + ... + // as applicable: + // "hasMMX", "hasSSE", "hasSSE2", "hasSSE3", "hasSSSE3", + // "hasSSE4A", "hasSSE4_1", "hasSSE4_2", "hasAVX", "hasAVX2", + // "hasAES", "hasEDSP", "hasARMv6", "hasARMv7", "hasNEON" + ], + }, + device: { // This section is only available on mobile devices. + model: <string>, // the "device" from FHR, null on failure + manufacturer: <string>, // null on failure + hardware: <string>, // null on failure + isTablet: <bool>, // null on failure + }, + os: { + name: <string>, // "Windows_NT" or null on failure + version: <string>, // e.g. "6.1", null on failure + kernelVersion: <string>, // android only or null on failure + servicePackMajor: <number>, // windows only or null on failure + servicePackMinor: <number>, // windows only or null on failure + windowsBuildNumber: <number>, // windows only or null on failure + windowsUBR: <number>, // windows 10 only or null on failure + installYear: <number>, // windows only or null on failure + locale: <string>, // "en" or null on failure + hasPrefetch: <bool>, // windows only, or null on failure + hasSuperfetch: <bool>, // windows only, or nul on failure + }, + hdd: { + profile: { // hdd where the profile folder is located + model: <string>, // windows only or null on failure + revision: <string>, // windows only or null on failure + type: <string>, // "SSD" or "HDD" windows only or null on failure + }, + binary: { // hdd where the application binary is located + model: <string>, // windows only or null on failure + revision: <string>, // windows only or null on failure + type: <string>, // "SSD" or "HDD" windows only or null on failure + }, + system: { // hdd where the system files are located + model: <string>, // windows only or null on failure + revision: <string>, // windows only or null on failure + type: <string>, // "SSD" or "HDD" windows only or null on failure + }, + }, + gfx: { + D2DEnabled: <bool>, // null on failure + DWriteEnabled: <bool>, // null on failure + ContentBackend: <string> // One of "Cairo", "Skia", or "Direct2D 1.1" + Headless: <bool>, // null on failure + //DWriteVersion: <string>, // temporarily removed, pending bug 1154500 + adapters: [ + { + description: <string>, // e.g. "Intel(R) HD Graphics 4600", null on failure + vendorID: <string>, // null on failure + deviceID: <string>, // null on failure + subsysID: <string>, // null on failure + RAM: <number>, // in MB, null on failure + driver: <string>, // null on failure + driverVendor: <string>, // null on failure + driverVersion: <string>, // null on failure + driverDate: <string>, // null on failure + GPUActive: <bool>, // currently always true for the first adapter + }, + ... + ], + // Note: currently only added on Desktop. On Linux, only a single + // monitor is returned for the primary screen. + monitors: [ + { + screenWidth: <number>, // screen width in pixels + screenHeight: <number>, // screen height in pixels + refreshRate: <number>, // refresh rate in hertz (present on Windows only). + // (values <= 1 indicate an unknown value) + pseudoDisplay: <bool>, // networked screen (present on Windows only) + scale: <number>, // backing scale factor (present on Mac only) + }, + ... + ], + features: { + compositor: <string>, // Layers backend for compositing (e.g. "d3d11", "none", "opengl", "webrender") + + // Each the following features can have one of the following statuses: + // "unused" - This feature has not been requested. + // "unavailable" - Safe Mode or OS restriction prevents use. + // "blocked" - Blocked due to an internal condition such as safe mode. + // "blacklisted" - Blocked due to a blacklist restriction. + // "denied" - Blocked due to allowlist restrictions. + // "disabled" - User explicitly disabled this default feature. + // "failed" - This feature was attempted but failed to initialize. + // "available" - User has this feature available. + // The status can also include a ":" followed by a reason + // e.g. "FEATURE_FAILURE_WEBRENDER_VIDEO_CRASH_INTEL_23.20.16.4973" + d3d11: { // This feature is Windows-only. + status: <string>, + warp: <bool>, // Software rendering (WARP) mode was chosen. + textureSharing: <bool> // Whether or not texture sharing works. + version: <number>, // The D3D11 device feature level. + blacklisted: <bool>, // Whether D3D11 is blacklisted; use to see whether WARP + // was blacklist induced or driver-failure induced. + }, + d2d: { // This feature is Windows-only. + status: <string>, + version: <string>, // Either "1.0" or "1.1". + }, + gpuProcess: { // Out-of-process compositing ("GPU process") feature + status: <string>, // "Available" means currently in use + }, + hwCompositing: { // hardware acceleration. i.e. whether we try using the GPU + status: <string> + }, + wrCompositor: { // native OS compositor (CA, DComp, etc.) + status: <string> + } + wrSoftware: { // Software backend for WebRender, only computed when 'compositor' is 'webrender' + status: <string> + } + openglCompositing: { // OpenGL compositing. + status: <string> + } + }, + }, + appleModelId: <string>, // Mac only or null on failure + sec: { // This feature is Windows 8+ only + antivirus: [ <string>, ... ], // null if unavailable on platform: Product name(s) of registered antivirus programs + antispyware: [ <string>, ... ], // null if unavailable on platform: Product name(s) of registered antispyware programs + firewall: [ <string>, ... ], // null if unavailable on platform: Product name(s) of registered firewall programs + }, + }, + addons: { + activeAddons: { // the currently enabled add-ons + <addon id>: { + blocklisted: <bool>, + description: <string>, // null if not available + name: <string>, + userDisabled: <bool>, + appDisabled: <bool>, + version: <string>, + scope: <integer>, + type: <string>, // "extension", "locale", ... + foreignInstall: <bool>, + hasBinaryComponents: <bool>, + installDay: <number>, // days since UNIX epoch, 0 on failure + updateDay: <number>, // days since UNIX epoch, 0 on failure + signedState: <integer>, // whether the add-on is signed by AMO, only present for extensions + isSystem: <bool>, // true if this is a System Add-on + isWebExtension: <bool>, // true if this is a WebExtension + multiprocessCompatible: <bool>, // true if this add-on does *not* require e10s shims + }, + ... + }, + theme: { // the active theme + id: <string>, + blocklisted: <bool>, + description: <string>, + name: <string>, + userDisabled: <bool>, + appDisabled: <bool>, + version: <string>, + scope: <integer>, + foreignInstall: <bool>, + hasBinaryComponents: <bool> + installDay: <number>, // days since UNIX epoch, 0 on failure + updateDay: <number>, // days since UNIX epoch, 0 on failure + }, + activeGMPlugins: { + <gmp id>: { + version: <string>, + userDisabled: <bool>, + applyBackgroundUpdates: <integer>, + }, + ... + }, + }, + experiments: { + "<experiment id>": { branch: "<branch>", type: "<type>", enrollmentId: "<id>" }, + // ... + } + } + +build +----- + +buildId +~~~~~~~ +Firefox builds downloaded from mozilla.org use a 14-digit buildId. Builds included in other distributions may have a different format (e.g. only 10 digits). + +Settings +-------- + +defaultSearchEngine +~~~~~~~~~~~~~~~~~~~ +Note: Deprecated, use defaultSearchEngineData instead. + +Contains the string identifier or name of the default search engine provider. This will not be present in environment data collected before the Search Service initialization. + +The special value ``NONE`` could occur if there is no default search engine. + +The special value ``UNDEFINED`` could occur if a default search engine exists but its identifier could not be determined. + +This field's contents are ``Services.search.defaultEngine.identifier`` (if defined) or ``"other-"`` + ``Services.search.defaultEngine.name`` if not. In other words, search engines without an ``.identifier`` are prefixed with ``other-``. + +defaultSearchEngineData +~~~~~~~~~~~~~~~~~~~~~~~ +Contains data identifying the engine currently set as the default. + +The object contains: + +- a ``name`` property with the name of the engine, or ``NONE`` if no + engine is currently set as the default. + +- a ``loadPath`` property: an anonymized path of the engine xml file, e.g. + jar:[app]/omni.ja!browser/engine.xml + (where 'browser' is the name of the chrome package, not a folder) + [profile]/searchplugins/engine.xml + [distribution]/searchplugins/common/engine.xml + [other]/engine.xml + [other]/addEngineWithDetails + [other]/addEngineWithDetails:extensionID + [http/https]example.com/engine-name.xml + [http/https]example.com/engine-name.xml:extensionID + +- an ``origin`` property: the value will be ``default`` for engines that are built-in or from distribution partners, ``verified`` for user-installed engines with valid verification hashes, ``unverified`` for non-default engines without verification hash, and ``invalid`` for engines with broken verification hashes. + +- a ``submissionURL`` property with the HTTP url we would use to search. + For privacy, we don't record this for user-installed engines. + +``loadPath`` and ``submissionURL`` are not present if ``name`` is ``NONE``. + +defaultPrivateSearchEngineData +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +This contains the data identifying the engine current set as the default for +private browsing mode. This may be the same engine as set for normal browsing +mode. + +This object contains the same information as ``defaultSearchEngineData``. It +is only reported if the ``browser.search.separatePrivateDefault`` preference is +set to ``true``. + +userPrefs +~~~~~~~~~ + +This object contains user preferences. + +Each key in the object is the name of a preference. A key's value depends on the policy with which the preference was collected. There are three such policies, "value", "state", and "default value". For preferences collected under the "value" policy, the value will be the preference's value. For preferences collected under the "state" policy, the value will be an opaque marker signifying only that the preference has a user value. The "state" policy is therefore used when user privacy is a concern. For preferences collected under the "default value" policy, the value will be the preference's default value, if the preference exists. If the preference does not exist, there is no key or value. + +The following is a partial list of `collected preferences <https://searchfox.org/mozilla-central/search?q=const+DEFAULT_ENVIRONMENT_PREFS&path=>`_. + +- ``browser.fixup.alternate.enabled``: Whether the browser should try to modify unknown hosts by adding a prefix (e.g. www) and a suffix (.com). Defaults to false. + +- ``browser.migrate.interactions.bookmarks``: True if the user has imported bookmarks from another browser before. This preference gets transferred during profile resets. + +- ``browser.migrate.interactions.history``: True if the user has imported history from another browser before. This preference gets transferred during profile resets. + +- ``browser.migrate.interactions.passwords``: True if the user has imported passwords from another browser before. This preference gets transferred during profile resets. + +- ``browser.search.suggest.enabled``: The "master switch" for search suggestions everywhere in Firefox (search bar, urlbar, etc.). Defaults to true. + +- ``browser.urlbar.autoFill``: The global preference for whether autofill in the urlbar is enabled. When false, all types of autofill are disabled. + +- ``browser.urlbar.autoFill.adaptiveHistory.enabled``: True if adaptive history autofill in the urlbar is enabled. + +- ``browser.urlbar.dnsResolveSingleWordsAfterSearch``: Controls when to DNS resolve single word search strings, after they were searched for. If the string is resolved as a valid host, show a "Did you mean to go to 'host'" prompt. 0: Never resolve, 1: Use heuristics, 2. Always resolve. Defaults to 0. + +- ``browser.urlbar.quicksuggest.onboardingDialogChoice``: The user's choice in the Firefox Suggest onboarding dialog. If the dialog was shown multiple times, this records the user's most recent choice. Values are the following. Empty string: The user has not made a choice (e.g., because the dialog hasn't been shown). ``accept_2`` is recorded when the user accepts the dialog and opts in, ``reject_2`` is recorded when the user rejects the dialog and opts out, ``learn_more_1`` is recorded when the user clicks "Learn more" on the introduction section (the user remains opted out), ``learn_more_2`` is recorded when the user clicks "Learn more" on the main section (the user remains opted out), ``close_1`` is recorded when the user clicks close button on the introduction section (the user remains opted out), ``not_now_2`` is recorded when the user clicks "Not now" link on main section (the user remains opted out), ``dismiss_1`` recorded when the user dismisses the dialog on the introduction section (the user remains opted out), ``dismiss_2`` recorded when the user dismisses the dialog on main (the user remains opted out). + +- ``browser.urlbar.quicksuggest.dataCollection.enabled``: Whether the user has opted in to data collection for Firefox Suggest. This pref is set to true when the user opts in to the Firefox Suggest onboarding dialog modal. The user can also toggle the pref using a toggle switch in the Firefox Suggest preferences UI. + +- ``browser.urlbar.showSearchTerms.enabled``: True if to show the search term in the urlbar while on a default search engine results page. + +- ``browser.urlbar.suggest.bestmatch``: True if to show best match result is enabled in the urlbar. + +- ``browser.urlbar.suggest.quicksuggest.nonsponsored``: True if non-sponsored Firefox Suggest suggestions are enabled in the urlbar. + +- ``browser.urlbar.suggest.quicksuggest.sponsored``: True if sponsored Firefox Suggest suggestions are enabled in the urlbar. + +- ``browser.urlbar.suggest.searches``: True if search suggestions are enabled in the urlbar. Defaults to false. + +- ``browser.zoom.full`` (deprecated): True if zoom is enabled for both text and images, that is if "Zoom Text Only" is not enabled. Defaults to true. This preference was collected in Firefox 50 to 52 (`Bug 979323 <https://bugzilla.mozilla.org/show_bug.cgi?id=979323>`_). + +- ``security.tls.version.enable-deprecated``: True if deprecated versions of TLS (1.0 and 1.1) have been enabled by the user. Defaults to false. + +- ``privacy.firstparty.isolate``: True if the user has changed the (unsupported, hidden) First Party Isolation preference. Defaults to false. + +- ``privacy.resistFingerprinting``: True if the user has changed the (unsupported, hidden) Resist Fingerprinting preference. Defaults to false. + +- ``toolkit.telemetry.pioneerId``: The state of the Pioneer ID. If set, then user is enrolled in Pioneer. Note that this does *not* collect the value. + +- ``app.normandy.test-prefs.bool``: Test pref that will help troubleshoot uneven unenrollment in experiments. Defaults to false. + +- ``app.normandy.test-prefs.integer``: Test pref that will help troubleshoot uneven unenrollment in experiments. Defaults to 0. + +- ``app.normandy.test-prefs.string``: Test pref that will help troubleshoot uneven unenrollment in experiments. Defaults to "". + +- ``network.trr.mode``: User-set DNS over HTTPS mode. Defaults to 0. + +- ``network.trr.strict_native_fallback``: Whether strict fallback mode is enabled for DoH mode 2. Defaults to true on Nightly, false elsewhere. + +- ``extensions.InstallTriggerImpl.enabled``: Whether the InstallTrigger implementation should be enabled (or hidden and none of its methods available). + +- ``extensions.InstallTrigger.enabled``: Whether the InstallTrigger implementation should be enabled (or completely hidden), separate from InstallTriggerImpl because InstallTrigger is improperly used also for UA detection. + +- ``extensions.eventPages.enabled``: Whether non-persistent background pages (also known as Event pages) should be enabled for `"manifest_version": 2` extensions. + +- ``extensions.quarantinedDomains.enabled``: Whether "Quarantined Domains" is enabled. + +- ``extensions.manifestV3.enabled``: Whether `"manifest_version": 3` extensions should be allowed to install successfully. + +- ``media.gmp-gmpopenh264.enabled``: Whether OpenH264 is enabled. + +- ``media.gmp-gmpopenh264.lastDownload``: When OpenH264 was last downloaded as seconds since Jan 1, 1970. + +- ``media.gmp-gmpopenh264.lastDownloadFailed``: When OpenH264 was last downloaded unsuccessfully as seconds since Jan 1, 1970. + +- ``media.gmp-gmpopenh264.lastDownloadFailReason``: The exception value when OpenH264 was last failed to downloaded. + +- ``media.gmp-gmpopenh264.lastInstallFailed``: When OpenH264 installation last failed as seconds since Jan 1, 1970. + +- ``media.gmp-gmpopenh264.lastInstallStart``: When OpenH264 installation was last started as seconds since Jan 1, 1970. + +- ``media.gmp-gmpopenh264.lastUpdate``: When OpenH264 was last updated as seconds since Jan 1, 1970. + +- ``media.gmp-gmpopenh264.visible``: Whether OpenH264 is visible. + +- ``media.gmp-manager.lastCheck``: When the gmp-manager last checked for updates as seconds since Jan 1, 1970. + +- ``media.gmp-manager.lastEmptyCheck``: When the gmp-manager last checked for updates and there was nothing to install as seconds since Jan 1, 1970. + +- ``nimbus.qa.pref-1``: Used to monitor the results of pref-setting test experiments. + +- ``nimbus.qa.pref-2``: Used to monitor the results of pref-setting test experiments. + +- ``signon.firefoxRelay.feature``: User choice regarding Firefox Relay integration with Firefox Password Manager. Can be one of undefined, "available", "offered", "enabled" or "disabled". + +attribution +~~~~~~~~~~~ + +This object contains the attribution data for the product installation. + +Attribution data is used to link installations of Firefox with the source that the user arrived at the Firefox download page from. It would indicate, for instance, when a user executed a web search for Firefox and arrived at the download page from there, directly navigated to the site, clicked on a link from a particular social media campaign, etc. + +The attribution data is included in some versions of the default Firefox installer for Windows (the "stub" installer) and stored as part of the installation. All platforms other than Windows and also Windows installations that did not use the stub installer do not have this data and will not include the ``attribution`` object. + +sandbox +~~~~~~~ + +This object contains data about the state of Firefox's sandbox. + +Specific keys are: + +- ``effectiveContentProcessLevel``: The meanings of the values are OS dependent. Details of the meanings can be found in the `Firefox prefs file <https://hg.mozilla.org/mozilla-central/file/tip/browser/app/profile/firefox.js>`_. The value here is the effective value, not the raw value, some platforms enforce a minimum sandbox level. If there is an error calculating this, it will be ``null``. +- ``contentWin32kLockdownState``: The status of Win32k Lockdown for Content process. + + - LockdownEnabled = 1 - After Firefox 98, this value will no longer appear in Telemetry. + - MissingWebRender = 2 + - OperatingSystemNotSupported = 3 + - PrefNotSet = 4 - After Firefox 98, this value will no longer appear in Telemetry. + - MissingRemoteWebGL = 5 + - MissingNonNativeTheming = 6 + - DisabledByEnvVar = 7 - MOZ_ENABLE_WIN32K is set + - DisabledBySafeMode = 8 + - DisabledByE10S = 9 - E10S is disabled for whatever reason + - DisabledByUserPref = 10 - The user manually set security.sandbox.content.win32k-disable to false + - EnabledByUserPref = 11 - The user manually set security.sandbox.content.win32k-disable to true + - DisabledByControlGroup = 12 - The user is in the Control Group, so it is disabled + - EnabledByTreatmentGroup = 13 - The user is in the Treatment Group, so it is enabled + - DisabledByDefault = 14 - The default value of the pref is false + - EnabledByDefault = 15 - The default value of the pref is true + - DecodersArentRemote = 16 - Some decoder is not remoted to RDD Process (checks PDMFactory::AllDecodersAreRemote) + - IncompatibleMitigationPolicy = 17 - Some incompatible Windows Exploit Mitigation policies are enabled + + +profile +------- + +creationDate +~~~~~~~~~~~~ + +The assumed creation date of this client's profile. +It's read from a file-stored timestamp from the client's profile directory. + +.. note:: + + If the timestamp file does not exist all files in the profile directory are scanned. + The oldest creation or modification date of the scanned files is then taken to be the profile creation date. + This has been shown to sometimes be inaccurate (`bug 1449739 <https://bugzilla.mozilla.org/show_bug.cgi?id=1449739>`_). + +resetDate +~~~~~~~~~~~~ + +The time of the last reset time for the profile. If the profile has never been +reset this field will not be present. +It's read from a file-stored timestamp from the client's profile directory. + +firstUseDate +~~~~~~~~~~~~ + +The time of the first use of profile. If this is an old profile where we can't +determine this this field will not be present. +It's read from a file-stored timestamp from the client's profile directory. + +partner +------- + +If the user is using a partner repack, this contains information identifying the repack being used, otherwise "partnerNames" will be an empty array and other entries will be null. The information may be missing when the profile just becomes available. In Firefox for desktop, the information along with other customizations defined in distribution.ini are processed later in the startup phase, and will be fully applied when "distribution-customization-complete" notification is sent. + +Distributions are most reliably identified by the ``distributionId`` field. Partner information can be found in the `partner repacks <https://github.com/mozilla-partners>`_ (`the old one <https://hg.mozilla.org/build/partner-repacks/>`_ is deprecated): it contains one private repository per partner. +Important values for ``distributionId`` include: + +- "MozillaOnline" for the Mozilla China repack. +- "canonical", for the `Ubuntu Firefox repack <http://bazaar.launchpad.net/~mozillateam/firefox/firefox.trusty/view/head:/debian/distribution.ini>`_. +- "yandex", for the Firefox Build by Yandex. + +system +------ + +os +~~ + +This object contains operating system information. + +- ``name``: the name of the OS. +- ``version``: a string representing the OS version. +- ``kernelVersion``: an Android only string representing the kernel version. +- ``servicePackMajor``: the Windows only major version number for the installed service pack. +- ``servicePackMinor``: the Windows only minor version number for the installed service pack. +- ``windowsBuildNumber``: the Windows build number. +- ``windowsUBR``: the Windows UBR number, only available for Windows >= 10. This value is incremented by Windows cumulative updates patches. +- ``installYear``: the Windows only integer representing the year the OS was installed. +- ``locale``: the string representing the OS locale. +- ``hasPrefetch``: the Windows-only boolean representing whether or not the OS-based prefetch application start-up optimization is set to use the default settings. +- ``hasSuperfetch``: the Windows-only boolean representing whether or not the OS-based superfetch application start-up optimization service is running and using the default settings. + +addons +------ + +activeAddons +~~~~~~~~~~~~ + +Starting from Firefox 44, the length of the following string fields: ``name``, ``description`` and ``version`` is limited to 100 characters. The same limitation applies to the same fields in ``theme``. + +Some of the fields in the record for each add-on are not available during startup. The fields that will always be present are ``id``, ``version``, ``type``, ``updateDate``, ``scope``, ``isSystem``, ``isWebExtension``, and ``multiprocessCompatible``. All the other fields documented above become present shortly after the ``sessionstore-windows-restored`` observer topic is notified. + +activeGMPPlugins +~~~~~~~~~~~~~~~~ + +Up-to-date information is not available immediately during startup. The field will be populated with dummy information until the blocklist is loaded. At the latest, this will happen just after the ``sessionstore-windows-restored`` observer topic is notified. + +experiments +----------- +For each experiment we collect the + +- ``id`` (Like ``hotfix-reset-xpi-verification-timestamp-1548973``, max length 100 characters) +- ``branch`` (Like ``control``, max length 100 characters) +- ``type`` (Optional. Like ``normandy-exp``, max length 20 characters) +- ``enrollmentId`` (Optional. Like ``5bae2134-e121-46c2-aa00-232f3f5855c5``, max length 40 characters) + +In the event any of these fields are truncated, a warning is printed to the console + +Note that this list includes other types of deliveries, including Normandy rollouts and Nimbus feature defaults. + +Version History +--------------- + +- Firefox 88: + + - Removed ``addons.activePlugins`` as part of removing NPAPI plugin support. (`bug 1682030 <https://bugzilla.mozilla.org/show_bug.cgi?id=1682030>`_) + +- Firefox 70: + + - Added ``experiments.<experiment id>.enrollmentId``. (`bug 1555172 <https://bugzilla.mozilla.org/show_bug.cgi?id=1555172>`_) + +- Firefox 67: + + - Removed ``persona``. The ``addons.activeAddons`` list should be used instead. (`bug 1525511 <https://bugzilla.mozilla.org/show_bug.cgi?id=1525511>`_) + +- Firefox 61: + + - Removed empty ``addons.activeExperiment`` (`bug 1452935 <https://bugzilla.mozilla.org/show_bug.cgi?id=1452935>`_). diff --git a/toolkit/components/telemetry/docs/data/event-ping.rst b/toolkit/components/telemetry/docs/data/event-ping.rst new file mode 100644 index 0000000000..05fdd60a0f --- /dev/null +++ b/toolkit/components/telemetry/docs/data/event-ping.rst @@ -0,0 +1,92 @@ + +"event" ping +============= + +This ping transmits :ref:`Telemetry Event <eventtelemetry>` records. + +The client id is submitted with this ping. +The :doc:`Telemetry Environment <../data/environment>` is submitted in this ping. + +.. code-block:: js + + { + "type": "event", + ... common ping data + "clientId": <UUID>, + "environment": { ... }, + "payload": { + "reason": {periodic, max, shutdown}, // Why the ping was submitted + "processStartTimestamp": <UNIX Timestamp>, // Minute precision, for calculating absolute time across pings + "sessionId": <UUID>, // For linking to "main" pings + "subsessionId": <UUID>, // For linking to "main" pings + "lostEventsCount": <number>, // How many events we had to drop. Valid only for reasons "max" and "shutdown" + "events": { + "parent": [ // process name, one of the keys from Processes.yaml + [timestamp, category, method, object, value, extra], + ... // At most 1000 + ] + } + } + } + +Send behavior +------------- + +The ping is submitted at most once per ten minute interval, and at least once per hour in +which an event was recorded. Upon reaching 1000 events, the ping is sent immediately +unless it would be within ten minutes of the previous ping, in which case some event +records may be lost. A count of these lost records is included in the ping. +to avoid losing collected data. + +On shutdown, during profile-before-change, a final ping is sent with any remaining event +records, regardless of frequency but obeying the event record limit. + +The 1000-record limit and one-hour and ten-minute frequencies are controlled by +:doc:`preferences <../internals/preferences>`. + +Field details +------------- + +reason +~~~~~~ +The ``reason`` field contains the information about why the "event" ping was submitted: + +* ``periodic``: The event ping was submitted because at least one event happened in the past hour. +* ``max``: The event ping was submitted because the 1000-record limit was reached. +* ``shutdown``: The event ping was submitted because Firefox is shutting down and some events + have yet to be submitted. + +processStartTimestamp +~~~~~~~~~~~~~~~~~~~~~ +The minute the user's Firefox main process was created. Event record timestamps are recorded +relative to Firefox's main process start. This provides the basis for reconstructing a user's full +session of events in order, as well as offering a mechanism for grouping event pings. + +sessionId +~~~~~~~~~ +The id of the session that was current when the ping was sent. + +subsessionId +~~~~~~~~~~~~ +The id of the subsession that was current when the ping was sent. + +.. note:: + + This may not be the same subsession that the events occurred in if a + :ref:`session split <sessionsplit>` happened in between. + +lostEventsCount +~~~~~~~~~~~~~~~ +The number of events we had to discard because we reached the 1000-per-ping limit before +we were able to send the ping. Should only have a non-zero value on "event" pings with +reason set to "max" or "shutdown". Which events are missing should be calculable via the +client's "main" pings using :ref:`Event Summary <events.event-summary>`. + +events +~~~~~~ +A map from process names to arrays of event records that have been :ref:`serialized <events.serializationformat>`. + +Version History +--------------- + +- Firefox 62: Started sending the "event" ping (`bug 1460595 <https://bugzilla.mozilla.org/show_bug.cgi?id=1460595>`_). diff --git a/toolkit/components/telemetry/docs/data/first-shutdown-ping.rst b/toolkit/components/telemetry/docs/data/first-shutdown-ping.rst new file mode 100644 index 0000000000..fe20c5df45 --- /dev/null +++ b/toolkit/components/telemetry/docs/data/first-shutdown-ping.rst @@ -0,0 +1,11 @@ + +"first-shutdown" ping +===================== + +This ping is a duplicate of the main-ping sent on first shutdown. Enabling pingsender +for first sessions will impact existing engagement metrics. The ``first-shutdown`` ping enables a +more accurate view of users that churn after the first session. This ping exists as a +stopgap until existing metrics are re-evaluated, allowing us to use the first session +``main-pings`` directly. + +See :doc:`main-ping` for details about this payload. diff --git a/toolkit/components/telemetry/docs/data/health-ping.rst b/toolkit/components/telemetry/docs/data/health-ping.rst new file mode 100644 index 0000000000..e5655924e1 --- /dev/null +++ b/toolkit/components/telemetry/docs/data/health-ping.rst @@ -0,0 +1,92 @@ + +"health" ping +============= + +This ping is intended to provide data about problems arise when submitting other pings. +The ping is submitted at most once per hour. On shutdown an additional ping is submitted +to avoid losing collected data. + +This ping is intended to be really small. +The client id is submitted with this ping. + +.. code-block:: js + + { + "type": "health", // type + ... common ping data + "clientId": <UUID>, // client id, e.g. + // "c641eacf-c30c-4171-b403-f077724e848a" + "payload": { + "os": { + "name": <string>, // OS name + "version": <string> // OS version + }, + "reason": <string>, // When ping was triggered, e.g. "immediate" or "shutdown". + "pingDiscardedForSize": { + "main": <number>, // Amount of occurrences for a specific ping type. + "core": <number> + ... + }, + "sendFailure": { + "timeout": <number>, // Amount of occurrences for a specific failure. + "abort": <number> + ... + } + } + } + +Send behavior +------------- + +``HealthPing.jsm`` tracks several problems: + +* The size of other assembled pings exceeds the ping limit. +* Failures while sending other pings. + +After recording the data, a health ping will be sent: + +* immediately, with the reason ``immediate`` , if it is first ping in the session or it passed at least one hour from the previous submission. +* after 1 hour minus the time passed from previous submission, with the reason ``delayed`` , if less than an hour passed from the previous submission. +* on shutdown, with the reason ``shutdown`` using :doc:`../internals/pingsender`, if recorded data is not empty. + +Field details +------------- + +reason +~~~~~~ +The ``reason`` field contains the information about why the "health" ping was submitted. It presently supports three reasons: + +* immediate: The health ping was submitted immediately after recording a failure. +* delayed: The health ping was submitted after a delay. +* shutdown: The health ping was submitted on shutdown. + +pingDiscardedForSize +~~~~~~~~~~~~~~~~~~~~ +The ``pingDiscardedForSize`` field contains the information about the top ten pings whose size exceeded the +ping size limit (1 MB). This field lists the number of discarded pings per ping type. + +The ping type "<unknown>" is used to indicate that a pending pings size exceeded the limit. This is because we don't have the pending pings type available cheaply at the moment. + +This field is optional. + +sendFailure +~~~~~~~~~~~ +The ``sendFailure`` field contains the information about pings, which had failures on sending. +This field lists the number of failed pings per ping send failure type. + +The recorded failure types are: + +* "eOK" - No error. +* "eRequest" - There was some error in the request before we started to service it. +* "eUnreachable" - The remote server was unreachable. +* "eChannelOpen" - The connection failed when we tried to open the channel. +* "eRedirect" - The connection failed when being redirected. +* "abort" - What XMLHttpRequest means by "abort" (see `MDN <https://developer.mozilla.org/en-US/docs/Web/Events/abort>`__) +* "timeout" - What XMLHttpRequest means by "timeout" (see `MDN <https://developer.mozilla.org/en-US/docs/Web/Events/timeout>`__) + +This field is optional. + +.. note:: + + Although both ``pingDiscardedForSize`` and ``sendFailure`` fields are optional, the health ping will only + be submitted if one of this field not empty. diff --git a/toolkit/components/telemetry/docs/data/heartbeat-ping.rst b/toolkit/components/telemetry/docs/data/heartbeat-ping.rst new file mode 100644 index 0000000000..f1920111fd --- /dev/null +++ b/toolkit/components/telemetry/docs/data/heartbeat-ping.rst @@ -0,0 +1,62 @@ + +"heartbeat" ping +================= + +This ping is submitted after a Firefox Heartbeat survey. Even if the user exits +the browser, closes the survey window, or ignores the survey, Heartbeat will +provide a ping to Telemetry for sending during the same session. + +The payload contains the user's survey response (if any) as well as timestamps +of various Heartbeat events (survey shown, survey closed, link clicked, etc). + +The ping will also report the "surveyId", "surveyVersion" and "testing" +Heartbeat survey parameters (if they are present in the survey config). +These "meta fields" will be repeated verbatim in the payload section. + +The environment block and client ID are submitted with this ping. + +Structure: + +.. code-block:: js + + { + type: "heartbeat", + version: 4, + clientId: <UUID>, + environment: { /* ... */ } + // ... common ping data + payload: { + version: 1, + flowId: <string>, + ... timestamps below ... + offeredTS: <integer epoch timestamp>, + learnMoreTS: <integer epoch timestamp>, + votedTS: <integer epoch timestamp>, + engagedTS: <integer epoch timestamp>, + closedTS: <integer epoch timestamp>, + expiredTS: <integer epoch timestamp>, + windowClosedTS: <integer epoch timestamp>, + // ... user's rating below + score: <integer>, + // ... survey meta fields below + surveyId: <string>, + surveyVersion: <integer>, + testing: <boolean> + } + } + +Notes: + +* Pings will **NOT** have all possible timestamps, timestamps are only reported for events that actually occurred. +* Timestamp meanings: + * offeredTS: when the survey was shown to the user + * learnMoreTS: when the user clicked on the "Learn More" link + * votedTS: when the user voted + * engagedTS: when the user clicked on the survey-provided button (alternative to voting feature) + * closedTS: when the Heartbeat notification bar was closed + * expiredTS: indicates that the survey expired after 2 hours of no interaction (threshold regulated by "browser.uitour.surveyDuration" pref) + * windowClosedTS: the user closed the entire Firefox window containing the survey, thus ending the survey. This timestamp will also be reported when the survey is ended by the browser being shut down. +* The surveyId/surveyVersion fields identify a specific survey (like a "1040EZ" tax paper form). The flowID is a UUID that uniquely identifies a single user's interaction with the survey. Think of it as a session token. +* The self-support page cannot include additional data in this payload. Only the the 4 flowId/surveyId/surveyVersion/testing fields are under the self-support page's control. + +See also: :doc:`common ping fields <common-ping>` diff --git a/toolkit/components/telemetry/docs/data/index.rst b/toolkit/components/telemetry/docs/data/index.rst new file mode 100644 index 0000000000..c03027d431 --- /dev/null +++ b/toolkit/components/telemetry/docs/data/index.rst @@ -0,0 +1,19 @@ +================== +Data documentation +================== + +.. toctree:: + :maxdepth: 2 + :titlesonly: + :glob: + + common-ping + environment + main-ping + crash-ping + backgroundhangmonitor-ping + anonymous-ping + first-shutdown-ping + *-ping + +The `mozilla-pipeline-schemas repository <https://github.com/mozilla-services/mozilla-pipeline-schemas/>`_ contains schemas for some of the pings. diff --git a/toolkit/components/telemetry/docs/data/install-ping.rst b/toolkit/components/telemetry/docs/data/install-ping.rst new file mode 100644 index 0000000000..32bde280b5 --- /dev/null +++ b/toolkit/components/telemetry/docs/data/install-ping.rst @@ -0,0 +1,234 @@ +============ +Install Ping +============ + +The install pings contain some data about the system and the installation process, sent whenever the installer exits [#earlyexit]_. + +--------- +Stub Ping +--------- + +The :doc:`Stub Installer </browser/installer/windows/installer/StubInstaller>` sends a ping just before it exits, in function SendPing of `stub.nsi <https://searchfox.org/mozilla-central/source/browser/installer/windows/nsis/stub.nsi>`_. This is sent as an HTTP GET request to DSMO (download-stats.mozilla.org). + +Ingestion is handled in `gcp-ingestion <https://mozilla.github.io/gcp-ingestion/>`_ at class StubUri within `ParseUri <https://github.com/mozilla/gcp-ingestion/blob/master/ingestion-beam/src/main/java/com/mozilla/telemetry/decoder/ParseUri.java>`_. Several of the fields are codes which are broken out into multiple boolean columns in the database table. + +----------------- +Full Install Ping +----------------- + +The :doc:`Full Installer </browser/installer/windows/installer/FullInstaller>` sends a ping just before it exits, in function SendPing of `installer.nsi <https://searchfox.org/mozilla-central/source/browser/installer/windows/nsis/installer.nsi>`_. This is an HTTP POST request with a JSON document, sent to the standard Telemetry endpoint (incoming.telemetry.mozilla.org). + +To avoid double counting, the full installer does not send a ping when it is launched from the stub installer, so pings where ``installer_type = "full"`` correspond to installs that did not use the stub. + +-------------------------- +Querying the install pings +-------------------------- + +The pings are recorded in the ``firefox_installer.install`` table, accessible in `Redash <https://sql.telemetry.mozilla.org>`_ [#redashlogin]_ using the default "Telemetry (BigQuery)" data source. + +Some of the columns are marked [DEPRECATED] because they involve features that were removed, mostly when the stub installer was `streamlined <https://bugzilla.mozilla.org/show_bug.cgi?id=1328445>`_ in Firefox 55. These columns were not removed to keep compatibility and so we could continue to use the old data, but they should no longer be used. + +The columns are annotated with "(stub)", "(full)", or "(both)" to indicate which types of installer provide meaningful values. + +See also the `JSON schema <https://github.com/mozilla-services/mozilla-pipeline-schemas/blob/master/templates/firefox-installer/install/install.1.schema.json>`_. + +submission_timestamp (both) + Time the ping was received + +installer_type (both) + Which type of installer generated this ping (full or stub) + +installer_version (full) + Version of the installer itself [#stubversion]_ + +build_channel (both) + Channel the installer was built with the branding for ("release", "beta", "nightly", or "default") + +update_channel (both) + Value of MOZ_UPDATE_CHANNEL for the installer build; should generally be the same as build_channel + +version, build_id (both) + Version number and Build ID of the installed product, from ``application.ini``. This is **not** the version of the installer itself. + + stub: 0 if the installation failed + + full: ``""`` if not found [#versionfailure]_ + +locale (both) + Locale of the installer and of the installed product, in AB_CD format + +from_msi (full) + True if the install was launched from an MSI wrapper. + +_64bit_build (both) + True if a 64-bit build was selected for installation. + + stub: This means the OS is 64-bit, the RAM requirement was met, and no third-party software that blocks 64-bit installations was found + + full: Hardcoded based on the architecture to be installed + +_64bit_os (both) + True if the version of Windows on the machine was 64-bit. + +os_version (both) + Version number of Windows in ``major.minor.build`` format [#win10build]_ + +service_pack (stub) + Latest Windows service pack installed on the machine. + +server_os (both) + True if the installed OS is a server version of Windows. + +admin_user (both) + True if the installer was run by a user with administrator privileges (and the UAC prompt was accepted). Specifically, this reports whether :abbr:`HKLM (HKEY_LOCAL_MACHINE)` was writeable. + +default_path (both) + True if the default installation path was not changed. + + stub: [DEPRECATED] [#stubdefaultpath]_ + +set_default (both) + True if the option to set the new installation as the default browser was left selected. + + stub: [DEPRECATED] [#stubsetdefault]_ + +new_default (both) + True if the new installation is now the default browser (registered to handle the http protocol). + + full: Checks the association using ``AppAssocReg::QueryCurrentDefault`` and :abbr:`HKCU (HKEY_CURRENT_USER)`. + + stub: [DEPRECATED] [#stubnewdefault]_ + +old_default (both) + True if firefox.exe in a different directory is now the default browser, mutually exclusive with new_default. The details are the same as new_default. + +had_old_install (both) + True if at least one existing installation of Firefox was found on the system prior to this installation. + + full: Checks for the installation directory given in the ``Software\Mozilla\${BrandFullName}`` registry keys, either :abbr:`HKLM` or :abbr:`HKCU` + + stub: Checks for the top level profile directory ``%LOCALAPPDATA%\Mozilla\Firefox`` + +old_version, old_build_id (stub) + Version number and Build ID (from ``application.ini``) of a previous installation of Firefox in the install directory, 0 if not found + +bytes_downloaded (stub) + Size of the full installer data that was transferred before the download ended (whether it failed, was cancelled, or completed normally) + +download_size (stub) + Expected size of the full installer download according to the HTTP response headers + +download_retries (stub) + Number of times the full installer download was retried or resumed. 10 retries is the maximum. + +download_time (stub) + Number of seconds spent downloading the full installer [#downloadphase]_ + +download_latency (stub) + Seconds between sending the full installer download request and receiving the first response data + +download_ip (stub) + IP address of the server the full installer was download from (can be either IPv4 or IPv6) + +manual_download (stub) + True if the user clicked on the button that opens the manual download page. The prompt to do that is shown after the installation fails or is cancelled. + +intro_time (both) + Seconds the user spent on the intro screen. + + stub: [DEPRECATED] The streamlined stub no longer has this screen, so this should always be 0. + +options_time (both) + Seconds the user spent on the options screen. + + stub: [DEPRECATED] The streamlined stub no longer has this screen, so this should always be 0. + +preinstall_time (stub) + Seconds spent verifying the downloaded full installer and preparing to run it + +install_time (both) + full: Seconds taken by the installation phase. + + stub: Seconds taken by the full installer. + +finish_time (both) + full: Seconds the user spent on the finish page. + + stub: Seconds spent waiting for the installed application to launch. + +succeeded (both) + True if a new installation was successfully created. False if that didn't happen for any reason, including when the user closed the installer window. + +disk_space_error (stub) + [DEPRECATED] True if the installation failed because the drive we're trying to install to does not have enough space. The streamlined stub no longer sends a ping in this case, because the installation drive can no longer be selected. + +no_write_access (stub) + [DEPRECATED] True if the installation failed because the user doesn't have permission to write to the path we're trying to install to. The streamlined stub no longer sends a ping in this case, because the installation directory can no longer be selected. + +user_cancelled (both) + True if the installation failed because the user cancelled it or closed the window. + +out_of_retries (stub) + True if the installation failed because the download had to be retried too many times (currently 10) + +file_error (stub) + True if the installation failed because the downloaded file couldn't be read from + +sig_not_trusted (stub) + True if the installation failed because the signature on the downloaded file wasn't valid or wasn't signed by a trusted authority + +sig_unexpected (stub) + True if the installation failed because the signature on the downloaded file didn't have the expected subject and issuer names + +install_timeout (stub) + True if the installation failed because running the full installer timed out. Currently that means it ran for more than 150 seconds for a new installation, or 165 seconds for a paveover installation. + +new_launched (both) + True if the installation succeeded and tried to launch the newly installed application. + +old_running (stub) + [DEPRECATED] True if the installation succeeded and we weren't able to launch the newly installed application because a copy of Firefox was already running. This should always be false since the check for a running copy was `removed <https://bugzilla.mozilla.org/show_bug.cgi?id=1601806>`_ in Firefox 74. + +attribution (both) + Any attribution data that was included with the installer + +profile_cleanup_prompt (stub) + 0: neither profile cleanup prompt was shown + + 1: the "reinstall" version of the profile cleanup prompt was shown (no existing installation was found, but the user did have an old Firefox profile) + + 2: the "paveover" version of the profile cleanup prompt was shown (an installation of Firefox was already present, but it's an older version) + +profile_cleanup_requested (stub) + True if either profile cleanup prompt was shown and the user accepted the prompt + +funnelcake (stub) + `Funnelcake <https://wiki.mozilla.org/Funnelcake>`_ ID + +ping_version (stub) + Version of the stub ping, currently 8 + +silent (full) + True if the install was silent (see :ref:`Full Installer Configuration`) + +--------- +Footnotes +--------- + +.. [#earlyexit] No ping is sent if the installer exits early because initial system requirements checks fail. + +.. [#redashlogin] A Mozilla LDAP login is required to access Redash. + +.. [#stubversion] The version of the installer would be useful for the stub, but it is not currently sent as part of the stub ping. + +.. [#versionfailure] If the installation failed or was cancelled, the full installer will still report the version number of whatever was in the installation directory, or ``""`` on if it couldn't be read. + +.. [#win10build] Previous versions of Windows have used a very small set of build numbers through their entire lifecycle. However, Windows 10 gets a new build number with every major update (about every 6 months), and many more builds have been released on its insider channels. So, to prevent a huge amount of noise, queries using this field should generally filter out the build number and only use the major and minor version numbers to differentiate Windows versions, unless the build number is specifically needed. + +.. [#stubdefaultpath] ``default_path`` should always be true in the stub, since we no longer support changing the path, but see `bug 1351697 <https://bugzilla.mozilla.org/show_bug.cgi?id=1351697>`_. + +.. [#stubsetdefault] We no longer attempt to change the default browser setting in the streamlined stub, so set_default should always be false. + +.. [#stubnewdefault] We no longer attempt to change the default browser setting in the streamlined stub, so new_default should usually be false, but the stub still checks the association at ``Software\Classes\http\shell\open\command`` in :abbr:`HKLM` or :abbr:`HKCU`. + +.. [#downloadphase] ``download_time`` was previously called ``download_phase_time``, this includes retries during the download phase. There was a different ``download_time`` field that specifically measured only the time of the last download, this is still submitted but it is ignored during ingestion. diff --git a/toolkit/components/telemetry/docs/data/launcher-process-failure-ping.rst b/toolkit/components/telemetry/docs/data/launcher-process-failure-ping.rst new file mode 100644 index 0000000000..f09e0422af --- /dev/null +++ b/toolkit/components/telemetry/docs/data/launcher-process-failure-ping.rst @@ -0,0 +1,96 @@ + +Launcher Process Failure ping +============================= + +This ping is generated by the Firefox launcher process when it fails to successfully start the browser, or generated by the browser process when it fails to start a sandboxed process. + +Structure: + +.. code-block:: js + + { + type: "launcher-process-failure", + "version": 1, + "id": <UUID>, + "creationDate": <Date string in ISO format>, + "update_channel": <string>, + "build_id": <string>, + "build_version": <string>, + // Windows version number in major.minor.build.UBR format (UBR is optional, only available on Win10) + "os_version": <string>, + // True if this build was running atop a Windows Server OS + "server_os": <bool>, + // The current user's OS locale setting + "os_locale": <string>, + // CPU Architecture. One of the values from the Windows SYSTEM_INFO::wProcessorArchitecture field + "cpu_arch": <int>, + "num_logical_cpus": <int>, + // True if the process was launched with Administrator privileges + // but without User Account Control (= UAC) + "is_admin_without_uac": <bool>, + // The type of the process which failed to start as a sandboxed process. + // If the launcher process fails to launch the browser process, this property is not set. + "process_type": <string>, + "memory": { + // Free space available in the page file, in bytes + "total_phys": <int>, + // Available physical memory on the machine, in bytes + "avail_phys": <int>, + // Free space available in the page file, in bytes + "avail_page_file": <int>, + // Available virtual memory on the machine, in bytes + "avail_virt": <int> + }, + "xpcom_abi": <string>, + "launcher_error": { + // The leaf name of the source file where the error was raised + "source_file": <string>, + // The line number of the source file where the error was raised + "source_line": <int>, + // The HRESULT error code of the error that was raised + "hresult": <int>, + // First sixteen bytes of a function that we failed to hook (Nightly-only). + // This field is added only on detour failures. + "detour_orig_bytes": <string> + }, + "security": { + // A list of names of installed antivirus products + "av": [ + <string>, + ... + ], + // A list of names of installed antispyware products + "antispyware": [ + <string>, + ... + ], + // A list of names of installed firewall products + "firewall": [ + <string>, + ... + ] + }, + // A mapping of all modules present in the failing process, including + // their version numbers and an optional index into the signatures array + "modules": { + <moduleName>: [ + <string>, + <int> + ], + ... + }, + // A list of all signatures that were used to sign the binaries that are + // listed in modules. + "signatures": [ + <string>, + ... + ] + } + +Version History +~~~~~~~~~~~~~~~ + +- Firefox 82: Added ``detour_orig_bytes`` (`bug 1588245 <https://bugzilla.mozilla.org/show_bug.cgi?id=1588245>`_). +- Firefox 82: Added ``process_type`` (`bug 1630444 <https://bugzilla.mozilla.org/show_bug.cgi?id=1630444>`_). +- Firefox 71: Added ``is_admin_without_uac`` (`bug 1567605 <https://bugzilla.mozilla.org/show_bug.cgi?id=1567605>`_). +- Firefox 67: Initial release (`bug 1460433 <https://bugzilla.mozilla.org/show_bug.cgi?id=1460433>`_). diff --git a/toolkit/components/telemetry/docs/data/main-ping.rst b/toolkit/components/telemetry/docs/data/main-ping.rst new file mode 100644 index 0000000000..23f7d8c600 --- /dev/null +++ b/toolkit/components/telemetry/docs/data/main-ping.rst @@ -0,0 +1,504 @@ + +"main" ping +=========== + +.. toctree:: + :maxdepth: 2 + +This is the "main" Telemetry ping type, whose payload contains most of the measurements that are used to track the performance and health of Firefox in the wild. +It includes histograms and other performance and diagnostic data. + +This ping may be triggered for one of many reasons documented by the ``reason`` field: + +* ``aborted-session`` - this ping is regularly saved to disk (every 5 minutes), overwriting itself, and deleted at shutdown. If a previous aborted session ping is found at startup, it gets sent to the server. The first aborted-session ping is generated as soon as Telemetry starts +* ``environment-change`` - the :doc:`environment` changed, so the session measurements got reset and a new subsession starts +* ``shutdown`` - triggered when the browser session ends. For the first browsing session, this ping is saved to disk and sent on the next browser restart. From the second browsing session on, this ping is sent immediately on shutdown using the :doc:`../internals/pingsender`, unless the OS is shutting down +* ``daily`` - a session split triggered in 24h hour intervals at local midnight. If an ``environment-change`` ping is generated by the time it should be sent, the daily ping is rescheduled for the next midnight +* ``saved-session`` - the *"classic"* Telemetry payload with measurements covering the whole browser session (only submitted on Android) + +.. _sessionsplit: + +Most reasons lead to a session split, initiating a new *subsession*. We reset important measurements for those subsessions. + +After a new subsession split, the ``internal-telemetry-after-subsession-split`` topic is notified to all the observers. *This is an internal topic and is only meant for internal Telemetry usage.* + +.. note:: + + ``saved-session`` is sent with a different ping type (``saved-session``, not ``main``), but otherwise has the same format as discussed here. As of Firefox 61 this is sent on Android only. + +Structure: + +.. code-block:: js + + { + version: 4, + + info: { + reason: <string>, // what triggered this ping: "saved-session", "environment-change", "shutdown", ... + revision: <string>, // the Histograms.json revision + timezoneOffset: <integer>, // time-zone offset from UTC, in minutes, for the current locale + previousBuildId: <string>, // null if this is the first run, or the previous build ID is unknown + + sessionId: <uuid>, // random session id, shared by subsessions + subsessionId: <uuid>, // random subsession id + previousSessionId: <uuid>, // session id of the previous session, null on first run. + previousSubsessionId: <uuid>, // subsession id of the previous subsession (even if it was in a different session), + // null on first run. + + subsessionCounter: <unsigned integer>, // the running no. of this subsession since the start of the browser session + profileSubsessionCounter: <unsigned integer>, // the running no. of all subsessions for the whole profile life time + + sessionStartDate: <ISO date>, // hourly precision, ISO date in local time + subsessionStartDate: <ISO date>, // hourly precision, ISO date in local time + sessionLength: <integer>, // the session length until now in seconds, monotonic + subsessionLength: <integer>, // the subsession length in seconds, monotonic + + addons: <string>, // obsolete, use ``environment.addons`` + }, + + processes: {...}, + simpleMeasurements: {...}, + + // The following properties may all be null if we fail to collect them. + histograms: {...}, + keyedHistograms: {...}, + chromeHangs: {...}, // removed in firefox 62 + threadHangStats: [...], // obsolete in firefox 57, use the 'bhr' ping + log: [...], // obsolete in firefox 61, use Event Telemetry or Scalars + gc: {...}, + fileIOReports: {...}, + lateWrites: {...}, + addonDetails: {...}, + UIMeasurements: [...], // Android only + slowSQL: {...}, + slowSQLstartup: {...}, + } + +info +---- + +sessionLength +~~~~~~~~~~~~~ +The length of the current session so far in seconds. +This uses a monotonic clock, so this may mismatch with other measurements that +are not monotonic like calculations based on ``Date.now()``. + +Note that this currently does not behave consistently over our supported platforms: + +* On Windows this uses ``GetTickCount64()``, which does increase over sleep periods +* On macOS this uses ``mach_absolute_time()``, which does not increase over sleep periods +* On POSIX/Linux this uses ``clock_gettime(CLOCK_MONOTONIC, &ts)``, which should not increase over sleep time + +See `bug 1204823 <https://bugzilla.mozilla.org/show_bug.cgi?id=1204823>`_ for details. + +subsessionLength +~~~~~~~~~~~~~~~~ +The length of this subsession in seconds. +This uses a monotonic clock, so this may mismatch with other measurements that are not monotonic (e.g. based on ``Date.now()``). + +Also see the remarks for ``sessionLength`` on platform consistency. + +processes +--------- +This section contains per-process data. + +Structure: + +.. code-block:: js + + "processes" : { + // ... other processes ... + "parent": { + scalars: {...}, + keyedScalars: {...}, + // parent process histograms and keyedHistograms are in main payload + }, + "content": { + scalars: {...}, + keyedScalars: {...}, + histograms: {...}, + keyedHistograms: {...}, + }, + "gpu": { + // ... + } + } + +histograms and keyedHistograms +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +This section contains histograms and keyed histograms accumulated on content processes. Histograms recorded on a content child process have different character than parent histograms. For instance, ``GC_MS`` will be much different in ``processes.content`` as it has to contend with web content, whereas the instance in ``payload.histograms`` has only to contend with browser JS. Also, some histograms may be absent if never recorded on a content child process (``EVENTLOOP_UI_ACTIVITY`` is parent-process-only). + +This format was adopted in Firefox 51 via bug 1218576. + +scalars and keyedScalars +~~~~~~~~~~~~~~~~~~~~~~~~ +This section contains the :doc:`../collection/scalars` that are valid for the current platform. Scalars are only submitted if data was added to them, and are only reported with subsession pings. The recorded scalars are described in the `Scalars.yaml <https://searchfox.org/mozilla-central/source/toolkit/components/telemetry/Scalars.yaml>`_ file. The ``info.revision`` field indicates the revision of the file that describes the reported scalars. + +simpleMeasurements +------------------ +This section contains a list of simple measurements, or counters. In addition to the ones highlighted below, Telemetry timestamps (see `here <https://searchfox.org/mozilla-central/search?q=TelemetryTimestamps.add&redirect=false&case=true>`__ and `here <https://searchfox.org/mozilla-central/search?q=recordTimestamp&redirect=false&case=true>`__) can be reported. + +totalTime +~~~~~~~~~ +A non-monotonic integer representing the number of seconds the session has been alive. + +addonManager +~~~~~~~~~~~~ +Only available in the extended set of measures, it contains a set of counters related to Addons. See `here <https://searchfox.org/mozilla-central/search?q=AddonManagerPrivate.recordSimpleMeasure&redirect=false&case=true>`__ for a list of recorded measures. + +UITelemetry +~~~~~~~~~~~ +As of Firefox 61 this section is no longer present. + +Only available in the extended set of measures. For more see :ref:`uitelemetry`. + +startupInterrupted +~~~~~~~~~~~~~~~~~~ +A boolean set to true if startup was interrupted by an interactive prompt. + +js +~~ +This section contains a series of counters from the JavaScript engine. + +Structure: + +.. code-block:: js + + "js" : { + // ... + } + +As of Firefox 59 this section no longer contains any entries, as of Firefox 61 this section is removed. + +maximalNumberOfConcurrentThreads +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +An integer representing the highest number of threads encountered so far during the session. + +startupSessionRestoreReadBytes +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Windows-only integer representing the number of bytes read by the main process up until the session store has finished restoring the windows. + +startupSessionRestoreWriteBytes +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Windows-only integer representing the number of bytes written by the main process up until the session store has finished restoring the windows. + +startupWindowVisibleReadBytes +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Windows-only integer representing the number of bytes read by the main process up until after a XUL window is made visible. + +startupWindowVisibleWriteBytes +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Windows-only integer representing the number of bytes written by the main process up until after a XUL window is made visible. + +debuggerAttached +~~~~~~~~~~~~~~~~ +A boolean set to true if a debugger is attached to the main process. + +shutdownDuration +~~~~~~~~~~~~~~~~ +The time, in milliseconds, it took to complete the last shutdown. + +failedProfileLockCount +~~~~~~~~~~~~~~~~~~~~~~ +The number of times the system failed to lock the user profile. + +activeTicks +~~~~~~~~~~~ +Integer count of the number of five-second intervals ('ticks') the user was considered 'active' (sending UI events to the window). An extra event is fired immediately when the user becomes active after being inactive. This is for some mouse and gamepad events, and all touch, keyboard, wheel, and pointer events (see `EventStateManager.cpp <https://searchfox.org/mozilla-central/source/dom/events/EventStateManager.cpp#504>`__). +This measure might be useful to give a trend of how much a user actually interacts with the browser when compared to overall session duration. It does not take into account whether or not the window has focus or is in the foreground. Just if it is receiving these interaction events. +Note that in ``main`` pings, this measure is reset on subsession splits, while in ``saved-session`` pings it covers the whole browser session. + +histograms +---------- +This section contains the histograms that are valid for the current platform. ``Flag`` histograms are always created and submitted with a default value of ``false`` if a value of ``true`` is not recorded during the time period. Other histogram types (see :ref:`choosing-histogram-type`) are not created nor submitted if no data was added to them. The type and format of the reported histograms is described by the ``Histograms.json`` file. Its most recent version is available `here <https://searchfox.org/mozilla-central/source/toolkit/components/telemetry/Histograms.json>`__. The ``info.revision`` field indicates the revision of the file that describes the reported histograms. + +keyedHistograms +--------------- +This section contains the keyed histograms available for the current platform. + +As of Firefox 48, this section does not contain empty keyed histograms anymore. + +threadHangStats +--------------- +As of Firefox 57 this section is no longer present, and has been replaced with the :doc:`bhr ping <backgroundhangmonitor-ping>`. + +Contains the statistics about the hangs in main and background threads. Note that hangs in this section capture the `label stack <https://developer.mozilla.org/en-US/docs/Mozilla/Performance/Profiling_with_the_Built-in_Profiler#Native_stack_vs._label_stack>`_ and an incomplete JS stack, which is not 100% precise. For particularly egregious hangs, and on nightly, an unsymbolicated native stack is also captured. The amount of time that is considered "egregious" is different from thread to thread, and is set when the BackgroundHangMonitor is constructed for that thread. In general though, hangs from 5 - 10 seconds are generally considered egregious. Shorter hangs (1 - 2s) are considered egregious for other threads (the compositor thread, and the hang monitor that is only enabled during tab switch). + +To avoid submitting overly large payloads, some limits are applied: + +* Identical, adjacent "(chrome script)" or "(content script)" stack entries are collapsed together. If a stack is reduced, the "(reduced stack)" frame marker is added as the oldest frame. +* The depth of the reported label stacks is limited to 11 entries. This value represents the 99.9th percentile of the thread hangs stack depths reported by Telemetry. +* The native stacks are limited to a depth of 25 stack frames. + +Structure: + +.. code-block:: js + + "threadHangStats" : [ + { + "name" : "Gecko", + "activity" : {...}, // a time histogram of all task run times + "nativeStacks": { // captured for all hangs on nightly, or egregious hangs on beta + "memoryMap": [ + ["wgdi32.pdb", "08A541B5942242BDB4AEABD8C87E4CFF2"], + ["igd10iumd32.pdb", "D36DEBF2E78149B5BE1856B772F1C3991"], + // ... other entries in the format ["module name", "breakpad identifier"] ... + ], + "stacks": [ + [ + [ + 0, // the module index or -1 for invalid module indices + 190649 // the offset of this program counter in its module or an absolute pc + ], + [1, 2540075], + // ... other frames ... + ], + // ... other stacks ... + ] + }, + "hangs" : [ + { + "stack" : [ + "Startup::XRE_Main", + "Timer::Fire", + "(content script)", + "IPDL::PPluginScriptableObject::SendGetChildProperty", + ... up to 11 frames ... + ], + "nativeStack": 0, // index into nativeStacks.stacks array + "histogram" : {...}, // the time histogram of the hang times + "annotations" : [ + { + "pluginName" : "Shockwave Flash", + "pluginVersion" : "18.0.0.209" + }, + ... other annotations ... + ] + }, + ], + }, + ... other threads ... + ] + +.. _chromeHangs: + +chromeHangs +----------- +As of Firefox 62, chromeHangs has been removed. Please look to the bhr ping for +similar functionality. + +Contains the statistics about the hangs happening exclusively on the main thread of the parent process. Precise C++ stacks are reported. This is only available on Nightly Release on Windows, when building using "--enable-profiling" switch. + +Some limits are applied: + +* Reported chrome hang stacks are limited in depth to 50 entries. +* The maximum number of reported stacks is 50. + +The module names can contain unicode characters. + +Structure: + +.. code-block:: js + + "chromeHangs" : { + "memoryMap" : [ + ["wgdi32.pdb", "08A541B5942242BDB4AEABD8C87E4CFF2"], + ["igd10iumd32.pdb", "D36DEBF2E78149B5BE1856B772F1C3991"], + ... other entries in the format ["module name", "breakpad identifier"] ... + ], + "stacks" : [ + [ + [ + 0, // the module index or -1 for invalid module indices + 190649 // the offset of this program counter in its module or an absolute pc + ], + [1, 2540075], + ... other frames, up to 50 ... + ], + ... other stacks, up to 50 ... + ], + "durations" : [8, ...], // the hang durations (in seconds) + "systemUptime" : [692, ...], // the system uptime (in minutes) at the time of the hang + "firefoxUptime" : [672, ...], // the Firefox uptime (in minutes) at the time of the hang + "annotations" : [ + [ + [0, ...], // the indices of the related hangs + { + "pluginName" : "Shockwave Flash", + "pluginVersion" : "18.0.0.209", + ... other annotations as key:value pairs ... + } + ], + ... + ] + }, + +log +--- +As of Firefox 61 this section is no longer present, use :ref:`eventtelemetry` or :doc:`../collection/scalars`. + +This section contains a log of important or unusual events reported through Telemetry. + +Structure: + +.. code-block:: js + + "log": [ + [ + "Event_ID", + 3785, // the timestamp (in milliseconds) for the log entry + ... other data ... + ], + ... + ] + +At present there is one known users of this section: Telemetry Experiments. + +Telemetry Experiments uses it to note when experiments are activated and terminated. + +fileIOReports +------------- +Contains the statistics of main-thread I/O recorded during the execution. Only the I/O stats for the XRE and the profile directories are currently reported, neither of them disclosing the full local path. + +Structure: + +.. code-block:: js + + "fileIOReports": { + "{xre}": [ + totalTime, // Accumulated duration of all operations + creates, // Number of create/open operations + reads, // Number of read operations + writes, // Number of write operations + fsyncs, // Number of fsync operations + stats, // Number of stat operations + ], + "{profile}": [ ... ], + ... + } + +lateWrites +---------- +This sections reports writes to the file system that happen during shutdown. The reported data contains the stack and the file names of the loaded libraries at the time the writes happened. + +The file names of the loaded libraries can contain unicode characters. + +Structure: + +.. code-block:: js + + "lateWrites" : { + "memoryMap" : [ + ["wgdi32.pdb", "08A541B5942242BDB4AEABD8C87E4CFF2"], + ... other entries in the format ["module name", "breakpad identifier"] ... + ], + "stacks" : [ + [ + [ + 0, // the module index or -1 for invalid module indices + 190649 // the offset of this program counter in its module or an absolute pc + ], + [1, 2540075], + ... other frames ... + ], + ... other stacks ... + ], + }, + +addonDetails +------------ +This section contains per add-on telemetry details, as reported by each add-on provider. The XPI provider is the only one reporting at the time of writing (`see Searchfox <https://searchfox.org/mozilla-central/search?q=setTelemetryDetails&case=true>`_). Telemetry does not manipulate or enforce a specific format for the supplied provider's data. + +Structure: + +.. code-block:: js + + "addonDetails": { + "XPI": { + "adbhelper@mozilla.org": { + "location": "app-profile", + "name": "ADB Helper", + "creator": "Mozilla & Android Open Source Project", + }, + ... + }, + ... + } + +slowSQL +------- +This section contains the information about the slow SQL queries for both the main and other threads. The execution of an SQL statement is considered slow if it takes 50ms or more on the main thread or 100ms or more on other threads. Slow SQL statements will be automatically trimmed to 1000 characters. This limit doesn't include the ellipsis and database name, that are appended at the end of the stored statement. + +Structure: + +.. code-block:: js + + "slowSQL": { + "mainThread": { + "Sanitized SQL Statement": [ + 1, // the number of times this statement was hit + 200 // the total time (in milliseconds) that was spent on this statement + ], + ... + }, + "otherThreads": { + "VACUUM /* places.sqlite */": [ + 1, + 330 + ], + ... + } + }, + +slowSQLStartup +-------------- +This section contains the slow SQL statements gathered at startup (until the "sessionstore-windows-restored" event is fired). The structure of this section resembles the one for `slowSQL`_. + +UIMeasurements +-------------- +This section is Android-only and contains UI specific Telemetry measurements and events (`see here <https://searchfox.org/mozilla-central/search?q=UITelemetry.%28addEvent|startSession|stopSession%29&redirect=false&case=false®exp=true>`_). + +Structure: + +.. code-block:: js + + "UIMeasurements": [ + { + "type": "event", // either "session" or "event" + "action": "action.1", + "method": "menu", + "sessions": [], + "timestamp": 12345, + "extras": "settings" + }, + { + "type": "session", + "name": "awesomescreen.1", + "reason": "commit", + "start": 123, + "end": 456 + } + ... + ], + +Version History +--------------- + +- Firefox 88: + + - Stopped reporting ``flashVersion`` since Flash is no longer supported. (`bug 1682030 <https://bugzilla.mozilla.org/show_bug.cgi?id=1682030>`_) + +- Firefox 61: + + - Stopped reporting ``childPayloads`` (`bug 1443599 <https://bugzilla.mozilla.org/show_bug.cgi?id=1443599>`_). + - Stopped reporting ``saved-session`` pings on Firefox Desktop (`bug 1443603 <https://bugzilla.mozilla.org/show_bug.cgi?id=1443603>`_). + - Stopped reporting ``simpleMeasurements.js`` (`bug 1278920 <https://bugzilla.mozilla.org/show_bug.cgi?id=1278920>`_). + - Stopped reporting ``UITelemetry`` (`bug 1443605 <https://bugzilla.mozilla.org/show_bug.cgi?id=1443605>`_) + +- Firefox 62: + + - ``events`` are now reported via the :doc:`../data/event-ping` (`bug 1460595 <https://bugzilla.mozilla.org/show_bug.cgi?id=1460595>`_). + +- Firefox 80: + + - Stopped reporting ``GCTelemetry`` (`bug 1482089 <https://bugzilla.mozilla.org/show_bug.cgi?id=1482089>`_). diff --git a/toolkit/components/telemetry/docs/data/modules-ping.rst b/toolkit/components/telemetry/docs/data/modules-ping.rst new file mode 100644 index 0000000000..c3b193f8dc --- /dev/null +++ b/toolkit/components/telemetry/docs/data/modules-ping.rst @@ -0,0 +1,46 @@ + +"modules" ping +============== + +This ping is sent once a week and includes the modules loaded in the Firefox process. + +The client ID and environment are submitted with this ping. + +Structure: + +.. code-block:: js + + { + type: "modules", + ... common ping data + clientId: <UUID>, + environment: { ... }, + payload: { + version: 1, + modules: [ + { + name: <string>, // Name of the module file (e.g. xul.dll) + version: <string>, // Version of the module + debugID: <string>, // ID of the debug information file + debugName: <string>, // Name of the debug information file + certSubject: <string>, // Name of the organization that signed the binary (Optional, only defined when present) + }, + ... + ], + } + } + +Notes +~~~~~ + +The version information is only available on Windows, it is null on other platforms. + +The debug name is the name of the PDB on Windows (which isn't always the same as the module name modulo the extension, e.g. the PDB for C:\Windows\SysWOW64\ntdll.dll is wntdll.pdb) and is the same as the module name on other platforms. + +The debug ID is platform-dependent. It is compatible with the Breakpad ID used on Socorro. + +Sometimes the debug name and debug ID are missing for Windows modules (often with malware). In this case, they will be "null". + +The length of the modules array is limited to 512 entries. + +The name and debug name are length limited, with a maximum of 64 characters. diff --git a/toolkit/components/telemetry/docs/data/new-profile-ping.rst b/toolkit/components/telemetry/docs/data/new-profile-ping.rst new file mode 100644 index 0000000000..67a95b45b4 --- /dev/null +++ b/toolkit/components/telemetry/docs/data/new-profile-ping.rst @@ -0,0 +1,83 @@ + +"new-profile" ping +================== + +This opt-out ping is sent from Firefox Desktop 30 minutes after the browser is started, on the first session +of a newly created profile. If the first session of a newly-created profile was shorter than 30 minutes, it +gets sent using the :doc:`../internals/pingsender` at shutdown. + +.. note:: + + We don't sent the ping immediately after Telemetry completes initialization to give the user enough + time to tweak their data collection preferences. + +Structure: + +.. code-block:: js + + { + type: "new-profile", + ... common ping data + clientId: <UUID>, + environment: { ... }, + payload: { + reason: "startup", // or "shutdown" + processes: { ... } + } + } + +payload.reason +-------------- +If this field contains ``startup``, then the ping was generated at the scheduled time after +startup. If it contains ``shutdown``, then the browser was closed before the time the +ping was scheduled. In the latter case, the ping is generated during shutdown and sent +using the :doc:`../internals/pingsender`. + +processes +--------- +This section contains per-process data. + +Structure: + +.. code-block:: js + + "processes" : { + "parent": { + "scalars": {...} + } + } + +scalars +~~~~~~~ +This section contains the :doc:`../collection/scalars` that are valid for the ``new-profile`` ping, +that is the ``record_into_store`` list contains ``new-profile``. +Scalars are only submitted if data was added to them. +The recorded scalars are described in the `Scalars.yaml <https://searchfox.org/mozilla-central/source/toolkit/components/telemetry/Scalars.yaml>`_ file. + +Duplicate pings +--------------- +We expect a low fraction of duplicates of this ping, mostly due to crashes happening +right after sending the ping and before the telemetry state gets flushed to the disk. This should +be fairly low in practice and manageable during the analysis phase. + +Expected behaviours +------------------- +The following is a list of conditions and expected behaviours for the ``new-profile`` ping: + +- **The ping is generated at the browser shutdown on a new profile, after the privacy policy is displayed:** + + - *for an user initiated browser shutdown*, ``new-profile`` is sent immediately using the :doc:`../internals/pingsender`; + - *for a browser shutdown triggered by OS shutdown*, ``new-profile`` is saved to disk and sent next time the browser restarts. +- **The ping is generated before the privacy policy is displayed**: ``new-profile`` is saved to disk and sent + next time the browser restarts. +- **The ping is set to be generated and Telemetry is disabled**: ``new-profile`` is never sent, even if Telemetry is + turned back on later. +- **Firefox crashes before the ping can be generated**: ``new-profile`` will be scheduled to be generated and + sent again on the next restart. +- **User performs a profile refresh**: + + - *the ping was already sent*: ``new-profile`` will not be sent again. + - *the ping was not sent*: ``new-profile`` will be generated and sent. + - *the refresh happens immediately after the profile creation, before the policy is shown*: ``new-profile`` will not be sent again. +- **Firefox is run with an old profile that already sent Telemetry data**: ``new-profile`` will not be generated + nor sent. diff --git a/toolkit/components/telemetry/docs/data/pioneer-study.rst b/toolkit/components/telemetry/docs/data/pioneer-study.rst new file mode 100644 index 0000000000..d592c692ab --- /dev/null +++ b/toolkit/components/telemetry/docs/data/pioneer-study.rst @@ -0,0 +1,58 @@ +============= +Pioneer Study +============= + +The `pioneer-study` ping is the main transport used by the Pioneer components. + +------- +Payload +------- + +It is made up of a clear text payload and an encrypted data payload, following the structure described below. + +Structure: + +.. code-block:: js + + "payload": { + "encryptedData": "<encrypted token>", + "schemaVersion": 1, + "schemaName": "debug", + "schemaNamespace": "<namespace>", + "encryptionKeyId": "<key id>", + "pioneerId": "<UUID>", + "studyName": "pioneer-v2-example" + } + +See also the `JSON schemas <https://github.com/mozilla-services/mozilla-pipeline-schemas/tree/master/schemas/pioneer-debug>`_. + +encryptedData + The encrypted data sent using the Pioneer platform. + +schemaVersion + The payload format version. + +schemaName + The name of the schema of the encrypted data. + +schemaNamespace + The namespace used to segregate data on the pipeline. + +encryptionKeyId + The id of the key used to encrypt the data. If `discarded` is used, then the `encryptedData` will be ignored and not decoded (only possible for `deletion-request` and `pioneer-enrollment` schemas). + +pioneerId + The id of the pioneer client. + +studyName (optional) + The id of the study for which the data is being collected. + +------------------------ +Special Pioneer Payloads +------------------------ + +This ping has two special Pioneer payload configurations, indicated by the different `schemaName`: `deletion-request` and `pioneer-enrollemnt`. + +The `deletion-request` is sent when a user opts out from a Pioneer study: it contains the `pioneerId` and the `studyName`. + +The `pioneer-enrollment` is sent when a user opts into the Pioneer program: in this case it reports `schemaNamespace: "pioneer-meta"` and will have no `studyName`. It is also sent when enrolling into a study, in which case it reports the same namespace as the `deletion-request` (i.e. the id the study making the request) and the `pioneer-enrollment` schema name. diff --git a/toolkit/components/telemetry/docs/data/sync-ping.rst b/toolkit/components/telemetry/docs/data/sync-ping.rst new file mode 100644 index 0000000000..4668b4e695 --- /dev/null +++ b/toolkit/components/telemetry/docs/data/sync-ping.rst @@ -0,0 +1,357 @@ + +"sync" ping +=========== + +This is an aggregated format that contains information about each sync that occurred during a timeframe. It is submitted every 12 hours, and on browser shutdown, but only if the ``syncs`` property would not be empty. The ping does not contain the environment block, nor the clientId. + +Each item in the ``syncs`` property is generated after a sync is completed, for both successful and failed syncs, and contains measurements pertaining to sync performance and error information. + +A JSON-schema document describing the exact format of the ping's payload property can be found at `services/sync/tests/unit/sync\_ping\_schema.json <https://searchfox.org/mozilla-central/source/services/sync/tests/unit/sync_ping_schema.json>`_. + +Structure: + +.. code-block:: js + + { + version: 4, + type: "sync", + ... common ping data + payload: { + version: 1, + os : { ... }, // os data from the current telemetry environment. OS specific, but typically includes name, version and locale. + discarded: <integer count> // Number of syncs discarded -- left out if zero. + why: <string>, // Why did we submit the ping? Either "shutdown", "schedule", or "idchanged". + uid: <string>, // Hashed FxA unique ID, or string of 32 zeros. If this changes between syncs, the payload is submitted. + deviceID: <string>, // Hashed FxA Device ID, hex string of 64 characters, not included if the user is not logged in. If this changes between syncs, the payload is submitted. + sessionStartDate: <ISO date>, // Hourly precision, ISO date in local time + // Array of recorded syncs. The ping is not submitted if this would be empty + syncs: [{ + when: <integer milliseconds since epoch>, + took: <integer duration in milliseconds>, + didLogin: <bool>, // Optional, is this the first sync after login? Excluded if we don't know. + why: <string>, // Optional, why the sync occurred, excluded if we don't know. + + // Optional, excluded if there was no error. + failureReason: { + name: <string>, // "httperror", "networkerror", "shutdownerror", etc. + code: <integer>, // Only present for "httperror" and "networkerror". + error: <string>, // Only present for "othererror" and "unexpectederror". + from: <string>, // Optional, and only present for "autherror". + }, + + // Optional, excluded if we couldn't get a valid uid or local device id + devices: [{ + os: <string>, // OS string as reported by Services.appinfo.OS, if known + version: <string>, // Firefox version, as reported by Services.appinfo.version if known + id: <string>, // Hashed FxA device id for device + type: <string>, // broad device "type", as reported by fxa ("mobile", "tv", etc). + syncID: <string>, // Hashed Sync device id for device, if the user is a sync user. + }], + + // Internal sync status information. Omitted if it would be empty. + status: { + sync: <string>, // The value of the Status.sync property, unless it indicates success. + service: <string>, // The value of the Status.service property, unless it indicates success. + }, + // Information about each engine's sync. + engines: [ + { + name: <string>, // "bookmarks", "tabs", etc. + took: <integer duration in milliseconds>, // Optional, values of 0 are omitted. + + status: <string>, // The value of Status.engines, if it holds a non-success value. + + // Optional, excluded if all items would be 0. A missing item indicates a value of 0. + incoming: { + applied: <integer>, // Number of records applied + succeeded: <integer>, // Number of records that applied without error + failed: <integer>, // Number of records that failed to apply + failedReasons: [ + name: <string> // Error message trying to apply the record + count: <integer> // count of how many times this error occurred + ], + }, + + // Optional, excluded if it would be empty. Records that would be + // empty (e.g. 0 sent and 0 failed) are omitted. + outgoing: [ + { + sent: <integer>, // Number of outgoing records sent. Zero values are omitted. + failed: <integer>, // Number that failed to send. Zero values are omitted. + failedReasons: [ + name: <string> // Error message trying to apply the record + count: <integer> // count of how many times this error occurred + ], + } + ], + // Optional, excluded if there were no errors + failureReason: { ... }, // Same as above. + + // Timings and counts for detailed steps that the engine reported + // as part of its sync. Optional; omitted if the engine didn't + // report any extra steps. + steps: { + name: <string>, // The step name. + took: <integer duration in milliseconds>, // Omitted if 0. + // Optional, extra named counts (e.g., number of items handled + // in this step). Omitted if the engine didn't report extra + // counts. + counts: [ + { + name: <string>, // The counter name. + count: <integer>, // The counter value. + }, + ], + }, + + // Optional, excluded if it would be empty or if the engine cannot + // or did not run validation on itself. + validation: { + // Optional validator version, default of 0. + version: <integer>, + checked: <integer>, + took: <non-monotonic integer duration in milliseconds>, + // Entries with a count of 0 are excluded, the array is excluded if no problems are found. + problems: [ + { + name: <string>, // The problem identified. + count: <integer>, // Number of times it occurred. + } + ], + // Format is same as above, this is only included if we tried and failed + // to run validation, and if it's present, all other fields in this object are optional. + failureReason: { ... }, + } + } + ], + // Information about any storage migrations that have occurred. Omitted if it would be empty. + migrations: [ + // See the section on the `migrations` array for detailed documentation on what may appear here. + { + type: <string identifier>, + // per-type data + } + ] + }], + // The "node type" as reported by the token server. This will not change + // from sync to sync, so is reported once per ping. Optional because it + // will not appear if the token server omits this information, but in + // general, we will expect all "new" pings to have it. + syncNodeType: <string>, + events: [ + event_array // See events below. + ], + histograms: { ... } // See histograms below + } + } + +info +---- + +discarded +~~~~~~~~~ + +The ping may only contain a certain number of entries in the ``"syncs"`` array, currently 500 (it is determined by the ``"services.sync.telemetry.maxPayloadCount"`` preference). Entries beyond this are discarded, and recorded in the discarded count. + +syncs.took +~~~~~~~~~~ + +These values should be monotonic. If we can't get a monotonic timestamp, -1 will be reported on the payload, and the values will be omitted from the engines. Additionally, the value will be omitted from an engine if it would be 0 (either due to timer inaccuracy or finishing instantaneously). + +uid +~~~~~~~~~ + +This property containing a hash of the FxA account identifier, which is a 32 character hexadecimal string. In the case that we are unable to authenticate with FxA and have never authenticated in the past, it will be a placeholder string consisting of 32 repeated ``0`` characters. + +syncs.why +~~~~~~~~~ + +One of the following values: + +- ``startup``: This is the first sync triggered after browser startup. +- ``schedule``: This is a sync triggered because it has been too long since the last sync. +- ``score``: This sync is triggered by a high score value one of sync's trackers, indicating that many changes have occurred since the last sync. +- ``user``: The user manually triggered the sync. +- ``tabs``: The user opened the synced tabs sidebar, which triggers a sync. + +syncs.status +~~~~~~~~~~~~ + +The ``engine.status``, ``payload.status.sync``, and ``payload.status.service`` properties are sync error codes, which are listed in `services/sync/modules/constants.js <https://searchfox.org/mozilla-central/source/services/sync/modules/constants.js>`_, and success values are not reported. + +syncs.failureReason +~~~~~~~~~~~~~~~~~~~ + +Stores error information, if any is present. Always contains the "name" property, which identifies the type of error it is. The types can be. + +- ``httperror``: Indicates that we received an HTTP error response code, but are unable to be more specific about the error. Contains the following properties: + + - ``code``: Integer HTTP status code. + +- ``nserror``: Indicates that an exception with the provided error code caused sync to fail. + + - ``code``: The nsresult error code (integer). + +- ``shutdownerror``: Indicates that the sync failed because we shut down before completion. + +- ``autherror``: Indicates an unrecoverable authentication error. + + - ``from``: Where the authentication error occurred, one of the following values: ``tokenserver``, ``fxaccounts``, or ``hawkclient``. + +- ``othererror``: Indicates that it is a sync error code that we are unable to give more specific information on. As with the ``syncStatus`` property, it is a sync error code, which are listed in `services/sync/modules/constants.js <https://searchfox.org/mozilla-central/source/services/sync/modules/constants.js>`_. + + - ``error``: String identifying which error was present. + +- ``unexpectederror``: Indicates that some other error caused sync to fail, typically an uncaught exception. + + - ``error``: The message provided by the error. + +- ``sqlerror``: Indicates that we received a ``mozIStorageError`` from a database query. + + - ``code``: Value of the ``error.result`` property, one of the constants listed `here <https://developer.mozilla.org/en-US/docs/Mozilla/Tech/XPCOM/Reference/Interface/MozIStorageError#Constants>`_. + +syncs.engine.name +~~~~~~~~~~~~~~~~~ + +Third-party engines are not reported, so only the following values are allowed: ``addons``, ``bookmarks``, ``clients``, ``forms``, ``history``, ``passwords``, ``prefs``, and ``tabs``. + +syncs.engine.validation.problems +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +For engines that can run validation on themselves, an array of objects describing validation errors that have occurred. Items that would have a count of 0 are excluded. Each engine will have its own set of items that it might put in the ``name`` field, but there are a finite number. See ``BookmarkProblemData.getSummary`` in `services/sync/modules/bookmark\_validator.js <https://searchfox.org/mozilla-central/source/services/sync/modules/bookmark_validator.js>`_ for an example. + +syncs.devices +~~~~~~~~~~~~~ + +The list of remote devices associated with this account, as reported by the clients collection. The ID of each device is hashed using the same algorithm as the local id. + +Events in the "sync" ping +------------------------- + +The sync ping includes events in the same format as they are included in the +main ping, see :ref:`eventtelemetry`. + +All events submitted as part of the sync ping which already include the "extra" +object (the 6th parameter of the event array described in the event telemetry +documentation) may also include a "serverTime" parameter, which the most recent +unix timestamp sent from the sync server (as a string). This arrives in the +``X-Weave-Timestamp`` HTTP header, and may be omitted in cases where the client +has not yet made a request to the server, or doesn't have it for any other +reason. It is included to improve flow analysis across multiple clients. + +Every event recorded in this ping will have a category of ``sync``. The following +events are defined, categorized by the event method. + +Histograms in the "sync" ping +----------------------------- + +The sync ping includes histograms relating to measurements of password manager usage. +These histograms are duplicated in the main ping. Histograms are only included in a ping if they have been set by the pwmgr code. +Currently, the histograms that can be included are: + +PWMGR_BLOCKLIST_NUM_SITES +PWMGR_FORM_AUTOFILL_RESULT +PWMGR_LOGIN_LAST_USED_DAYS +PWMGR_LOGIN_PAGE_SAFETY +PWMGR_NUM_PASSWORDS_PER_HOSTNAME +PWMGR_NUM_SAVED_PASSWORDS +PWMGR_PROMPT_REMEMBER_ACTION +PWMGR_PROMPT_UPDATE_ACTION +PWMGR_SAVING_ENABLED + +Histograms are objects with the following 6 properties: +- min - minimum bucket size +- max - maximum bucket size +- histogram_type +- counts - array representing contents of the buckets in the histogram +- ranges - array with calculated bucket sizes + +sendcommand +~~~~~~~~~~~ + +Records that Sync wrote a remote "command" to another client. These commands +cause that other client to take some action, such as resetting Sync on that +client, or opening a new URL. + +- object: The specific command being written. +- value: Not used (ie, ``null``) +- extra: An object with the following attributes: + + - deviceID: A GUID which identifies the device the command is being sent to. + - flowID: A GUID which uniquely identifies this command invocation. This GUID + is the same for every device the tab is sent to. + - streamID: A GUID which uniquely identifies this command invocation's + specific target. This GUID is unique for every device the tab is + sent to (new in Firefox 79). + - serverTime: (optional) Most recent server timestamp, as described above. + +processcommand +~~~~~~~~~~~~~~ + +Records that Sync processed a remote "command" previously sent by another +client. This is logically the "other end" of ``sendcommand``. + +- object: The specific command being processed. +- value: Not used (ie, ``null``) +- extra: An object with the following attributes: + + - flowID: A GUID which uniquely identifies this command invocation. The value + for this GUID will be the same as the flowID sent to the client via + ``sendcommand``. + - streamID: A GUID which uniquely identifies this command invocation's + specific target. The value for this GUID will be the same as the + streamID sent to the client via ``sendcommand`` (new in Firefox 79). + - reason: A string value of either ``"poll"``, ``"push"``, or ``"push-missed"`` + representing an explanation for why the command is being processed. + - serverTime: (optional) Most recent server timestamp, as described above. + +The ``migrations`` Array +------------------------ + +The application-services developers are in the process of oxidizing parts of firefox sync and the related data storage code, which typically requires migrating the old storage into a new database and/or format. + +When a migration like this occurs, a record is reported in this list the next time the sync ping is submitted. + +Because the format of each data store may be drastically different, we are not attempting to come up with a generic representation here, and currently planning on allowing each migration record to vary independently (at least for now). These records will be distinctly identified by their ``"type"`` field. + +They should only appear once per migration (that is, we'd rather fail to report a record than report them multiple times). + +migrations.type: ``"webext-storage"`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +This indicates a migration was performed from the legacy kinto-based extension-storage database into the new webext-storage rust implementation. + +It contains the following fields: + +- ``type``: Always the string ``"webext-storage"``. + +- ``entries``: The number of entries/preferences in the source (legacy) database, including ones we failed to read. See below for information on the distinction between ``entries`` and ``extensions`` in this record. + +- ``entriesSuccessful``: The number of entries/preferences (see below) which we have successfully migrated into the destination database.. + +- ``extensions``: The number of distinct extensions which have at least one preference in the source (legacy) database. + +- ``extensionsSuccessful``: The number of distinct extensions which have at least one preference in the destination (migrated) database. + +- ``openFailure``: A boolean flag that is true if we hit a read error prior to . This likely indicates complete corruption, or a bug in an underlying library like rusqlite. + + +Note: "entries" vs "extensions" +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The ``webext-storage`` migration record detailed above contains counts for both: + +- The number of "entries" detected vs successfully migrated. +- The number of "extensions" detected vs successfully migrated. + +This may seem redundant, but these refer to different (but related) things. The distinction here has to do with the way the two databases store extension-storage data: + +* The legacy database stores one row for each (``extension_id``, ``preference_name``, ``preference_value``) triple. These are referred to as ``entries``. + +* Conversely, the new database stores one row per extension, which is a pair containing both the ``extension_id``, as well as a dictionary holding all preference data, and so are equivalent to extensions. + +(The description above is a somewhat simplified view of things, as it ignores a number values each database stores which is irrelevant for migration) + +That is, ``entries`` represent each individual preference setting, and ``extensions`` represent the collected set of preferences for a given extension. + +Counts for are *both* of these are present as it's likely that the disparity would point to different kinds of issues with the migration code. diff --git a/toolkit/components/telemetry/docs/data/third-party-modules-ping.rst b/toolkit/components/telemetry/docs/data/third-party-modules-ping.rst new file mode 100644 index 0000000000..3871b2e4b3 --- /dev/null +++ b/toolkit/components/telemetry/docs/data/third-party-modules-ping.rst @@ -0,0 +1,135 @@ +.. _third-party-modules-ping: + +"third-party-modules" ping +========================== + +This ping contains information about events whereby third-party modules +were loaded into Firefox processes. + +.. code-block:: js + + { + "type": "third-party-modules", + ... common ping data + "clientId": <UUID>, + "environment": { ... }, + "payload": { + "structVersion": 1, + // List of DLL filenames that are on the dynamic blocklist + "blockedModules": [<string>], + "modules": [ + { + // The sanitized name of the module as resolved by the Windows loader. + "resolvedDllName": <string>, + // Version of the DLL as contained in its resources's fixed version information. + "fileVersion": <string>, + // The value of the CompanyName field as extracted from the DLL's version information. This property is only present when such version info is present, and when the 'signedBy' property is absent. + "companyName": <string>, + // The organization whose certificate was used to sign the DLL. Only present for signed modules. + "signedBy": <string>, + // Flags that indicate this module's level of trustworthiness. This corresponds to one or more mozilla::ModuleTrustFlags OR'd together. + "trustFlags": <unsigned int> + }, + ... Additional modules (maximum 100) + ], + "processes": { + <string containing processType and pid, formatted as `${processType}.0x${pid}">`: { + // Except for Default (which is remapped to "browser"), one of the process string names specified in xpcom/build/GeckoProcessTypes.h. + "processType": <string>, + // Elapsed time since process creation that this object was generated, in seconds. + "elapsed": <number>, + // Time spent loading xul.dll in milliseconds. + "xulLoadDurationMS": <number>, + // Number of dropped events due to failures sanitizing file paths. + "sanitizationFailures": <int>, + // Number of dropped events due to failures computing trust levels. + "trustTestFailures": <int>, + // Array of module load events for this process. The entries of this array are ordered to be in sync with the combinedStacks.stacks array (see below) + "events": [ + { + // Elapsed time since process creation that this event was generated, in milliseconds. + "processUptimeMS": <int>, + // Time spent loading this module, in milliseconds. + "loadDurationMS": <number>, + // Thread ID for the thread that loaded the module. + "threadID": <int>, + // Name of the thread that loaded the module, when applicable. + "threadName": <string>, + // The sanitized name of the module that was requested by the invoking code. Only exists when it is different from resolvedDllName. + "requestedDllName": <string>, + // The base address to which the loader mapped the module. + "baseAddress": <string formatted as "0x%x">, + // Index of the element in the modules array that contains details about the module that was loaded during this event. + "moduleIndex": <int>, + // True if the module is included in the executable's Import Directory Table. + "isDependent": <bool>, + // The status of DLL load. This corresponds to enum ModuleLoadInfo::Status. + "loadStatus": <int> + }, + ... Additional events (maximum 50) + ], + "combinedStacks": [ + "memoryMap": [ + [ + // Name of the module symbol file, e.g. ``xul.pdb`` + <string>, + // Breakpad identifier of the module, e.g. ``08A541B5942242BDB4AEABD8C87E4CFF2`` + <string> + ], + ... Additional modules + ], + // Array of stacks for this process. These entries are ordered to be in sync with the events array + "stacks": [ + [ + [ + // The module index or -1 for invalid module indices + <integer>, + // The program counter relative to its module base, or an absolute pc if the module index is -1 + <unsigned integer> + ], + ... Additional stack frames (maximum 512) + ], + ... Additional stack traces (maximum 50) + ] + ] + }, + ... Additional processes (maximum 100) + } + } + } + +payload.processes[...].events[...].resolvedDllName +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +The effective path to the module file, sanitized to remove any potentially +sensitive information. In most cases, the directory path is removed leaving only +the leaf name, e.g. ``foo.dll``. There are three exceptions: + +* Paths under ``%ProgramFiles%`` are preserved, e.g. ``%ProgramFiles%\FooApplication\foo.dll`` +* Paths under ``%SystemRoot%`` are preserved, e.g. ``%SystemRoot%\System32\DriverStore\FileRepository\nvlt.inf_amd64_97992900c592012e\nvinitx.dll`` +* Paths under the temporary path are preserved, e.g. ``%TEMP%\bin\foo.dll`` + +payload.processes[...].events[...].requestedDllName +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +The name of the module as it was requested from the OS. This string is also +sanitized in a similar fashion to to ``resolvedDllName``. This string is +omitted from the ping when it is identical to ``resolvedDllName``. + +Notes +~~~~~ +* The client id is submitted with this ping. +* The :doc:`Telemetry Environment <../data/environment>` is submitted in this ping. +* String fields within ``payload`` are limited in length to 260 characters. +* This ping is sent once daily. +* If there are no events to report, this ping is not sent. + +Version History +~~~~~~~~~~~~~~~ +- Firefox 110: Added ``blockedModules`` (`bug 1808158 <https://bugzilla.mozilla.org/show_bug.cgi?id=1808158>`_). +- Firefox 77: Added ``isDependent`` (`bug 1620118 <https://bugzilla.mozilla.org/show_bug.cgi?id=1620118>`_). +- Firefox 71: Renamed from untrustedModules to third-party-modules with a revised schema (`bug 1542830 <https://bugzilla.mozilla.org/show_bug.cgi?id=1542830>`_). +- Firefox 70: Added ``%SystemRoot%`` as an exemption to path sanitization (`bug 1573275 <https://bugzilla.mozilla.org/show_bug.cgi?id=1573275>`_). +- Firefox 66: + - Added Windows Side-by-side directory trust flag (`bug 1514694 <https://bugzilla.mozilla.org/show_bug.cgi?id=1514694>`_). + - Added module load times (``xulLoadDurationMS``, ``loadDurationMS``) and xul.dll trust flag (`bug 1518490 <https://bugzilla.mozilla.org/show_bug.cgi?id=1518490>`_). + - Added SysWOW64 trust flag (`bug 1518798 <https://bugzilla.mozilla.org/show_bug.cgi?id=1518798>`_). +- Firefox 65: Initial support (`bug 1435827 <https://bugzilla.mozilla.org/show_bug.cgi?id=1435827>`_). diff --git a/toolkit/components/telemetry/docs/data/uitour-ping.rst b/toolkit/components/telemetry/docs/data/uitour-ping.rst new file mode 100644 index 0000000000..06e4a9f49b --- /dev/null +++ b/toolkit/components/telemetry/docs/data/uitour-ping.rst @@ -0,0 +1,25 @@ + +"uitour-tag" ping +================= + +This ping is submitted via the UITour ``setTreatmentTag`` API. It may be used by +the tour to record what settings were made by a user or to track the result of +A/B experiments. + +The client ID is submitted with this ping. + +Structure: + +.. code-block:: js + + { + version: 1, + type: "uitour-tag", + clientId: <string>, + payload: { + tagName: <string>, + tagValue: <string> + } + } + +See also: :doc:`common ping fields <common-ping>` diff --git a/toolkit/components/telemetry/docs/data/uninstall-ping.rst b/toolkit/components/telemetry/docs/data/uninstall-ping.rst new file mode 100644 index 0000000000..281cc3fcec --- /dev/null +++ b/toolkit/components/telemetry/docs/data/uninstall-ping.rst @@ -0,0 +1,36 @@ + +"uninstall" ping +================ + +This opt-out ping is sent from the Windows uninstaller when the uninstall finishes. Notably it includes ``clientId`` and the :doc:`Telemetry Environment <environment>`. It follows the :doc:`common ping format <common-ping>`. + +Structure: + +.. code-block:: js + + { + type: "uninstall", + ... common ping data + clientId: <UUID>, + environment: { ... }, + payload: { + otherInstalls: <integer>, // Optional, number of other installs on the system, max 11. + } + } + +See also the `JSON schema <https://github.com/mozilla-services/mozilla-pipeline-schemas/blob/master/templates/telemetry/uninstall/uninstall.4.schema.json>`_. These pings are recorded in the ``telemetry.uninstall`` table in Redash, using the default "Telemetry (BigQuery)" data source. + +payload.otherInstalls +--------------------- +This is a count of how many other installs of Firefox were present on the system at the time the ping was written. It is the number of values in the ``Software\Mozilla\Firefox\TaskBarIDs`` registry key, for both 32-bit and 64-bit architectures, for both HKCU and HKLM, excluding duplicates, and excluding a value for this install (if present). For example, if this is the only install on the system, the value will be 0. It may be missing in case of an error. + +This count is capped at 11. This avoids introducing a high-resolution identifier in case of a system with a large, unique number of installs. + +Uninstall Ping storage and lifetime +----------------------------------- + +On delayed Telemetry init (about 1 minute into each run of Firefox), if opt-out telemetry is enabled, this ping is written to disk. There is a single ping for each install, any uninstall pings from the same install are removed before the new ping is written. + +The ping is removed if Firefox notices that opt-out telemetry has been disabled, either when the ``datareporting.healthreport.uploadEnabled`` pref goes false or when it is false on delayed init. Conversely, when opt-out telemetry is re-enabled, the ping is written as Telemetry is setting itself up again. + +The ping is sent by the uninstaller some arbitrary time after it is written to disk by Firefox, so it could be significantly out of date when it is submitted. There should be little impact from stale data, since analysis is likely to focus on clients that uninstalled soon after running Firefox, and this ping mostly changes when Firefox itself is updated. diff --git a/toolkit/components/telemetry/docs/data/update-ping.rst b/toolkit/components/telemetry/docs/data/update-ping.rst new file mode 100644 index 0000000000..037d521019 --- /dev/null +++ b/toolkit/components/telemetry/docs/data/update-ping.rst @@ -0,0 +1,79 @@ + +"update" ping +================== + +This opt-out ping is sent from Firefox Desktop when a browser update is ready to be applied and after it was correctly applied. + +Structure: + +.. code-block:: js + + { + type: "update", + ... common ping data + clientId: <UUID>, + environment: { ... }, + payload: { + reason: <string>, // "ready", "success" + targetChannel: <string>, // "nightly" (only present for reason = "ready") + targetVersion: <string>, // "56.0a1" (only present for reason = "ready") + targetBuildId: <string>, // "20080811053724" (only present for reason = "ready") + targetDisplayVersion: <string>, // "56.0a1" (only present for reason = "ready") + previousChannel: <string>, // "nightly" or null (only present for reason = "success") + previousVersion: <string>, // "55.0a1" (only present for reason = "success") + previousBuildId: <string>, // "20080810053724" (only present for reason = "success") + } + } + +payload.reason +-------------- +This field supports the following values: + +- ``ready`` meaning that the ping was generated after an update was downloaded and marked as ready to be processed. For *non-staged* updates this happens as soon as the download finishes and is verified while for *staged* updates this happens before the staging step is started. +- ``success`` the ping was generated after the browser was restarted and the update correctly applied. + +payload.targetChannel +----------------------- +The Firefox channel the update was fetched from (only valid for pings with reason "ready"). + +payload.targetVersion +----------------------- +The Firefox version the browser is updating to. Follows the same format as application.version (only valid for pings with reason "ready"). + +payload.targetBuildId +----------------------- +The Firefox build id the browser is updating to. Follows the same format as application.buildId (only valid for pings with reason "ready"). + +payload.targetDisplayVersion +---------------------------- +The Firefox display version the browser is updating to. This may contain a different value than ``targetVersion``, e.g. for the ``Beta`` channel this field will report the beta suffix while ``targetVersion`` will only report the version number. + +payload.previousChannel +----------------------- +The Firefox channel the profile was on before the update was applied (only valid for pings with reason "success"). +This can be ``null``. + +payload.previousVersion +----------------------- +The Firefox version the browser is updating from. Follows the same format as application.version (only valid for pings with reason "success"). + +payload.previousBuildId +----------------------- +The Firefox build id the browser is updating from. Follows the same format as application.buildId (only valid for pings with reason "success"). + +Expected behaviours +------------------- +The following is a list of conditions and expected behaviours for the ``update`` ping: + +- **The ping is generated once every time an update is downloaded, after it was verified:** + + - *for users who saw the privacy policy*, the ``update`` ping is sent immediately; + - *for users who did not see the privacy policy*, the ``update`` ping is saved to disk and sent after the policy is displayed. +- **If the download of the update retries or other fallback occurs**: the ``update`` ping will not be generated + multiple times, but only one time once the download is complete and verified. +- **If automatic updates are disabled**: when the user forces a manual update, no ``update`` ping will be generated. +- **If updates fail to apply**: in some cases the client will download the same update blob and generate a new ``update`` ping for the same target version and build id, with a different document id. +- **If the build update channel contains the CCK keyword**, the update ping will not report it but rather report a vanilla channel name (e.g. ``mozilla-cck-test-beta`` gets reported as ``beta``). +- **If a profile refresh occurs before the update is applied**, the update ping with ``reason = success`` will not be generated. +- **If the update is applied on a new profile, different then the one it was downloaded in**, the update ping with ``reason = success`` will not be generated. +- **If a newer browser version is installed over an older**, the update ping with ``reason = success`` will not be generated. diff --git a/toolkit/components/telemetry/docs/data/xfocsp-error-report-ping.rst b/toolkit/components/telemetry/docs/data/xfocsp-error-report-ping.rst new file mode 100644 index 0000000000..f1b6f928a3 --- /dev/null +++ b/toolkit/components/telemetry/docs/data/xfocsp-error-report-ping.rst @@ -0,0 +1,69 @@ + +"xfocsp-error-report" ping +========================== + +This opt-in ping is sent when an X-Frame-Options error or a CSP: frame-ancestors +happens to report the error. Users can opt-in this by checking the reporting +checkbox. After users opt-in, this ping will be sent every time the error +happens. Users can opt-out this by un-checking the reporting checkbox on the +error page. The client_id and environment are not sent with this ping. + +Structure: + +.. code-block:: js + + { + "type": "xfocsp-error-report", + ... common ping data + "payload": { + "error_type": <string>, + "xfo_header": <string>, + "csp_header": <string>, + "frame_hostname": <string>, + "top_hostname": <string>, + "frame_uri": <string>, + "top_uri": <string>, + } + } + +info +---- + +error_type +~~~~~~~~~~ + +The type of what error triggers this ping. This could be either "xfo" or "csp". + +xfo_header +~~~~~~~~~~ + +The X-Frame-Options value in the response HTTP header. + +csp_header +~~~~~~~~~~ + +The CSP: frame-ancestors value in the response HTTP header. + +frame_hostname +~~~~~~~~~~~~~~ + +The hostname of the frame which triggers the error. + +top_hostname +~~~~~~~~~~~~ + +The hostname of the top-level page which loads the frame. + +frame_uri +~~~~~~~~~ + +The uri of the frame which triggers the error. This excludes the query strings. + +top_uri +~~~~~~~ + +The uri of the top-level page which loads the frame. This excludes the query +strings. + + +See also: :doc:`common ping fields <common-ping>` |