From 5f8de423f190bbb79a62f804151bc24824fa32d8 Mon Sep 17 00:00:00 2001 From: "Matt A. Tobin" Date: Fri, 2 Feb 2018 04:16:08 -0500 Subject: Add m-esr52 at 52.6.0 --- .../telemetry/docs/data/addons-malware-ping.rst | 42 ++ .../components/telemetry/docs/data/common-ping.rst | 42 ++ .../components/telemetry/docs/data/core-ping.rst | 191 +++++++ .../components/telemetry/docs/data/crash-ping.rst | 144 +++++ .../telemetry/docs/data/deletion-ping.rst | 19 + .../components/telemetry/docs/data/environment.rst | 373 +++++++++++++ .../telemetry/docs/data/heartbeat-ping.rst | 63 +++ toolkit/components/telemetry/docs/data/index.rst | 18 + .../components/telemetry/docs/data/main-ping.rst | 609 +++++++++++++++++++++ .../components/telemetry/docs/data/sync-ping.rst | 182 ++++++ .../components/telemetry/docs/data/uitour-ping.rst | 26 + 11 files changed, 1709 insertions(+) create mode 100644 toolkit/components/telemetry/docs/data/addons-malware-ping.rst create mode 100644 toolkit/components/telemetry/docs/data/common-ping.rst create mode 100644 toolkit/components/telemetry/docs/data/core-ping.rst create mode 100644 toolkit/components/telemetry/docs/data/crash-ping.rst create mode 100644 toolkit/components/telemetry/docs/data/deletion-ping.rst create mode 100644 toolkit/components/telemetry/docs/data/environment.rst create mode 100644 toolkit/components/telemetry/docs/data/heartbeat-ping.rst create mode 100644 toolkit/components/telemetry/docs/data/index.rst create mode 100644 toolkit/components/telemetry/docs/data/main-ping.rst create mode 100644 toolkit/components/telemetry/docs/data/sync-ping.rst create mode 100644 toolkit/components/telemetry/docs/data/uitour-ping.rst (limited to 'toolkit/components/telemetry/docs/data') diff --git a/toolkit/components/telemetry/docs/data/addons-malware-ping.rst b/toolkit/components/telemetry/docs/data/addons-malware-ping.rst new file mode 100644 index 000000000..18502d748 --- /dev/null +++ b/toolkit/components/telemetry/docs/data/addons-malware-ping.rst @@ -0,0 +1,42 @@ + +Add-ons malware ping +==================== + +This ping is generated by an add-on created by Mozilla and shipped to users on older versions of Firefox (44-46). The ping contains information about the profile that might have been altered by a third party malicious add-on. + +Structure: + +.. code-block:: js + + { + type: "malware-addon-states", + ... + clientId: , + environment: { ... }, + // Common ping data. + payload: { + // True if the blocklist was disabled at startup time. + blocklistDisabled: , + // True if the malicious add-on exists and is enabled. False if it + // exists and is disabled or null if the add-on was not found. + mainAddonActive: , + // A value of the malicious add-on block list state, or null if the + // add-on was not found. + mainAddonBlocked: , + // True if a malicious user.js file was found in the profile. + foundUserJS: , + // If a malicious secmodd.db file was found the extension ID that the // file contained.. + secmoddAddon: , . + // A list of IDs for extensions which were hidden by malicious CSS. + hiddenAddons: [ + , + ... + ], + // A mapping of installed add-on IDs with known malicious + // update URL patterns to their exact update URLs. + updateURLs: { + : , + ... + } + } + } diff --git a/toolkit/components/telemetry/docs/data/common-ping.rst b/toolkit/components/telemetry/docs/data/common-ping.rst new file mode 100644 index 000000000..445557efd --- /dev/null +++ b/toolkit/components/telemetry/docs/data/common-ping.rst @@ -0,0 +1,42 @@ + +Common ping format +================== + +This defines the top-level structure of a Telemetry ping. +It contains basic information shared between different ping types, which enables proper storage and processing of the raw pings server-side. + +It also contains optional further information: + +* the :doc:`environment data <../data/environment>`, which contains important info to correlate the measurements against +* the ``clientId``, a UUID identifying a profile and allowing user-oriented correlation of data + +*Note:* Both are not submitted with all ping types due to privacy concerns. This and the data it that can be correlated against is inspected under the `data collection policy `_. + +Finally, the structure also contains the `payload`, which is the specific data submitted for the respective *ping type*. + +Structure: + +.. code-block:: js + + { + type: , // "main", "activation", "deletion", "saved-session", ... + id: , // a UUID that identifies this ping + creationDate: , // the date the ping was generated + version: , // the version of the ping format, currently 4 + + application: { + architecture: , // build architecture, e.g. x86 + buildId: , // "20141126041045" + name: , // "Firefox" + version: , // "35.0" + displayVersion: , // "35.0b3" + vendor: , // "Mozilla" + platformVersion: , // "35.0" + xpcomAbi: , // e.g. "x86-msvc" + channel: , // "beta" + }, + + clientId: , // optional + environment: { ... }, // optional, not all pings contain the environment + payload: { ... }, // the actual payload data for this ping type + } diff --git a/toolkit/components/telemetry/docs/data/core-ping.rst b/toolkit/components/telemetry/docs/data/core-ping.rst new file mode 100644 index 000000000..7f38f2f7e --- /dev/null +++ b/toolkit/components/telemetry/docs/data/core-ping.rst @@ -0,0 +1,191 @@ + +"core" ping +============ + +This mobile-specific ping is intended to provide the most critical +data in a concise format, allowing for frequent uploads. + +Since this ping is used to measure retention, it should be sent +each time the browser is opened. + +Submission will be per the Edge server specification:: + + /submit/telemetry/docId/docType/appName/appVersion/appUpdateChannel/appBuildID + +* ``docId`` is a UUID for deduping +* ``docType`` is “core” +* ``appName`` is “Fennec” +* ``appVersion`` is the version of the application (e.g. "46.0a1") +* ``appUpdateChannel`` is “release”, “beta”, etc. +* ``appBuildID`` is the build number + +Note: Counts below (e.g. search & usage times) are “since the last +ping”, not total for the whole application lifetime. + +Structure: + +.. code-block:: js + + { + "v": 7, // ping format version + "clientId": , // client id, e.g. + // "c641eacf-c30c-4171-b403-f077724e848a" + "seq": , // running ping counter, e.g. 3 + "locale": , // application locale, e.g. "en-US" + "os": , // OS name. + "osversion": , // OS version. + "device": , // Build.MANUFACTURER + " - " + Build.MODEL + // where manufacturer is truncated to 12 characters + // & model is truncated to 19 characters + "arch": , // e.g. "arm", "x86" + "profileDate": , // Profile creation date in days since + // UNIX epoch. + "defaultSearch": , // Identifier of the default search engine, + // e.g. "yahoo". + "distributionId": , // Distribution identifier (optional) + "created": , // date the ping was created + // in local time, "yyyy-mm-dd" + "tz": , // timezone offset (in minutes) of the + // device when the ping was created + "sessions": , // number of sessions since last upload + "durations": , // combined duration, in seconds, of all + // sessions since last upload + "searches": , // Optional, object of search use counts in the + // format: { "engine.source": } + // e.g.: { "yahoo.suggestion": 3, "other.listitem": 1 } + "experiments": [, /* … */], // Optional, array of identifiers + // for the active experiments + } + +Field details +------------- + +device +~~~~~~ +The ``device`` field is filled in with information specified by the hardware +manufacturer. As such, it could be excessively long and use excessive amounts +of limited user data. To avoid this, we limit the length of the field. We're +more likely have collisions for models within a manufacturer (e.g. "Galaxy S5" +vs. "Galaxy Note") than we are for shortened manufacturer names so we provide +more characters for the model than the manufacturer. + +distributionId +~~~~~~~~~~~~~~ +The ``distributionId`` contains the distribution ID as specified by +preferences.json for a given distribution. More information on distributions +can be found `here `_. + +It is optional. + +defaultSearch +~~~~~~~~~~~~~ +On Android, this field may be ``null``. To get the engine, we rely on +``SearchEngineManager#getDefaultEngine``, which searches in several places in +order to find the search engine identifier: + +* Shared Preferences +* The distribution (if it exists) +* The localized default engine + +If the identifier could not be retrieved, this field is ``null``. If the +identifier is retrieved, we attempt to create an instance of the search +engine from the search plugins (in order): + +* In the distribution +* From the localized plugins shipped with the browser +* The third-party plugins that are installed in the profile directory + +If the plugins fail to create a search engine instance, this field is also +``null``. + +This field can also be ``null`` when a custom search engine is set as the +default. + +sessions & durations +~~~~~~~~~~~~~~~~~~~~ +On Android, a session is the time when Firefox is focused in the foreground. +`sessions` tracks the number of sessions since the last upload and +`durations` is the accumulated duration in seconds of all of these +sessions. Note that showing a dialog (including a Firefox dialog) will +take Firefox out of focus & end the current session. + +An implementation that records a session when Firefox is completely hidden is +preferrable (e.g. to avoid the dialog issue above), however, it's more complex +to implement and so we chose not to, at least for the initial implementation. + +profileDate +~~~~~~~~~~~ +On Android, this value is created at profile creation time and retrieved or, +for legacy profiles, taken from the package install time (note: this is not the +same exact metric as profile creation time but we compromised in favor of ease +of implementation). + +Additionally on Android, this field may be ``null`` in the unlikely event that +all of the following events occur: + +#. The times.json file does not exist +#. The package install date could not be persisted to disk + +The reason we don't just return the package install time even if the date could +not be persisted to disk is to ensure the value doesn't change once we start +sending it: we only want to send consistent values. + +searches +~~~~~~~~ +In the case a search engine is added by a user, the engine identifier "other" is used, e.g. "other.". + +Sources in Android are based on the existing UI telemetry values and are as +follows: + +* actionbar: the user types in the url bar and hits enter to use the default + search engine +* listitem: the user selects a search engine from the list of secondary search + engines at the bottom of the screen +* suggestion: the user clicks on a search suggestion or, in the case that + suggestions are disabled, the row corresponding with the main engine + +Other parameters +---------------- + +HTTP "Date" header +~~~~~~~~~~~~~~~~~~ +This header is used to track the submission date of the core ping in the format +specified by +`rfc 2616 sec 14.18 `_, +et al (e.g. "Tue, 01 Feb 2011 14:00:00 GMT"). + + +Version history +--------------- +* v7: added ``sessionCount`` & ``sessionDuration`` +* v6: added ``searches`` +* v5: added ``created`` & ``tz`` +* v4: ``profileDate`` will return package install time when times.json is not available +* v3: added ``defaultSearch`` +* v2: added ``distributionId`` +* v1: initial version + +Notes +~~~~~ + +* ``distributionId`` (v2) actually landed after ``profileDate`` (v4) but was + uplifted to 46, whereas ``profileDate`` landed on 47. The version numbers in + code were updated to be increasing (bug 1264492) and the version history docs + rearranged accordingly. + +Android implementation notes +---------------------------- +On Android, the uploader has a high probability of delivering the complete data +for a given client but not a 100% probability. This was a conscious decision to +keep the code simple. The cases where we can lose data: + +* Resetting the field measurements (including incrementing the sequence number) + and storing a ping for upload are not atomic. Android can kill our process + for memory pressure in between these distinct operations so we can just lose + a ping's worth of data. That sequence number will be missing on the server. +* If we exceed some number of pings on disk that have not yet been uploaded, + we remove old pings to save storage space. For those pings, we will lose + their data and their sequence numbers will be missing on the server. + +Note: we never expect to drop data without also dropping a sequence number so +we are able to determine when data loss occurs. diff --git a/toolkit/components/telemetry/docs/data/crash-ping.rst b/toolkit/components/telemetry/docs/data/crash-ping.rst new file mode 100644 index 000000000..3cdbc6030 --- /dev/null +++ b/toolkit/components/telemetry/docs/data/crash-ping.rst @@ -0,0 +1,144 @@ + +"crash" ping +============ + +This ping is captured after the main Firefox process crashes, whether or not the crash report is submitted to crash-stats.mozilla.org. It includes non-identifying metadata about the crash. + +The environment block that is sent with this ping varies: if Firefox was running long enough to record the environment block before the crash, then the environment at the time of the crash will be recorded and ``hasCrashEnvironment`` will be true. If Firefox crashed before the environment was recorded, ``hasCrashEnvironment`` will be false and the recorded environment will be the environment at time of submission. + +The client ID is submitted with this ping. + +Structure: + +.. code-block:: js + + { + version: 1, + type: "crash", + ... common ping data + clientId: , + environment: { ... }, + payload: { + crashDate: "YYYY-MM-DD", + sessionId: , // may be missing for crashes that happen early + // in startup. Added in Firefox 48 with the + // intention of uplifting to Firefox 46 + crashId: , // Optional, ID of the associated crash + stackTraces: { ... }, // Optional, see below + metadata: { // Annotations saved while Firefox was running. See nsExceptionHandler.cpp for more information + ProductName: "Firefox", + ReleaseChannel: , + Version: , + BuildID: "YYYYMMDDHHMMSS", + AvailablePageFile: , // Windows-only, available paging file + AvailablePhysicalMemory: , // Windows-only, available physical memory + AvailableVirtualMemory: , // Windows-only, available virtual memory + BlockedDllList: , // Windows-only, see WindowsDllBlocklist.cpp for details + BlocklistInitFailed: 1, // Windows-only, present only if the DLL blocklist initialization failed + CrashTime: