From 5f8de423f190bbb79a62f804151bc24824fa32d8 Mon Sep 17 00:00:00 2001 From: "Matt A. Tobin" Date: Fri, 2 Feb 2018 04:16:08 -0500 Subject: Add m-esr52 at 52.6.0 --- .../telemetry/docs/collection/custom-pings.rst | 74 + .../telemetry/docs/collection/histograms.rst | 5 + .../components/telemetry/docs/collection/index.rst | 35 + .../telemetry/docs/collection/measuring-time.rst | 74 + .../telemetry/docs/collection/scalars.rst | 140 ++ .../telemetry/docs/concepts/archiving.rst | 12 + .../components/telemetry/docs/concepts/crashes.rst | 23 + .../components/telemetry/docs/concepts/index.rst | 23 + .../components/telemetry/docs/concepts/pings.rst | 32 + .../telemetry/docs/concepts/sessions.rst | 40 + .../telemetry/docs/concepts/submission.rst | 34 + .../docs/concepts/subsession_triggers.png | Bin 0 -> 1219295 bytes .../telemetry/docs/data/addons-malware-ping.rst | 42 + .../components/telemetry/docs/data/common-ping.rst | 42 + .../components/telemetry/docs/data/core-ping.rst | 191 ++ .../components/telemetry/docs/data/crash-ping.rst | 144 ++ .../telemetry/docs/data/deletion-ping.rst | 19 + .../components/telemetry/docs/data/environment.rst | 373 ++++ .../telemetry/docs/data/heartbeat-ping.rst | 63 + toolkit/components/telemetry/docs/data/index.rst | 18 + .../components/telemetry/docs/data/main-ping.rst | 609 ++++++ .../components/telemetry/docs/data/sync-ping.rst | 182 ++ .../components/telemetry/docs/data/uitour-ping.rst | 26 + .../components/telemetry/docs/fhr/architecture.rst | 226 +++ .../components/telemetry/docs/fhr/dataformat.rst | 1997 ++++++++++++++++++++ .../components/telemetry/docs/fhr/identifiers.rst | 83 + toolkit/components/telemetry/docs/fhr/index.rst | 34 + toolkit/components/telemetry/docs/index.rst | 25 + .../components/telemetry/docs/internals/index.rst | 9 + .../telemetry/docs/internals/preferences.rst | 119 ++ 30 files changed, 4694 insertions(+) create mode 100644 toolkit/components/telemetry/docs/collection/custom-pings.rst create mode 100644 toolkit/components/telemetry/docs/collection/histograms.rst create mode 100644 toolkit/components/telemetry/docs/collection/index.rst create mode 100644 toolkit/components/telemetry/docs/collection/measuring-time.rst create mode 100644 toolkit/components/telemetry/docs/collection/scalars.rst create mode 100644 toolkit/components/telemetry/docs/concepts/archiving.rst create mode 100644 toolkit/components/telemetry/docs/concepts/crashes.rst create mode 100644 toolkit/components/telemetry/docs/concepts/index.rst create mode 100644 toolkit/components/telemetry/docs/concepts/pings.rst create mode 100644 toolkit/components/telemetry/docs/concepts/sessions.rst create mode 100644 toolkit/components/telemetry/docs/concepts/submission.rst create mode 100644 toolkit/components/telemetry/docs/concepts/subsession_triggers.png create mode 100644 toolkit/components/telemetry/docs/data/addons-malware-ping.rst create mode 100644 toolkit/components/telemetry/docs/data/common-ping.rst create mode 100644 toolkit/components/telemetry/docs/data/core-ping.rst create mode 100644 toolkit/components/telemetry/docs/data/crash-ping.rst create mode 100644 toolkit/components/telemetry/docs/data/deletion-ping.rst create mode 100644 toolkit/components/telemetry/docs/data/environment.rst create mode 100644 toolkit/components/telemetry/docs/data/heartbeat-ping.rst create mode 100644 toolkit/components/telemetry/docs/data/index.rst create mode 100644 toolkit/components/telemetry/docs/data/main-ping.rst create mode 100644 toolkit/components/telemetry/docs/data/sync-ping.rst create mode 100644 toolkit/components/telemetry/docs/data/uitour-ping.rst create mode 100644 toolkit/components/telemetry/docs/fhr/architecture.rst create mode 100644 toolkit/components/telemetry/docs/fhr/dataformat.rst create mode 100644 toolkit/components/telemetry/docs/fhr/identifiers.rst create mode 100644 toolkit/components/telemetry/docs/fhr/index.rst create mode 100644 toolkit/components/telemetry/docs/index.rst create mode 100644 toolkit/components/telemetry/docs/internals/index.rst create mode 100644 toolkit/components/telemetry/docs/internals/preferences.rst (limited to 'toolkit/components/telemetry/docs') diff --git a/toolkit/components/telemetry/docs/collection/custom-pings.rst b/toolkit/components/telemetry/docs/collection/custom-pings.rst new file mode 100644 index 000000000..daad87bfe --- /dev/null +++ b/toolkit/components/telemetry/docs/collection/custom-pings.rst @@ -0,0 +1,74 @@ +======================= +Submitting custom pings +======================= + +Custom pings can be submitted from JavaScript using: + +.. code-block:: js + + TelemetryController.submitExternalPing(type, payload, options) + +- ``type`` - a ``string`` that is the type of the ping, limited to ``/^[a-z0-9][a-z0-9-]+[a-z0-9]$/i``. +- ``payload`` - the actual payload data for the ping, has to be a JSON style object. +- ``options`` - optional, an object containing additional options: + - ``addClientId``- whether to add the client id to the ping, defaults to ``false`` + - ``addEnvironment`` - whether to add the environment data to the ping, defaults to ``false`` + - ``overrideEnvironment`` - a JSON style object that overrides the environment data + +``TelemetryController`` will assemble a ping with the passed payload and the specified options. +That ping will be archived locally for use with Shield and inspection in ``about:telemetry``. +If the preferences allow upload of Telemetry pings, the ping will be uploaded at the next opportunity (this is subject to throttling, retry-on-failure, etc.). + +Submission constraints +---------------------- + +When submitting pings on shutdown, they should not be submitted after Telemetry shutdown. +Pings should be submitted at the latest within: + +- the `observer notification `_ ``"profile-before-change"`` +- the :ref:`AsyncShutdown phase ` ``sendTelemetry`` + +There are other constraints that can lead to a ping submission getting dropped: + +- invalid ping type strings +- invalid payload types: E.g. strings instead of objects. +- oversized payloads: We currently only drop pings >1MB, but targetting sizes of <=10KB is recommended. + +Tools +===== + +Helpful tools for designing new pings include: + +- `gzipServer `_ - a Python script that can run locally and receives and saves Telemetry pings. Making Firefox send to it allows inspecting outgoing pings easily. +- ``about:telemetry`` - allows inspecting submitted pings from the local archive, including all custom ones. + +Designing custom pings +====================== + +In general, creating a new custom ping means you don't benefit automatically from the existing tooling. Further work is needed to make data show up in re:dash or other analysis tools. + +In addition to the `data collection review `_, questions to guide a new pings design are: + +- Submission interval & triggers: + - What events trigger ping submission? + - What interval is the ping submitted in? + - Is there a throttling mechanism? + - What is the desired latency? (submitting "at least daily" still leads to certain latency tails) + - Are pings submitted on a clock schedule? Or based on "time since session start", "time since last ping" etc.? (I.e. will we get sharp spikes in submission volume?) +- Size and volume: + - What’s the size of the submitted payload? + - What's the full ping size including metadata in the pipeline? + - What’s the target population? + - What's the overall estimated volume? +- Dataset: + - Is it opt-out? + - Does it need to be opt-out? + - Does it need to be in a separate ping? (why can’t the data live in probes?) +- Privacy: + - Is there risk to leak PII? + - How is that risk mitigated? +- Data contents: + - Does the submitted data answer the posed product questions? + - Does the shape of the data allow to answer the questions efficiently? + - Is the data limited to whats needed to answer the questions? + - Does the data use common formats? (i.e. can we re-use tooling or analysis know-how) diff --git a/toolkit/components/telemetry/docs/collection/histograms.rst b/toolkit/components/telemetry/docs/collection/histograms.rst new file mode 100644 index 000000000..8d0233dbf --- /dev/null +++ b/toolkit/components/telemetry/docs/collection/histograms.rst @@ -0,0 +1,5 @@ +========== +Histograms +========== + +Recording into histograms is currently documented in `a MDN article `_. diff --git a/toolkit/components/telemetry/docs/collection/index.rst b/toolkit/components/telemetry/docs/collection/index.rst new file mode 100644 index 000000000..e4084e62a --- /dev/null +++ b/toolkit/components/telemetry/docs/collection/index.rst @@ -0,0 +1,35 @@ +=============== +Data collection +=============== + +There are different APIs and formats to collect data in Firefox, all suiting different use cases. + +In general, we aim to submit data in a common format where possible. This has several advantages; from common code and tooling to sharing analysis know-how. + +In cases where this isn't possible and more flexibility is needed, we can submit custom pings or consider adding different data formats to existing pings. + +*Note:* Every new data collection must go through a `data collection review `_. + +The current data collection possibilities include: + +* :doc:`scalars` allow recording of a single value (string, boolean, a number) +* :doc:`histograms` can efficiently record multiple data points +* ``environment`` data records information about the system and settings a session occurs in +* ``TelemetryLog`` allows collecting ordered event entries (note: this does not have supporting analysis tools) +* :doc:`measuring elapsed time ` +* :doc:`custom pings ` + +.. toctree:: + :maxdepth: 2 + :titlesonly: + :hidden: + :glob: + + scalars + histograms + measuring-time + custom-pings + +Browser Usage Telemetry +~~~~~~~~~~~~~~~~~~~~~~~ +For more information, see :ref:`browserusagetelemetry`. diff --git a/toolkit/components/telemetry/docs/collection/measuring-time.rst b/toolkit/components/telemetry/docs/collection/measuring-time.rst new file mode 100644 index 000000000..918c8a85a --- /dev/null +++ b/toolkit/components/telemetry/docs/collection/measuring-time.rst @@ -0,0 +1,74 @@ +====================== +Measuring elapsed time +====================== + +To make it easier to measure how long operations take, we have helpers for both JavaScript and C++. +These helpers record the elapsed time into histograms, so you have to create suitable histograms for them first. + +From JavaScript +=============== +JavaScript can measure elapsed time using `TelemetryStopwatch.jsm `_. + +``TelemetryStopwatch`` is a helper that simplifies recording elapsed time (in milliseconds) into histograms (plain or keyed). + +API: + +.. code-block:: js + + TelemetryStopwatch = { + // Start, cancel & finish recording elapsed time into a histogram. + // |aObject| is optional. If specificied, the timer is associated with this + // object, so multiple time measurements can be done concurrently. + start(histogramId, aObject); + cancel(histogramId, aObject); + finish(histogramId, aObject); + // Start, cancel & finished recording elapsed time into a keyed histogram. + // |key| specificies the key to record into. + // |aObject| is optional and used as above. + startKeyed(histogramId, key, aObject); + cancelKeyed(histogramId, key, aObject); + finishKeyed(histogramId, key, aObject); + }; + +Example: + +.. code-block:: js + + TelemetryStopwatch.start("SAMPLE_FILE_LOAD_TIME_MS"); + // ... start loading file. + if (failedToOpenFile) { + // Cancel this if the operation failed early etc. + TelemetryStopwatch.cancel("SAMPLE_FILE_LOAD_TIME_MS"); + return; + } + // ... do more work. + TelemetryStopwatch.finish("SAMPLE_FILE_LOAD_TIME_MS"); + +From C++ +======== + +API: + +.. code-block:: cpp + + // This helper class is the preferred way to record elapsed time. + template + class AutoTimer { + // Record into a plain histogram. + explicit AutoTimer(TimeStamp aStart = TimeStamp::Now()); + // Record into a keyed histogram, with key |aKey|. + explicit AutoTimer(const nsCString& aKey, + TimeStamp aStart = TimeStamp::Now()); + }; + + void AccumulateTimeDelta(ID id, TimeStamp start, TimeStamp end = TimeStamp::Now()); + +Example: + +.. code-block:: cpp + + { + Telemetry::AutoTimer telemetry; + // ... scan disk for plugins. + } + // When leaving the scope, AutoTimers destructor will record the time that passed. diff --git a/toolkit/components/telemetry/docs/collection/scalars.rst b/toolkit/components/telemetry/docs/collection/scalars.rst new file mode 100644 index 000000000..2c48601a4 --- /dev/null +++ b/toolkit/components/telemetry/docs/collection/scalars.rst @@ -0,0 +1,140 @@ +======= +Scalars +======= + +Historically we started to overload our histogram mechanism to also collect scalar data, +such as flag values, counts, labels and others. +The scalar measurement types are the suggested way to collect that kind of scalar data. +We currently only support recording of scalars from the parent process. +The serialized scalar data is submitted with the :doc:`main pings <../data/main-ping>`. + +The API +======= +Scalar probes can be managed either through the `nsITelemetry interface `_ +or the `C++ API `_. + +JS API +------ +Probes in privileged JavaScript code can use the following functions to manipulate scalars: + +.. code-block:: js + + Services.telemetry.scalarAdd(aName, aValue); + Services.telemetry.scalarSet(aName, aValue); + Services.telemetry.scalarSetMaximum(aName, aValue); + + Services.telemetry.keyedScalarAdd(aName, aKey, aValue); + Services.telemetry.keyedScalarSet(aName, aKey, aValue); + Services.telemetry.keyedScalarSetMaximum(aName, aKey, aValue); + +These functions can throw if, for example, an operation is performed on a scalar type that doesn't support it +(e.g. calling scalarSetMaximum on a scalar of the string kind). Please look at the `code documentation `_ for +additional information. + +C++ API +------- +Probes in native code can use the more convenient helper functions declared in `Telemetry.h `_: + +.. code-block:: cpp + + void ScalarAdd(mozilla::Telemetry::ScalarID aId, uint32_t aValue); + void ScalarSet(mozilla::Telemetry::ScalarID aId, uint32_t aValue); + void ScalarSet(mozilla::Telemetry::ScalarID aId, const nsAString& aValue); + void ScalarSet(mozilla::Telemetry::ScalarID aId, bool aValue); + void ScalarSetMaximum(mozilla::Telemetry::ScalarID aId, uint32_t aValue); + + void ScalarAdd(mozilla::Telemetry::ScalarID aId, const nsAString& aKey, uint32_t aValue); + void ScalarSet(mozilla::Telemetry::ScalarID aId, const nsAString& aKey, uint32_t aValue); + void ScalarSet(mozilla::Telemetry::ScalarID aId, const nsAString& aKey, bool aValue); + void ScalarSetMaximum(mozilla::Telemetry::ScalarID aId, const nsAString& aKey, uint32_t aValue); + +The YAML definition file +======================== +Scalar probes are required to be registered, both for validation and transparency reasons, +in the `Scalars.yaml `_ +definition file. + +The probes in the definition file are represented in a fixed-depth, two-level structure: + +.. code-block:: yaml + + # The following is a group. + a.group.hierarchy: + a_probe_name: + kind: uint + ... + another_probe: + kind: string + ... + ... + group2: + probe: + kind: int + ... + +Group and probe names need to follow a few rules: + +- they cannot exceed 40 characters each; +- group names must be alpha-numeric + ``.``, with no leading/trailing digit or ``.``; +- probe names must be alpha-numeric + ``_``, with no leading/trailing digit or ``_``. + +A probe can be defined as follows: + +.. code-block:: yaml + + a.group.hierarchy: + a_scalar: + bug_numbers: + - 1276190 + description: A nice one-line description. + expires: never + kind: uint + notification_emails: + - telemetry-client-dev@mozilla.com + +Required Fields +--------------- + +- ``bug_numbers``: A list of unsigned integers representing the number of the bugs the probe was introduced in. +- ``description``: A single or multi-line string describing what data the probe collects and when it gets collected. +- ``expires``: The version number in which the scalar expires, e.g. "30"; a version number of type "N" and "N.0" is automatically converted to "N.0a1" in order to expire the scalar also in the development channels. A telemetry probe acting on an expired scalar will print a warning into the browser console. For scalars that never expire the value ``never`` can be used. +- ``kind``: A string representing the scalar type. Allowed values are ``uint``, ``string`` and ``boolean``. +- ``notification_emails``: A list of email addresses to notify with alerts of expiring probes. More importantly, these are used by the data steward to verify that the probe is still useful. + +Optional Fields +--------------- + +- ``cpp_guard``: A string that gets inserted as an ``#ifdef`` directive around the automatically generated C++ declaration. This is typically used for platform-specific scalars, e.g. ``ANDROID``. +- ``release_channel_collection``: This can be either ``opt-in`` (default) or ``opt-out``. With the former the scalar is submitted by default on pre-release channels; on the release channel only if the user opted into additional data collection. With the latter the scalar is submitted by default on release and pre-release channels, unless the user opted out. +- ``keyed``: A boolean that determines whether this is a keyed scalar. It defaults to ``False``. + +String type restrictions +------------------------ +To prevent abuses, the content of a string scalar is limited to 50 characters in length. Trying +to set a longer string will result in an error and no string being set. + +Keyed Scalars +------------- +Keyed scalars are collections of one of the available scalar types, indexed by a string key that can contain UTF8 characters and cannot be longer than 70 characters. Keyed scalars can contain up to 100 keys. This scalar type is for example useful when you want to break down certain counts by a name, like how often searches happen with which search engine. + +Keyed scalars should only be used if the set of keys are not known beforehand. If the keys are from a known set of strings, other options are preferred if suitable, like categorical histograms or splitting measurements up into separate scalars. + +The processor scripts +===================== +The scalar definition file is processed and checked for correctness at compile time. If it +conforms to the specification, the processor scripts generate two C++ headers files, included +by the Telemetry C++ core. + +gen-scalar-data.py +------------------ +This script is called by the build system to generate the ``TelemetryScalarData.h`` C++ header +file out of the scalar definitions. +This header file contains an array holding the scalar names and version strings, in addition +to an array of ``ScalarInfo`` structures representing all the scalars. + +gen-scalar-enum.py +------------------ +This script is called by the build system to generate the ``TelemetryScalarEnums.h`` C++ header +file out of the scalar definitions. +This header file contains an enum class with all the scalar identifiers used to access them +from code through the C++ API. diff --git a/toolkit/components/telemetry/docs/concepts/archiving.rst b/toolkit/components/telemetry/docs/concepts/archiving.rst new file mode 100644 index 000000000..a2c57de43 --- /dev/null +++ b/toolkit/components/telemetry/docs/concepts/archiving.rst @@ -0,0 +1,12 @@ +========= +Archiving +========= + +When archiving is enabled through the relevant pref (``toolkit.telemetry.archive.enabled``), pings submitted to ``TelemetryController`` are also stored locally in the user profile directory, in ``/datareporting/archived``. + +To allow for cheaper lookup of archived pings, storage follows a specific naming scheme for both the directory and the ping file name: `/...jsonlz4`. + +* ```` - The subdirectory name, generated from the ping creation date. +* ```` - Timestamp of the ping creation date. +* ```` - The ping identifier. +* ```` - The ping type. diff --git a/toolkit/components/telemetry/docs/concepts/crashes.rst b/toolkit/components/telemetry/docs/concepts/crashes.rst new file mode 100644 index 000000000..c9f69a23b --- /dev/null +++ b/toolkit/components/telemetry/docs/concepts/crashes.rst @@ -0,0 +1,23 @@ +======= +Crashes +======= + +There are many different kinds of crashes for Firefox, there is not a single system used to record all of them. + +Main process crashes +==================== + +If the Firefox main process dies, that should be recorded as an aborted session. We would submit a :doc:`main ping <../data/main-ping>` with the reason ``aborted-session``. +If we have a crash dump for that crash, we should also submit a :doc:`crash ping <../data/crash-ping>`. + +The ``aborted-session`` information is first written to disk 60 seconds after startup, any earlier crashes will not trigger an ``aborted-session`` ping. +Also, the ``aborted-session`` is updated at least every 5 minutes, so it may lag behind the last session state. + +Crashes during startup should be recorded in the next sessions main ping in the ``STARTUP_CRASH_DETECTED`` histogram. + +Child process crashes +===================== + +If a Firefox plugin, content or gmplugin process dies unexpectedly, this is recorded in the main pings ``SUBPROCESS_ABNORMAL_ABORT`` keyed histogram. + +If we catch a crash report for this, then additionally the ``SUBPROCESS_CRASHES_WITH_DUMP`` keyed histogram is incremented. diff --git a/toolkit/components/telemetry/docs/concepts/index.rst b/toolkit/components/telemetry/docs/concepts/index.rst new file mode 100644 index 000000000..a49466f8d --- /dev/null +++ b/toolkit/components/telemetry/docs/concepts/index.rst @@ -0,0 +1,23 @@ +======== +Concepts +======== + +There are common concepts used throughout Telemetry: + +* :doc:`pings ` - the packets we use to submit data +* :doc:`sessions & subsessions ` - how we slice a users' time in the browser +* *measurements* - how we :doc:`collect data <../collection/index>` +* *opt-in* & *opt-out* - the different sets of data we collect +* :doc:`submission ` - how we send data to the servers +* :doc:`archiving ` - retaining ping data locally +* :doc:`crashes ` - the different data crashes generate + +.. toctree:: + :maxdepth: 2 + :titlesonly: + :glob: + :hidden: + + pings + crashes + * diff --git a/toolkit/components/telemetry/docs/concepts/pings.rst b/toolkit/components/telemetry/docs/concepts/pings.rst new file mode 100644 index 000000000..db7371b32 --- /dev/null +++ b/toolkit/components/telemetry/docs/concepts/pings.rst @@ -0,0 +1,32 @@ +.. _telemetry_pings: + +===================== +Telemetry pings +===================== + +A *Telemetry ping* is the data that we send to Mozillas Telemetry servers. + +That data is stored as a JSON object client-side and contains common information to all pings and a payload specific to a certain *ping types*. + +The top-level structure is defined by the :doc:`common ping format <../data/common-ping>` format. +It contains: + +* some basic information shared between different ping types +* the :doc:`environment data <../data/environment>` (optional) +* the data specific to the *ping type*, the *payload*. + +Ping types +========== + +We send Telemetry with different ping types. The :doc:`main <../data/main-ping>` ping is the ping that contains the bulk of the Telemetry measurements for Firefox. For more specific use-cases, we send other ping types. + +Pings sent from code that ships with Firefox are listed in the :doc:`data documentation <../data/index>`. + +Important examples are: + +* :doc:`main <../data/main-ping>` - contains the information collected by Telemetry (Histograms, hang stacks, ...) +* :doc:`saved-session <../data/main-ping>` - has the same format as a main ping, but it contains the *"classic"* Telemetry payload with measurements covering the whole browser session. This is only a separate type to make storage of saved-session easier server-side. This is temporary and will be removed soon. +* :doc:`crash <../data/crash-ping>` - a ping that is captured and sent after Firefox crashes. +* ``activation`` - *planned* - sent right after installation or profile creation +* ``upgrade`` - *planned* - sent right after an upgrade +* :doc:`deletion <../data/deletion-ping>` - sent when FHR upload is disabled, requesting deletion of the data associated with this user diff --git a/toolkit/components/telemetry/docs/concepts/sessions.rst b/toolkit/components/telemetry/docs/concepts/sessions.rst new file mode 100644 index 000000000..088556978 --- /dev/null +++ b/toolkit/components/telemetry/docs/concepts/sessions.rst @@ -0,0 +1,40 @@ +======== +Sessions +======== + +A *session* is the time from when Firefox starts until it shut down. +A session can be very long-running. E.g. for Mac users that are used to always put their laptops into sleep-mode, Firefox may run for weeks. +We slice the sessions into smaller logical units called *subsessions*. + +Subsessions +=========== + +The first subsession starts when the browser starts. After that, we split the subsession for different reasons: + +* ``daily``, when crossing local midnight. This keeps latency acceptable by triggering a ping at least daily for most active users. +* ``environment-change``, when a change to the *environment* happens. This happens for important changes to the Firefox settings and when addons activate or deactivate. + +On a subsession split, a :doc:`main ping <../data/main-ping>` with that reason will be submitted. We store the reason in the pings payload, to see what triggered it. + +A session always ends with a subsession with one of two reason: + +* ``shutdown``, when the browser was cleanly shut down. To avoid delaying shutdown, we only save this ping to disk and send it at the next opportunity (typically the next browsing session). +* ``aborted-session``, when the browser crashed. While Firefox is active, we write the current ``main`` ping data to disk every 5 minutes. If the browser crashes, we find this data on disk on the next start and send it with this reason. + +.. image:: subsession_triggers.png + +Subsession data +=============== + +A subsessions data consists of: + +* general information: the date the subsession started, how long it lasted, etc. +* specific measurements: histogram & scalar data, etc. + +This has some advantages: + +* Latency - Sending a ping with all the data of a subsession immediately after it ends means we get the data from installs faster. For ``main`` pings, we aim to send a ping at least daily by starting a new subsession at local midnight. +* Correlation - By starting new subsessions when fundamental settings change (i.e. changes to the *environment*), we can correlate a subsessions data better to those settings. + + + diff --git a/toolkit/components/telemetry/docs/concepts/submission.rst b/toolkit/components/telemetry/docs/concepts/submission.rst new file mode 100644 index 000000000..165917d40 --- /dev/null +++ b/toolkit/components/telemetry/docs/concepts/submission.rst @@ -0,0 +1,34 @@ +========== +Submission +========== + +*Note:* The server-side behaviour is documented in the `HTTP Edge Server specification `_. + +Pings are submitted via a common API on ``TelemetryController``. +If a ping fails to successfully submit to the server immediately (e.g. because +of missing internet connection), Telemetry will store it on disk and retry to +send it until the maximum ping age is exceeded (14 days). + +*Note:* the :doc:`main pings <../data/main-ping>` are kept locally even after successful submission to enable the HealthReport and SelfSupport features. They will be deleted after their retention period of 180 days. + +Submission logic +================ + +Sending of pending pings starts as soon as the delayed startup is finished. They are sent in batches, newest-first, with up +to 10 persisted pings per batch plus all unpersisted pings. +The send logic then waits for each batch to complete. + +If it succeeds we trigger the next send of a ping batch. This is delayed as needed to only trigger one batch send per minute. + +If ping sending encounters an error that means retrying later, a backoff timeout behavior is +triggered, exponentially increasing the timeout for the next try from 1 minute up to a limit of 120 minutes. +Any new ping submissions and "idle-daily" events reset this behavior as a safety mechanism and trigger immediate ping sending. + +Status codes +============ + +The telemetry server team is working towards `the common services status codes `_, but for now the following logic is sufficient for Telemetry: + +* `2XX` - success, don't resubmit +* `4XX` - there was some problem with the request - the client should not try to resubmit as it would just receive the same response +* `5XX` - there was a server-side error, the client should try to resubmit later diff --git a/toolkit/components/telemetry/docs/concepts/subsession_triggers.png b/toolkit/components/telemetry/docs/concepts/subsession_triggers.png new file mode 100644 index 000000000..5717b00a9 Binary files /dev/null and b/toolkit/components/telemetry/docs/concepts/subsession_triggers.png differ diff --git a/toolkit/components/telemetry/docs/data/addons-malware-ping.rst b/toolkit/components/telemetry/docs/data/addons-malware-ping.rst new file mode 100644 index 000000000..18502d748 --- /dev/null +++ b/toolkit/components/telemetry/docs/data/addons-malware-ping.rst @@ -0,0 +1,42 @@ + +Add-ons malware ping +==================== + +This ping is generated by an add-on created by Mozilla and shipped to users on older versions of Firefox (44-46). The ping contains information about the profile that might have been altered by a third party malicious add-on. + +Structure: + +.. code-block:: js + + { + type: "malware-addon-states", + ... + clientId: , + environment: { ... }, + // Common ping data. + payload: { + // True if the blocklist was disabled at startup time. + blocklistDisabled: , + // True if the malicious add-on exists and is enabled. False if it + // exists and is disabled or null if the add-on was not found. + mainAddonActive: , + // A value of the malicious add-on block list state, or null if the + // add-on was not found. + mainAddonBlocked: , + // True if a malicious user.js file was found in the profile. + foundUserJS: , + // If a malicious secmodd.db file was found the extension ID that the // file contained.. + secmoddAddon: , . + // A list of IDs for extensions which were hidden by malicious CSS. + hiddenAddons: [ + , + ... + ], + // A mapping of installed add-on IDs with known malicious + // update URL patterns to their exact update URLs. + updateURLs: { + : , + ... + } + } + } diff --git a/toolkit/components/telemetry/docs/data/common-ping.rst b/toolkit/components/telemetry/docs/data/common-ping.rst new file mode 100644 index 000000000..445557efd --- /dev/null +++ b/toolkit/components/telemetry/docs/data/common-ping.rst @@ -0,0 +1,42 @@ + +Common ping format +================== + +This defines the top-level structure of a Telemetry ping. +It contains basic information shared between different ping types, which enables proper storage and processing of the raw pings server-side. + +It also contains optional further information: + +* the :doc:`environment data <../data/environment>`, which contains important info to correlate the measurements against +* the ``clientId``, a UUID identifying a profile and allowing user-oriented correlation of data + +*Note:* Both are not submitted with all ping types due to privacy concerns. This and the data it that can be correlated against is inspected under the `data collection policy `_. + +Finally, the structure also contains the `payload`, which is the specific data submitted for the respective *ping type*. + +Structure: + +.. code-block:: js + + { + type: , // "main", "activation", "deletion", "saved-session", ... + id: , // a UUID that identifies this ping + creationDate: , // the date the ping was generated + version: , // the version of the ping format, currently 4 + + application: { + architecture: , // build architecture, e.g. x86 + buildId: , // "20141126041045" + name: , // "Firefox" + version: , // "35.0" + displayVersion: , // "35.0b3" + vendor: , // "Mozilla" + platformVersion: , // "35.0" + xpcomAbi: , // e.g. "x86-msvc" + channel: , // "beta" + }, + + clientId: , // optional + environment: { ... }, // optional, not all pings contain the environment + payload: { ... }, // the actual payload data for this ping type + } diff --git a/toolkit/components/telemetry/docs/data/core-ping.rst b/toolkit/components/telemetry/docs/data/core-ping.rst new file mode 100644 index 000000000..7f38f2f7e --- /dev/null +++ b/toolkit/components/telemetry/docs/data/core-ping.rst @@ -0,0 +1,191 @@ + +"core" ping +============ + +This mobile-specific ping is intended to provide the most critical +data in a concise format, allowing for frequent uploads. + +Since this ping is used to measure retention, it should be sent +each time the browser is opened. + +Submission will be per the Edge server specification:: + + /submit/telemetry/docId/docType/appName/appVersion/appUpdateChannel/appBuildID + +* ``docId`` is a UUID for deduping +* ``docType`` is “core” +* ``appName`` is “Fennec” +* ``appVersion`` is the version of the application (e.g. "46.0a1") +* ``appUpdateChannel`` is “release”, “beta”, etc. +* ``appBuildID`` is the build number + +Note: Counts below (e.g. search & usage times) are “since the last +ping”, not total for the whole application lifetime. + +Structure: + +.. code-block:: js + + { + "v": 7, // ping format version + "clientId": , // client id, e.g. + // "c641eacf-c30c-4171-b403-f077724e848a" + "seq": , // running ping counter, e.g. 3 + "locale": , // application locale, e.g. "en-US" + "os": , // OS name. + "osversion": , // OS version. + "device": , // Build.MANUFACTURER + " - " + Build.MODEL + // where manufacturer is truncated to 12 characters + // & model is truncated to 19 characters + "arch": , // e.g. "arm", "x86" + "profileDate": , // Profile creation date in days since + // UNIX epoch. + "defaultSearch": , // Identifier of the default search engine, + // e.g. "yahoo". + "distributionId": , // Distribution identifier (optional) + "created": , // date the ping was created + // in local time, "yyyy-mm-dd" + "tz": , // timezone offset (in minutes) of the + // device when the ping was created + "sessions": , // number of sessions since last upload + "durations": , // combined duration, in seconds, of all + // sessions since last upload + "searches": , // Optional, object of search use counts in the + // format: { "engine.source": } + // e.g.: { "yahoo.suggestion": 3, "other.listitem": 1 } + "experiments": [, /* … */], // Optional, array of identifiers + // for the active experiments + } + +Field details +------------- + +device +~~~~~~ +The ``device`` field is filled in with information specified by the hardware +manufacturer. As such, it could be excessively long and use excessive amounts +of limited user data. To avoid this, we limit the length of the field. We're +more likely have collisions for models within a manufacturer (e.g. "Galaxy S5" +vs. "Galaxy Note") than we are for shortened manufacturer names so we provide +more characters for the model than the manufacturer. + +distributionId +~~~~~~~~~~~~~~ +The ``distributionId`` contains the distribution ID as specified by +preferences.json for a given distribution. More information on distributions +can be found `here `_. + +It is optional. + +defaultSearch +~~~~~~~~~~~~~ +On Android, this field may be ``null``. To get the engine, we rely on +``SearchEngineManager#getDefaultEngine``, which searches in several places in +order to find the search engine identifier: + +* Shared Preferences +* The distribution (if it exists) +* The localized default engine + +If the identifier could not be retrieved, this field is ``null``. If the +identifier is retrieved, we attempt to create an instance of the search +engine from the search plugins (in order): + +* In the distribution +* From the localized plugins shipped with the browser +* The third-party plugins that are installed in the profile directory + +If the plugins fail to create a search engine instance, this field is also +``null``. + +This field can also be ``null`` when a custom search engine is set as the +default. + +sessions & durations +~~~~~~~~~~~~~~~~~~~~ +On Android, a session is the time when Firefox is focused in the foreground. +`sessions` tracks the number of sessions since the last upload and +`durations` is the accumulated duration in seconds of all of these +sessions. Note that showing a dialog (including a Firefox dialog) will +take Firefox out of focus & end the current session. + +An implementation that records a session when Firefox is completely hidden is +preferrable (e.g. to avoid the dialog issue above), however, it's more complex +to implement and so we chose not to, at least for the initial implementation. + +profileDate +~~~~~~~~~~~ +On Android, this value is created at profile creation time and retrieved or, +for legacy profiles, taken from the package install time (note: this is not the +same exact metric as profile creation time but we compromised in favor of ease +of implementation). + +Additionally on Android, this field may be ``null`` in the unlikely event that +all of the following events occur: + +#. The times.json file does not exist +#. The package install date could not be persisted to disk + +The reason we don't just return the package install time even if the date could +not be persisted to disk is to ensure the value doesn't change once we start +sending it: we only want to send consistent values. + +searches +~~~~~~~~ +In the case a search engine is added by a user, the engine identifier "other" is used, e.g. "other.". + +Sources in Android are based on the existing UI telemetry values and are as +follows: + +* actionbar: the user types in the url bar and hits enter to use the default + search engine +* listitem: the user selects a search engine from the list of secondary search + engines at the bottom of the screen +* suggestion: the user clicks on a search suggestion or, in the case that + suggestions are disabled, the row corresponding with the main engine + +Other parameters +---------------- + +HTTP "Date" header +~~~~~~~~~~~~~~~~~~ +This header is used to track the submission date of the core ping in the format +specified by +`rfc 2616 sec 14.18 `_, +et al (e.g. "Tue, 01 Feb 2011 14:00:00 GMT"). + + +Version history +--------------- +* v7: added ``sessionCount`` & ``sessionDuration`` +* v6: added ``searches`` +* v5: added ``created`` & ``tz`` +* v4: ``profileDate`` will return package install time when times.json is not available +* v3: added ``defaultSearch`` +* v2: added ``distributionId`` +* v1: initial version + +Notes +~~~~~ + +* ``distributionId`` (v2) actually landed after ``profileDate`` (v4) but was + uplifted to 46, whereas ``profileDate`` landed on 47. The version numbers in + code were updated to be increasing (bug 1264492) and the version history docs + rearranged accordingly. + +Android implementation notes +---------------------------- +On Android, the uploader has a high probability of delivering the complete data +for a given client but not a 100% probability. This was a conscious decision to +keep the code simple. The cases where we can lose data: + +* Resetting the field measurements (including incrementing the sequence number) + and storing a ping for upload are not atomic. Android can kill our process + for memory pressure in between these distinct operations so we can just lose + a ping's worth of data. That sequence number will be missing on the server. +* If we exceed some number of pings on disk that have not yet been uploaded, + we remove old pings to save storage space. For those pings, we will lose + their data and their sequence numbers will be missing on the server. + +Note: we never expect to drop data without also dropping a sequence number so +we are able to determine when data loss occurs. diff --git a/toolkit/components/telemetry/docs/data/crash-ping.rst b/toolkit/components/telemetry/docs/data/crash-ping.rst new file mode 100644 index 000000000..3cdbc6030 --- /dev/null +++ b/toolkit/components/telemetry/docs/data/crash-ping.rst @@ -0,0 +1,144 @@ + +"crash" ping +============ + +This ping is captured after the main Firefox process crashes, whether or not the crash report is submitted to crash-stats.mozilla.org. It includes non-identifying metadata about the crash. + +The environment block that is sent with this ping varies: if Firefox was running long enough to record the environment block before the crash, then the environment at the time of the crash will be recorded and ``hasCrashEnvironment`` will be true. If Firefox crashed before the environment was recorded, ``hasCrashEnvironment`` will be false and the recorded environment will be the environment at time of submission. + +The client ID is submitted with this ping. + +Structure: + +.. code-block:: js + + { + version: 1, + type: "crash", + ... common ping data + clientId: , + environment: { ... }, + payload: { + crashDate: "YYYY-MM-DD", + sessionId: , // may be missing for crashes that happen early + // in startup. Added in Firefox 48 with the + // intention of uplifting to Firefox 46 + crashId: , // Optional, ID of the associated crash + stackTraces: { ... }, // Optional, see below + metadata: { // Annotations saved while Firefox was running. See nsExceptionHandler.cpp for more information + ProductName: "Firefox", + ReleaseChannel: , + Version: , + BuildID: "YYYYMMDDHHMMSS", + AvailablePageFile: , // Windows-only, available paging file + AvailablePhysicalMemory: , // Windows-only, available physical memory + AvailableVirtualMemory: , // Windows-only, available virtual memory + BlockedDllList: , // Windows-only, see WindowsDllBlocklist.cpp for details + BlocklistInitFailed: 1, // Windows-only, present only if the DLL blocklist initialization failed + CrashTime: