summaryrefslogtreecommitdiffstats
path: root/dom/media/MediaStreamListener.h
blob: eefd4adac8e9049f4efac9de04339f721638f54d (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
/* vim: set ts=8 sts=2 et sw=2 tw=80: */
/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

#ifndef MOZILLA_MEDIASTREAMLISTENER_h_
#define MOZILLA_MEDIASTREAMLISTENER_h_

#include "MediaStreamGraph.h"
#include "StreamTracks.h"
#include "MediaStreamGraph.h"

namespace mozilla {

class AudioSegment;
class MediaStream;
class MediaStreamGraph;
class MediaStreamVideoSink;
class VideoSegment;

enum MediaStreamGraphEvent : uint32_t {
  EVENT_FINISHED,
  EVENT_REMOVED,
  EVENT_HAS_DIRECT_LISTENERS, // transition from no direct listeners
  EVENT_HAS_NO_DIRECT_LISTENERS,  // transition to no direct listeners
};

// maskable flags, not a simple enumerated value
enum TrackEventCommand : uint32_t {
  TRACK_EVENT_NONE = 0x00,
  TRACK_EVENT_CREATED = 0x01,
  TRACK_EVENT_ENDED = 0x02,
  TRACK_EVENT_UNUSED = ~(TRACK_EVENT_ENDED | TRACK_EVENT_CREATED),
};

/**
 * This is a base class for media graph thread listener callbacks.
 * Override methods to be notified of audio or video data or changes in stream
 * state.
 *
 * This can be used by stream recorders or network connections that receive
 * stream input. It could also be used for debugging.
 *
 * All notification methods are called from the media graph thread. Overriders
 * of these methods are responsible for all synchronization. Beware!
 * These methods are called without the media graph monitor held, so
 * reentry into media graph methods is possible, although very much discouraged!
 * You should do something non-blocking and non-reentrant (e.g. dispatch an
 * event to some thread) and return.
 * The listener is not allowed to add/remove any listeners from the stream.
 *
 * When a listener is first attached, we guarantee to send a NotifyBlockingChanged
 * callback to notify of the initial blocking state. Also, if a listener is
 * attached to a stream that has already finished, we'll call NotifyFinished.
 */
class MediaStreamListener {
protected:
  // Protected destructor, to discourage deletion outside of Release():
  virtual ~MediaStreamListener() {}

public:
  NS_INLINE_DECL_THREADSAFE_REFCOUNTING(MediaStreamListener)

  /**
   * When a SourceMediaStream has pulling enabled, and the MediaStreamGraph
   * control loop is ready to pull, this gets called. A NotifyPull implementation
   * is allowed to call the SourceMediaStream methods that alter track
   * data. It is not allowed to make other MediaStream API calls, including
   * calls to add or remove MediaStreamListeners. It is not allowed to block
   * for any length of time.
   * aDesiredTime is the stream time we would like to get data up to. Data
   * beyond this point will not be played until NotifyPull runs again, so there's
   * not much point in providing it. Note that if the stream is blocked for
   * some reason, then data before aDesiredTime may not be played immediately.
   */
  virtual void NotifyPull(MediaStreamGraph* aGraph, StreamTime aDesiredTime) {}

  enum Blocking {
    BLOCKED,
    UNBLOCKED
  };
  /**
   * Notify that the blocking status of the stream changed. The initial state
   * is assumed to be BLOCKED.
   */
  virtual void NotifyBlockingChanged(MediaStreamGraph* aGraph, Blocking aBlocked) {}

  /**
   * Notify that the stream has data in each track
   * for the stream's current time. Once this state becomes true, it will
   * always be true since we block stream time from progressing to times where
   * there isn't data in each track.
   */
  virtual void NotifyHasCurrentData(MediaStreamGraph* aGraph) {}

  /**
   * Notify that the stream output is advancing. aCurrentTime is the graph's
   * current time. MediaStream::GraphTimeToStreamTime can be used to get the
   * stream time.
   */
  virtual void NotifyOutput(MediaStreamGraph* aGraph, GraphTime aCurrentTime) {}

  /**
   * Notify that an event has occurred on the Stream
   */
  virtual void NotifyEvent(MediaStreamGraph* aGraph, MediaStreamGraphEvent aEvent) {}

  /**
   * Notify that changes to one of the stream tracks have been queued.
   * aTrackEvents can be any combination of TRACK_EVENT_CREATED and
   * TRACK_EVENT_ENDED. aQueuedMedia is the data being added to the track
   * at aTrackOffset (relative to the start of the stream).
   * aInputStream and aInputTrackID will be set if the changes originated
   * from an input stream's track. In practice they will only be used for
   * ProcessedMediaStreams.
   */
  virtual void NotifyQueuedTrackChanges(MediaStreamGraph* aGraph, TrackID aID,
                                        StreamTime aTrackOffset,
                                        TrackEventCommand aTrackEvents,
                                        const MediaSegment& aQueuedMedia,
                                        MediaStream* aInputStream = nullptr,
                                        TrackID aInputTrackID = TRACK_INVALID) {}

  /**
   * Notify queued audio data. Only audio data need to be queued. The video data
   * will be notified by MediaStreamVideoSink::SetCurrentFrame.
   */
  virtual void NotifyQueuedAudioData(MediaStreamGraph* aGraph, TrackID aID,
                                     StreamTime aTrackOffset,
                                     const AudioSegment& aQueuedMedia,
                                     MediaStream* aInputStream = nullptr,
                                     TrackID aInputTrackID = TRACK_INVALID) {}

  /**
   * Notify that all new tracks this iteration have been created.
   * This is to ensure that tracks added atomically to MediaStreamGraph
   * are also notified of atomically to MediaStreamListeners.
   */
  virtual void NotifyFinishedTrackCreation(MediaStreamGraph* aGraph) {}
};

/**
 * This is a base class for media graph thread listener callbacks locked to
 * specific tracks. Override methods to be notified of audio or video data or
 * changes in track state.
 *
 * All notification methods are called from the media graph thread. Overriders
 * of these methods are responsible for all synchronization. Beware!
 * These methods are called without the media graph monitor held, so
 * reentry into media graph methods is possible, although very much discouraged!
 * You should do something non-blocking and non-reentrant (e.g. dispatch an
 * event to some thread) and return.
 * The listener is not allowed to add/remove any listeners from the parent
 * stream.
 *
 * If a listener is attached to a track that has already ended, we guarantee
 * to call NotifyEnded.
 */
class MediaStreamTrackListener
{
  NS_INLINE_DECL_THREADSAFE_REFCOUNTING(MediaStreamTrackListener)

public:
  virtual void NotifyQueuedChanges(MediaStreamGraph* aGraph,
                                   StreamTime aTrackOffset,
                                   const MediaSegment& aQueuedMedia) {}

  virtual void NotifyPrincipalHandleChanged(MediaStreamGraph* aGraph,
                                            const PrincipalHandle& aNewPrincipalHandle) {}

  virtual void NotifyEnded() {}

  virtual void NotifyRemoved() {}

protected:
  virtual ~MediaStreamTrackListener() {}
};


/**
 * This is a base class for media graph thread listener direct callbacks
 * from within AppendToTrack(). Note that your regular listener will
 * still get NotifyQueuedTrackChanges() callbacks from the MSG thread, so
 * you must be careful to ignore them if AddDirectListener was successful.
 */
class DirectMediaStreamListener : public MediaStreamListener
{
public:
  virtual ~DirectMediaStreamListener() {}

  /*
   * This will be called on any DirectMediaStreamListener added to a
   * a SourceMediaStream when AppendToTrack() is called.  The MediaSegment
   * will be the RawSegment (unresampled) if available in AppendToTrack().
   * Note that NotifyQueuedTrackChanges() calls will also still occur.
   */
  virtual void NotifyRealtimeData(MediaStreamGraph* aGraph, TrackID aID,
                                  StreamTime aTrackOffset,
                                  uint32_t aTrackEvents,
                                  const MediaSegment& aMedia) {}
};

/**
 * This is a base class for media graph thread listener direct callbacks from
 * within AppendToTrack(). It is bound to a certain track and can only be
 * installed on audio tracks. Once added to a track on any stream in the graph,
 * the graph will try to install it at that track's source of media data.
 *
 * This works for TrackUnionStreams, which will forward the listener to the
 * track's input track if it exists, or wait for it to be created before
 * forwarding if it doesn't.
 * Once it reaches a SourceMediaStream, it can be successfully installed.
 * Other types of streams will fail installation since they are not supported.
 *
 * Note that this listener and others for the same track will still get
 * NotifyQueuedChanges() callbacks from the MSG tread, so you must be careful
 * to ignore them if this listener was successfully installed.
 */
class DirectMediaStreamTrackListener : public MediaStreamTrackListener
{
  friend class SourceMediaStream;
  friend class TrackUnionStream;

public:
  /*
   * This will be called on any DirectMediaStreamTrackListener added to a
   * SourceMediaStream when AppendToTrack() is called for the listener's bound
   * track, using the thread of the AppendToTrack() caller. The MediaSegment
   * will be the RawSegment (unresampled) if available in AppendToTrack().
   * If the track is enabled at the source but has been disabled in one of the
   * streams in between the source and where it was originally added, aMedia
   * will be a disabled version of the one passed to AppendToTrack() as well.
   * Note that NotifyQueuedTrackChanges() calls will also still occur.
   */
  virtual void NotifyRealtimeTrackData(MediaStreamGraph* aGraph,
                                       StreamTime aTrackOffset,
                                       const MediaSegment& aMedia) {}

  /**
   * When a direct listener is processed for installation by the
   * MediaStreamGraph it will be notified with whether the installation was
   * successful or not. The results of this installation are the following:
   * TRACK_NOT_FOUND_AT_SOURCE
   *    We found the source stream of media data for this track, but the track
   *    didn't exist. This should only happen if you try to install the listener
   *    directly to a SourceMediaStream that doesn't contain the given TrackID.
   * STREAM_NOT_SUPPORTED
   *    While looking for the data source of this track, we found a MediaStream
   *    that is not a SourceMediaStream or a TrackUnionStream.
   * ALREADY_EXIST
   *    This DirectMediaStreamTrackListener already exists in the
   *    SourceMediaStream.
   * SUCCESS
   *    Installation was successful and this listener will start receiving
   *    NotifyRealtimeData on the next AppendToTrack().
   */
  enum class InstallationResult {
    TRACK_NOT_FOUND_AT_SOURCE,
    TRACK_TYPE_NOT_SUPPORTED,
    STREAM_NOT_SUPPORTED,
    ALREADY_EXISTS,
    SUCCESS
  };
  virtual void NotifyDirectListenerInstalled(InstallationResult aResult) {}
  virtual void NotifyDirectListenerUninstalled() {}

  virtual MediaStreamVideoSink* AsMediaStreamVideoSink() { return nullptr; }

protected:
  virtual ~DirectMediaStreamTrackListener() {}

  void MirrorAndDisableSegment(AudioSegment& aFrom, AudioSegment& aTo);
  void MirrorAndDisableSegment(VideoSegment& aFrom,
                               VideoSegment& aTo,
                               DisabledTrackMode aMode);
  void NotifyRealtimeTrackDataAndApplyTrackDisabling(MediaStreamGraph* aGraph,
                                                     StreamTime aTrackOffset,
                                                     MediaSegment& aMedia);

  void IncreaseDisabled(DisabledTrackMode aMode);
  void DecreaseDisabled(DisabledTrackMode aMode);

  // Matches the number of disabled streams to which this listener is attached.
  // The number of streams are those between the stream the listener was added
  // and the SourceMediaStream that is the input of the data.
  Atomic<int32_t> mDisabledFreezeCount;
  Atomic<int32_t> mDisabledBlackCount;

  nsAutoPtr<MediaSegment> mMedia;
};

} // namespace mozilla

#endif // MOZILLA_MEDIASTREAMLISTENER_h_