summaryrefslogtreecommitdiffstats
path: root/dom/media/MediaDecoderReader.h
blob: a31687be8710a998901ae40d0927a16581df8d1b (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
/* vim: set ts=8 sts=2 et sw=2 tw=80: */
/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
#if !defined(MediaDecoderReader_h_)
#define MediaDecoderReader_h_

#include "mozilla/EnumSet.h"
#include "mozilla/MozPromise.h"
#include "nsAutoPtr.h"

#include "AbstractMediaDecoder.h"
#include "MediaInfo.h"
#include "MediaData.h"
#include "MediaResult.h"
#include "MediaMetadataManager.h"
#include "MediaQueue.h"
#include "MediaTimer.h"
#include "MP3Demuxer.h"
#include "AudioCompactor.h"
#include "Intervals.h"
#include "TimeUnits.h"
#include "SeekTarget.h"

namespace mozilla {

#ifdef MOZ_EME
class CDMProxy;
#endif
class MediaDecoderReader;

struct WaitForDataRejectValue
{
  enum Reason {
    SHUTDOWN,
    CANCELED
  };

  WaitForDataRejectValue(MediaData::Type aType, Reason aReason)
    :mType(aType), mReason(aReason) {}
  MediaData::Type mType;
  Reason mReason;
};

class MetadataHolder
{
public:
  NS_INLINE_DECL_THREADSAFE_REFCOUNTING(MetadataHolder)
  MediaInfo mInfo;
  nsAutoPtr<MetadataTags> mTags;

private:
  virtual ~MetadataHolder() {}
};

// Encapsulates the decoding and reading of media data. Reading can either
// synchronous and done on the calling "decode" thread, or asynchronous and
// performed on a background thread, with the result being returned by
// callback. Never hold the decoder monitor when calling into this class.
// Unless otherwise specified, methods and fields of this class can only
// be accessed on the decode task queue.
class MediaDecoderReader {
  friend class ReRequestVideoWithSkipTask;
  friend class ReRequestAudioTask;

  static const bool IsExclusive = true;

public:
  using TrackSet = EnumSet<TrackInfo::TrackType>;

  using MetadataPromise =
    MozPromise<RefPtr<MetadataHolder>, MediaResult, IsExclusive>;
  using MediaDataPromise =
    MozPromise<RefPtr<MediaData>, MediaResult, IsExclusive>;
  using SeekPromise = MozPromise<media::TimeUnit, nsresult, IsExclusive>;

  // Note that, conceptually, WaitForData makes sense in a non-exclusive sense.
  // But in the current architecture it's only ever used exclusively (by MDSM),
  // so we mark it that way to verify our assumptions. If you have a use-case
  // for multiple WaitForData consumers, feel free to flip the exclusivity here.
  using WaitForDataPromise =
    MozPromise<MediaData::Type, WaitForDataRejectValue, IsExclusive>;

  using BufferedUpdatePromise = MozPromise<bool, bool, IsExclusive>;

  NS_INLINE_DECL_THREADSAFE_REFCOUNTING(MediaDecoderReader)

  // The caller must ensure that Shutdown() is called before aDecoder is
  // destroyed.
  explicit MediaDecoderReader(AbstractMediaDecoder* aDecoder);

  // Initializes the reader, returns NS_OK on success, or NS_ERROR_FAILURE
  // on failure.
  nsresult Init();

  // Called by MDSM in dormant state to release resources allocated by this
  // reader. The reader can resume decoding by calling Seek() to a specific
  // position.
  virtual void ReleaseResources() {}

  // Destroys the decoding state. The reader cannot be made usable again.
  // This is different from ReleaseMediaResources() as it is irreversable,
  // whereas ReleaseMediaResources() is.  Must be called on the decode
  // thread.
  virtual RefPtr<ShutdownPromise> Shutdown();

  virtual bool OnTaskQueue() const
  {
    return OwnerThread()->IsCurrentThreadIn();
  }

  // Resets all state related to decoding, emptying all buffers etc.
  // Cancels all pending Request*Data() request callbacks, rejects any
  // outstanding seek promises, and flushes the decode pipeline. The
  // decoder must not call any of the callbacks for outstanding
  // Request*Data() calls after this is called. Calls to Request*Data()
  // made after this should be processed as usual.
  //
  // Normally this call preceedes a Seek() call, or shutdown.
  //
  // The first samples of every stream produced after a ResetDecode() call
  // *must* be marked as "discontinuities". If it's not, seeking work won't
  // properly!
  //
  // aParam is a set of TrackInfo::TrackType enums specifying which
  // queues need to be reset, defaulting to both audio and video tracks.
  virtual nsresult ResetDecode(TrackSet aTracks = TrackSet(TrackInfo::kAudioTrack,
                                                           TrackInfo::kVideoTrack));

  // Requests one audio sample from the reader.
  //
  // The decode should be performed asynchronously, and the promise should
  // be resolved when it is complete. Don't hold the decoder
  // monitor while calling this, as the implementation may try to wait
  // on something that needs the monitor and deadlock.
  virtual RefPtr<MediaDataPromise> RequestAudioData();

  // Requests one video sample from the reader.
  //
  // Don't hold the decoder monitor while calling this, as the implementation
  // may try to wait on something that needs the monitor and deadlock.
  // If aSkipToKeyframe is true, the decode should skip ahead to the
  // the next keyframe at or after aTimeThreshold microseconds.
  virtual RefPtr<MediaDataPromise>
  RequestVideoData(bool aSkipToNextKeyframe, int64_t aTimeThreshold);

  // By default, the state machine polls the reader once per second when it's
  // in buffering mode. Some readers support a promise-based mechanism by which
  // they notify the state machine when the data arrives.
  virtual bool IsWaitForDataSupported() const { return false; }

  virtual RefPtr<WaitForDataPromise> WaitForData(MediaData::Type aType)
  {
    MOZ_CRASH();
  }

  // By default, the reader return the decoded data. Some readers support
  // retuning demuxed data.
  virtual bool IsDemuxOnlySupported() const { return false; }

  // Configure the reader to return demuxed or decoded data
  // upon calls to Request{Audio,Video}Data.
  virtual void SetDemuxOnly(bool /*aDemuxedOnly*/) {}

  // The default implementation of AsyncReadMetadata is implemented in terms of
  // synchronous ReadMetadata() calls. Implementations may also
  // override AsyncReadMetadata to create a more proper async implementation.
  virtual RefPtr<MetadataPromise> AsyncReadMetadata();

  // Fills aInfo with the latest cached data required to present the media,
  // ReadUpdatedMetadata will always be called once ReadMetadata has succeeded.
  virtual void ReadUpdatedMetadata(MediaInfo* aInfo) {}

  // Moves the decode head to aTime microseconds. aEndTime denotes the end
  // time of the media in usecs. This is only needed for OggReader, and should
  // probably be removed somehow.
  virtual RefPtr<SeekPromise> Seek(SeekTarget aTarget, int64_t aEndTime) = 0;

  // Called to move the reader into idle state. When the reader is
  // created it is assumed to be active (i.e. not idle). When the media
  // element is paused and we don't need to decode any more data, the state
  // machine calls SetIdle() to inform the reader that its decoder won't be
  // needed for a while. The reader can use these notifications to enter
  // a low power state when the decoder isn't needed, if desired.
  // This is most useful on mobile.
  // Note: DecodeVideoFrame, DecodeAudioData, ReadMetadata and Seek should
  // activate the decoder if necessary. The state machine only needs to know
  // when to call SetIdle().
  virtual void SetIdle() {}

#ifdef MOZ_EME
  virtual void SetCDMProxy(CDMProxy* aProxy) {}
#endif

  // Tell the reader that the data decoded are not for direct playback, so it
  // can accept more files, in particular those which have more channels than
  // available in the audio output.
  void SetIgnoreAudioOutputFormat()
  {
    mIgnoreAudioOutputFormat = true;
  }

  // MediaSourceReader opts out of the start-time-guessing mechanism.
  virtual bool ForceZeroStartTime() const { return false; }

  // The MediaDecoderStateMachine uses various heuristics that assume that
  // raw media data is arriving sequentially from a network channel. This
  // makes sense in the <video src="foo"> case, but not for more advanced use
  // cases like MSE.
  virtual bool UseBufferingHeuristics() const { return true; }

  // Returns the number of bytes of memory allocated by structures/frames in
  // the video queue.
  size_t SizeOfVideoQueueInBytes() const;

  // Returns the number of bytes of memory allocated by structures/frames in
  // the audio queue.
  size_t SizeOfAudioQueueInBytes() const;

  virtual size_t SizeOfVideoQueueInFrames();
  virtual size_t SizeOfAudioQueueInFrames();

  void NotifyDataArrived()
  {
    MOZ_ASSERT(OnTaskQueue());
    NS_ENSURE_TRUE_VOID(!mShutdown);
    NotifyDataArrivedInternal();
    UpdateBuffered();
  }

  // Update the buffered ranges and upon doing so return a promise
  // to indicate success. Overrides may need to do extra work to ensure
  // buffered is up to date.
  virtual RefPtr<BufferedUpdatePromise> UpdateBufferedWithPromise()
  {
    MOZ_ASSERT(OnTaskQueue());
    UpdateBuffered();
    return BufferedUpdatePromise::CreateAndResolve(true, __func__);
  }

  virtual MediaQueue<AudioData>& AudioQueue() { return mAudioQueue; }
  virtual MediaQueue<VideoData>& VideoQueue() { return mVideoQueue; }

  AbstractCanonical<media::TimeIntervals>* CanonicalBuffered()
  {
    return &mBuffered;
  }

  void DispatchSetStartTime(int64_t aStartTime)
  {
    RefPtr<MediaDecoderReader> self = this;
    nsCOMPtr<nsIRunnable> r =
      NS_NewRunnableFunction([self, aStartTime] () -> void
    {
      MOZ_ASSERT(self->OnTaskQueue());
      MOZ_ASSERT(!self->HaveStartTime());
      self->mStartTime.emplace(aStartTime);
      self->UpdateBuffered();
    });
    OwnerThread()->Dispatch(r.forget());
  }

  TaskQueue* OwnerThread() const
  {
    return mTaskQueue;
  }

  // Returns true if the reader implements RequestAudioData()
  // and RequestVideoData() asynchronously, rather than using the
  // implementation in this class to adapt the old synchronous to
  // the newer async model.
  virtual bool IsAsync() const { return false; }

  // Returns true if this decoder reader uses hardware accelerated video
  // decoding.
  virtual bool VideoIsHardwareAccelerated() const { return false; }

  TimedMetadataEventSource& TimedMetadataEvent()
  {
    return mTimedMetadataEvent;
  }

  // Notified by the OggReader during playback when chained ogg is detected.
  MediaEventSource<void>& OnMediaNotSeekable() { return mOnMediaNotSeekable; }

  TimedMetadataEventProducer& TimedMetadataProducer()
  {
    return mTimedMetadataEvent;
  }

  MediaEventProducer<void>& MediaNotSeekableProducer()
  {
    return mOnMediaNotSeekable;
  }

  // Switch the video decoder to BlankDecoderModule. It might takes effective
  // since a few samples later depends on how much demuxed samples are already
  // queued in the original video decoder.
  virtual void SetVideoBlankDecode(bool aIsBlankDecode) {}

protected:
  virtual ~MediaDecoderReader();

  // Populates aBuffered with the time ranges which are buffered. This may only
  // be called on the decode task queue, and should only be used internally by
  // UpdateBuffered - mBuffered (or mirrors of it) should be used for everything
  // else.
  //
  // This base implementation in MediaDecoderReader estimates the time ranges
  // buffered by interpolating the cached byte ranges with the duration
  // of the media. Reader subclasses should override this method if they
  // can quickly calculate the buffered ranges more accurately.
  //
  // The primary advantage of this implementation in the reader base class
  // is that it's a fast approximation, which does not perform any I/O.
  //
  // The OggReader relies on this base implementation not performing I/O,
  // since in FirefoxOS we can't do I/O on the main thread, where this is
  // called.
  virtual media::TimeIntervals GetBuffered();

  RefPtr<MediaDataPromise> DecodeToFirstVideoData();

  bool HaveStartTime()
  {
    MOZ_ASSERT(OnTaskQueue());
    return mStartTime.isSome();
  }

  int64_t StartTime() { MOZ_ASSERT(HaveStartTime()); return mStartTime.ref(); }

  // Queue of audio frames. This queue is threadsafe, and is accessed from
  // the audio, decoder, state machine, and main threads.
  MediaQueue<AudioData> mAudioQueue;

  // Queue of video frames. This queue is threadsafe, and is accessed from
  // the decoder, state machine, and main threads.
  MediaQueue<VideoData> mVideoQueue;

  // An adapter to the audio queue which first copies data to buffers with
  // minimal allocation slop and then pushes them to the queue.  This is
  // useful for decoders working with formats that give awkward numbers of
  // frames such as mp3.
  AudioCompactor mAudioCompactor;

  // Reference to the owning decoder object.
  AbstractMediaDecoder* mDecoder;

  // Decode task queue.
  RefPtr<TaskQueue> mTaskQueue;

  // State-watching manager.
  WatchManager<MediaDecoderReader> mWatchManager;

  // Buffered range.
  Canonical<media::TimeIntervals> mBuffered;

  // Stores presentation info required for playback.
  MediaInfo mInfo;

  // Duration, mirrored from the state machine task queue.
  Mirror<media::NullableTimeUnit> mDuration;

  // Whether we should accept media that we know we can't play
  // directly, because they have a number of channel higher than
  // what we support.
  bool mIgnoreAudioOutputFormat;

  // The start time of the media, in microseconds. This is the presentation
  // time of the first frame decoded from the media. This is initialized to -1,
  // and then set to a value >= by MediaDecoderStateMachine::SetStartTime(),
  // after which point it never changes (though SetStartTime may be called
  // multiple times with the same value).
  //
  // This is an ugly breach of abstractions - it's currently necessary for the
  // readers to return the correct value of GetBuffered. We should refactor
  // things such that all GetBuffered calls go through the MDSM, which would
  // offset the range accordingly.
  Maybe<int64_t> mStartTime;

  // This is a quick-and-dirty way for DecodeAudioData implementations to
  // communicate the presence of a decoding error to RequestAudioData. We should
  // replace this with a promise-y mechanism as we make this stuff properly
  // async.
  bool mHitAudioDecodeError;
  bool mShutdown;

  // Used to send TimedMetadata to the listener.
  TimedMetadataEventProducer mTimedMetadataEvent;

  // Notify if this media is not seekable.
  MediaEventProducer<void> mOnMediaNotSeekable;

private:
  virtual nsresult InitInternal() { return NS_OK; }

  // Does any spinup that needs to happen on this task queue. This runs on a
  // different thread than Init, and there should not be ordering dependencies
  // between the two (even though in practice, Init will always run first right
  // now thanks to the tail dispatcher).
  void InitializationTask();

  // Read header data for all bitstreams in the file. Fills aInfo with
  // the data required to present the media, and optionally fills *aTags
  // with tag metadata from the file.
  // Returns NS_OK on success, or NS_ERROR_FAILURE on failure.
  virtual nsresult ReadMetadata(MediaInfo* aInfo, MetadataTags** aTags)
  {
    MOZ_CRASH();
  }

  // Recomputes mBuffered.
  virtual void UpdateBuffered();

  virtual void VisibilityChanged();

  virtual void NotifyDataArrivedInternal() {}

  // Overrides of this function should decodes an unspecified amount of
  // audio data, enqueuing the audio data in mAudioQueue. Returns true
  // when there's more audio to decode, false if the audio is finished,
  // end of file has been reached, or an un-recoverable read error has
  // occured. This function blocks until the decode is complete.
  virtual bool DecodeAudioData()
  {
    return false;
  }

  // Overrides of this function should read and decodes one video frame.
  // Packets with a timestamp less than aTimeThreshold will be decoded
  // (unless they're not keyframes and aKeyframeSkip is true), but will
  // not be added to the queue. This function blocks until the decode
  // is complete.
  virtual bool DecodeVideoFrame(bool &aKeyframeSkip, int64_t aTimeThreshold)
  {
    return false;
  }

  // Promises used only for the base-class (sync->async adapter) implementation
  // of Request{Audio,Video}Data.
  MozPromiseHolder<MediaDataPromise> mBaseAudioPromise;
  MozPromiseHolder<MediaDataPromise> mBaseVideoPromise;

  MediaEventListener mDataArrivedListener;
};

} // namespace mozilla

#endif