summaryrefslogtreecommitdiffstats
path: root/dom/media/MediaFormatReader.h
blob: 4d05ca201c01160627dc9aadf4a97fc9f41020c3 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
/* vim:set ts=2 sw=2 sts=2 et cindent: */
/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

#if !defined(MediaFormatReader_h_)
#define MediaFormatReader_h_

#include "mozilla/Atomics.h"
#include "mozilla/Maybe.h"
#include "mozilla/TaskQueue.h"
#include "mozilla/Monitor.h"

#include "MediaEventSource.h"
#include "MediaDataDemuxer.h"
#include "MediaDecoderReader.h"
#include "nsAutoPtr.h"
#include "PDMFactory.h"

namespace mozilla {

class CDMProxy;

class MediaFormatReader final : public MediaDecoderReader
{
  typedef TrackInfo::TrackType TrackType;

public:
  MediaFormatReader(AbstractMediaDecoder* aDecoder,
                    MediaDataDemuxer* aDemuxer,
                    VideoFrameContainer* aVideoFrameContainer = nullptr);

  virtual ~MediaFormatReader();

  size_t SizeOfVideoQueueInFrames() override;
  size_t SizeOfAudioQueueInFrames() override;

  RefPtr<MediaDataPromise>
  RequestVideoData(bool aSkipToNextKeyframe, int64_t aTimeThreshold) override;

  RefPtr<MediaDataPromise> RequestAudioData() override;

  RefPtr<MetadataPromise> AsyncReadMetadata() override;

  void ReadUpdatedMetadata(MediaInfo* aInfo) override;

  RefPtr<SeekPromise>
  Seek(SeekTarget aTarget, int64_t aUnused) override;

protected:
  void NotifyDataArrivedInternal() override;

public:
  media::TimeIntervals GetBuffered() override;

  RefPtr<BufferedUpdatePromise> UpdateBufferedWithPromise() override;

  bool ForceZeroStartTime() const override;

  // For Media Resource Management
  void ReleaseResources() override;

  nsresult ResetDecode(TrackSet aTracks) override;

  RefPtr<ShutdownPromise> Shutdown() override;

  bool IsAsync() const override { return true; }

  bool VideoIsHardwareAccelerated() const override;

  bool IsWaitForDataSupported() const override { return true; }
  RefPtr<WaitForDataPromise> WaitForData(MediaData::Type aType) override;

  // MediaFormatReader supports demuxed-only mode.
  bool IsDemuxOnlySupported() const override { return true; }

  void SetDemuxOnly(bool aDemuxedOnly) override
  {
    if (OnTaskQueue()) {
      mDemuxOnly = aDemuxedOnly;
      return;
    }
    nsCOMPtr<nsIRunnable> r = NewRunnableMethod<bool>(
      this, &MediaDecoderReader::SetDemuxOnly, aDemuxedOnly);
    OwnerThread()->Dispatch(r.forget());
  }

  bool UseBufferingHeuristics() const override
  {
    return mTrackDemuxersMayBlock;
  }

  void SetCDMProxy(CDMProxy* aProxy) override;

  // Returns a string describing the state of the decoder data.
  // Used for debugging purposes.
  void GetMozDebugReaderData(nsAString& aString);

  void SetVideoBlankDecode(bool aIsBlankDecode) override;

private:
  nsresult InitInternal() override;

  bool HasVideo() const { return mVideo.mTrackDemuxer; }
  bool HasAudio() const { return mAudio.mTrackDemuxer; }

  bool IsWaitingOnCDMResource();

  bool InitDemuxer();
  // Notify the demuxer that new data has been received.
  // The next queued task calling GetBuffered() is guaranteed to have up to date
  // buffered ranges.
  void NotifyDemuxer();
  void ReturnOutput(MediaData* aData, TrackType aTrack);

  // Enqueues a task to call Update(aTrack) on the decoder task queue.
  // Lock for corresponding track must be held.
  void ScheduleUpdate(TrackType aTrack);
  void Update(TrackType aTrack);
  // Handle actions should more data be received.
  // Returns true if no more action is required.
  bool UpdateReceivedNewData(TrackType aTrack);
  // Called when new samples need to be demuxed.
  void RequestDemuxSamples(TrackType aTrack);
  // Handle demuxed samples by the input behavior.
  void HandleDemuxedSamples(TrackType aTrack,
                            AbstractMediaDecoder::AutoNotifyDecoded& aA);
  // Decode any pending already demuxed samples.
  void DecodeDemuxedSamples(TrackType aTrack,
                            MediaRawData* aSample);

  struct InternalSeekTarget {
    InternalSeekTarget(const media::TimeInterval& aTime, bool aDropTarget)
      : mTime(aTime)
      , mDropTarget(aDropTarget)
      , mWaiting(false)
      , mHasSeeked(false)
    {}

    media::TimeUnit Time() const { return mTime.mStart; }
    media::TimeUnit EndTime() const { return mTime.mEnd; }
    bool Contains(const media::TimeUnit& aTime) const
    {
      return mTime.Contains(aTime);
    }

    media::TimeInterval mTime;
    bool mDropTarget;
    bool mWaiting;
    bool mHasSeeked;
  };

  // Perform an internal seek to aTime. If aDropTarget is true then
  // the first sample past the target will be dropped.
  void InternalSeek(TrackType aTrack, const InternalSeekTarget& aTarget);

  // Drain the current decoder.
  void DrainDecoder(TrackType aTrack);
  void NotifyNewOutput(TrackType aTrack, MediaData* aSample);
  void NotifyInputExhausted(TrackType aTrack);
  void NotifyDrainComplete(TrackType aTrack);
  void NotifyError(TrackType aTrack, const MediaResult& aError);
  void NotifyWaitingForData(TrackType aTrack);
  void NotifyWaitingForKey(TrackType aTrack);
  void NotifyEndOfStream(TrackType aTrack);

  void ExtractCryptoInitData(nsTArray<uint8_t>& aInitData);

  // Initializes mLayersBackendType if possible.
  void InitLayersBackendType();

  // DecoderCallback proxies the MediaDataDecoderCallback calls to these
  // functions.
  void Output(TrackType aType, MediaData* aSample);
  void InputExhausted(TrackType aTrack);
  void Error(TrackType aTrack, const MediaResult& aError);
  void Reset(TrackType aTrack);
  void DrainComplete(TrackType aTrack);
  void DropDecodedSamples(TrackType aTrack);
  void WaitingForKey(TrackType aTrack);

  bool ShouldSkip(bool aSkipToNextKeyframe, media::TimeUnit aTimeThreshold);

  void SetVideoDecodeThreshold();

  size_t SizeOfQueue(TrackType aTrack);

  RefPtr<PDMFactory> mPlatform;

  class DecoderCallback : public MediaDataDecoderCallback {
  public:
    DecoderCallback(MediaFormatReader* aReader, TrackType aType)
      : mReader(aReader)
      , mType(aType)
    {
    }
    void Output(MediaData* aSample) override {
      mReader->Output(mType, aSample);
    }
    void InputExhausted() override {
      mReader->InputExhausted(mType);
    }
    void Error(const MediaResult& aError) override {
      mReader->Error(mType, aError);
    }
    void DrainComplete() override {
      mReader->DrainComplete(mType);
    }
    void ReleaseMediaResources() override {
      mReader->ReleaseResources();
    }
    bool OnReaderTaskQueue() override {
      return mReader->OnTaskQueue();
    }
    void WaitingForKey() override {
      mReader->WaitingForKey(mType);
    }

  private:
    MediaFormatReader* mReader;
    TrackType mType;
  };

  struct DecoderData {
    DecoderData(MediaFormatReader* aOwner,
                MediaData::Type aType,
                uint32_t aNumOfMaxError)
      : mOwner(aOwner)
      , mType(aType)
      , mMonitor("DecoderData")
      , mDescription("shutdown")
      , mUpdateScheduled(false)
      , mDemuxEOS(false)
      , mWaitingForData(false)
      , mWaitingForKey(false)
      , mReceivedNewData(false)
      , mOutputRequested(false)
      , mDecodePending(false)
      , mNeedDraining(false)
      , mDraining(false)
      , mDrainComplete(false)
      , mNumOfConsecutiveError(0)
      , mMaxConsecutiveError(aNumOfMaxError)
      , mNumSamplesInput(0)
      , mNumSamplesOutput(0)
      , mNumSamplesOutputTotal(0)
      , mNumSamplesSkippedTotal(0)
      , mSizeOfQueue(0)
      , mIsHardwareAccelerated(false)
      , mLastStreamSourceID(UINT32_MAX)
      , mIsBlankDecode(false)
    {}

    MediaFormatReader* mOwner;
    // Disambiguate Audio vs Video.
    MediaData::Type mType;
    RefPtr<MediaTrackDemuxer> mTrackDemuxer;
    // TaskQueue on which decoder can choose to decode.
    // Only non-null up until the decoder is created.
    RefPtr<TaskQueue> mTaskQueue;
    // Callback that receives output and error notifications from the decoder.
    nsAutoPtr<DecoderCallback> mCallback;

    // Monitor protecting mDescription and mDecoder.
    Monitor mMonitor;
    // The platform decoder.
    RefPtr<MediaDataDecoder> mDecoder;
    const char* mDescription;
    void ShutdownDecoder()
    {
      MonitorAutoLock mon(mMonitor);
      if (mDecoder) {
        mDecoder->Shutdown();
      }
      mDescription = "shutdown";
      mDecoder = nullptr;
    }

    // Only accessed from reader's task queue.
    bool mUpdateScheduled;
    bool mDemuxEOS;
    bool mWaitingForData;
    bool mWaitingForKey;
    bool mReceivedNewData;

    // Pending seek.
    MozPromiseRequestHolder<MediaTrackDemuxer::SeekPromise> mSeekRequest;

    // Queued demux samples waiting to be decoded.
    nsTArray<RefPtr<MediaRawData>> mQueuedSamples;
    MozPromiseRequestHolder<MediaTrackDemuxer::SamplesPromise> mDemuxRequest;
    // A WaitingPromise is pending if the demuxer is waiting for data or
    // if the decoder is waiting for a key.
    MozPromiseHolder<WaitForDataPromise> mWaitingPromise;
    bool HasWaitingPromise() const
    {
      MOZ_ASSERT(mOwner->OnTaskQueue());
      return !mWaitingPromise.IsEmpty();
    }
    bool IsWaiting() const
    {
      MOZ_ASSERT(mOwner->OnTaskQueue());
      return mWaitingForData || mWaitingForKey;
    }

    // MediaDataDecoder handler's variables.
    bool mOutputRequested;
    // Set to true once the MediaDataDecoder has been fed a compressed sample.
    // No more samples will be passed to the decoder while true.
    // mDecodePending is reset when:
    // 1- The decoder calls InputExhausted
    // 2- The decoder is Flushed or Reset.
    bool mDecodePending;
    bool mNeedDraining;
    bool mDraining;
    bool mDrainComplete;

    bool HasPendingDrain() const
    {
      return mDraining || mDrainComplete;
    }

    uint32_t mNumOfConsecutiveError;
    uint32_t mMaxConsecutiveError;

    Maybe<MediaResult> mError;
    bool HasFatalError() const
    {
      if (!mError.isSome()) {
        return false;
      }
      if (mError.ref() == NS_ERROR_DOM_MEDIA_DECODE_ERR) {
        // Allow decode errors to be non-fatal, but give up
        // if we have too many.
        return mNumOfConsecutiveError > mMaxConsecutiveError;
      } else if (mError.ref() == NS_ERROR_DOM_MEDIA_NEED_NEW_DECODER) {
        // If the caller asked for a new decoder we shouldn't treat
        // it as fatal.
        return false;
      } else {
        // All other error types are fatal
        return true;
      }
    }

    // If set, all decoded samples prior mTimeThreshold will be dropped.
    // Used for internal seeking when a change of stream is detected or when
    // encountering data discontinuity.
    Maybe<InternalSeekTarget> mTimeThreshold;
    // Time of last sample returned.
    Maybe<media::TimeInterval> mLastSampleTime;

    // Decoded samples returned my mDecoder awaiting being returned to
    // state machine upon request.
    nsTArray<RefPtr<MediaData>> mOutput;
    uint64_t mNumSamplesInput;
    uint64_t mNumSamplesOutput;
    uint64_t mNumSamplesOutputTotal;
    uint64_t mNumSamplesSkippedTotal;

    // These get overridden in the templated concrete class.
    // Indicate if we have a pending promise for decoded frame.
    // Rejecting the promise will stop the reader from decoding ahead.
    virtual bool HasPromise() const = 0;
    virtual RefPtr<MediaDataPromise> EnsurePromise(const char* aMethodName) = 0;
    virtual void ResolvePromise(MediaData* aData, const char* aMethodName) = 0;
    virtual void RejectPromise(const MediaResult& aError,
                               const char* aMethodName) = 0;

    // Clear track demuxer related data.
    void ResetDemuxer()
    {
      mDemuxRequest.DisconnectIfExists();
      mSeekRequest.DisconnectIfExists();
      mTrackDemuxer->Reset();
      mQueuedSamples.Clear();
    }

    // Flush the decoder if present and reset decoding related data.
    // Decoding will be suspended until mInputRequested is set again.
    // Following a flush, the decoder is ready to accept any new data.
    void Flush()
    {
      if (mDecoder) {
        mDecoder->Flush();
      }
      mOutputRequested = false;
      mDecodePending = false;
      mOutput.Clear();
      mNumSamplesInput = 0;
      mNumSamplesOutput = 0;
      mSizeOfQueue = 0;
      mDraining = false;
      mDrainComplete = false;
    }

    // Reset the state of the DecoderData, clearing all queued frames
    // (pending demuxed and decoded).
    // Decoding will be suspended until mInputRequested is set again.
    // The track demuxer is *not* reset.
    void ResetState()
    {
      MOZ_ASSERT(mOwner->OnTaskQueue());
      mDemuxEOS = false;
      mWaitingForData = false;
      mWaitingForKey = false;
      mQueuedSamples.Clear();
      mOutputRequested = false;
      mNeedDraining = false;
      mDecodePending = false;
      mDraining = false;
      mDrainComplete = false;
      mTimeThreshold.reset();
      mLastSampleTime.reset();
      mOutput.Clear();
      mNumSamplesInput = 0;
      mNumSamplesOutput = 0;
      mSizeOfQueue = 0;
      mNextStreamSourceID.reset();
      if (!HasFatalError()) {
        mError.reset();
      }
    }

    bool HasInternalSeekPending() const
    {
      return mTimeThreshold && !mTimeThreshold.ref().mHasSeeked;
    }

    // Used by the MDSM for logging purposes.
    Atomic<size_t> mSizeOfQueue;
    // Used by the MDSM to determine if video decoding is hardware accelerated.
    // This value is updated after a frame is successfully decoded.
    Atomic<bool> mIsHardwareAccelerated;
    // Sample format monitoring.
    uint32_t mLastStreamSourceID;
    Maybe<uint32_t> mNextStreamSourceID;
    media::TimeIntervals mTimeRanges;
    Maybe<media::TimeUnit> mLastTimeRangesEnd;
    // TrackInfo as first discovered during ReadMetadata.
    UniquePtr<TrackInfo> mOriginalInfo;
    RefPtr<SharedTrackInfo> mInfo;
    Maybe<media::TimeUnit> mFirstDemuxedSampleTime;
    // Use BlankDecoderModule or not.
    bool mIsBlankDecode;

  };

  class DecoderDataWithPromise : public DecoderData {
  public:
    DecoderDataWithPromise(MediaFormatReader* aOwner,
                           MediaData::Type aType,
                           uint32_t aNumOfMaxError)
      : DecoderData(aOwner, aType, aNumOfMaxError)
      , mHasPromise(false)

    {}

    bool HasPromise() const override
    {
      return mHasPromise;
    }

    RefPtr<MediaDataPromise> EnsurePromise(const char* aMethodName) override
    {
      MOZ_ASSERT(mOwner->OnTaskQueue());
      mHasPromise = true;
      return mPromise.Ensure(aMethodName);
    }

    void ResolvePromise(MediaData* aData, const char* aMethodName) override
    {
      MOZ_ASSERT(mOwner->OnTaskQueue());
      mPromise.Resolve(aData, aMethodName);
      mHasPromise = false;
    }

    void RejectPromise(const MediaResult& aError,
                       const char* aMethodName) override
    {
      MOZ_ASSERT(mOwner->OnTaskQueue());
      mPromise.Reject(aError, aMethodName);
      mHasPromise = false;
    }

  private:
    MozPromiseHolder<MediaDataPromise> mPromise;
    Atomic<bool> mHasPromise;
  };

  DecoderDataWithPromise mAudio;
  DecoderDataWithPromise mVideo;

  // Returns true when the decoder for this track needs input.
  bool NeedInput(DecoderData& aDecoder);

  DecoderData& GetDecoderData(TrackType aTrack);

  // Demuxer objects.
  RefPtr<MediaDataDemuxer> mDemuxer;
  bool mDemuxerInitDone;
  void OnDemuxerInitDone(nsresult);
  void OnDemuxerInitFailed(const MediaResult& aError);
  MozPromiseRequestHolder<MediaDataDemuxer::InitPromise> mDemuxerInitRequest;
  void OnDemuxFailed(TrackType aTrack, const MediaResult& aError);

  void DoDemuxVideo();
  void OnVideoDemuxCompleted(RefPtr<MediaTrackDemuxer::SamplesHolder> aSamples);
  void OnVideoDemuxFailed(const MediaResult& aError)
  {
    OnDemuxFailed(TrackType::kVideoTrack, aError);
  }

  void DoDemuxAudio();
  void OnAudioDemuxCompleted(RefPtr<MediaTrackDemuxer::SamplesHolder> aSamples);
  void OnAudioDemuxFailed(const MediaResult& aError)
  {
    OnDemuxFailed(TrackType::kAudioTrack, aError);
  }

  void SkipVideoDemuxToNextKeyFrame(media::TimeUnit aTimeThreshold);
  MozPromiseRequestHolder<MediaTrackDemuxer::SkipAccessPointPromise> mSkipRequest;
  void VideoSkipReset(uint32_t aSkipped);
  void OnVideoSkipCompleted(uint32_t aSkipped);
  void OnVideoSkipFailed(MediaTrackDemuxer::SkipFailureHolder aFailure);

  // The last number of decoded output frames that we've reported to
  // MediaDecoder::NotifyDecoded(). We diff the number of output video
  // frames every time that DecodeVideoData() is called, and report the
  // delta there.
  uint64_t mLastReportedNumDecodedFrames;

  // Timestamp of the previous decoded keyframe, in microseconds.
  int64_t mPreviousDecodedKeyframeTime_us;
  // Default mLastDecodedKeyframeTime_us value, must be bigger than anything.
  static const int64_t sNoPreviousDecodedKeyframe = INT64_MAX;

  RefPtr<layers::KnowsCompositor> mKnowsCompositor;

  // Metadata objects
  // True if we've read the streams' metadata.
  bool mInitDone;
  MozPromiseHolder<MetadataPromise> mMetadataPromise;
  bool IsEncrypted() const;

  // Set to true if any of our track buffers may be blocking.
  bool mTrackDemuxersMayBlock;

  // Set the demuxed-only flag.
  Atomic<bool> mDemuxOnly;

  // Seeking objects.
  void SetSeekTarget(const SeekTarget& aTarget);
  media::TimeUnit DemuxStartTime();
  bool IsSeeking() const { return mPendingSeekTime.isSome(); }
  bool IsVideoSeeking() const
  {
    return IsSeeking() && mOriginalSeekTarget.IsVideoOnly();
  }
  void ScheduleSeek();
  void AttemptSeek();
  void OnSeekFailed(TrackType aTrack, const MediaResult& aError);
  void DoVideoSeek();
  void OnVideoSeekCompleted(media::TimeUnit aTime);
  void OnVideoSeekFailed(const MediaResult& aError);
  bool mSeekScheduled;

  void NotifyCompositorUpdated(RefPtr<layers::KnowsCompositor> aKnowsCompositor)
  {
    mKnowsCompositor = aKnowsCompositor;
  }

  void DoAudioSeek();
  void OnAudioSeekCompleted(media::TimeUnit aTime);
  void OnAudioSeekFailed(const MediaResult& aError);
  // The SeekTarget that was last given to Seek()
  SeekTarget mOriginalSeekTarget;
  // Temporary seek information while we wait for the data
  Maybe<media::TimeUnit> mFallbackSeekTime;
  Maybe<media::TimeUnit> mPendingSeekTime;
  MozPromiseHolder<SeekPromise> mSeekPromise;

  RefPtr<VideoFrameContainer> mVideoFrameContainer;
  layers::ImageContainer* GetImageContainer();

  RefPtr<CDMProxy> mCDMProxy;

  RefPtr<GMPCrashHelper> mCrashHelper;

  void SetBlankDecode(TrackType aTrack, bool aIsBlankDecode);

  class DecoderFactory;
  UniquePtr<DecoderFactory> mDecoderFactory;

  MediaEventListener mCompositorUpdatedListener;
};

} // namespace mozilla

#endif