summaryrefslogtreecommitdiffstats
path: root/dom/media/MediaFormatReader.h
diff options
context:
space:
mode:
Diffstat (limited to 'dom/media/MediaFormatReader.h')
-rw-r--r--dom/media/MediaFormatReader.h601
1 files changed, 601 insertions, 0 deletions
diff --git a/dom/media/MediaFormatReader.h b/dom/media/MediaFormatReader.h
new file mode 100644
index 000000000..4d05ca201
--- /dev/null
+++ b/dom/media/MediaFormatReader.h
@@ -0,0 +1,601 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim:set ts=2 sw=2 sts=2 et cindent: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#if !defined(MediaFormatReader_h_)
+#define MediaFormatReader_h_
+
+#include "mozilla/Atomics.h"
+#include "mozilla/Maybe.h"
+#include "mozilla/TaskQueue.h"
+#include "mozilla/Monitor.h"
+
+#include "MediaEventSource.h"
+#include "MediaDataDemuxer.h"
+#include "MediaDecoderReader.h"
+#include "nsAutoPtr.h"
+#include "PDMFactory.h"
+
+namespace mozilla {
+
+class CDMProxy;
+
+class MediaFormatReader final : public MediaDecoderReader
+{
+ typedef TrackInfo::TrackType TrackType;
+
+public:
+ MediaFormatReader(AbstractMediaDecoder* aDecoder,
+ MediaDataDemuxer* aDemuxer,
+ VideoFrameContainer* aVideoFrameContainer = nullptr);
+
+ virtual ~MediaFormatReader();
+
+ size_t SizeOfVideoQueueInFrames() override;
+ size_t SizeOfAudioQueueInFrames() override;
+
+ RefPtr<MediaDataPromise>
+ RequestVideoData(bool aSkipToNextKeyframe, int64_t aTimeThreshold) override;
+
+ RefPtr<MediaDataPromise> RequestAudioData() override;
+
+ RefPtr<MetadataPromise> AsyncReadMetadata() override;
+
+ void ReadUpdatedMetadata(MediaInfo* aInfo) override;
+
+ RefPtr<SeekPromise>
+ Seek(SeekTarget aTarget, int64_t aUnused) override;
+
+protected:
+ void NotifyDataArrivedInternal() override;
+
+public:
+ media::TimeIntervals GetBuffered() override;
+
+ RefPtr<BufferedUpdatePromise> UpdateBufferedWithPromise() override;
+
+ bool ForceZeroStartTime() const override;
+
+ // For Media Resource Management
+ void ReleaseResources() override;
+
+ nsresult ResetDecode(TrackSet aTracks) override;
+
+ RefPtr<ShutdownPromise> Shutdown() override;
+
+ bool IsAsync() const override { return true; }
+
+ bool VideoIsHardwareAccelerated() const override;
+
+ bool IsWaitForDataSupported() const override { return true; }
+ RefPtr<WaitForDataPromise> WaitForData(MediaData::Type aType) override;
+
+ // MediaFormatReader supports demuxed-only mode.
+ bool IsDemuxOnlySupported() const override { return true; }
+
+ void SetDemuxOnly(bool aDemuxedOnly) override
+ {
+ if (OnTaskQueue()) {
+ mDemuxOnly = aDemuxedOnly;
+ return;
+ }
+ nsCOMPtr<nsIRunnable> r = NewRunnableMethod<bool>(
+ this, &MediaDecoderReader::SetDemuxOnly, aDemuxedOnly);
+ OwnerThread()->Dispatch(r.forget());
+ }
+
+ bool UseBufferingHeuristics() const override
+ {
+ return mTrackDemuxersMayBlock;
+ }
+
+ void SetCDMProxy(CDMProxy* aProxy) override;
+
+ // Returns a string describing the state of the decoder data.
+ // Used for debugging purposes.
+ void GetMozDebugReaderData(nsAString& aString);
+
+ void SetVideoBlankDecode(bool aIsBlankDecode) override;
+
+private:
+ nsresult InitInternal() override;
+
+ bool HasVideo() const { return mVideo.mTrackDemuxer; }
+ bool HasAudio() const { return mAudio.mTrackDemuxer; }
+
+ bool IsWaitingOnCDMResource();
+
+ bool InitDemuxer();
+ // Notify the demuxer that new data has been received.
+ // The next queued task calling GetBuffered() is guaranteed to have up to date
+ // buffered ranges.
+ void NotifyDemuxer();
+ void ReturnOutput(MediaData* aData, TrackType aTrack);
+
+ // Enqueues a task to call Update(aTrack) on the decoder task queue.
+ // Lock for corresponding track must be held.
+ void ScheduleUpdate(TrackType aTrack);
+ void Update(TrackType aTrack);
+ // Handle actions should more data be received.
+ // Returns true if no more action is required.
+ bool UpdateReceivedNewData(TrackType aTrack);
+ // Called when new samples need to be demuxed.
+ void RequestDemuxSamples(TrackType aTrack);
+ // Handle demuxed samples by the input behavior.
+ void HandleDemuxedSamples(TrackType aTrack,
+ AbstractMediaDecoder::AutoNotifyDecoded& aA);
+ // Decode any pending already demuxed samples.
+ void DecodeDemuxedSamples(TrackType aTrack,
+ MediaRawData* aSample);
+
+ struct InternalSeekTarget {
+ InternalSeekTarget(const media::TimeInterval& aTime, bool aDropTarget)
+ : mTime(aTime)
+ , mDropTarget(aDropTarget)
+ , mWaiting(false)
+ , mHasSeeked(false)
+ {}
+
+ media::TimeUnit Time() const { return mTime.mStart; }
+ media::TimeUnit EndTime() const { return mTime.mEnd; }
+ bool Contains(const media::TimeUnit& aTime) const
+ {
+ return mTime.Contains(aTime);
+ }
+
+ media::TimeInterval mTime;
+ bool mDropTarget;
+ bool mWaiting;
+ bool mHasSeeked;
+ };
+
+ // Perform an internal seek to aTime. If aDropTarget is true then
+ // the first sample past the target will be dropped.
+ void InternalSeek(TrackType aTrack, const InternalSeekTarget& aTarget);
+
+ // Drain the current decoder.
+ void DrainDecoder(TrackType aTrack);
+ void NotifyNewOutput(TrackType aTrack, MediaData* aSample);
+ void NotifyInputExhausted(TrackType aTrack);
+ void NotifyDrainComplete(TrackType aTrack);
+ void NotifyError(TrackType aTrack, const MediaResult& aError);
+ void NotifyWaitingForData(TrackType aTrack);
+ void NotifyWaitingForKey(TrackType aTrack);
+ void NotifyEndOfStream(TrackType aTrack);
+
+ void ExtractCryptoInitData(nsTArray<uint8_t>& aInitData);
+
+ // Initializes mLayersBackendType if possible.
+ void InitLayersBackendType();
+
+ // DecoderCallback proxies the MediaDataDecoderCallback calls to these
+ // functions.
+ void Output(TrackType aType, MediaData* aSample);
+ void InputExhausted(TrackType aTrack);
+ void Error(TrackType aTrack, const MediaResult& aError);
+ void Reset(TrackType aTrack);
+ void DrainComplete(TrackType aTrack);
+ void DropDecodedSamples(TrackType aTrack);
+ void WaitingForKey(TrackType aTrack);
+
+ bool ShouldSkip(bool aSkipToNextKeyframe, media::TimeUnit aTimeThreshold);
+
+ void SetVideoDecodeThreshold();
+
+ size_t SizeOfQueue(TrackType aTrack);
+
+ RefPtr<PDMFactory> mPlatform;
+
+ class DecoderCallback : public MediaDataDecoderCallback {
+ public:
+ DecoderCallback(MediaFormatReader* aReader, TrackType aType)
+ : mReader(aReader)
+ , mType(aType)
+ {
+ }
+ void Output(MediaData* aSample) override {
+ mReader->Output(mType, aSample);
+ }
+ void InputExhausted() override {
+ mReader->InputExhausted(mType);
+ }
+ void Error(const MediaResult& aError) override {
+ mReader->Error(mType, aError);
+ }
+ void DrainComplete() override {
+ mReader->DrainComplete(mType);
+ }
+ void ReleaseMediaResources() override {
+ mReader->ReleaseResources();
+ }
+ bool OnReaderTaskQueue() override {
+ return mReader->OnTaskQueue();
+ }
+ void WaitingForKey() override {
+ mReader->WaitingForKey(mType);
+ }
+
+ private:
+ MediaFormatReader* mReader;
+ TrackType mType;
+ };
+
+ struct DecoderData {
+ DecoderData(MediaFormatReader* aOwner,
+ MediaData::Type aType,
+ uint32_t aNumOfMaxError)
+ : mOwner(aOwner)
+ , mType(aType)
+ , mMonitor("DecoderData")
+ , mDescription("shutdown")
+ , mUpdateScheduled(false)
+ , mDemuxEOS(false)
+ , mWaitingForData(false)
+ , mWaitingForKey(false)
+ , mReceivedNewData(false)
+ , mOutputRequested(false)
+ , mDecodePending(false)
+ , mNeedDraining(false)
+ , mDraining(false)
+ , mDrainComplete(false)
+ , mNumOfConsecutiveError(0)
+ , mMaxConsecutiveError(aNumOfMaxError)
+ , mNumSamplesInput(0)
+ , mNumSamplesOutput(0)
+ , mNumSamplesOutputTotal(0)
+ , mNumSamplesSkippedTotal(0)
+ , mSizeOfQueue(0)
+ , mIsHardwareAccelerated(false)
+ , mLastStreamSourceID(UINT32_MAX)
+ , mIsBlankDecode(false)
+ {}
+
+ MediaFormatReader* mOwner;
+ // Disambiguate Audio vs Video.
+ MediaData::Type mType;
+ RefPtr<MediaTrackDemuxer> mTrackDemuxer;
+ // TaskQueue on which decoder can choose to decode.
+ // Only non-null up until the decoder is created.
+ RefPtr<TaskQueue> mTaskQueue;
+ // Callback that receives output and error notifications from the decoder.
+ nsAutoPtr<DecoderCallback> mCallback;
+
+ // Monitor protecting mDescription and mDecoder.
+ Monitor mMonitor;
+ // The platform decoder.
+ RefPtr<MediaDataDecoder> mDecoder;
+ const char* mDescription;
+ void ShutdownDecoder()
+ {
+ MonitorAutoLock mon(mMonitor);
+ if (mDecoder) {
+ mDecoder->Shutdown();
+ }
+ mDescription = "shutdown";
+ mDecoder = nullptr;
+ }
+
+ // Only accessed from reader's task queue.
+ bool mUpdateScheduled;
+ bool mDemuxEOS;
+ bool mWaitingForData;
+ bool mWaitingForKey;
+ bool mReceivedNewData;
+
+ // Pending seek.
+ MozPromiseRequestHolder<MediaTrackDemuxer::SeekPromise> mSeekRequest;
+
+ // Queued demux samples waiting to be decoded.
+ nsTArray<RefPtr<MediaRawData>> mQueuedSamples;
+ MozPromiseRequestHolder<MediaTrackDemuxer::SamplesPromise> mDemuxRequest;
+ // A WaitingPromise is pending if the demuxer is waiting for data or
+ // if the decoder is waiting for a key.
+ MozPromiseHolder<WaitForDataPromise> mWaitingPromise;
+ bool HasWaitingPromise() const
+ {
+ MOZ_ASSERT(mOwner->OnTaskQueue());
+ return !mWaitingPromise.IsEmpty();
+ }
+ bool IsWaiting() const
+ {
+ MOZ_ASSERT(mOwner->OnTaskQueue());
+ return mWaitingForData || mWaitingForKey;
+ }
+
+ // MediaDataDecoder handler's variables.
+ bool mOutputRequested;
+ // Set to true once the MediaDataDecoder has been fed a compressed sample.
+ // No more samples will be passed to the decoder while true.
+ // mDecodePending is reset when:
+ // 1- The decoder calls InputExhausted
+ // 2- The decoder is Flushed or Reset.
+ bool mDecodePending;
+ bool mNeedDraining;
+ bool mDraining;
+ bool mDrainComplete;
+
+ bool HasPendingDrain() const
+ {
+ return mDraining || mDrainComplete;
+ }
+
+ uint32_t mNumOfConsecutiveError;
+ uint32_t mMaxConsecutiveError;
+
+ Maybe<MediaResult> mError;
+ bool HasFatalError() const
+ {
+ if (!mError.isSome()) {
+ return false;
+ }
+ if (mError.ref() == NS_ERROR_DOM_MEDIA_DECODE_ERR) {
+ // Allow decode errors to be non-fatal, but give up
+ // if we have too many.
+ return mNumOfConsecutiveError > mMaxConsecutiveError;
+ } else if (mError.ref() == NS_ERROR_DOM_MEDIA_NEED_NEW_DECODER) {
+ // If the caller asked for a new decoder we shouldn't treat
+ // it as fatal.
+ return false;
+ } else {
+ // All other error types are fatal
+ return true;
+ }
+ }
+
+ // If set, all decoded samples prior mTimeThreshold will be dropped.
+ // Used for internal seeking when a change of stream is detected or when
+ // encountering data discontinuity.
+ Maybe<InternalSeekTarget> mTimeThreshold;
+ // Time of last sample returned.
+ Maybe<media::TimeInterval> mLastSampleTime;
+
+ // Decoded samples returned my mDecoder awaiting being returned to
+ // state machine upon request.
+ nsTArray<RefPtr<MediaData>> mOutput;
+ uint64_t mNumSamplesInput;
+ uint64_t mNumSamplesOutput;
+ uint64_t mNumSamplesOutputTotal;
+ uint64_t mNumSamplesSkippedTotal;
+
+ // These get overridden in the templated concrete class.
+ // Indicate if we have a pending promise for decoded frame.
+ // Rejecting the promise will stop the reader from decoding ahead.
+ virtual bool HasPromise() const = 0;
+ virtual RefPtr<MediaDataPromise> EnsurePromise(const char* aMethodName) = 0;
+ virtual void ResolvePromise(MediaData* aData, const char* aMethodName) = 0;
+ virtual void RejectPromise(const MediaResult& aError,
+ const char* aMethodName) = 0;
+
+ // Clear track demuxer related data.
+ void ResetDemuxer()
+ {
+ mDemuxRequest.DisconnectIfExists();
+ mSeekRequest.DisconnectIfExists();
+ mTrackDemuxer->Reset();
+ mQueuedSamples.Clear();
+ }
+
+ // Flush the decoder if present and reset decoding related data.
+ // Decoding will be suspended until mInputRequested is set again.
+ // Following a flush, the decoder is ready to accept any new data.
+ void Flush()
+ {
+ if (mDecoder) {
+ mDecoder->Flush();
+ }
+ mOutputRequested = false;
+ mDecodePending = false;
+ mOutput.Clear();
+ mNumSamplesInput = 0;
+ mNumSamplesOutput = 0;
+ mSizeOfQueue = 0;
+ mDraining = false;
+ mDrainComplete = false;
+ }
+
+ // Reset the state of the DecoderData, clearing all queued frames
+ // (pending demuxed and decoded).
+ // Decoding will be suspended until mInputRequested is set again.
+ // The track demuxer is *not* reset.
+ void ResetState()
+ {
+ MOZ_ASSERT(mOwner->OnTaskQueue());
+ mDemuxEOS = false;
+ mWaitingForData = false;
+ mWaitingForKey = false;
+ mQueuedSamples.Clear();
+ mOutputRequested = false;
+ mNeedDraining = false;
+ mDecodePending = false;
+ mDraining = false;
+ mDrainComplete = false;
+ mTimeThreshold.reset();
+ mLastSampleTime.reset();
+ mOutput.Clear();
+ mNumSamplesInput = 0;
+ mNumSamplesOutput = 0;
+ mSizeOfQueue = 0;
+ mNextStreamSourceID.reset();
+ if (!HasFatalError()) {
+ mError.reset();
+ }
+ }
+
+ bool HasInternalSeekPending() const
+ {
+ return mTimeThreshold && !mTimeThreshold.ref().mHasSeeked;
+ }
+
+ // Used by the MDSM for logging purposes.
+ Atomic<size_t> mSizeOfQueue;
+ // Used by the MDSM to determine if video decoding is hardware accelerated.
+ // This value is updated after a frame is successfully decoded.
+ Atomic<bool> mIsHardwareAccelerated;
+ // Sample format monitoring.
+ uint32_t mLastStreamSourceID;
+ Maybe<uint32_t> mNextStreamSourceID;
+ media::TimeIntervals mTimeRanges;
+ Maybe<media::TimeUnit> mLastTimeRangesEnd;
+ // TrackInfo as first discovered during ReadMetadata.
+ UniquePtr<TrackInfo> mOriginalInfo;
+ RefPtr<SharedTrackInfo> mInfo;
+ Maybe<media::TimeUnit> mFirstDemuxedSampleTime;
+ // Use BlankDecoderModule or not.
+ bool mIsBlankDecode;
+
+ };
+
+ class DecoderDataWithPromise : public DecoderData {
+ public:
+ DecoderDataWithPromise(MediaFormatReader* aOwner,
+ MediaData::Type aType,
+ uint32_t aNumOfMaxError)
+ : DecoderData(aOwner, aType, aNumOfMaxError)
+ , mHasPromise(false)
+
+ {}
+
+ bool HasPromise() const override
+ {
+ return mHasPromise;
+ }
+
+ RefPtr<MediaDataPromise> EnsurePromise(const char* aMethodName) override
+ {
+ MOZ_ASSERT(mOwner->OnTaskQueue());
+ mHasPromise = true;
+ return mPromise.Ensure(aMethodName);
+ }
+
+ void ResolvePromise(MediaData* aData, const char* aMethodName) override
+ {
+ MOZ_ASSERT(mOwner->OnTaskQueue());
+ mPromise.Resolve(aData, aMethodName);
+ mHasPromise = false;
+ }
+
+ void RejectPromise(const MediaResult& aError,
+ const char* aMethodName) override
+ {
+ MOZ_ASSERT(mOwner->OnTaskQueue());
+ mPromise.Reject(aError, aMethodName);
+ mHasPromise = false;
+ }
+
+ private:
+ MozPromiseHolder<MediaDataPromise> mPromise;
+ Atomic<bool> mHasPromise;
+ };
+
+ DecoderDataWithPromise mAudio;
+ DecoderDataWithPromise mVideo;
+
+ // Returns true when the decoder for this track needs input.
+ bool NeedInput(DecoderData& aDecoder);
+
+ DecoderData& GetDecoderData(TrackType aTrack);
+
+ // Demuxer objects.
+ RefPtr<MediaDataDemuxer> mDemuxer;
+ bool mDemuxerInitDone;
+ void OnDemuxerInitDone(nsresult);
+ void OnDemuxerInitFailed(const MediaResult& aError);
+ MozPromiseRequestHolder<MediaDataDemuxer::InitPromise> mDemuxerInitRequest;
+ void OnDemuxFailed(TrackType aTrack, const MediaResult& aError);
+
+ void DoDemuxVideo();
+ void OnVideoDemuxCompleted(RefPtr<MediaTrackDemuxer::SamplesHolder> aSamples);
+ void OnVideoDemuxFailed(const MediaResult& aError)
+ {
+ OnDemuxFailed(TrackType::kVideoTrack, aError);
+ }
+
+ void DoDemuxAudio();
+ void OnAudioDemuxCompleted(RefPtr<MediaTrackDemuxer::SamplesHolder> aSamples);
+ void OnAudioDemuxFailed(const MediaResult& aError)
+ {
+ OnDemuxFailed(TrackType::kAudioTrack, aError);
+ }
+
+ void SkipVideoDemuxToNextKeyFrame(media::TimeUnit aTimeThreshold);
+ MozPromiseRequestHolder<MediaTrackDemuxer::SkipAccessPointPromise> mSkipRequest;
+ void VideoSkipReset(uint32_t aSkipped);
+ void OnVideoSkipCompleted(uint32_t aSkipped);
+ void OnVideoSkipFailed(MediaTrackDemuxer::SkipFailureHolder aFailure);
+
+ // The last number of decoded output frames that we've reported to
+ // MediaDecoder::NotifyDecoded(). We diff the number of output video
+ // frames every time that DecodeVideoData() is called, and report the
+ // delta there.
+ uint64_t mLastReportedNumDecodedFrames;
+
+ // Timestamp of the previous decoded keyframe, in microseconds.
+ int64_t mPreviousDecodedKeyframeTime_us;
+ // Default mLastDecodedKeyframeTime_us value, must be bigger than anything.
+ static const int64_t sNoPreviousDecodedKeyframe = INT64_MAX;
+
+ RefPtr<layers::KnowsCompositor> mKnowsCompositor;
+
+ // Metadata objects
+ // True if we've read the streams' metadata.
+ bool mInitDone;
+ MozPromiseHolder<MetadataPromise> mMetadataPromise;
+ bool IsEncrypted() const;
+
+ // Set to true if any of our track buffers may be blocking.
+ bool mTrackDemuxersMayBlock;
+
+ // Set the demuxed-only flag.
+ Atomic<bool> mDemuxOnly;
+
+ // Seeking objects.
+ void SetSeekTarget(const SeekTarget& aTarget);
+ media::TimeUnit DemuxStartTime();
+ bool IsSeeking() const { return mPendingSeekTime.isSome(); }
+ bool IsVideoSeeking() const
+ {
+ return IsSeeking() && mOriginalSeekTarget.IsVideoOnly();
+ }
+ void ScheduleSeek();
+ void AttemptSeek();
+ void OnSeekFailed(TrackType aTrack, const MediaResult& aError);
+ void DoVideoSeek();
+ void OnVideoSeekCompleted(media::TimeUnit aTime);
+ void OnVideoSeekFailed(const MediaResult& aError);
+ bool mSeekScheduled;
+
+ void NotifyCompositorUpdated(RefPtr<layers::KnowsCompositor> aKnowsCompositor)
+ {
+ mKnowsCompositor = aKnowsCompositor;
+ }
+
+ void DoAudioSeek();
+ void OnAudioSeekCompleted(media::TimeUnit aTime);
+ void OnAudioSeekFailed(const MediaResult& aError);
+ // The SeekTarget that was last given to Seek()
+ SeekTarget mOriginalSeekTarget;
+ // Temporary seek information while we wait for the data
+ Maybe<media::TimeUnit> mFallbackSeekTime;
+ Maybe<media::TimeUnit> mPendingSeekTime;
+ MozPromiseHolder<SeekPromise> mSeekPromise;
+
+ RefPtr<VideoFrameContainer> mVideoFrameContainer;
+ layers::ImageContainer* GetImageContainer();
+
+ RefPtr<CDMProxy> mCDMProxy;
+
+ RefPtr<GMPCrashHelper> mCrashHelper;
+
+ void SetBlankDecode(TrackType aTrack, bool aIsBlankDecode);
+
+ class DecoderFactory;
+ UniquePtr<DecoderFactory> mDecoderFactory;
+
+ MediaEventListener mCompositorUpdatedListener;
+};
+
+} // namespace mozilla
+
+#endif