diff options
Diffstat (limited to 'dom/media/mediasource/TrackBuffersManager.h')
| -rw-r--r-- | dom/media/mediasource/TrackBuffersManager.h | 500 |
1 files changed, 500 insertions, 0 deletions
diff --git a/dom/media/mediasource/TrackBuffersManager.h b/dom/media/mediasource/TrackBuffersManager.h new file mode 100644 index 000000000..de6b115f5 --- /dev/null +++ b/dom/media/mediasource/TrackBuffersManager.h @@ -0,0 +1,500 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef MOZILLA_TRACKBUFFERSMANAGER_H_ +#define MOZILLA_TRACKBUFFERSMANAGER_H_ + +#include "mozilla/Atomics.h" +#include "mozilla/Maybe.h" +#include "mozilla/Monitor.h" +#include "AutoTaskQueue.h" +#include "mozilla/dom/SourceBufferBinding.h" + +#include "MediaData.h" +#include "MediaDataDemuxer.h" +#include "MediaResult.h" +#include "MediaSourceDecoder.h" +#include "SourceBufferTask.h" +#include "TimeUnits.h" +#include "nsAutoPtr.h" +#include "nsProxyRelease.h" +#include "nsString.h" +#include "nsTArray.h" + +namespace mozilla { + +class ContainerParser; +class MediaByteBuffer; +class MediaRawData; +class MediaSourceDemuxer; +class SourceBufferResource; + +class SourceBufferTaskQueue +{ +public: + SourceBufferTaskQueue() + : mMonitor("SourceBufferTaskQueue") + {} + ~SourceBufferTaskQueue() + { + MOZ_ASSERT(mQueue.IsEmpty(), "All tasks must have been processed"); + } + + void Push(SourceBufferTask* aTask) + { + MonitorAutoLock mon(mMonitor); + mQueue.AppendElement(aTask); + } + + already_AddRefed<SourceBufferTask> Pop() + { + MonitorAutoLock mon(mMonitor); + if (!mQueue.Length()) { + return nullptr; + } + RefPtr<SourceBufferTask> task = Move(mQueue[0]); + mQueue.RemoveElementAt(0); + return task.forget(); + } + + nsTArray<SourceBufferTask>::size_type Length() const + { + MonitorAutoLock mon(mMonitor); + return mQueue.Length(); + } + +private: + mutable Monitor mMonitor; + nsTArray<RefPtr<SourceBufferTask>> mQueue; +}; + +class TrackBuffersManager +{ +public: + NS_INLINE_DECL_THREADSAFE_REFCOUNTING(TrackBuffersManager); + + enum class EvictDataResult : int8_t + { + NO_DATA_EVICTED, + CANT_EVICT, + BUFFER_FULL, + }; + + typedef TrackInfo::TrackType TrackType; + typedef MediaData::Type MediaType; + typedef nsTArray<RefPtr<MediaRawData>> TrackBuffer; + typedef SourceBufferTask::AppendPromise AppendPromise; + typedef SourceBufferTask::RangeRemovalPromise RangeRemovalPromise; + + // Interface for SourceBuffer + TrackBuffersManager(MediaSourceDecoder* aParentDecoder, + const nsACString& aType); + + // Queue a task to add data to the end of the input buffer and run the MSE + // Buffer Append Algorithm + // 3.5.5 Buffer Append Algorithm. + // http://w3c.github.io/media-source/index.html#sourcebuffer-buffer-append + RefPtr<AppendPromise> AppendData(MediaByteBuffer* aData, + const SourceBufferAttributes& aAttributes); + + // Queue a task to abort any pending AppendData. + // Does nothing at this stage. + void AbortAppendData(); + + // Queue a task to run MSE Reset Parser State Algorithm. + // 3.5.2 Reset Parser State + void ResetParserState(SourceBufferAttributes& aAttributes); + + // Queue a task to run the MSE range removal algorithm. + // http://w3c.github.io/media-source/#sourcebuffer-coded-frame-removal + RefPtr<RangeRemovalPromise> RangeRemoval(media::TimeUnit aStart, + media::TimeUnit aEnd); + + // Schedule data eviction if necessary as the next call to AppendData will + // add aSize bytes. + // Eviction is done in two steps, first remove data up to aPlaybackTime + // and if still more space is needed remove from the end. + EvictDataResult EvictData(const media::TimeUnit& aPlaybackTime, int64_t aSize); + + // Returns the buffered range currently managed. + // This may be called on any thread. + // Buffered must conform to http://w3c.github.io/media-source/index.html#widl-SourceBuffer-buffered + media::TimeIntervals Buffered() const; + media::TimeUnit HighestStartTime() const; + media::TimeUnit HighestEndTime() const; + + // Return the size of the data managed by this SourceBufferContentManager. + int64_t GetSize() const; + + // Indicate that the MediaSource parent object got into "ended" state. + void Ended(); + + // The parent SourceBuffer is about to be destroyed. + void Detach(); + + int64_t EvictionThreshold() const; + + // Interface for MediaSourceDemuxer + MediaInfo GetMetadata() const; + const TrackBuffer& GetTrackBuffer(TrackInfo::TrackType aTrack) const; + const media::TimeIntervals& Buffered(TrackInfo::TrackType) const; + const media::TimeUnit& HighestStartTime(TrackInfo::TrackType) const; + media::TimeIntervals SafeBuffered(TrackInfo::TrackType) const; + bool IsEnded() const + { + return mEnded; + } + uint32_t Evictable(TrackInfo::TrackType aTrack) const; + media::TimeUnit Seek(TrackInfo::TrackType aTrack, + const media::TimeUnit& aTime, + const media::TimeUnit& aFuzz); + uint32_t SkipToNextRandomAccessPoint(TrackInfo::TrackType aTrack, + const media::TimeUnit& aTimeThreadshold, + const media::TimeUnit& aFuzz, + bool& aFound); + + already_AddRefed<MediaRawData> GetSample(TrackInfo::TrackType aTrack, + const media::TimeUnit& aFuzz, + MediaResult& aResult); + int32_t FindCurrentPosition(TrackInfo::TrackType aTrack, + const media::TimeUnit& aFuzz) const; + media::TimeUnit GetNextRandomAccessPoint(TrackInfo::TrackType aTrack, + const media::TimeUnit& aFuzz); + + void AddSizeOfResources(MediaSourceDecoder::ResourceSizes* aSizes) const; + +private: + typedef MozPromise<bool, MediaResult, /* IsExclusive = */ true> CodedFrameProcessingPromise; + + // for MediaSourceDemuxer::GetMozDebugReaderData + friend class MediaSourceDemuxer; + ~TrackBuffersManager(); + // All following functions run on the taskqueue. + RefPtr<AppendPromise> DoAppendData(RefPtr<MediaByteBuffer> aData, + SourceBufferAttributes aAttributes); + void ScheduleSegmentParserLoop(); + void SegmentParserLoop(); + void InitializationSegmentReceived(); + void ShutdownDemuxers(); + void CreateDemuxerforMIMEType(); + void ResetDemuxingState(); + void NeedMoreData(); + void RejectAppend(const MediaResult& aRejectValue, const char* aName); + // Will return a promise that will be resolved once all frames of the current + // media segment have been processed. + RefPtr<CodedFrameProcessingPromise> CodedFrameProcessing(); + void CompleteCodedFrameProcessing(); + // Called by ResetParserState. + void CompleteResetParserState(); + RefPtr<RangeRemovalPromise> + CodedFrameRemovalWithPromise(media::TimeInterval aInterval); + bool CodedFrameRemoval(media::TimeInterval aInterval); + void SetAppendState(SourceBufferAttributes::AppendState aAppendState); + + bool HasVideo() const + { + return mVideoTracks.mNumTracks > 0; + } + bool HasAudio() const + { + return mAudioTracks.mNumTracks > 0; + } + + // The input buffer as per http://w3c.github.io/media-source/index.html#sourcebuffer-input-buffer + RefPtr<MediaByteBuffer> mInputBuffer; + // Buffer full flag as per https://w3c.github.io/media-source/#sourcebuffer-buffer-full-flag. + // Accessed on both the main thread and the task queue. + Atomic<bool> mBufferFull; + bool mFirstInitializationSegmentReceived; + // Set to true once a new segment is started. + bool mNewMediaSegmentStarted; + bool mActiveTrack; + nsCString mType; + + // ContainerParser objects and methods. + // Those are used to parse the incoming input buffer. + + // Recreate the ContainerParser and if aReuseInitData is true then + // feed it with the previous init segment found. + void RecreateParser(bool aReuseInitData); + nsAutoPtr<ContainerParser> mParser; + + // Demuxer objects and methods. + void AppendDataToCurrentInputBuffer(MediaByteBuffer* aData); + RefPtr<MediaByteBuffer> mInitData; + // Temporary input buffer to handle partial media segment header. + // We store the current input buffer content into it should we need to + // reinitialize the demuxer once we have some samples and a discontinuity is + // detected. + RefPtr<MediaByteBuffer> mPendingInputBuffer; + RefPtr<SourceBufferResource> mCurrentInputBuffer; + RefPtr<MediaDataDemuxer> mInputDemuxer; + // Length already processed in current media segment. + uint64_t mProcessedInput; + Maybe<media::TimeUnit> mLastParsedEndTime; + + void OnDemuxerInitDone(nsresult); + void OnDemuxerInitFailed(const MediaResult& aFailure); + void OnDemuxerResetDone(nsresult); + MozPromiseRequestHolder<MediaDataDemuxer::InitPromise> mDemuxerInitRequest; + + void OnDemuxFailed(TrackType aTrack, const MediaResult& aError); + void DoDemuxVideo(); + void OnVideoDemuxCompleted(RefPtr<MediaTrackDemuxer::SamplesHolder> aSamples); + void OnVideoDemuxFailed(const MediaResult& aError) + { + mVideoTracks.mDemuxRequest.Complete(); + OnDemuxFailed(TrackType::kVideoTrack, aError); + } + void DoDemuxAudio(); + void OnAudioDemuxCompleted(RefPtr<MediaTrackDemuxer::SamplesHolder> aSamples); + void OnAudioDemuxFailed(const MediaResult& aError) + { + mAudioTracks.mDemuxRequest.Complete(); + OnDemuxFailed(TrackType::kAudioTrack, aError); + } + + void DoEvictData(const media::TimeUnit& aPlaybackTime, int64_t aSizeToEvict); + + struct TrackData + { + TrackData() + : mNumTracks(0) + , mNeedRandomAccessPoint(true) + , mSizeBuffer(0) + {} + uint32_t mNumTracks; + // Definition of variables: + // https://w3c.github.io/media-source/#track-buffers + // Last decode timestamp variable that stores the decode timestamp of the + // last coded frame appended in the current coded frame group. + // The variable is initially unset to indicate that no coded frames have + // been appended yet. + Maybe<media::TimeUnit> mLastDecodeTimestamp; + // Last frame duration variable that stores the coded frame duration of the + // last coded frame appended in the current coded frame group. + // The variable is initially unset to indicate that no coded frames have + // been appended yet. + Maybe<media::TimeUnit> mLastFrameDuration; + // Highest end timestamp variable that stores the highest coded frame end + // timestamp across all coded frames in the current coded frame group that + // were appended to this track buffer. + // The variable is initially unset to indicate that no coded frames have + // been appended yet. + Maybe<media::TimeUnit> mHighestEndTimestamp; + // Highest presentation timestamp in track buffer. + // Protected by global monitor, except when reading on the task queue as it + // is only written there. + media::TimeUnit mHighestStartTimestamp; + // Longest frame duration seen since last random access point. + // Only ever accessed when mLastDecodeTimestamp and mLastFrameDuration are + // set. + media::TimeUnit mLongestFrameDuration; + // Need random access point flag variable that keeps track of whether the + // track buffer is waiting for a random access point coded frame. + // The variable is initially set to true to indicate that random access + // point coded frame is needed before anything can be added to the track + // buffer. + bool mNeedRandomAccessPoint; + RefPtr<MediaTrackDemuxer> mDemuxer; + MozPromiseRequestHolder<MediaTrackDemuxer::SamplesPromise> mDemuxRequest; + // Highest end timestamp of the last media segment demuxed. + media::TimeUnit mLastParsedEndTime; + + // If set, position where the next contiguous frame will be inserted. + // If a discontinuity is detected, it will be unset and recalculated upon + // the next insertion. + Maybe<uint32_t> mNextInsertionIndex; + // Samples just demuxed, but not yet parsed. + TrackBuffer mQueuedSamples; + const TrackBuffer& GetTrackBuffer() const + { + MOZ_RELEASE_ASSERT(mBuffers.Length(), + "TrackBuffer must have been created"); + return mBuffers.LastElement(); + } + TrackBuffer& GetTrackBuffer() + { + MOZ_RELEASE_ASSERT(mBuffers.Length(), + "TrackBuffer must have been created"); + return mBuffers.LastElement(); + } + // We only manage a single track of each type at this time. + nsTArray<TrackBuffer> mBuffers; + // Track buffer ranges variable that represents the presentation time ranges + // occupied by the coded frames currently stored in the track buffer. + media::TimeIntervals mBufferedRanges; + // Sanitized mBufferedRanges with a fuzz of half a sample's duration applied + // This buffered ranges is the basis of what is exposed to the JS. + media::TimeIntervals mSanitizedBufferedRanges; + // Byte size of all samples contained in this track buffer. + uint32_t mSizeBuffer; + // TrackInfo of the first metadata received. + RefPtr<SharedTrackInfo> mInfo; + // TrackInfo of the last metadata parsed (updated with each init segment. + RefPtr<SharedTrackInfo> mLastInfo; + + // If set, position of the next sample to be retrieved by GetSample(). + // If the position is equal to the TrackBuffer's length, it indicates that + // we've reached EOS. + Maybe<uint32_t> mNextGetSampleIndex; + // Approximation of the next sample's decode timestamp. + media::TimeUnit mNextSampleTimecode; + // Approximation of the next sample's presentation timestamp. + media::TimeUnit mNextSampleTime; + + struct EvictionIndex + { + EvictionIndex() { Reset(); } + void Reset() + { + mEvictable = 0; + mLastIndex = 0; + } + uint32_t mEvictable; + uint32_t mLastIndex; + }; + // Size of data that can be safely evicted during the next eviction + // cycle. + // We consider as evictable all frames up to the last keyframe prior to + // mNextGetSampleIndex. If mNextGetSampleIndex isn't set, then we assume + // that we can't yet evict data. + // Protected by global monitor, except when reading on the task queue as it + // is only written there. + EvictionIndex mEvictionIndex; + + void ResetAppendState() + { + mLastDecodeTimestamp.reset(); + mLastFrameDuration.reset(); + mHighestEndTimestamp.reset(); + mNeedRandomAccessPoint = true; + + mNextInsertionIndex.reset(); + } + + void AddSizeOfResources(MediaSourceDecoder::ResourceSizes* aSizes) const; + }; + + void CheckSequenceDiscontinuity(const media::TimeUnit& aPresentationTime); + void ProcessFrames(TrackBuffer& aSamples, TrackData& aTrackData); + media::TimeInterval PresentationInterval(const TrackBuffer& aSamples) const; + bool CheckNextInsertionIndex(TrackData& aTrackData, + const media::TimeUnit& aSampleTime); + void InsertFrames(TrackBuffer& aSamples, + const media::TimeIntervals& aIntervals, + TrackData& aTrackData); + void UpdateHighestTimestamp(TrackData& aTrackData, + const media::TimeUnit& aHighestTime); + // Remove all frames and their dependencies contained in aIntervals. + // Return the index at which frames were first removed or 0 if no frames + // removed. + uint32_t RemoveFrames(const media::TimeIntervals& aIntervals, + TrackData& aTrackData, + uint32_t aStartIndex); + // Recalculate track's evictable amount. + void ResetEvictionIndex(TrackData& aTrackData); + void UpdateEvictionIndex(TrackData& aTrackData, uint32_t aCurrentIndex); + // Find index of sample. Return a negative value if not found. + uint32_t FindSampleIndex(const TrackBuffer& aTrackBuffer, + const media::TimeInterval& aInterval); + const MediaRawData* GetSample(TrackInfo::TrackType aTrack, + uint32_t aIndex, + const media::TimeUnit& aExpectedDts, + const media::TimeUnit& aExpectedPts, + const media::TimeUnit& aFuzz); + void UpdateBufferedRanges(); + void RejectProcessing(const MediaResult& aRejectValue, const char* aName); + void ResolveProcessing(bool aResolveValue, const char* aName); + MozPromiseRequestHolder<CodedFrameProcessingPromise> mProcessingRequest; + MozPromiseHolder<CodedFrameProcessingPromise> mProcessingPromise; + + // Trackbuffers definition. + nsTArray<const TrackData*> GetTracksList() const; + nsTArray<TrackData*> GetTracksList(); + TrackData& GetTracksData(TrackType aTrack) + { + switch(aTrack) { + case TrackType::kVideoTrack: + return mVideoTracks; + case TrackType::kAudioTrack: + default: + return mAudioTracks; + } + } + const TrackData& GetTracksData(TrackType aTrack) const + { + switch(aTrack) { + case TrackType::kVideoTrack: + return mVideoTracks; + case TrackType::kAudioTrack: + default: + return mAudioTracks; + } + } + TrackData mVideoTracks; + TrackData mAudioTracks; + + // TaskQueue methods and objects. + AbstractThread* GetTaskQueue() const + { + return mTaskQueue; + } + bool OnTaskQueue() const + { + return !GetTaskQueue() || GetTaskQueue()->IsCurrentThreadIn(); + } + RefPtr<AutoTaskQueue> mTaskQueue; + + // SourceBuffer Queues and running context. + SourceBufferTaskQueue mQueue; + void QueueTask(SourceBufferTask* aTask); + void ProcessTasks(); + // Set if the TrackBuffersManager is currently processing a task. + // At this stage, this task is always a AppendBufferTask. + RefPtr<SourceBufferTask> mCurrentTask; + // Current SourceBuffer state for ongoing task. + // Its content is returned to the SourceBuffer once the AppendBufferTask has + // completed. + UniquePtr<SourceBufferAttributes> mSourceBufferAttributes; + // The current sourcebuffer append window. It's content is equivalent to + // mSourceBufferAttributes.mAppendWindowStart/End + media::TimeInterval mAppendWindow; + + // Strong references to external objects. + nsMainThreadPtrHandle<MediaSourceDecoder> mParentDecoder; + + // Return public highest end time across all aTracks. + // Monitor must be held. + media::TimeUnit HighestEndTime(nsTArray<const media::TimeIntervals*>& aTracks) const; + + // Set to true if mediasource state changed to ended. + Atomic<bool> mEnded; + + // Global size of this source buffer content. + Atomic<int64_t> mSizeSourceBuffer; + const int64_t mVideoEvictionThreshold; + const int64_t mAudioEvictionThreshold; + enum class EvictionState + { + NO_EVICTION_NEEDED, + EVICTION_NEEDED, + EVICTION_COMPLETED, + }; + Atomic<EvictionState> mEvictionState; + + // Monitor to protect following objects accessed across multiple threads. + mutable Monitor mMonitor; + // Stable audio and video track time ranges. + media::TimeIntervals mVideoBufferedRanges; + media::TimeIntervals mAudioBufferedRanges; + // MediaInfo of the first init segment read. + MediaInfo mInfo; +}; + +} // namespace mozilla + +#endif /* MOZILLA_TRACKBUFFERSMANAGER_H_ */ |