/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ /* vim:set ts=2 sw=2 sts=2 et cindent: */ /* This Source Code Form is subject to the terms of the Mozilla Public * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ #include #include #include #include "MediaSourceDemuxer.h" #include "MediaSourceUtils.h" #include "SourceBufferList.h" #include "nsPrintfCString.h" #include "OpusDecoder.h" namespace mozilla { typedef TrackInfo::TrackType TrackType; using media::TimeUnit; using media::TimeIntervals; MediaSourceDemuxer::MediaSourceDemuxer() : mTaskQueue(new AutoTaskQueue(GetMediaThreadPool(MediaThreadType::PLAYBACK), /* aSupportsTailDispatch = */ false)) , mMonitor("MediaSourceDemuxer") { MOZ_ASSERT(NS_IsMainThread()); } // Due to inaccuracies in determining buffer end // frames (Bug 1065207). This value is based on videos seen in the wild. const TimeUnit MediaSourceDemuxer::EOS_FUZZ = media::TimeUnit::FromMicroseconds(500000); RefPtr MediaSourceDemuxer::Init() { return InvokeAsync(GetTaskQueue(), this, __func__, &MediaSourceDemuxer::AttemptInit); } RefPtr MediaSourceDemuxer::AttemptInit() { MOZ_ASSERT(OnTaskQueue()); if (ScanSourceBuffersForContent()) { return InitPromise::CreateAndResolve(NS_OK, __func__); } RefPtr p = mInitPromise.Ensure(__func__); return p; } void MediaSourceDemuxer::AddSizeOfResources(MediaSourceDecoder::ResourceSizes* aSizes) { MOZ_ASSERT(NS_IsMainThread()); // NB: The track buffers must only be accessed on the TaskQueue. RefPtr self = this; RefPtr sizes = aSizes; nsCOMPtr task = NS_NewRunnableFunction([self, sizes] () { for (TrackBuffersManager* manager : self->mSourceBuffers) { manager->AddSizeOfResources(sizes); } }); GetTaskQueue()->Dispatch(task.forget()); } void MediaSourceDemuxer::NotifyDataArrived() { RefPtr self = this; nsCOMPtr task = NS_NewRunnableFunction([self] () { if (self->mInitPromise.IsEmpty()) { return; } if (self->ScanSourceBuffersForContent()) { self->mInitPromise.ResolveIfExists(NS_OK, __func__); } }); GetTaskQueue()->Dispatch(task.forget()); } bool MediaSourceDemuxer::ScanSourceBuffersForContent() { MOZ_ASSERT(OnTaskQueue()); if (mSourceBuffers.IsEmpty()) { return false; } MonitorAutoLock mon(mMonitor); bool haveEmptySourceBuffer = false; for (const auto& sourceBuffer : mSourceBuffers) { MediaInfo info = sourceBuffer->GetMetadata(); if (!info.HasAudio() && !info.HasVideo()) { haveEmptySourceBuffer = true; } if (info.HasAudio() && !mAudioTrack) { mInfo.mAudio = info.mAudio; mAudioTrack = sourceBuffer; } if (info.HasVideo() && !mVideoTrack) { mInfo.mVideo = info.mVideo; mVideoTrack = sourceBuffer; } if (info.IsEncrypted() && !mInfo.IsEncrypted()) { mInfo.mCrypto = info.mCrypto; } } if (mInfo.HasAudio() && mInfo.HasVideo()) { // We have both audio and video. We can ignore non-ready source buffer. return true; } return !haveEmptySourceBuffer; } bool MediaSourceDemuxer::HasTrackType(TrackType aType) const { MonitorAutoLock mon(mMonitor); switch (aType) { case TrackType::kAudioTrack: return mInfo.HasAudio(); case TrackType::kVideoTrack: return mInfo.HasVideo(); default: return false; } } uint32_t MediaSourceDemuxer::GetNumberTracks(TrackType aType) const { return HasTrackType(aType) ? 1u : 0; } already_AddRefed MediaSourceDemuxer::GetTrackDemuxer(TrackType aType, uint32_t aTrackNumber) { RefPtr manager = GetManager(aType); if (!manager) { return nullptr; } RefPtr e = new MediaSourceTrackDemuxer(this, aType, manager); mDemuxers.AppendElement(e); return e.forget(); } bool MediaSourceDemuxer::IsSeekable() const { return true; } UniquePtr MediaSourceDemuxer::GetCrypto() { MonitorAutoLock mon(mMonitor); auto crypto = MakeUnique(); *crypto = mInfo.mCrypto; return crypto; } void MediaSourceDemuxer::AttachSourceBuffer(TrackBuffersManager* aSourceBuffer) { nsCOMPtr task = NewRunnableMethod( this, &MediaSourceDemuxer::DoAttachSourceBuffer, aSourceBuffer); GetTaskQueue()->Dispatch(task.forget()); } void MediaSourceDemuxer::DoAttachSourceBuffer(mozilla::TrackBuffersManager* aSourceBuffer) { MOZ_ASSERT(OnTaskQueue()); mSourceBuffers.AppendElement(aSourceBuffer); ScanSourceBuffersForContent(); } void MediaSourceDemuxer::DetachSourceBuffer(TrackBuffersManager* aSourceBuffer) { nsCOMPtr task = NewRunnableMethod( this, &MediaSourceDemuxer::DoDetachSourceBuffer, aSourceBuffer); GetTaskQueue()->Dispatch(task.forget()); } void MediaSourceDemuxer::DoDetachSourceBuffer(TrackBuffersManager* aSourceBuffer) { MOZ_ASSERT(OnTaskQueue()); for (uint32_t i = 0; i < mSourceBuffers.Length(); i++) { if (mSourceBuffers[i].get() == aSourceBuffer) { mSourceBuffers.RemoveElementAt(i); } } if (aSourceBuffer == mAudioTrack) { mAudioTrack = nullptr; } if (aSourceBuffer == mVideoTrack) { mVideoTrack = nullptr; } ScanSourceBuffersForContent(); } TrackInfo* MediaSourceDemuxer::GetTrackInfo(TrackType aTrack) { MonitorAutoLock mon(mMonitor); switch (aTrack) { case TrackType::kAudioTrack: return &mInfo.mAudio; case TrackType::kVideoTrack: return &mInfo.mVideo; default: return nullptr; } } TrackBuffersManager* MediaSourceDemuxer::GetManager(TrackType aTrack) { MonitorAutoLock mon(mMonitor); switch (aTrack) { case TrackType::kAudioTrack: return mAudioTrack; case TrackType::kVideoTrack: return mVideoTrack; default: return nullptr; } } MediaSourceDemuxer::~MediaSourceDemuxer() { mInitPromise.RejectIfExists(NS_ERROR_DOM_MEDIA_CANCELED, __func__); } void MediaSourceDemuxer::GetMozDebugReaderData(nsAString& aString) { MonitorAutoLock mon(mMonitor); nsAutoCString result; result += nsPrintfCString("Dumping data for demuxer %p:\n", this); if (mAudioTrack) { result += nsPrintfCString("\tDumping Audio Track Buffer(%s): - mLastAudioTime: %f\n" "\t\tNumSamples:%u Size:%u Evictable:%u NextGetSampleIndex:%u NextInsertionIndex:%d\n", mAudioTrack->mAudioTracks.mInfo->mMimeType.get(), mAudioTrack->mAudioTracks.mNextSampleTime.ToSeconds(), mAudioTrack->mAudioTracks.mBuffers[0].Length(), mAudioTrack->mAudioTracks.mSizeBuffer, mAudioTrack->Evictable(TrackInfo::kAudioTrack), mAudioTrack->mAudioTracks.mNextGetSampleIndex.valueOr(-1), mAudioTrack->mAudioTracks.mNextInsertionIndex.valueOr(-1)); result += nsPrintfCString("\t\tBuffered: ranges=%s\n", DumpTimeRanges(mAudioTrack->SafeBuffered(TrackInfo::kAudioTrack)).get()); } if (mVideoTrack) { result += nsPrintfCString("\tDumping Video Track Buffer(%s) - mLastVideoTime: %f\n" "\t\tNumSamples:%u Size:%u Evictable:%u NextGetSampleIndex:%u NextInsertionIndex:%d\n", mVideoTrack->mVideoTracks.mInfo->mMimeType.get(), mVideoTrack->mVideoTracks.mNextSampleTime.ToSeconds(), mVideoTrack->mVideoTracks.mBuffers[0].Length(), mVideoTrack->mVideoTracks.mSizeBuffer, mVideoTrack->Evictable(TrackInfo::kVideoTrack), mVideoTrack->mVideoTracks.mNextGetSampleIndex.valueOr(-1), mVideoTrack->mVideoTracks.mNextInsertionIndex.valueOr(-1)); result += nsPrintfCString("\t\tBuffered: ranges=%s\n", DumpTimeRanges(mVideoTrack->SafeBuffered(TrackInfo::kVideoTrack)).get()); } aString += NS_ConvertUTF8toUTF16(result); } MediaSourceTrackDemuxer::MediaSourceTrackDemuxer(MediaSourceDemuxer* aParent, TrackInfo::TrackType aType, TrackBuffersManager* aManager) : mParent(aParent) , mManager(aManager) , mType(aType) , mMonitor("MediaSourceTrackDemuxer") , mReset(true) , mPreRoll( TimeUnit::FromMicroseconds( OpusDataDecoder::IsOpus(mParent->GetTrackInfo(mType)->mMimeType) ? 80000 : 0)) { } UniquePtr MediaSourceTrackDemuxer::GetInfo() const { return mParent->GetTrackInfo(mType)->Clone(); } RefPtr MediaSourceTrackDemuxer::Seek(media::TimeUnit aTime) { MOZ_ASSERT(mParent, "Called after BreackCycle()"); return InvokeAsync(mParent->GetTaskQueue(), this, __func__, &MediaSourceTrackDemuxer::DoSeek, aTime); } RefPtr MediaSourceTrackDemuxer::GetSamples(int32_t aNumSamples) { MOZ_ASSERT(mParent, "Called after BreackCycle()"); return InvokeAsync(mParent->GetTaskQueue(), this, __func__, &MediaSourceTrackDemuxer::DoGetSamples, aNumSamples); } void MediaSourceTrackDemuxer::Reset() { MOZ_ASSERT(mParent, "Called after BreackCycle()"); RefPtr self = this; nsCOMPtr task = NS_NewRunnableFunction([self] () { self->mNextSample.reset(); self->mReset = true; self->mManager->Seek(self->mType, TimeUnit(), TimeUnit()); { MonitorAutoLock mon(self->mMonitor); self->mNextRandomAccessPoint = self->mManager->GetNextRandomAccessPoint(self->mType, MediaSourceDemuxer::EOS_FUZZ); } }); mParent->GetTaskQueue()->Dispatch(task.forget()); } nsresult MediaSourceTrackDemuxer::GetNextRandomAccessPoint(media::TimeUnit* aTime) { MonitorAutoLock mon(mMonitor); *aTime = mNextRandomAccessPoint; return NS_OK; } RefPtr MediaSourceTrackDemuxer::SkipToNextRandomAccessPoint(media::TimeUnit aTimeThreshold) { return InvokeAsync(mParent->GetTaskQueue(), this, __func__, &MediaSourceTrackDemuxer::DoSkipToNextRandomAccessPoint, aTimeThreshold); } media::TimeIntervals MediaSourceTrackDemuxer::GetBuffered() { return mManager->Buffered(); } void MediaSourceTrackDemuxer::BreakCycles() { RefPtr self = this; nsCOMPtr task = NS_NewRunnableFunction([self]() { self->mParent = nullptr; self->mManager = nullptr; } ); mParent->GetTaskQueue()->Dispatch(task.forget()); } RefPtr MediaSourceTrackDemuxer::DoSeek(media::TimeUnit aTime) { TimeIntervals buffered = mManager->Buffered(mType); // Fuzz factor represents a +/- threshold. So when seeking it allows the gap // to be twice as big as the fuzz value. We only want to allow EOS_FUZZ gap. buffered.SetFuzz(MediaSourceDemuxer::EOS_FUZZ / 2); TimeUnit seekTime = std::max(aTime - mPreRoll, TimeUnit::FromMicroseconds(0)); if (mManager->IsEnded() && seekTime >= buffered.GetEnd()) { // We're attempting to seek past the end time. Cap seekTime so that we seek // to the last sample instead. seekTime = std::max(mManager->HighestStartTime(mType) - mPreRoll, TimeUnit::FromMicroseconds(0)); } if (!buffered.ContainsWithStrictEnd(seekTime)) { if (!buffered.ContainsWithStrictEnd(aTime)) { // We don't have the data to seek to. return SeekPromise::CreateAndReject(NS_ERROR_DOM_MEDIA_WAITING_FOR_DATA, __func__); } // Theoretically we should reject the promise with WAITING_FOR_DATA, // however, to avoid unwanted regressions we assume that if at this time // we don't have the wanted data it won't come later. // Instead of using the pre-rolled time, use the earliest time available in // the interval. TimeIntervals::IndexType index = buffered.Find(aTime); MOZ_ASSERT(index != TimeIntervals::NoIndex); seekTime = buffered[index].mStart; } seekTime = mManager->Seek(mType, seekTime, MediaSourceDemuxer::EOS_FUZZ); MediaResult result = NS_OK; RefPtr sample = mManager->GetSample(mType, media::TimeUnit(), result); MOZ_ASSERT(NS_SUCCEEDED(result) && sample); mNextSample = Some(sample); mReset = false; { MonitorAutoLock mon(mMonitor); mNextRandomAccessPoint = mManager->GetNextRandomAccessPoint(mType, MediaSourceDemuxer::EOS_FUZZ); } return SeekPromise::CreateAndResolve(seekTime, __func__); } RefPtr MediaSourceTrackDemuxer::DoGetSamples(int32_t aNumSamples) { if (mReset) { // If a seek (or reset) was recently performed, we ensure that the data // we are about to retrieve is still available. TimeIntervals buffered = mManager->Buffered(mType); buffered.SetFuzz(MediaSourceDemuxer::EOS_FUZZ / 2); if (!buffered.Length() && mManager->IsEnded()) { return SamplesPromise::CreateAndReject(NS_ERROR_DOM_MEDIA_END_OF_STREAM, __func__); } if (!buffered.ContainsWithStrictEnd(TimeUnit::FromMicroseconds(0))) { return SamplesPromise::CreateAndReject(NS_ERROR_DOM_MEDIA_WAITING_FOR_DATA, __func__); } mReset = false; } RefPtr sample; if (mNextSample) { sample = mNextSample.ref(); mNextSample.reset(); } else { MediaResult result = NS_OK; sample = mManager->GetSample(mType, MediaSourceDemuxer::EOS_FUZZ, result); if (!sample) { if (result == NS_ERROR_DOM_MEDIA_END_OF_STREAM || result == NS_ERROR_DOM_MEDIA_WAITING_FOR_DATA) { return SamplesPromise::CreateAndReject( (result == NS_ERROR_DOM_MEDIA_END_OF_STREAM && mManager->IsEnded()) ? NS_ERROR_DOM_MEDIA_END_OF_STREAM : NS_ERROR_DOM_MEDIA_WAITING_FOR_DATA, __func__); } return SamplesPromise::CreateAndReject(result, __func__); } } RefPtr samples = new SamplesHolder; samples->mSamples.AppendElement(sample); if (mNextRandomAccessPoint.ToMicroseconds() <= sample->mTime) { MonitorAutoLock mon(mMonitor); mNextRandomAccessPoint = mManager->GetNextRandomAccessPoint(mType, MediaSourceDemuxer::EOS_FUZZ); } return SamplesPromise::CreateAndResolve(samples, __func__); } RefPtr MediaSourceTrackDemuxer::DoSkipToNextRandomAccessPoint(media::TimeUnit aTimeThreadshold) { uint32_t parsed = 0; // Ensure that the data we are about to skip to is still available. TimeIntervals buffered = mManager->Buffered(mType); buffered.SetFuzz(MediaSourceDemuxer::EOS_FUZZ / 2); if (buffered.ContainsWithStrictEnd(aTimeThreadshold)) { bool found; parsed = mManager->SkipToNextRandomAccessPoint(mType, aTimeThreadshold, MediaSourceDemuxer::EOS_FUZZ, found); if (found) { return SkipAccessPointPromise::CreateAndResolve(parsed, __func__); } } SkipFailureHolder holder( mManager->IsEnded() ? NS_ERROR_DOM_MEDIA_END_OF_STREAM : NS_ERROR_DOM_MEDIA_WAITING_FOR_DATA, parsed); return SkipAccessPointPromise::CreateAndReject(holder, __func__); } } // namespace mozilla