summaryrefslogtreecommitdiffstats
path: root/dom/media/MediaFormatReader.cpp
diff options
context:
space:
mode:
Diffstat (limited to 'dom/media/MediaFormatReader.cpp')
-rw-r--r--dom/media/MediaFormatReader.cpp2372
1 files changed, 2372 insertions, 0 deletions
diff --git a/dom/media/MediaFormatReader.cpp b/dom/media/MediaFormatReader.cpp
new file mode 100644
index 000000000..2093803ad
--- /dev/null
+++ b/dom/media/MediaFormatReader.cpp
@@ -0,0 +1,2372 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim:set ts=2 sw=2 sts=2 et cindent: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "mozilla/CDMProxy.h"
+#include "mozilla/ClearOnShutdown.h"
+#include "mozilla/dom/HTMLMediaElement.h"
+#include "mozilla/Preferences.h"
+#include "mozilla/Telemetry.h"
+#include "nsContentUtils.h"
+#include "nsPrintfCString.h"
+#include "nsSize.h"
+#include "Layers.h"
+#include "MediaData.h"
+#include "MediaInfo.h"
+#include "MediaFormatReader.h"
+#include "MediaPrefs.h"
+#include "MediaResource.h"
+#include "mozilla/SharedThreadPool.h"
+#include "VideoUtils.h"
+#include "VideoFrameContainer.h"
+#include "mozilla/layers/ShadowLayers.h"
+
+#include <algorithm>
+#include <queue>
+
+using namespace mozilla::media;
+
+using mozilla::layers::Image;
+using mozilla::layers::LayerManager;
+using mozilla::layers::LayersBackend;
+
+static mozilla::LazyLogModule sFormatDecoderLog("MediaFormatReader");
+mozilla::LazyLogModule gMediaDemuxerLog("MediaDemuxer");
+
+#define LOG(arg, ...) MOZ_LOG(sFormatDecoderLog, mozilla::LogLevel::Debug, ("MediaFormatReader(%p)::%s: " arg, this, __func__, ##__VA_ARGS__))
+#define LOGV(arg, ...) MOZ_LOG(sFormatDecoderLog, mozilla::LogLevel::Verbose, ("MediaFormatReader(%p)::%s: " arg, this, __func__, ##__VA_ARGS__))
+
+namespace mozilla {
+
+/**
+ * This is a singleton which controls the number of decoders that can be
+ * created concurrently. Before calling PDMFactory::CreateDecoder(), Alloc()
+ * must be called to get a token object as a permission to create a decoder.
+ * The token should stay alive until Shutdown() is called on the decoder.
+ * The destructor of the token will restore the decoder count so it is available
+ * for next calls of Alloc().
+ */
+class DecoderAllocPolicy
+{
+ using TrackType = TrackInfo::TrackType;
+
+public:
+ class Token
+ {
+ NS_INLINE_DECL_THREADSAFE_REFCOUNTING(Token)
+ protected:
+ virtual ~Token() {}
+ };
+
+ using Promise = MozPromise<RefPtr<Token>, bool, true>;
+
+ // Acquire a token for decoder creation. Thread-safe.
+ auto Alloc() -> RefPtr<Promise>;
+
+ // Called by ClearOnShutdown() to delete the singleton.
+ void operator=(decltype(nullptr));
+
+ // Get the singleton for the given track type. Thread-safe.
+ static DecoderAllocPolicy& Instance(TrackType aTrack);
+
+private:
+ class AutoDeallocToken;
+ using PromisePrivate = Promise::Private;
+ explicit DecoderAllocPolicy(TrackType aTrack);
+ ~DecoderAllocPolicy();
+ // Called by the destructor of TokenImpl to restore the decoder limit.
+ void Dealloc();
+ // Decrement the decoder limit and resolve a promise if available.
+ void ResolvePromise(ReentrantMonitorAutoEnter& aProofOfLock);
+
+ // Protect access to Instance().
+ static StaticMutex sMutex;
+
+ ReentrantMonitor mMonitor;
+ // The number of decoders available for creation.
+ int mDecoderLimit;
+ // Track type.
+ const TrackType mTrack;
+ // Requests to acquire tokens.
+ std::queue<RefPtr<PromisePrivate>> mPromises;
+};
+
+StaticMutex DecoderAllocPolicy::sMutex;
+
+class DecoderAllocPolicy::AutoDeallocToken : public Token
+{
+public:
+ explicit AutoDeallocToken(TrackType aTrack)
+ : mTrack(aTrack)
+ {}
+
+private:
+ ~AutoDeallocToken()
+ {
+ DecoderAllocPolicy::Instance(mTrack).Dealloc();
+ }
+
+ const TrackType mTrack;
+};
+
+DecoderAllocPolicy::DecoderAllocPolicy(TrackType aTrack)
+ : mMonitor("DecoderAllocPolicy::mMonitor")
+ , mDecoderLimit(MediaPrefs::MediaDecoderLimit())
+ , mTrack(aTrack)
+{
+ AbstractThread::MainThread()->Dispatch(NS_NewRunnableFunction([this] () {
+ ClearOnShutdown(this, ShutdownPhase::ShutdownThreads);
+ }));
+}
+
+DecoderAllocPolicy::~DecoderAllocPolicy()
+{
+ while (!mPromises.empty()) {
+ RefPtr<PromisePrivate> p = mPromises.front().forget();
+ mPromises.pop();
+ p->Reject(true, __func__);
+ }
+}
+
+DecoderAllocPolicy&
+DecoderAllocPolicy::Instance(TrackType aTrack)
+{
+ StaticMutexAutoLock lock(sMutex);
+ if (aTrack == TrackType::kAudioTrack) {
+ static auto sAudioPolicy = new DecoderAllocPolicy(TrackType::kAudioTrack);
+ return *sAudioPolicy;
+ } else {
+ static auto sVideoPolicy = new DecoderAllocPolicy(TrackType::kVideoTrack);
+ return *sVideoPolicy;
+ }
+}
+
+auto
+DecoderAllocPolicy::Alloc() -> RefPtr<Promise>
+{
+ // No decoder limit set.
+ if (mDecoderLimit < 0) {
+ return Promise::CreateAndResolve(new Token(), __func__);
+ }
+
+ ReentrantMonitorAutoEnter mon(mMonitor);
+ RefPtr<PromisePrivate> p = new PromisePrivate(__func__);
+ mPromises.push(p);
+ ResolvePromise(mon);
+ return p.forget();
+}
+
+void
+DecoderAllocPolicy::Dealloc()
+{
+ ReentrantMonitorAutoEnter mon(mMonitor);
+ ++mDecoderLimit;
+ ResolvePromise(mon);
+}
+
+void
+DecoderAllocPolicy::ResolvePromise(ReentrantMonitorAutoEnter& aProofOfLock)
+{
+ MOZ_ASSERT(mDecoderLimit >= 0);
+
+ if (mDecoderLimit > 0 && !mPromises.empty()) {
+ --mDecoderLimit;
+ RefPtr<PromisePrivate> p = mPromises.front().forget();
+ mPromises.pop();
+ p->Resolve(new AutoDeallocToken(mTrack), __func__);
+ }
+}
+
+void
+DecoderAllocPolicy::operator=(std::nullptr_t)
+{
+ delete this;
+}
+
+class MediaFormatReader::DecoderFactory
+{
+ using InitPromise = MediaDataDecoder::InitPromise;
+ using TokenPromise = DecoderAllocPolicy::Promise;
+ using Token = DecoderAllocPolicy::Token;
+
+public:
+ explicit DecoderFactory(MediaFormatReader* aOwner) : mOwner(aOwner) {}
+ void CreateDecoder(TrackType aTrack);
+
+private:
+ class Wrapper;
+
+ enum class Stage : int8_t
+ {
+ None,
+ WaitForToken,
+ CreateDecoder,
+ WaitForInit
+ };
+
+ struct Data
+ {
+ Stage mStage = Stage::None;
+ RefPtr<Token> mToken;
+ RefPtr<MediaDataDecoder> mDecoder;
+ MozPromiseRequestHolder<TokenPromise> mTokenPromise;
+ MozPromiseRequestHolder<InitPromise> mInitPromise;
+ ~Data()
+ {
+ mTokenPromise.DisconnectIfExists();
+ mInitPromise.DisconnectIfExists();
+ if (mDecoder) {
+ mDecoder->Flush();
+ mDecoder->Shutdown();
+ }
+ }
+ } mAudio, mVideo;
+
+ void RunStage(TrackType aTrack);
+ MediaResult DoCreateDecoder(TrackType aTrack);
+ void DoInitDecoder(TrackType aTrack);
+
+ MediaFormatReader* const mOwner; // guaranteed to be valid by the owner.
+};
+
+void
+MediaFormatReader::DecoderFactory::CreateDecoder(TrackType aTrack)
+{
+ MOZ_ASSERT(aTrack == TrackInfo::kAudioTrack ||
+ aTrack == TrackInfo::kVideoTrack);
+ RunStage(aTrack);
+}
+
+class MediaFormatReader::DecoderFactory::Wrapper : public MediaDataDecoder
+{
+ using Token = DecoderAllocPolicy::Token;
+
+public:
+ Wrapper(already_AddRefed<MediaDataDecoder> aDecoder,
+ already_AddRefed<Token> aToken)
+ : mDecoder(aDecoder), mToken(aToken) {}
+
+ RefPtr<InitPromise> Init() override { return mDecoder->Init(); }
+ void Input(MediaRawData* aSample) override { mDecoder->Input(aSample); }
+ void Flush() override { mDecoder->Flush(); }
+ void Drain() override { mDecoder->Drain(); }
+ bool IsHardwareAccelerated(nsACString& aFailureReason) const override
+ {
+ return mDecoder->IsHardwareAccelerated(aFailureReason);
+ }
+ const char* GetDescriptionName() const override
+ {
+ return mDecoder->GetDescriptionName();
+ }
+ void SetSeekThreshold(const media::TimeUnit& aTime) override
+ {
+ mDecoder->SetSeekThreshold(aTime);
+ }
+ void Shutdown() override
+ {
+ mDecoder->Shutdown();
+ mDecoder = nullptr;
+ mToken = nullptr;
+ }
+
+private:
+ RefPtr<MediaDataDecoder> mDecoder;
+ RefPtr<Token> mToken;
+};
+
+void
+MediaFormatReader::DecoderFactory::RunStage(TrackType aTrack)
+{
+ auto& data = aTrack == TrackInfo::kAudioTrack ? mAudio : mVideo;
+
+ switch (data.mStage) {
+ case Stage::None: {
+ MOZ_ASSERT(!data.mToken);
+ data.mTokenPromise.Begin(DecoderAllocPolicy::Instance(aTrack).Alloc()->Then(
+ mOwner->OwnerThread(), __func__,
+ [this, &data, aTrack] (Token* aToken) {
+ data.mTokenPromise.Complete();
+ data.mToken = aToken;
+ data.mStage = Stage::CreateDecoder;
+ RunStage(aTrack);
+ },
+ [&data] () {
+ data.mTokenPromise.Complete();
+ data.mStage = Stage::None;
+ }));
+ data.mStage = Stage::WaitForToken;
+ break;
+ }
+
+ case Stage::WaitForToken: {
+ MOZ_ASSERT(!data.mToken);
+ MOZ_ASSERT(data.mTokenPromise.Exists());
+ break;
+ }
+
+ case Stage::CreateDecoder: {
+ MOZ_ASSERT(data.mToken);
+ MOZ_ASSERT(!data.mDecoder);
+ MOZ_ASSERT(!data.mInitPromise.Exists());
+
+ MediaResult rv = DoCreateDecoder(aTrack);
+ if (NS_FAILED(rv)) {
+ NS_WARNING("Error constructing decoders");
+ data.mToken = nullptr;
+ data.mStage = Stage::None;
+ mOwner->NotifyError(aTrack, rv);
+ return;
+ }
+
+ data.mDecoder = new Wrapper(data.mDecoder.forget(), data.mToken.forget());
+ DoInitDecoder(aTrack);
+ data.mStage = Stage::WaitForInit;
+ break;
+ }
+
+ case Stage::WaitForInit: {
+ MOZ_ASSERT(data.mDecoder);
+ MOZ_ASSERT(data.mInitPromise.Exists());
+ break;
+ }
+ }
+}
+
+MediaResult
+MediaFormatReader::DecoderFactory::DoCreateDecoder(TrackType aTrack)
+{
+ auto& ownerData = mOwner->GetDecoderData(aTrack);
+ auto& data = aTrack == TrackInfo::kAudioTrack ? mAudio : mVideo;
+
+ auto decoderCreatingError = "error creating audio decoder";
+ MediaResult result = MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR, decoderCreatingError);
+
+ if (!mOwner->mPlatform) {
+ mOwner->mPlatform = new PDMFactory();
+ if (mOwner->IsEncrypted()) {
+ MOZ_ASSERT(mOwner->mCDMProxy);
+ mOwner->mPlatform->SetCDMProxy(mOwner->mCDMProxy);
+ }
+ }
+
+ switch (aTrack) {
+ case TrackInfo::kAudioTrack: {
+ data.mDecoder = mOwner->mPlatform->CreateDecoder({
+ ownerData.mInfo
+ ? *ownerData.mInfo->GetAsAudioInfo()
+ : *ownerData.mOriginalInfo->GetAsAudioInfo(),
+ ownerData.mTaskQueue,
+ ownerData.mCallback.get(),
+ mOwner->mCrashHelper,
+ ownerData.mIsBlankDecode,
+ &result
+ });
+ break;
+ }
+
+ case TrackType::kVideoTrack: {
+ // Decoders use the layers backend to decide if they can use hardware decoding,
+ // so specify LAYERS_NONE if we want to forcibly disable it.
+ data.mDecoder = mOwner->mPlatform->CreateDecoder({
+ ownerData.mInfo
+ ? *ownerData.mInfo->GetAsVideoInfo()
+ : *ownerData.mOriginalInfo->GetAsVideoInfo(),
+ ownerData.mTaskQueue,
+ ownerData.mCallback.get(),
+ mOwner->mKnowsCompositor,
+ mOwner->GetImageContainer(),
+ mOwner->mCrashHelper,
+ ownerData.mIsBlankDecode,
+ &result
+ });
+ break;
+ }
+
+ default:
+ break;
+ }
+
+ if (data.mDecoder) {
+ result = MediaResult(NS_OK);
+ return result;
+ }
+
+ ownerData.mDescription = decoderCreatingError;
+ return result;
+}
+
+void
+MediaFormatReader::DecoderFactory::DoInitDecoder(TrackType aTrack)
+{
+ auto& ownerData = mOwner->GetDecoderData(aTrack);
+ auto& data = aTrack == TrackInfo::kAudioTrack ? mAudio : mVideo;
+
+ data.mInitPromise.Begin(data.mDecoder->Init()->Then(
+ mOwner->OwnerThread(), __func__,
+ [this, &data, &ownerData] (TrackType aTrack) {
+ data.mInitPromise.Complete();
+ data.mStage = Stage::None;
+ MonitorAutoLock mon(ownerData.mMonitor);
+ ownerData.mDecoder = data.mDecoder.forget();
+ ownerData.mDescription = ownerData.mDecoder->GetDescriptionName();
+ mOwner->SetVideoDecodeThreshold();
+ mOwner->ScheduleUpdate(aTrack);
+ },
+ [this, &data, aTrack] (MediaResult aError) {
+ data.mInitPromise.Complete();
+ data.mStage = Stage::None;
+ data.mDecoder->Shutdown();
+ data.mDecoder = nullptr;
+ mOwner->NotifyError(aTrack, aError);
+ }));
+}
+
+static const char*
+TrackTypeToStr(TrackInfo::TrackType aTrack)
+{
+ MOZ_ASSERT(aTrack == TrackInfo::kAudioTrack ||
+ aTrack == TrackInfo::kVideoTrack ||
+ aTrack == TrackInfo::kTextTrack);
+ switch (aTrack) {
+ case TrackInfo::kAudioTrack:
+ return "Audio";
+ case TrackInfo::kVideoTrack:
+ return "Video";
+ case TrackInfo::kTextTrack:
+ return "Text";
+ default:
+ return "Unknown";
+ }
+}
+
+MediaFormatReader::MediaFormatReader(AbstractMediaDecoder* aDecoder,
+ MediaDataDemuxer* aDemuxer,
+ VideoFrameContainer* aVideoFrameContainer)
+ : MediaDecoderReader(aDecoder)
+ , mAudio(this, MediaData::AUDIO_DATA,
+ Preferences::GetUint("media.audio-max-decode-error", 3))
+ , mVideo(this, MediaData::VIDEO_DATA,
+ Preferences::GetUint("media.video-max-decode-error", 2))
+ , mDemuxer(aDemuxer)
+ , mDemuxerInitDone(false)
+ , mLastReportedNumDecodedFrames(0)
+ , mPreviousDecodedKeyframeTime_us(sNoPreviousDecodedKeyframe)
+ , mInitDone(false)
+ , mTrackDemuxersMayBlock(false)
+ , mDemuxOnly(false)
+ , mSeekScheduled(false)
+ , mVideoFrameContainer(aVideoFrameContainer)
+ , mDecoderFactory(new DecoderFactory(this))
+{
+ MOZ_ASSERT(aDemuxer);
+ MOZ_COUNT_CTOR(MediaFormatReader);
+
+ if (aDecoder && aDecoder->CompositorUpdatedEvent()) {
+ mCompositorUpdatedListener =
+ aDecoder->CompositorUpdatedEvent()->Connect(
+ mTaskQueue, this, &MediaFormatReader::NotifyCompositorUpdated);
+ }
+}
+
+MediaFormatReader::~MediaFormatReader()
+{
+ MOZ_COUNT_DTOR(MediaFormatReader);
+}
+
+RefPtr<ShutdownPromise>
+MediaFormatReader::Shutdown()
+{
+ MOZ_ASSERT(OnTaskQueue());
+
+ mDecoderFactory = nullptr;
+ mDemuxerInitRequest.DisconnectIfExists();
+ mMetadataPromise.RejectIfExists(NS_ERROR_DOM_MEDIA_CANCELED, __func__);
+ mSeekPromise.RejectIfExists(NS_ERROR_DOM_MEDIA_CANCELED, __func__);
+ mSkipRequest.DisconnectIfExists();
+
+ if (mAudio.mDecoder) {
+ Reset(TrackInfo::kAudioTrack);
+ if (mAudio.HasPromise()) {
+ mAudio.RejectPromise(NS_ERROR_DOM_MEDIA_CANCELED, __func__);
+ }
+ mAudio.ShutdownDecoder();
+ }
+ if (mAudio.mTrackDemuxer) {
+ mAudio.ResetDemuxer();
+ mAudio.mTrackDemuxer->BreakCycles();
+ mAudio.mTrackDemuxer = nullptr;
+ }
+ if (mAudio.mTaskQueue) {
+ mAudio.mTaskQueue->BeginShutdown();
+ mAudio.mTaskQueue->AwaitShutdownAndIdle();
+ mAudio.mTaskQueue = nullptr;
+ }
+ MOZ_ASSERT(!mAudio.HasPromise());
+
+ if (mVideo.mDecoder) {
+ Reset(TrackInfo::kVideoTrack);
+ if (mVideo.HasPromise()) {
+ mVideo.RejectPromise(NS_ERROR_DOM_MEDIA_CANCELED, __func__);
+ }
+ mVideo.ShutdownDecoder();
+ }
+ if (mVideo.mTrackDemuxer) {
+ mVideo.ResetDemuxer();
+ mVideo.mTrackDemuxer->BreakCycles();
+ mVideo.mTrackDemuxer = nullptr;
+ }
+ if (mVideo.mTaskQueue) {
+ mVideo.mTaskQueue->BeginShutdown();
+ mVideo.mTaskQueue->AwaitShutdownAndIdle();
+ mVideo.mTaskQueue = nullptr;
+ }
+ MOZ_ASSERT(!mVideo.HasPromise());
+
+ mDemuxer = nullptr;
+ mPlatform = nullptr;
+ mVideoFrameContainer = nullptr;
+
+ mCompositorUpdatedListener.DisconnectIfExists();
+
+ return MediaDecoderReader::Shutdown();
+}
+
+void
+MediaFormatReader::InitLayersBackendType()
+{
+ // Extract the layer manager backend type so that platform decoders
+ // can determine whether it's worthwhile using hardware accelerated
+ // video decoding.
+ if (!mDecoder) {
+ return;
+ }
+ MediaDecoderOwner* owner = mDecoder->GetOwner();
+ if (!owner) {
+ NS_WARNING("MediaFormatReader without a decoder owner, can't get HWAccel");
+ return;
+ }
+
+ dom::HTMLMediaElement* element = owner->GetMediaElement();
+ NS_ENSURE_TRUE_VOID(element);
+
+ RefPtr<LayerManager> layerManager =
+ nsContentUtils::LayerManagerForDocument(element->OwnerDoc());
+ NS_ENSURE_TRUE_VOID(layerManager);
+
+ mKnowsCompositor = layerManager->AsShadowForwarder();
+}
+
+nsresult
+MediaFormatReader::InitInternal()
+{
+ MOZ_ASSERT(NS_IsMainThread(), "Must be on main thread.");
+
+ InitLayersBackendType();
+
+ mAudio.mTaskQueue =
+ new TaskQueue(GetMediaThreadPool(MediaThreadType::PLATFORM_DECODER));
+ mVideo.mTaskQueue =
+ new TaskQueue(GetMediaThreadPool(MediaThreadType::PLATFORM_DECODER));
+
+ // Note: GMPCrashHelper must be created on main thread, as it may use
+ // weak references, which aren't threadsafe.
+ mCrashHelper = mDecoder->GetCrashHelper();
+
+ return NS_OK;
+}
+
+class DispatchKeyNeededEvent : public Runnable {
+public:
+ DispatchKeyNeededEvent(AbstractMediaDecoder* aDecoder,
+ nsTArray<uint8_t>& aInitData,
+ const nsString& aInitDataType)
+ : mDecoder(aDecoder)
+ , mInitData(aInitData)
+ , mInitDataType(aInitDataType)
+ {
+ }
+ NS_IMETHOD Run() override {
+ // Note: Null check the owner, as the decoder could have been shutdown
+ // since this event was dispatched.
+ MediaDecoderOwner* owner = mDecoder->GetOwner();
+ if (owner) {
+ owner->DispatchEncrypted(mInitData, mInitDataType);
+ }
+ mDecoder = nullptr;
+ return NS_OK;
+ }
+private:
+ RefPtr<AbstractMediaDecoder> mDecoder;
+ nsTArray<uint8_t> mInitData;
+ nsString mInitDataType;
+};
+
+void
+MediaFormatReader::SetCDMProxy(CDMProxy* aProxy)
+{
+ RefPtr<CDMProxy> proxy = aProxy;
+ RefPtr<MediaFormatReader> self = this;
+ nsCOMPtr<nsIRunnable> r = NS_NewRunnableFunction([=] () {
+ MOZ_ASSERT(self->OnTaskQueue());
+ self->mCDMProxy = proxy;
+ });
+ OwnerThread()->Dispatch(r.forget());
+}
+
+bool
+MediaFormatReader::IsWaitingOnCDMResource() {
+ MOZ_ASSERT(OnTaskQueue());
+ return IsEncrypted() && !mCDMProxy;
+}
+
+RefPtr<MediaDecoderReader::MetadataPromise>
+MediaFormatReader::AsyncReadMetadata()
+{
+ MOZ_ASSERT(OnTaskQueue());
+
+ MOZ_DIAGNOSTIC_ASSERT(mMetadataPromise.IsEmpty());
+
+ if (mInitDone) {
+ // We are returning from dormant.
+ RefPtr<MetadataHolder> metadata = new MetadataHolder();
+ metadata->mInfo = mInfo;
+ metadata->mTags = nullptr;
+ return MetadataPromise::CreateAndResolve(metadata, __func__);
+ }
+
+ RefPtr<MetadataPromise> p = mMetadataPromise.Ensure(__func__);
+
+ mDemuxerInitRequest.Begin(mDemuxer->Init()
+ ->Then(OwnerThread(), __func__, this,
+ &MediaFormatReader::OnDemuxerInitDone,
+ &MediaFormatReader::OnDemuxerInitFailed));
+ return p;
+}
+
+void
+MediaFormatReader::OnDemuxerInitDone(nsresult)
+{
+ MOZ_ASSERT(OnTaskQueue());
+ mDemuxerInitRequest.Complete();
+
+ mDemuxerInitDone = true;
+
+ UniquePtr<MetadataTags> tags(MakeUnique<MetadataTags>());
+
+ RefPtr<PDMFactory> platform;
+ if (!IsWaitingOnCDMResource()) {
+ platform = new PDMFactory();
+ }
+
+ // To decode, we need valid video and a place to put it.
+ bool videoActive = !!mDemuxer->GetNumberTracks(TrackInfo::kVideoTrack) &&
+ GetImageContainer();
+
+ if (videoActive) {
+ // We currently only handle the first video track.
+ mVideo.mTrackDemuxer = mDemuxer->GetTrackDemuxer(TrackInfo::kVideoTrack, 0);
+ if (!mVideo.mTrackDemuxer) {
+ mMetadataPromise.Reject(NS_ERROR_DOM_MEDIA_METADATA_ERR, __func__);
+ return;
+ }
+
+ UniquePtr<TrackInfo> videoInfo = mVideo.mTrackDemuxer->GetInfo();
+ videoActive = videoInfo && videoInfo->IsValid();
+ if (videoActive) {
+ if (platform && !platform->SupportsMimeType(videoInfo->mMimeType, nullptr)) {
+ // We have no decoder for this track. Error.
+ mMetadataPromise.Reject(NS_ERROR_DOM_MEDIA_METADATA_ERR, __func__);
+ return;
+ }
+ mInfo.mVideo = *videoInfo->GetAsVideoInfo();
+ for (const MetadataTag& tag : videoInfo->mTags) {
+ tags->Put(tag.mKey, tag.mValue);
+ }
+ mVideo.mOriginalInfo = Move(videoInfo);
+ mVideo.mCallback = new DecoderCallback(this, TrackInfo::kVideoTrack);
+ mVideo.mTimeRanges = mVideo.mTrackDemuxer->GetBuffered();
+ mTrackDemuxersMayBlock |= mVideo.mTrackDemuxer->GetSamplesMayBlock();
+ } else {
+ mVideo.mTrackDemuxer->BreakCycles();
+ mVideo.mTrackDemuxer = nullptr;
+ }
+ }
+
+ bool audioActive = !!mDemuxer->GetNumberTracks(TrackInfo::kAudioTrack);
+ if (audioActive) {
+ mAudio.mTrackDemuxer = mDemuxer->GetTrackDemuxer(TrackInfo::kAudioTrack, 0);
+ if (!mAudio.mTrackDemuxer) {
+ mMetadataPromise.Reject(NS_ERROR_DOM_MEDIA_METADATA_ERR, __func__);
+ return;
+ }
+
+ UniquePtr<TrackInfo> audioInfo = mAudio.mTrackDemuxer->GetInfo();
+ // We actively ignore audio tracks that we know we can't play.
+ audioActive = audioInfo && audioInfo->IsValid() &&
+ (!platform ||
+ platform->SupportsMimeType(audioInfo->mMimeType, nullptr));
+
+ if (audioActive) {
+ mInfo.mAudio = *audioInfo->GetAsAudioInfo();
+ for (const MetadataTag& tag : audioInfo->mTags) {
+ tags->Put(tag.mKey, tag.mValue);
+ }
+ mAudio.mOriginalInfo = Move(audioInfo);
+ mAudio.mCallback = new DecoderCallback(this, TrackInfo::kAudioTrack);
+ mAudio.mTimeRanges = mAudio.mTrackDemuxer->GetBuffered();
+ mTrackDemuxersMayBlock |= mAudio.mTrackDemuxer->GetSamplesMayBlock();
+ } else {
+ mAudio.mTrackDemuxer->BreakCycles();
+ mAudio.mTrackDemuxer = nullptr;
+ }
+ }
+
+ UniquePtr<EncryptionInfo> crypto = mDemuxer->GetCrypto();
+ if (mDecoder && crypto && crypto->IsEncrypted()) {
+ // Try and dispatch 'encrypted'. Won't go if ready state still HAVE_NOTHING.
+ for (uint32_t i = 0; i < crypto->mInitDatas.Length(); i++) {
+ NS_DispatchToMainThread(
+ new DispatchKeyNeededEvent(mDecoder, crypto->mInitDatas[i].mInitData, crypto->mInitDatas[i].mType));
+ }
+ mInfo.mCrypto = *crypto;
+ }
+
+ int64_t videoDuration = HasVideo() ? mInfo.mVideo.mDuration : 0;
+ int64_t audioDuration = HasAudio() ? mInfo.mAudio.mDuration : 0;
+
+ int64_t duration = std::max(videoDuration, audioDuration);
+ if (duration != -1) {
+ mInfo.mMetadataDuration = Some(TimeUnit::FromMicroseconds(duration));
+ }
+
+ mInfo.mMediaSeekable = mDemuxer->IsSeekable();
+ mInfo.mMediaSeekableOnlyInBufferedRanges =
+ mDemuxer->IsSeekableOnlyInBufferedRanges();
+
+ if (!videoActive && !audioActive) {
+ mMetadataPromise.Reject(NS_ERROR_DOM_MEDIA_METADATA_ERR, __func__);
+ return;
+ }
+
+ mInitDone = true;
+ RefPtr<MetadataHolder> metadata = new MetadataHolder();
+ metadata->mInfo = mInfo;
+ metadata->mTags = tags->Count() ? tags.release() : nullptr;
+ mMetadataPromise.Resolve(metadata, __func__);
+}
+
+bool
+MediaFormatReader::IsEncrypted() const
+{
+ return (HasAudio() && mInfo.mAudio.mCrypto.mValid) ||
+ (HasVideo() && mInfo.mVideo.mCrypto.mValid);
+}
+
+void
+MediaFormatReader::OnDemuxerInitFailed(const MediaResult& aError)
+{
+ mDemuxerInitRequest.Complete();
+ mMetadataPromise.Reject(aError, __func__);
+}
+
+void
+MediaFormatReader::ReadUpdatedMetadata(MediaInfo* aInfo)
+{
+ *aInfo = mInfo;
+}
+
+MediaFormatReader::DecoderData&
+MediaFormatReader::GetDecoderData(TrackType aTrack)
+{
+ MOZ_ASSERT(aTrack == TrackInfo::kAudioTrack ||
+ aTrack == TrackInfo::kVideoTrack);
+ if (aTrack == TrackInfo::kAudioTrack) {
+ return mAudio;
+ }
+ return mVideo;
+}
+
+bool
+MediaFormatReader::ShouldSkip(bool aSkipToNextKeyframe, media::TimeUnit aTimeThreshold)
+{
+ MOZ_ASSERT(HasVideo());
+ media::TimeUnit nextKeyframe;
+ nsresult rv = mVideo.mTrackDemuxer->GetNextRandomAccessPoint(&nextKeyframe);
+ if (NS_FAILED(rv)) {
+ return aSkipToNextKeyframe;
+ }
+ return (nextKeyframe < aTimeThreshold ||
+ (mVideo.mTimeThreshold &&
+ mVideo.mTimeThreshold.ref().EndTime() < aTimeThreshold)) &&
+ nextKeyframe.ToMicroseconds() >= 0 && !nextKeyframe.IsInfinite();
+}
+
+RefPtr<MediaDecoderReader::MediaDataPromise>
+MediaFormatReader::RequestVideoData(bool aSkipToNextKeyframe,
+ int64_t aTimeThreshold)
+{
+ MOZ_ASSERT(OnTaskQueue());
+ MOZ_DIAGNOSTIC_ASSERT(mSeekPromise.IsEmpty(), "No sample requests allowed while seeking");
+ MOZ_DIAGNOSTIC_ASSERT(!mVideo.HasPromise(), "No duplicate sample requests");
+ MOZ_DIAGNOSTIC_ASSERT(!mVideo.mSeekRequest.Exists() ||
+ mVideo.mTimeThreshold.isSome());
+ MOZ_DIAGNOSTIC_ASSERT(!IsSeeking(), "called mid-seek");
+ LOGV("RequestVideoData(%d, %lld)", aSkipToNextKeyframe, aTimeThreshold);
+
+ if (!HasVideo()) {
+ LOG("called with no video track");
+ return MediaDataPromise::CreateAndReject(NS_ERROR_DOM_MEDIA_FATAL_ERR, __func__);
+ }
+
+ if (IsSeeking()) {
+ LOG("called mid-seek. Rejecting.");
+ return MediaDataPromise::CreateAndReject(NS_ERROR_DOM_MEDIA_CANCELED, __func__);
+ }
+
+ if (mShutdown) {
+ NS_WARNING("RequestVideoData on shutdown MediaFormatReader!");
+ return MediaDataPromise::CreateAndReject(NS_ERROR_DOM_MEDIA_CANCELED, __func__);
+ }
+
+ media::TimeUnit timeThreshold{media::TimeUnit::FromMicroseconds(aTimeThreshold)};
+ // Ensure we have no pending seek going as ShouldSkip could return out of date
+ // information.
+ if (!mVideo.HasInternalSeekPending() &&
+ ShouldSkip(aSkipToNextKeyframe, timeThreshold)) {
+ RefPtr<MediaDataPromise> p = mVideo.EnsurePromise(__func__);
+ SkipVideoDemuxToNextKeyFrame(timeThreshold);
+ return p;
+ }
+
+ RefPtr<MediaDataPromise> p = mVideo.EnsurePromise(__func__);
+ ScheduleUpdate(TrackInfo::kVideoTrack);
+
+ return p;
+}
+
+void
+MediaFormatReader::OnDemuxFailed(TrackType aTrack, const MediaResult& aError)
+{
+ MOZ_ASSERT(OnTaskQueue());
+ LOG("Failed to demux %s, failure:%u",
+ aTrack == TrackType::kVideoTrack ? "video" : "audio", aError.Code());
+ auto& decoder = GetDecoderData(aTrack);
+ decoder.mDemuxRequest.Complete();
+ switch (aError.Code()) {
+ case NS_ERROR_DOM_MEDIA_END_OF_STREAM:
+ if (!decoder.mWaitingForData) {
+ decoder.mNeedDraining = true;
+ }
+ NotifyEndOfStream(aTrack);
+ break;
+ case NS_ERROR_DOM_MEDIA_WAITING_FOR_DATA:
+ if (!decoder.mWaitingForData) {
+ decoder.mNeedDraining = true;
+ }
+ NotifyWaitingForData(aTrack);
+ break;
+ case NS_ERROR_DOM_MEDIA_CANCELED:
+ if (decoder.HasPromise()) {
+ decoder.RejectPromise(NS_ERROR_DOM_MEDIA_CANCELED, __func__);
+ }
+ break;
+ default:
+ NotifyError(aTrack, aError);
+ break;
+ }
+}
+
+void
+MediaFormatReader::DoDemuxVideo()
+{
+ mVideo.mDemuxRequest.Begin(mVideo.mTrackDemuxer->GetSamples(1)
+ ->Then(OwnerThread(), __func__, this,
+ &MediaFormatReader::OnVideoDemuxCompleted,
+ &MediaFormatReader::OnVideoDemuxFailed));
+}
+
+void
+MediaFormatReader::OnVideoDemuxCompleted(RefPtr<MediaTrackDemuxer::SamplesHolder> aSamples)
+{
+ LOGV("%d video samples demuxed (sid:%d)",
+ aSamples->mSamples.Length(),
+ aSamples->mSamples[0]->mTrackInfo ? aSamples->mSamples[0]->mTrackInfo->GetID() : 0);
+ mVideo.mDemuxRequest.Complete();
+ mVideo.mQueuedSamples.AppendElements(aSamples->mSamples);
+ ScheduleUpdate(TrackInfo::kVideoTrack);
+}
+
+RefPtr<MediaDecoderReader::MediaDataPromise>
+MediaFormatReader::RequestAudioData()
+{
+ MOZ_ASSERT(OnTaskQueue());
+ MOZ_DIAGNOSTIC_ASSERT(!mAudio.HasPromise(), "No duplicate sample requests");
+ MOZ_DIAGNOSTIC_ASSERT(IsVideoSeeking() || mSeekPromise.IsEmpty(),
+ "No sample requests allowed while seeking");
+ MOZ_DIAGNOSTIC_ASSERT(IsVideoSeeking() ||
+ !mAudio.mSeekRequest.Exists() ||
+ mAudio.mTimeThreshold.isSome());
+ MOZ_DIAGNOSTIC_ASSERT(IsVideoSeeking() || !IsSeeking(), "called mid-seek");
+ LOGV("");
+
+ if (!HasAudio()) {
+ LOG("called with no audio track");
+ return MediaDataPromise::CreateAndReject(NS_ERROR_DOM_MEDIA_FATAL_ERR, __func__);
+ }
+
+ if (IsSeeking()) {
+ LOG("called mid-seek. Rejecting.");
+ return MediaDataPromise::CreateAndReject(NS_ERROR_DOM_MEDIA_CANCELED, __func__);
+ }
+
+ if (mShutdown) {
+ NS_WARNING("RequestAudioData on shutdown MediaFormatReader!");
+ return MediaDataPromise::CreateAndReject(NS_ERROR_DOM_MEDIA_CANCELED, __func__);
+ }
+
+ RefPtr<MediaDataPromise> p = mAudio.EnsurePromise(__func__);
+ ScheduleUpdate(TrackInfo::kAudioTrack);
+
+ return p;
+}
+
+void
+MediaFormatReader::DoDemuxAudio()
+{
+ mAudio.mDemuxRequest.Begin(mAudio.mTrackDemuxer->GetSamples(1)
+ ->Then(OwnerThread(), __func__, this,
+ &MediaFormatReader::OnAudioDemuxCompleted,
+ &MediaFormatReader::OnAudioDemuxFailed));
+}
+
+void
+MediaFormatReader::OnAudioDemuxCompleted(RefPtr<MediaTrackDemuxer::SamplesHolder> aSamples)
+{
+ LOGV("%d audio samples demuxed (sid:%d)",
+ aSamples->mSamples.Length(),
+ aSamples->mSamples[0]->mTrackInfo ? aSamples->mSamples[0]->mTrackInfo->GetID() : 0);
+ mAudio.mDemuxRequest.Complete();
+ mAudio.mQueuedSamples.AppendElements(aSamples->mSamples);
+ ScheduleUpdate(TrackInfo::kAudioTrack);
+}
+
+void
+MediaFormatReader::NotifyNewOutput(TrackType aTrack, MediaData* aSample)
+{
+ MOZ_ASSERT(OnTaskQueue());
+ LOGV("Received new %s sample time:%lld duration:%lld",
+ TrackTypeToStr(aTrack), aSample->mTime, aSample->mDuration);
+ auto& decoder = GetDecoderData(aTrack);
+ if (!decoder.mOutputRequested) {
+ LOG("MediaFormatReader produced output while flushing, discarding.");
+ return;
+ }
+ decoder.mOutput.AppendElement(aSample);
+ decoder.mNumSamplesOutput++;
+ decoder.mNumOfConsecutiveError = 0;
+ ScheduleUpdate(aTrack);
+}
+
+void
+MediaFormatReader::NotifyInputExhausted(TrackType aTrack)
+{
+ MOZ_ASSERT(OnTaskQueue());
+ LOGV("Decoder has requested more %s data", TrackTypeToStr(aTrack));
+ auto& decoder = GetDecoderData(aTrack);
+ decoder.mDecodePending = false;
+ ScheduleUpdate(aTrack);
+}
+
+void
+MediaFormatReader::NotifyDrainComplete(TrackType aTrack)
+{
+ MOZ_ASSERT(OnTaskQueue());
+ auto& decoder = GetDecoderData(aTrack);
+ LOG("%s", TrackTypeToStr(aTrack));
+ if (!decoder.mOutputRequested) {
+ LOG("MediaFormatReader called DrainComplete() before flushing, ignoring.");
+ return;
+ }
+ decoder.mDrainComplete = true;
+ ScheduleUpdate(aTrack);
+}
+
+void
+MediaFormatReader::NotifyError(TrackType aTrack, const MediaResult& aError)
+{
+ MOZ_ASSERT(OnTaskQueue());
+ NS_WARNING(aError.Description().get());
+ LOGV("%s Decoding error", TrackTypeToStr(aTrack));
+ auto& decoder = GetDecoderData(aTrack);
+ decoder.mError = decoder.HasFatalError() ? decoder.mError : Some(aError);
+ ScheduleUpdate(aTrack);
+}
+
+void
+MediaFormatReader::NotifyWaitingForData(TrackType aTrack)
+{
+ MOZ_ASSERT(OnTaskQueue());
+ auto& decoder = GetDecoderData(aTrack);
+ decoder.mWaitingForData = true;
+ if (decoder.mTimeThreshold) {
+ decoder.mTimeThreshold.ref().mWaiting = true;
+ }
+ ScheduleUpdate(aTrack);
+}
+
+void
+MediaFormatReader::NotifyWaitingForKey(TrackType aTrack)
+{
+ MOZ_ASSERT(OnTaskQueue());
+ auto& decoder = GetDecoderData(aTrack);
+ if (mDecoder) {
+ mDecoder->NotifyWaitingForKey();
+ }
+ if (!decoder.mDecodePending) {
+ LOGV("WaitingForKey received while no pending decode. Ignoring");
+ }
+ decoder.mWaitingForKey = true;
+ ScheduleUpdate(aTrack);
+}
+
+void
+MediaFormatReader::NotifyEndOfStream(TrackType aTrack)
+{
+ MOZ_ASSERT(OnTaskQueue());
+ auto& decoder = GetDecoderData(aTrack);
+ decoder.mDemuxEOS = true;
+ ScheduleUpdate(aTrack);
+}
+
+bool
+MediaFormatReader::NeedInput(DecoderData& aDecoder)
+{
+ // To account for H.264 streams which may require a longer
+ // run of input than we input, decoders fire an "input exhausted" callback.
+ // The decoder will not be fed a new raw sample until InputExhausted
+ // has been called.
+ return
+ (aDecoder.HasPromise() || aDecoder.mTimeThreshold.isSome()) &&
+ !aDecoder.HasPendingDrain() &&
+ !aDecoder.HasFatalError() &&
+ !aDecoder.mDemuxRequest.Exists() &&
+ !aDecoder.mOutput.Length() &&
+ !aDecoder.HasInternalSeekPending() &&
+ !aDecoder.mDecodePending;
+}
+
+void
+MediaFormatReader::ScheduleUpdate(TrackType aTrack)
+{
+ MOZ_ASSERT(OnTaskQueue());
+ if (mShutdown) {
+ return;
+ }
+ auto& decoder = GetDecoderData(aTrack);
+ if (decoder.mUpdateScheduled) {
+ return;
+ }
+ LOGV("SchedulingUpdate(%s)", TrackTypeToStr(aTrack));
+ decoder.mUpdateScheduled = true;
+ RefPtr<nsIRunnable> task(
+ NewRunnableMethod<TrackType>(this, &MediaFormatReader::Update, aTrack));
+ OwnerThread()->Dispatch(task.forget());
+}
+
+bool
+MediaFormatReader::UpdateReceivedNewData(TrackType aTrack)
+{
+ MOZ_ASSERT(OnTaskQueue());
+ auto& decoder = GetDecoderData(aTrack);
+
+ if (!decoder.mReceivedNewData) {
+ return false;
+ }
+
+ // Update our cached TimeRange.
+ decoder.mTimeRanges = decoder.mTrackDemuxer->GetBuffered();
+
+ // We do not want to clear mWaitingForData while there are pending
+ // demuxing or seeking operations that could affect the value of this flag.
+ // This is in order to ensure that we will retry once they complete as we may
+ // now have new data that could potentially allow those operations to
+ // successfully complete if tried again.
+ if (decoder.mSeekRequest.Exists()) {
+ // Nothing more to do until this operation complete.
+ return true;
+ }
+
+ if (aTrack == TrackType::kVideoTrack && mSkipRequest.Exists()) {
+ LOGV("Skipping in progress, nothing more to do");
+ return true;
+ }
+
+ if (decoder.mDemuxRequest.Exists()) {
+ // We may have pending operations to process, so we want to continue
+ // after UpdateReceivedNewData returns.
+ return false;
+ }
+
+ if (decoder.HasPendingDrain()) {
+ // We do not want to clear mWaitingForData or mDemuxEOS while
+ // a drain is in progress in order to properly complete the operation.
+ return false;
+ }
+
+ bool hasLastEnd;
+ media::TimeUnit lastEnd = decoder.mTimeRanges.GetEnd(&hasLastEnd);
+ if (hasLastEnd) {
+ if (decoder.mLastTimeRangesEnd && decoder.mLastTimeRangesEnd.ref() < lastEnd) {
+ // New data was added after our previous end, we can clear the EOS flag.
+ decoder.mDemuxEOS = false;
+ }
+ decoder.mLastTimeRangesEnd = Some(lastEnd);
+ }
+
+ decoder.mReceivedNewData = false;
+ if (decoder.mTimeThreshold) {
+ decoder.mTimeThreshold.ref().mWaiting = false;
+ }
+ decoder.mWaitingForData = false;
+
+ if (decoder.HasFatalError()) {
+ return false;
+ }
+
+ if (!mSeekPromise.IsEmpty() &&
+ (!IsVideoSeeking() || aTrack == TrackInfo::kVideoTrack)) {
+ MOZ_ASSERT(!decoder.HasPromise());
+ MOZ_DIAGNOSTIC_ASSERT((IsVideoSeeking() || !mAudio.mTimeThreshold) &&
+ !mVideo.mTimeThreshold,
+ "InternalSeek must have been aborted when Seek was first called");
+ MOZ_DIAGNOSTIC_ASSERT((IsVideoSeeking() || !mAudio.HasWaitingPromise()) &&
+ !mVideo.HasWaitingPromise(),
+ "Waiting promises must have been rejected when Seek was first called");
+ if (mVideo.mSeekRequest.Exists() ||
+ (!IsVideoSeeking() && mAudio.mSeekRequest.Exists())) {
+ // Already waiting for a seek to complete. Nothing more to do.
+ return true;
+ }
+ LOG("Attempting Seek");
+ ScheduleSeek();
+ return true;
+ }
+ if (decoder.HasInternalSeekPending() || decoder.HasWaitingPromise()) {
+ if (decoder.HasInternalSeekPending()) {
+ LOG("Attempting Internal Seek");
+ InternalSeek(aTrack, decoder.mTimeThreshold.ref());
+ }
+ if (decoder.HasWaitingPromise() && !decoder.IsWaiting()) {
+ MOZ_ASSERT(!decoder.HasPromise());
+ LOG("We have new data. Resolving WaitingPromise");
+ decoder.mWaitingPromise.Resolve(decoder.mType, __func__);
+ }
+ return true;
+ }
+ return false;
+}
+
+void
+MediaFormatReader::RequestDemuxSamples(TrackType aTrack)
+{
+ MOZ_ASSERT(OnTaskQueue());
+ auto& decoder = GetDecoderData(aTrack);
+ MOZ_ASSERT(!decoder.mDemuxRequest.Exists());
+
+ if (!decoder.mQueuedSamples.IsEmpty()) {
+ // No need to demux new samples.
+ return;
+ }
+
+ if (decoder.mDemuxEOS) {
+ // Nothing left to demux.
+ // We do not want to attempt to demux while in waiting for data mode
+ // as it would retrigger an unecessary drain.
+ return;
+ }
+
+ LOGV("Requesting extra demux %s", TrackTypeToStr(aTrack));
+ if (aTrack == TrackInfo::kVideoTrack) {
+ DoDemuxVideo();
+ } else {
+ DoDemuxAudio();
+ }
+}
+
+void
+MediaFormatReader::DecodeDemuxedSamples(TrackType aTrack,
+ MediaRawData* aSample)
+{
+ MOZ_ASSERT(OnTaskQueue());
+ auto& decoder = GetDecoderData(aTrack);
+ decoder.mDecoder->Input(aSample);
+ decoder.mDecodePending = true;
+}
+
+void
+MediaFormatReader::HandleDemuxedSamples(TrackType aTrack,
+ AbstractMediaDecoder::AutoNotifyDecoded& aA)
+{
+ MOZ_ASSERT(OnTaskQueue());
+
+ auto& decoder = GetDecoderData(aTrack);
+
+ if (decoder.mQueuedSamples.IsEmpty()) {
+ return;
+ }
+
+ if (!decoder.mDecoder) {
+ mDecoderFactory->CreateDecoder(aTrack);
+ return;
+ }
+
+ if (!ForceZeroStartTime() && decoder.mFirstDemuxedSampleTime.isNothing()) {
+ decoder.mFirstDemuxedSampleTime.emplace(
+ media::TimeUnit::FromMicroseconds(decoder.mQueuedSamples[0]->mTime));
+ }
+
+ LOGV("Giving %s input to decoder", TrackTypeToStr(aTrack));
+
+ // Decode all our demuxed frames.
+ bool samplesPending = false;
+ while (decoder.mQueuedSamples.Length()) {
+ RefPtr<MediaRawData> sample = decoder.mQueuedSamples[0];
+ RefPtr<SharedTrackInfo> info = sample->mTrackInfo;
+
+ if (info && decoder.mLastStreamSourceID != info->GetID()) {
+ if (samplesPending) {
+ // Let existing samples complete their decoding. We'll resume later.
+ return;
+ }
+
+ if (decoder.mNextStreamSourceID.isNothing() ||
+ decoder.mNextStreamSourceID.ref() != info->GetID()) {
+ LOG("%s stream id has changed from:%d to:%d, draining decoder.",
+ TrackTypeToStr(aTrack), decoder.mLastStreamSourceID,
+ info->GetID());
+ decoder.mNeedDraining = true;
+ decoder.mNextStreamSourceID = Some(info->GetID());
+ ScheduleUpdate(aTrack);
+ return;
+ }
+
+ LOG("%s stream id has changed from:%d to:%d, recreating decoder.",
+ TrackTypeToStr(aTrack), decoder.mLastStreamSourceID,
+ info->GetID());
+ decoder.mLastStreamSourceID = info->GetID();
+ decoder.mNextStreamSourceID.reset();
+ // Reset will clear our array of queued samples. So make a copy now.
+ nsTArray<RefPtr<MediaRawData>> samples{decoder.mQueuedSamples};
+ Reset(aTrack);
+ decoder.ShutdownDecoder();
+ decoder.mInfo = info;
+ if (sample->mKeyframe) {
+ decoder.mQueuedSamples.AppendElements(Move(samples));
+ ScheduleUpdate(aTrack);
+ } else {
+ TimeInterval time =
+ TimeInterval(TimeUnit::FromMicroseconds(sample->mTime),
+ TimeUnit::FromMicroseconds(sample->GetEndTime()));
+ InternalSeekTarget seekTarget =
+ decoder.mTimeThreshold.refOr(InternalSeekTarget(time, false));
+ LOG("Stream change occurred on a non-keyframe. Seeking to:%lld",
+ sample->mTime);
+ InternalSeek(aTrack, seekTarget);
+ }
+ return;
+ }
+
+ LOGV("Input:%lld (dts:%lld kf:%d)",
+ sample->mTime, sample->mTimecode, sample->mKeyframe);
+ decoder.mOutputRequested = true;
+ decoder.mNumSamplesInput++;
+ decoder.mSizeOfQueue++;
+ if (aTrack == TrackInfo::kVideoTrack) {
+ aA.mStats.mParsedFrames++;
+ }
+
+ if (mDemuxOnly) {
+ ReturnOutput(sample, aTrack);
+ } else {
+ DecodeDemuxedSamples(aTrack, sample);
+ }
+
+ decoder.mQueuedSamples.RemoveElementAt(0);
+ if (mDemuxOnly) {
+ // If demuxed-only case, ReturnOutput will resolve with one demuxed data.
+ // Then we should stop doing the iteration.
+ return;
+ }
+ samplesPending = true;
+ }
+}
+
+void
+MediaFormatReader::InternalSeek(TrackType aTrack, const InternalSeekTarget& aTarget)
+{
+ MOZ_ASSERT(OnTaskQueue());
+ LOG("%s internal seek to %f",
+ TrackTypeToStr(aTrack), aTarget.Time().ToSeconds());
+
+ auto& decoder = GetDecoderData(aTrack);
+ decoder.Flush();
+ decoder.ResetDemuxer();
+ decoder.mTimeThreshold = Some(aTarget);
+ RefPtr<MediaFormatReader> self = this;
+ decoder.mSeekRequest.Begin(decoder.mTrackDemuxer->Seek(decoder.mTimeThreshold.ref().Time())
+ ->Then(OwnerThread(), __func__,
+ [self, aTrack] (media::TimeUnit aTime) {
+ auto& decoder = self->GetDecoderData(aTrack);
+ decoder.mSeekRequest.Complete();
+ MOZ_ASSERT(decoder.mTimeThreshold,
+ "Seek promise must be disconnected when timethreshold is reset");
+ decoder.mTimeThreshold.ref().mHasSeeked = true;
+ self->SetVideoDecodeThreshold();
+ self->ScheduleUpdate(aTrack);
+ },
+ [self, aTrack] (const MediaResult& aError) {
+ auto& decoder = self->GetDecoderData(aTrack);
+ decoder.mSeekRequest.Complete();
+ switch (aError.Code()) {
+ case NS_ERROR_DOM_MEDIA_WAITING_FOR_DATA:
+ self->NotifyWaitingForData(aTrack);
+ break;
+ case NS_ERROR_DOM_MEDIA_END_OF_STREAM:
+ decoder.mTimeThreshold.reset();
+ self->NotifyEndOfStream(aTrack);
+ break;
+ case NS_ERROR_DOM_MEDIA_CANCELED:
+ decoder.mTimeThreshold.reset();
+ break;
+ default:
+ decoder.mTimeThreshold.reset();
+ self->NotifyError(aTrack, aError);
+ break;
+ }
+ }));
+}
+
+void
+MediaFormatReader::DrainDecoder(TrackType aTrack)
+{
+ MOZ_ASSERT(OnTaskQueue());
+
+ auto& decoder = GetDecoderData(aTrack);
+ if (!decoder.mNeedDraining || decoder.mDraining) {
+ return;
+ }
+ decoder.mNeedDraining = false;
+ // mOutputRequest must be set, otherwise NotifyDrainComplete()
+ // may reject the drain if a Flush recently occurred.
+ decoder.mOutputRequested = true;
+ if (!decoder.mDecoder ||
+ decoder.mNumSamplesInput == decoder.mNumSamplesOutput) {
+ // No frames to drain.
+ NotifyDrainComplete(aTrack);
+ return;
+ }
+ decoder.mDecoder->Drain();
+ decoder.mDraining = true;
+ LOG("Requesting %s decoder to drain", TrackTypeToStr(aTrack));
+}
+
+void
+MediaFormatReader::Update(TrackType aTrack)
+{
+ MOZ_ASSERT(OnTaskQueue());
+
+ if (mShutdown) {
+ return;
+ }
+
+ LOGV("Processing update for %s", TrackTypeToStr(aTrack));
+
+ bool needOutput = false;
+ auto& decoder = GetDecoderData(aTrack);
+ decoder.mUpdateScheduled = false;
+
+ if (!mInitDone) {
+ return;
+ }
+
+ if (aTrack == TrackType::kVideoTrack && mSkipRequest.Exists()) {
+ LOGV("Skipping in progress, nothing more to do");
+ return;
+ }
+
+ if (decoder.HasWaitingPromise() && decoder.mDrainComplete) {
+ // This situation will occur when a change of stream ID occurred during
+ // internal seeking following a gap encountered in the data, a drain was
+ // requested and has now completed. We need to complete the draining process
+ // so that the new data can be processed.
+ // We can complete the draining operation now as we have no pending
+ // operation when a waiting promise is pending.
+ decoder.mDrainComplete = false;
+ decoder.mDraining = false;
+ }
+
+ if (UpdateReceivedNewData(aTrack)) {
+ LOGV("Nothing more to do");
+ return;
+ }
+
+ if (decoder.mSeekRequest.Exists()) {
+ LOGV("Seeking hasn't completed, nothing more to do");
+ return;
+ }
+
+ MOZ_DIAGNOSTIC_ASSERT(!decoder.HasInternalSeekPending() ||
+ (!decoder.mOutput.Length() &&
+ !decoder.mQueuedSamples.Length()),
+ "No frames can be demuxed or decoded while an internal seek is pending");
+
+ // Record number of frames decoded and parsed. Automatically update the
+ // stats counters using the AutoNotifyDecoded stack-based class.
+ AbstractMediaDecoder::AutoNotifyDecoded a(mDecoder);
+
+ // Drop any frames found prior our internal seek target.
+ while (decoder.mTimeThreshold && decoder.mOutput.Length()) {
+ RefPtr<MediaData>& output = decoder.mOutput[0];
+ InternalSeekTarget target = decoder.mTimeThreshold.ref();
+ media::TimeUnit time = media::TimeUnit::FromMicroseconds(output->mTime);
+ if (time >= target.Time()) {
+ // We have reached our internal seek target.
+ decoder.mTimeThreshold.reset();
+ // We might have dropped some keyframes.
+ mPreviousDecodedKeyframeTime_us = sNoPreviousDecodedKeyframe;
+ }
+ if (time < target.Time() || (target.mDropTarget && target.Contains(time))) {
+ LOGV("Internal Seeking: Dropping %s frame time:%f wanted:%f (kf:%d)",
+ TrackTypeToStr(aTrack),
+ media::TimeUnit::FromMicroseconds(output->mTime).ToSeconds(),
+ target.Time().ToSeconds(),
+ output->mKeyframe);
+ decoder.mOutput.RemoveElementAt(0);
+ decoder.mSizeOfQueue -= 1;
+ }
+ }
+
+ while (decoder.mOutput.Length() && decoder.mOutput[0]->mType == MediaData::NULL_DATA) {
+ LOGV("Dropping null data. Time: %lld", decoder.mOutput[0]->mTime);
+ decoder.mOutput.RemoveElementAt(0);
+ decoder.mSizeOfQueue -= 1;
+ }
+
+ if (decoder.HasPromise()) {
+ needOutput = true;
+ if (decoder.mOutput.Length()) {
+ RefPtr<MediaData> output = decoder.mOutput[0];
+ decoder.mOutput.RemoveElementAt(0);
+ decoder.mSizeOfQueue -= 1;
+ decoder.mLastSampleTime =
+ Some(TimeInterval(TimeUnit::FromMicroseconds(output->mTime),
+ TimeUnit::FromMicroseconds(output->GetEndTime())));
+ decoder.mNumSamplesOutputTotal++;
+ ReturnOutput(output, aTrack);
+ // We have a decoded sample ready to be returned.
+ if (aTrack == TrackType::kVideoTrack) {
+ uint64_t delta =
+ decoder.mNumSamplesOutputTotal - mLastReportedNumDecodedFrames;
+ a.mStats.mDecodedFrames = static_cast<uint32_t>(delta);
+ mLastReportedNumDecodedFrames = decoder.mNumSamplesOutputTotal;
+ if (output->mKeyframe) {
+ if (mPreviousDecodedKeyframeTime_us < output->mTime) {
+ // There is a previous keyframe -> Record inter-keyframe stats.
+ uint64_t segment_us = output->mTime - mPreviousDecodedKeyframeTime_us;
+ a.mStats.mInterKeyframeSum_us += segment_us;
+ a.mStats.mInterKeyframeCount += 1;
+ if (a.mStats.mInterKeyFrameMax_us < segment_us) {
+ a.mStats.mInterKeyFrameMax_us = segment_us;
+ }
+ }
+ mPreviousDecodedKeyframeTime_us = output->mTime;
+ }
+ nsCString error;
+ mVideo.mIsHardwareAccelerated =
+ mVideo.mDecoder && mVideo.mDecoder->IsHardwareAccelerated(error);
+ }
+ } else if (decoder.HasFatalError()) {
+ LOG("Rejecting %s promise: DECODE_ERROR", TrackTypeToStr(aTrack));
+ decoder.RejectPromise(decoder.mError.ref(), __func__);
+ return;
+ } else if (decoder.mDrainComplete) {
+ bool wasDraining = decoder.mDraining;
+ decoder.mDrainComplete = false;
+ decoder.mDraining = false;
+ if (decoder.mDemuxEOS) {
+ LOG("Rejecting %s promise: EOS", TrackTypeToStr(aTrack));
+ decoder.RejectPromise(NS_ERROR_DOM_MEDIA_END_OF_STREAM, __func__);
+ } else if (decoder.mWaitingForData) {
+ if (wasDraining && decoder.mLastSampleTime &&
+ !decoder.mNextStreamSourceID) {
+ // We have completed draining the decoder following WaitingForData.
+ // Set up the internal seek machinery to be able to resume from the
+ // last sample decoded.
+ LOG("Seeking to last sample time: %lld",
+ decoder.mLastSampleTime.ref().mStart.ToMicroseconds());
+ InternalSeek(aTrack, InternalSeekTarget(decoder.mLastSampleTime.ref(), true));
+ }
+ if (!decoder.mReceivedNewData) {
+ LOG("Rejecting %s promise: WAITING_FOR_DATA", TrackTypeToStr(aTrack));
+ decoder.RejectPromise(NS_ERROR_DOM_MEDIA_WAITING_FOR_DATA, __func__);
+ }
+ }
+ // Now that draining has completed, we check if we have received
+ // new data again as the result may now be different from the earlier
+ // run.
+ if (UpdateReceivedNewData(aTrack) || decoder.mSeekRequest.Exists()) {
+ LOGV("Nothing more to do");
+ return;
+ }
+ } else if (decoder.mDemuxEOS && !decoder.mNeedDraining &&
+ !decoder.HasPendingDrain() && decoder.mQueuedSamples.IsEmpty()) {
+ // It is possible to transition from WAITING_FOR_DATA directly to EOS
+ // state during the internal seek; in which case no draining would occur.
+ // There is no more samples left to be decoded and we are already in
+ // EOS state. We can immediately reject the data promise.
+ LOG("Rejecting %s promise: EOS", TrackTypeToStr(aTrack));
+ decoder.RejectPromise(NS_ERROR_DOM_MEDIA_END_OF_STREAM, __func__);
+ } else if (decoder.mWaitingForKey) {
+ LOG("Rejecting %s promise: WAITING_FOR_DATA due to waiting for key",
+ TrackTypeToStr(aTrack));
+ decoder.RejectPromise(NS_ERROR_DOM_MEDIA_WAITING_FOR_DATA, __func__);
+ }
+ }
+
+ if (decoder.mNeedDraining) {
+ DrainDecoder(aTrack);
+ return;
+ }
+
+ if (decoder.mError && !decoder.HasFatalError()) {
+ decoder.mDecodePending = false;
+ bool needsNewDecoder = decoder.mError.ref() == NS_ERROR_DOM_MEDIA_NEED_NEW_DECODER;
+ if (!needsNewDecoder && ++decoder.mNumOfConsecutiveError > decoder.mMaxConsecutiveError) {
+ NotifyError(aTrack, decoder.mError.ref());
+ return;
+ }
+ decoder.mError.reset();
+ LOG("%s decoded error count %d", TrackTypeToStr(aTrack),
+ decoder.mNumOfConsecutiveError);
+ media::TimeUnit nextKeyframe;
+ if (aTrack == TrackType::kVideoTrack && !decoder.HasInternalSeekPending() &&
+ NS_SUCCEEDED(decoder.mTrackDemuxer->GetNextRandomAccessPoint(&nextKeyframe))) {
+ if (needsNewDecoder) {
+ decoder.ShutdownDecoder();
+ }
+ SkipVideoDemuxToNextKeyFrame(decoder.mLastSampleTime.refOr(TimeInterval()).Length());
+ return;
+ } else if (aTrack == TrackType::kAudioTrack) {
+ decoder.Flush();
+ }
+ }
+
+ bool needInput = NeedInput(decoder);
+
+ LOGV("Update(%s) ni=%d no=%d ie=%d, in:%llu out:%llu qs=%u pending:%u waiting:%d promise:%d wfk:%d sid:%u",
+ TrackTypeToStr(aTrack), needInput, needOutput, decoder.mDecodePending,
+ decoder.mNumSamplesInput, decoder.mNumSamplesOutput,
+ uint32_t(size_t(decoder.mSizeOfQueue)), uint32_t(decoder.mOutput.Length()),
+ decoder.mWaitingForData, decoder.HasPromise(),
+ decoder.mWaitingForKey, decoder.mLastStreamSourceID);
+
+ if ((decoder.mWaitingForData &&
+ (!decoder.mTimeThreshold || decoder.mTimeThreshold.ref().mWaiting)) ||
+ (decoder.mWaitingForKey && decoder.mDecodePending)) {
+ // Nothing more we can do at present.
+ LOGV("Still waiting for data or key.");
+ return;
+ }
+
+ if (decoder.mWaitingForKey) {
+ decoder.mWaitingForKey = false;
+ if (decoder.HasWaitingPromise() && !decoder.IsWaiting()) {
+ LOGV("No longer waiting for key. Resolving waiting promise");
+ decoder.mWaitingPromise.Resolve(decoder.mType, __func__);
+ return;
+ }
+ }
+
+ if (!needInput) {
+ LOGV("No need for additional input (pending:%u)",
+ uint32_t(decoder.mOutput.Length()));
+ return;
+ }
+
+ // Demux samples if we don't have some.
+ RequestDemuxSamples(aTrack);
+
+ HandleDemuxedSamples(aTrack, a);
+}
+
+void
+MediaFormatReader::ReturnOutput(MediaData* aData, TrackType aTrack)
+{
+ MOZ_ASSERT(GetDecoderData(aTrack).HasPromise());
+ MOZ_DIAGNOSTIC_ASSERT(aData->mType != MediaData::NULL_DATA);
+ LOG("Resolved data promise for %s [%lld, %lld]", TrackTypeToStr(aTrack),
+ aData->mTime, aData->GetEndTime());
+
+ if (aTrack == TrackInfo::kAudioTrack) {
+ if (aData->mType != MediaData::RAW_DATA) {
+ AudioData* audioData = static_cast<AudioData*>(aData);
+
+ if (audioData->mChannels != mInfo.mAudio.mChannels ||
+ audioData->mRate != mInfo.mAudio.mRate) {
+ LOG("change of audio format (rate:%d->%d). "
+ "This is an unsupported configuration",
+ mInfo.mAudio.mRate, audioData->mRate);
+ mInfo.mAudio.mRate = audioData->mRate;
+ mInfo.mAudio.mChannels = audioData->mChannels;
+ }
+ }
+ mAudio.ResolvePromise(aData, __func__);
+ } else if (aTrack == TrackInfo::kVideoTrack) {
+ if (aData->mType != MediaData::RAW_DATA) {
+ VideoData* videoData = static_cast<VideoData*>(aData);
+
+ if (videoData->mDisplay != mInfo.mVideo.mDisplay) {
+ LOG("change of video display size (%dx%d->%dx%d)",
+ mInfo.mVideo.mDisplay.width, mInfo.mVideo.mDisplay.height,
+ videoData->mDisplay.width, videoData->mDisplay.height);
+ mInfo.mVideo.mDisplay = videoData->mDisplay;
+ }
+ }
+ mVideo.ResolvePromise(aData, __func__);
+ }
+}
+
+size_t
+MediaFormatReader::SizeOfVideoQueueInFrames()
+{
+ return SizeOfQueue(TrackInfo::kVideoTrack);
+}
+
+size_t
+MediaFormatReader::SizeOfAudioQueueInFrames()
+{
+ return SizeOfQueue(TrackInfo::kAudioTrack);
+}
+
+size_t
+MediaFormatReader::SizeOfQueue(TrackType aTrack)
+{
+ auto& decoder = GetDecoderData(aTrack);
+ return decoder.mSizeOfQueue;
+}
+
+RefPtr<MediaDecoderReader::WaitForDataPromise>
+MediaFormatReader::WaitForData(MediaData::Type aType)
+{
+ MOZ_ASSERT(OnTaskQueue());
+ TrackType trackType = aType == MediaData::VIDEO_DATA ?
+ TrackType::kVideoTrack : TrackType::kAudioTrack;
+ auto& decoder = GetDecoderData(trackType);
+ if (!decoder.IsWaiting()) {
+ // We aren't waiting for anything.
+ return WaitForDataPromise::CreateAndResolve(decoder.mType, __func__);
+ }
+ RefPtr<WaitForDataPromise> p = decoder.mWaitingPromise.Ensure(__func__);
+ ScheduleUpdate(trackType);
+ return p;
+}
+
+nsresult
+MediaFormatReader::ResetDecode(TrackSet aTracks)
+{
+ MOZ_ASSERT(OnTaskQueue());
+ LOGV("");
+
+ mSeekPromise.RejectIfExists(NS_OK, __func__);
+ mSkipRequest.DisconnectIfExists();
+
+ // Do the same for any data wait promises.
+ if (aTracks.contains(TrackInfo::kAudioTrack)) {
+ mAudio.mWaitingPromise.RejectIfExists(
+ WaitForDataRejectValue(MediaData::AUDIO_DATA,
+ WaitForDataRejectValue::CANCELED), __func__);
+ }
+
+ if (aTracks.contains(TrackInfo::kVideoTrack)) {
+ mVideo.mWaitingPromise.RejectIfExists(
+ WaitForDataRejectValue(MediaData::VIDEO_DATA,
+ WaitForDataRejectValue::CANCELED), __func__);
+ }
+
+ // Reset miscellaneous seeking state.
+ mPendingSeekTime.reset();
+
+ if (HasVideo() && aTracks.contains(TrackInfo::kVideoTrack)) {
+ mVideo.ResetDemuxer();
+ Reset(TrackInfo::kVideoTrack);
+ if (mVideo.HasPromise()) {
+ mVideo.RejectPromise(NS_ERROR_DOM_MEDIA_CANCELED, __func__);
+ }
+ }
+
+ if (HasAudio() && aTracks.contains(TrackInfo::kAudioTrack)) {
+ mAudio.ResetDemuxer();
+ Reset(TrackInfo::kAudioTrack);
+ if (mAudio.HasPromise()) {
+ mAudio.RejectPromise(NS_ERROR_DOM_MEDIA_CANCELED, __func__);
+ }
+ }
+
+ return MediaDecoderReader::ResetDecode(aTracks);
+}
+
+void
+MediaFormatReader::Output(TrackType aTrack, MediaData* aSample)
+{
+ if (!aSample) {
+ NS_WARNING("MediaFormatReader::Output() passed a null sample");
+ Error(aTrack, MediaResult(NS_ERROR_DOM_MEDIA_DECODE_ERR, __func__));
+ return;
+ }
+
+ LOGV("Decoded %s sample time=%lld timecode=%lld kf=%d dur=%lld",
+ TrackTypeToStr(aTrack), aSample->mTime, aSample->mTimecode,
+ aSample->mKeyframe, aSample->mDuration);
+
+ RefPtr<nsIRunnable> task =
+ NewRunnableMethod<TrackType, MediaData*>(
+ this, &MediaFormatReader::NotifyNewOutput, aTrack, aSample);
+ OwnerThread()->Dispatch(task.forget());
+}
+
+void
+MediaFormatReader::DrainComplete(TrackType aTrack)
+{
+ RefPtr<nsIRunnable> task =
+ NewRunnableMethod<TrackType>(
+ this, &MediaFormatReader::NotifyDrainComplete, aTrack);
+ OwnerThread()->Dispatch(task.forget());
+}
+
+void
+MediaFormatReader::InputExhausted(TrackType aTrack)
+{
+ RefPtr<nsIRunnable> task =
+ NewRunnableMethod<TrackType>(
+ this, &MediaFormatReader::NotifyInputExhausted, aTrack);
+ OwnerThread()->Dispatch(task.forget());
+}
+
+void
+MediaFormatReader::Error(TrackType aTrack, const MediaResult& aError)
+{
+ RefPtr<nsIRunnable> task =
+ NewRunnableMethod<TrackType, MediaResult>(
+ this, &MediaFormatReader::NotifyError, aTrack, aError);
+ OwnerThread()->Dispatch(task.forget());
+}
+
+void
+MediaFormatReader::WaitingForKey(TrackType aTrack)
+{
+ RefPtr<nsIRunnable> task =
+ NewRunnableMethod<TrackType>(
+ this, &MediaFormatReader::NotifyWaitingForKey, aTrack);
+ OwnerThread()->Dispatch(task.forget());
+}
+
+void
+MediaFormatReader::Reset(TrackType aTrack)
+{
+ MOZ_ASSERT(OnTaskQueue());
+ LOG("Reset(%s) BEGIN", TrackTypeToStr(aTrack));
+
+ auto& decoder = GetDecoderData(aTrack);
+
+ decoder.ResetState();
+ decoder.Flush();
+
+ LOG("Reset(%s) END", TrackTypeToStr(aTrack));
+}
+
+void
+MediaFormatReader::DropDecodedSamples(TrackType aTrack)
+{
+ MOZ_ASSERT(OnTaskQueue());
+ auto& decoder = GetDecoderData(aTrack);
+ size_t lengthDecodedQueue = decoder.mOutput.Length();
+ if (lengthDecodedQueue && decoder.mTimeThreshold.isSome()) {
+ TimeUnit time =
+ TimeUnit::FromMicroseconds(decoder.mOutput.LastElement()->mTime);
+ if (time >= decoder.mTimeThreshold.ref().Time()) {
+ // We would have reached our internal seek target.
+ decoder.mTimeThreshold.reset();
+ }
+ }
+ decoder.mOutput.Clear();
+ decoder.mSizeOfQueue -= lengthDecodedQueue;
+ if (aTrack == TrackInfo::kVideoTrack && mDecoder) {
+ mDecoder->NotifyDecodedFrames({ 0, 0, lengthDecodedQueue });
+ }
+}
+
+void
+MediaFormatReader::SkipVideoDemuxToNextKeyFrame(media::TimeUnit aTimeThreshold)
+{
+ MOZ_ASSERT(OnTaskQueue());
+ LOG("Skipping up to %lld", aTimeThreshold.ToMicroseconds());
+
+ // We've reached SkipVideoDemuxToNextKeyFrame when our decoding is late.
+ // As such we can drop all already decoded samples and discard all pending
+ // samples.
+ // TODO: Ideally we should set mOutputRequested to false so that all pending
+ // frames are dropped too. However, we can't do such thing as the code assumes
+ // that the decoder just got flushed. Once bug 1257107 land, we could set the
+ // decoder threshold to the value of currentTime.
+ DropDecodedSamples(TrackInfo::kVideoTrack);
+
+ mSkipRequest.Begin(mVideo.mTrackDemuxer->SkipToNextRandomAccessPoint(aTimeThreshold)
+ ->Then(OwnerThread(), __func__, this,
+ &MediaFormatReader::OnVideoSkipCompleted,
+ &MediaFormatReader::OnVideoSkipFailed));
+ return;
+}
+
+void
+MediaFormatReader::VideoSkipReset(uint32_t aSkipped)
+{
+ MOZ_ASSERT(OnTaskQueue());
+
+ // Some frames may have been output by the decoder since we initiated the
+ // videoskip process and we know they would be late.
+ DropDecodedSamples(TrackInfo::kVideoTrack);
+ // Report the pending frames as dropped.
+ if (mDecoder) {
+ mDecoder->NotifyDecodedFrames({ 0, 0, SizeOfVideoQueueInFrames() });
+ }
+
+ // Cancel any pending demux request and pending demuxed samples.
+ mVideo.mDemuxRequest.DisconnectIfExists();
+ Reset(TrackType::kVideoTrack);
+
+ if (mDecoder) {
+ mDecoder->NotifyDecodedFrames({ aSkipped, 0, aSkipped });
+ }
+
+ mVideo.mNumSamplesSkippedTotal += aSkipped;
+}
+
+void
+MediaFormatReader::OnVideoSkipCompleted(uint32_t aSkipped)
+{
+ MOZ_ASSERT(OnTaskQueue());
+ LOG("Skipping succeeded, skipped %u frames", aSkipped);
+ mSkipRequest.Complete();
+
+ VideoSkipReset(aSkipped);
+
+ ScheduleUpdate(TrackInfo::kVideoTrack);
+}
+
+void
+MediaFormatReader::OnVideoSkipFailed(MediaTrackDemuxer::SkipFailureHolder aFailure)
+{
+ MOZ_ASSERT(OnTaskQueue());
+ LOG("Skipping failed, skipped %u frames", aFailure.mSkipped);
+ mSkipRequest.Complete();
+
+ switch (aFailure.mFailure.Code()) {
+ case NS_ERROR_DOM_MEDIA_END_OF_STREAM:
+ case NS_ERROR_DOM_MEDIA_WAITING_FOR_DATA:
+ // Some frames may have been output by the decoder since we initiated the
+ // videoskip process and we know they would be late.
+ DropDecodedSamples(TrackInfo::kVideoTrack);
+ // We can't complete the skip operation, will just service a video frame
+ // normally.
+ ScheduleUpdate(TrackInfo::kVideoTrack);
+ break;
+ case NS_ERROR_DOM_MEDIA_CANCELED:
+ if (mVideo.HasPromise()) {
+ mVideo.RejectPromise(aFailure.mFailure, __func__);
+ }
+ break;
+ default:
+ NotifyError(TrackType::kVideoTrack, aFailure.mFailure);
+ break;
+ }
+}
+
+RefPtr<MediaDecoderReader::SeekPromise>
+MediaFormatReader::Seek(SeekTarget aTarget, int64_t aUnused)
+{
+ MOZ_ASSERT(OnTaskQueue());
+
+ LOG("aTarget=(%lld)", aTarget.GetTime().ToMicroseconds());
+
+ MOZ_DIAGNOSTIC_ASSERT(mSeekPromise.IsEmpty());
+ MOZ_DIAGNOSTIC_ASSERT(!mVideo.HasPromise());
+ MOZ_DIAGNOSTIC_ASSERT(aTarget.IsVideoOnly() || !mAudio.HasPromise());
+ MOZ_DIAGNOSTIC_ASSERT(mPendingSeekTime.isNothing());
+ MOZ_DIAGNOSTIC_ASSERT(mVideo.mTimeThreshold.isNothing());
+ MOZ_DIAGNOSTIC_ASSERT(aTarget.IsVideoOnly() || mAudio.mTimeThreshold.isNothing());
+
+ if (!mInfo.mMediaSeekable && !mInfo.mMediaSeekableOnlyInBufferedRanges) {
+ LOG("Seek() END (Unseekable)");
+ return SeekPromise::CreateAndReject(NS_ERROR_FAILURE, __func__);
+ }
+
+ if (mShutdown) {
+ return SeekPromise::CreateAndReject(NS_ERROR_FAILURE, __func__);
+ }
+
+ SetSeekTarget(Move(aTarget));
+
+ RefPtr<SeekPromise> p = mSeekPromise.Ensure(__func__);
+
+ ScheduleSeek();
+
+ return p;
+}
+
+void
+MediaFormatReader::SetSeekTarget(const SeekTarget& aTarget)
+{
+ MOZ_ASSERT(OnTaskQueue());
+
+ SeekTarget target = aTarget;
+
+ // Transform the seek target time to the demuxer timeline.
+ if (!ForceZeroStartTime()) {
+ target.SetTime(aTarget.GetTime() - TimeUnit::FromMicroseconds(StartTime())
+ + DemuxStartTime());
+ }
+
+ mOriginalSeekTarget = target;
+ mFallbackSeekTime = mPendingSeekTime = Some(target.GetTime());
+}
+
+TimeUnit
+MediaFormatReader::DemuxStartTime()
+{
+ MOZ_ASSERT(OnTaskQueue());
+ MOZ_ASSERT(!ForceZeroStartTime());
+ MOZ_ASSERT(HasAudio() || HasVideo());
+
+ const TimeUnit startTime =
+ std::min(mAudio.mFirstDemuxedSampleTime.refOr(TimeUnit::FromInfinity()),
+ mVideo.mFirstDemuxedSampleTime.refOr(TimeUnit::FromInfinity()));
+
+ return startTime.IsInfinite() ? TimeUnit::FromMicroseconds(0) : startTime;
+}
+
+void
+MediaFormatReader::ScheduleSeek()
+{
+ if (mSeekScheduled) {
+ return;
+ }
+ mSeekScheduled = true;
+ OwnerThread()->Dispatch(NewRunnableMethod(this, &MediaFormatReader::AttemptSeek));
+}
+
+void
+MediaFormatReader::AttemptSeek()
+{
+ MOZ_ASSERT(OnTaskQueue());
+
+ mSeekScheduled = false;
+
+ if (mPendingSeekTime.isNothing()) {
+ return;
+ }
+
+ if (HasVideo()) {
+ mVideo.ResetDemuxer();
+ mVideo.ResetState();
+ }
+
+ // Don't reset the audio demuxer not state when seeking video only
+ // as it will cause the audio to seek back to the beginning
+ // resulting in out-of-sync audio from video.
+ if (HasAudio() && !mOriginalSeekTarget.IsVideoOnly()) {
+ mAudio.ResetDemuxer();
+ mAudio.ResetState();
+ }
+
+ if (HasVideo()) {
+ DoVideoSeek();
+ } else if (HasAudio()) {
+ DoAudioSeek();
+ } else {
+ MOZ_CRASH();
+ }
+}
+
+void
+MediaFormatReader::OnSeekFailed(TrackType aTrack, const MediaResult& aError)
+{
+ MOZ_ASSERT(OnTaskQueue());
+ LOGV("%s failure:%u", TrackTypeToStr(aTrack), aError.Code());
+ if (aTrack == TrackType::kVideoTrack) {
+ mVideo.mSeekRequest.Complete();
+ } else {
+ mAudio.mSeekRequest.Complete();
+ }
+
+ if (aError == NS_ERROR_DOM_MEDIA_WAITING_FOR_DATA) {
+ if (HasVideo() && aTrack == TrackType::kAudioTrack &&
+ mFallbackSeekTime.isSome() &&
+ mPendingSeekTime.ref() != mFallbackSeekTime.ref()) {
+ // We have failed to seek audio where video seeked to earlier.
+ // Attempt to seek instead to the closest point that we know we have in
+ // order to limit A/V sync discrepency.
+
+ // Ensure we have the most up to date buffered ranges.
+ UpdateReceivedNewData(TrackType::kAudioTrack);
+ Maybe<media::TimeUnit> nextSeekTime;
+ // Find closest buffered time found after video seeked time.
+ for (const auto& timeRange : mAudio.mTimeRanges) {
+ if (timeRange.mStart >= mPendingSeekTime.ref()) {
+ nextSeekTime.emplace(timeRange.mStart);
+ break;
+ }
+ }
+ if (nextSeekTime.isNothing() ||
+ nextSeekTime.ref() > mFallbackSeekTime.ref()) {
+ nextSeekTime = Some(mFallbackSeekTime.ref());
+ LOG("Unable to seek audio to video seek time. A/V sync may be broken");
+ } else {
+ mFallbackSeekTime.reset();
+ }
+ mPendingSeekTime = nextSeekTime;
+ DoAudioSeek();
+ return;
+ }
+ NotifyWaitingForData(aTrack);
+ return;
+ }
+ MOZ_ASSERT(!mVideo.mSeekRequest.Exists() && !mAudio.mSeekRequest.Exists());
+ mPendingSeekTime.reset();
+ mSeekPromise.Reject(aError, __func__);
+}
+
+void
+MediaFormatReader::DoVideoSeek()
+{
+ MOZ_ASSERT(mPendingSeekTime.isSome());
+ LOGV("Seeking video to %lld", mPendingSeekTime.ref().ToMicroseconds());
+ media::TimeUnit seekTime = mPendingSeekTime.ref();
+ mVideo.mSeekRequest.Begin(mVideo.mTrackDemuxer->Seek(seekTime)
+ ->Then(OwnerThread(), __func__, this,
+ &MediaFormatReader::OnVideoSeekCompleted,
+ &MediaFormatReader::OnVideoSeekFailed));
+}
+
+void
+MediaFormatReader::OnVideoSeekCompleted(media::TimeUnit aTime)
+{
+ MOZ_ASSERT(OnTaskQueue());
+ LOGV("Video seeked to %lld", aTime.ToMicroseconds());
+ mVideo.mSeekRequest.Complete();
+
+ mPreviousDecodedKeyframeTime_us = sNoPreviousDecodedKeyframe;
+
+ SetVideoDecodeThreshold();
+
+ if (HasAudio() && !mOriginalSeekTarget.IsVideoOnly()) {
+ MOZ_ASSERT(mPendingSeekTime.isSome());
+ if (mOriginalSeekTarget.IsFast()) {
+ // We are performing a fast seek. We need to seek audio to where the
+ // video seeked to, to ensure proper A/V sync once playback resume.
+ mPendingSeekTime = Some(aTime);
+ }
+ DoAudioSeek();
+ } else {
+ mPendingSeekTime.reset();
+ mSeekPromise.Resolve(aTime, __func__);
+ }
+}
+
+void
+MediaFormatReader::OnVideoSeekFailed(const MediaResult& aError)
+{
+ mPreviousDecodedKeyframeTime_us = sNoPreviousDecodedKeyframe;
+ OnSeekFailed(TrackType::kVideoTrack, aError);
+}
+
+void
+MediaFormatReader::SetVideoDecodeThreshold()
+{
+ MOZ_ASSERT(OnTaskQueue());
+
+ if (!HasVideo() || !mVideo.mDecoder) {
+ return;
+ }
+
+ if (!mVideo.mTimeThreshold && !IsSeeking()) {
+ return;
+ }
+
+ TimeUnit threshold;
+ if (mVideo.mTimeThreshold) {
+ // For internalSeek.
+ threshold = mVideo.mTimeThreshold.ref().Time();
+ } else if (IsSeeking()) {
+ // If IsSeeking() is true, then video seek must have completed already.
+ TimeUnit keyframe;
+ if (NS_FAILED(mVideo.mTrackDemuxer->GetNextRandomAccessPoint(&keyframe))) {
+ return;
+ }
+
+ // If the key frame is invalid/infinite, it means the target position is
+ // closing to end of stream. We don't want to skip any frame at this point.
+ if (!keyframe.IsValid() || keyframe.IsInfinite()) {
+ return;
+ }
+ threshold = mOriginalSeekTarget.GetTime();
+ } else {
+ return;
+ }
+
+ LOG("Set seek threshold to %lld", threshold.ToMicroseconds());
+ mVideo.mDecoder->SetSeekThreshold(threshold);
+}
+
+void
+MediaFormatReader::DoAudioSeek()
+{
+ MOZ_ASSERT(mPendingSeekTime.isSome());
+ LOGV("Seeking audio to %lld", mPendingSeekTime.ref().ToMicroseconds());
+ media::TimeUnit seekTime = mPendingSeekTime.ref();
+ mAudio.mSeekRequest.Begin(mAudio.mTrackDemuxer->Seek(seekTime)
+ ->Then(OwnerThread(), __func__, this,
+ &MediaFormatReader::OnAudioSeekCompleted,
+ &MediaFormatReader::OnAudioSeekFailed));
+}
+
+void
+MediaFormatReader::OnAudioSeekCompleted(media::TimeUnit aTime)
+{
+ MOZ_ASSERT(OnTaskQueue());
+ LOGV("Audio seeked to %lld", aTime.ToMicroseconds());
+ mAudio.mSeekRequest.Complete();
+ mPendingSeekTime.reset();
+ mSeekPromise.Resolve(aTime, __func__);
+}
+
+void
+MediaFormatReader::OnAudioSeekFailed(const MediaResult& aError)
+{
+ OnSeekFailed(TrackType::kAudioTrack, aError);
+}
+
+media::TimeIntervals
+MediaFormatReader::GetBuffered()
+{
+ MOZ_ASSERT(OnTaskQueue());
+ media::TimeIntervals videoti;
+ media::TimeIntervals audioti;
+ media::TimeIntervals intervals;
+
+ if (!mInitDone) {
+ return intervals;
+ }
+ int64_t startTime = 0;
+ if (!ForceZeroStartTime()) {
+ if (!HaveStartTime()) {
+ return intervals;
+ }
+ startTime = StartTime();
+ }
+ // Ensure we have up to date buffered time range.
+ if (HasVideo()) {
+ UpdateReceivedNewData(TrackType::kVideoTrack);
+ }
+ if (HasAudio()) {
+ UpdateReceivedNewData(TrackType::kAudioTrack);
+ }
+ if (HasVideo()) {
+ videoti = mVideo.mTimeRanges;
+ }
+ if (HasAudio()) {
+ audioti = mAudio.mTimeRanges;
+ }
+ if (HasAudio() && HasVideo()) {
+ intervals = media::Intersection(Move(videoti), Move(audioti));
+ } else if (HasAudio()) {
+ intervals = Move(audioti);
+ } else if (HasVideo()) {
+ intervals = Move(videoti);
+ }
+
+ if (!intervals.Length() ||
+ intervals.GetStart() == media::TimeUnit::FromMicroseconds(0)) {
+ // IntervalSet already starts at 0 or is empty, nothing to shift.
+ return intervals;
+ }
+ return intervals.Shift(media::TimeUnit::FromMicroseconds(-startTime));
+}
+
+// For the MediaFormatReader override we need to force an update to the
+// buffered ranges, so we call NotifyDataArrive
+RefPtr<MediaDecoderReader::BufferedUpdatePromise>
+MediaFormatReader::UpdateBufferedWithPromise() {
+ MOZ_ASSERT(OnTaskQueue());
+ // Call NotifyDataArrive to force a recalculation of the buffered
+ // ranges. UpdateBuffered alone will not force a recalculation, so we
+ // use NotifyDataArrived which sets flags to force this recalculation.
+ // See MediaFormatReader::UpdateReceivedNewData for an example of where
+ // the new data flag is used.
+ NotifyDataArrived();
+ return BufferedUpdatePromise::CreateAndResolve(true, __func__);
+}
+
+void MediaFormatReader::ReleaseResources()
+{
+ mVideo.ShutdownDecoder();
+ mAudio.ShutdownDecoder();
+}
+
+bool
+MediaFormatReader::VideoIsHardwareAccelerated() const
+{
+ return mVideo.mIsHardwareAccelerated;
+}
+
+void
+MediaFormatReader::NotifyDemuxer()
+{
+ MOZ_ASSERT(OnTaskQueue());
+
+ if (mShutdown || !mDemuxer ||
+ (!mDemuxerInitDone && !mDemuxerInitRequest.Exists())) {
+ return;
+ }
+
+ LOGV("");
+
+ mDemuxer->NotifyDataArrived();
+
+ if (!mInitDone) {
+ return;
+ }
+ if (HasVideo()) {
+ mVideo.mReceivedNewData = true;
+ ScheduleUpdate(TrackType::kVideoTrack);
+ }
+ if (HasAudio()) {
+ mAudio.mReceivedNewData = true;
+ ScheduleUpdate(TrackType::kAudioTrack);
+ }
+}
+
+void
+MediaFormatReader::NotifyDataArrivedInternal()
+{
+ MOZ_ASSERT(OnTaskQueue());
+ NotifyDemuxer();
+}
+
+bool
+MediaFormatReader::ForceZeroStartTime() const
+{
+ return !mDemuxer->ShouldComputeStartTime();
+}
+
+layers::ImageContainer*
+MediaFormatReader::GetImageContainer()
+{
+ return mVideoFrameContainer
+ ? mVideoFrameContainer->GetImageContainer() : nullptr;
+}
+
+void
+MediaFormatReader::GetMozDebugReaderData(nsAString& aString)
+{
+ nsAutoCString result;
+ const char* audioName = "unavailable";
+ const char* videoName = audioName;
+
+ if (HasAudio()) {
+ MonitorAutoLock mon(mAudio.mMonitor);
+ audioName = mAudio.mDescription;
+ }
+ if (HasVideo()) {
+ MonitorAutoLock mon(mVideo.mMonitor);
+ videoName = mVideo.mDescription;
+ }
+
+ result += nsPrintfCString("audio decoder: %s\n", audioName);
+ result += nsPrintfCString("audio frames decoded: %lld\n",
+ mAudio.mNumSamplesOutputTotal);
+ if (HasAudio()) {
+ result += nsPrintfCString("audio state: ni=%d no=%d ie=%d demuxr:%d demuxq:%d tt:%f tths:%d in:%llu out:%llu qs=%u pending:%u waiting:%d wfk:%d sid:%u\n",
+ NeedInput(mAudio), mAudio.HasPromise(),
+ mAudio.mDecodePending,
+ mAudio.mDemuxRequest.Exists(),
+ int(mAudio.mQueuedSamples.Length()),
+ mAudio.mTimeThreshold
+ ? mAudio.mTimeThreshold.ref().Time().ToSeconds()
+ : -1.0,
+ mAudio.mTimeThreshold
+ ? mAudio.mTimeThreshold.ref().mHasSeeked
+ : -1,
+ mAudio.mNumSamplesInput, mAudio.mNumSamplesOutput,
+ unsigned(size_t(mAudio.mSizeOfQueue)),
+ unsigned(mAudio.mOutput.Length()),
+ mAudio.mWaitingForData, mAudio.mWaitingForKey,
+ mAudio.mLastStreamSourceID);
+ }
+ result += nsPrintfCString("video decoder: %s\n", videoName);
+ result += nsPrintfCString("hardware video decoding: %s\n",
+ VideoIsHardwareAccelerated() ? "enabled" : "disabled");
+ result += nsPrintfCString("video frames decoded: %lld (skipped:%lld)\n",
+ mVideo.mNumSamplesOutputTotal,
+ mVideo.mNumSamplesSkippedTotal);
+ if (HasVideo()) {
+ result += nsPrintfCString("video state: ni=%d no=%d ie=%d demuxr:%d demuxq:%d tt:%f tths:%d in:%llu out:%llu qs=%u pending:%u waiting:%d wfk:%d, sid:%u\n",
+ NeedInput(mVideo), mVideo.HasPromise(),
+ mVideo.mDecodePending,
+ mVideo.mDemuxRequest.Exists(),
+ int(mVideo.mQueuedSamples.Length()),
+ mVideo.mTimeThreshold
+ ? mVideo.mTimeThreshold.ref().Time().ToSeconds()
+ : -1.0,
+ mVideo.mTimeThreshold
+ ? mVideo.mTimeThreshold.ref().mHasSeeked
+ : -1,
+ mVideo.mNumSamplesInput, mVideo.mNumSamplesOutput,
+ unsigned(size_t(mVideo.mSizeOfQueue)),
+ unsigned(mVideo.mOutput.Length()),
+ mVideo.mWaitingForData, mVideo.mWaitingForKey,
+ mVideo.mLastStreamSourceID);
+ }
+ aString += NS_ConvertUTF8toUTF16(result);
+}
+
+void
+MediaFormatReader::SetVideoBlankDecode(bool aIsBlankDecode)
+{
+ MOZ_ASSERT(OnTaskQueue());
+ return SetBlankDecode(TrackType::kVideoTrack, aIsBlankDecode);
+}
+
+void
+MediaFormatReader::SetBlankDecode(TrackType aTrack, bool aIsBlankDecode)
+{
+ MOZ_ASSERT(OnTaskQueue());
+ auto& decoder = GetDecoderData(aTrack);
+
+ LOG("%s, decoder.mIsBlankDecode = %d => aIsBlankDecode = %d",
+ TrackTypeToStr(aTrack), decoder.mIsBlankDecode, aIsBlankDecode);
+
+ if (decoder.mIsBlankDecode == aIsBlankDecode) {
+ return;
+ }
+
+ decoder.mIsBlankDecode = aIsBlankDecode;
+ decoder.Flush();
+ decoder.ShutdownDecoder();
+ ScheduleUpdate(TrackInfo::kVideoTrack);
+
+ return;
+}
+
+} // namespace mozilla