From d7beb75aa889967780067ade8219b4a662f22e38 Mon Sep 17 00:00:00 2001 From: wolfbeast Date: Thu, 7 Jun 2018 16:04:56 +0200 Subject: Media: harden TrackID handling. --- dom/html/HTMLMediaElement.cpp | 47 ++++++++++++++++-- dom/html/HTMLMediaElement.h | 12 ++++- dom/media/MediaDecoder.cpp | 14 +++++- dom/media/MediaDecoder.h | 6 ++- dom/media/MediaDecoderStateMachine.cpp | 15 +++++- dom/media/MediaDecoderStateMachine.h | 5 +- dom/media/mediasink/DecodedStream.cpp | 11 +++-- dom/media/mediasink/OutputStreamManager.cpp | 75 +++++++++++++++++++++++------ dom/media/mediasink/OutputStreamManager.h | 35 ++++++++++---- 9 files changed, 183 insertions(+), 37 deletions(-) diff --git a/dom/html/HTMLMediaElement.cpp b/dom/html/HTMLMediaElement.cpp index 1f1a545fa..0b9f606f1 100644 --- a/dom/html/HTMLMediaElement.cpp +++ b/dom/html/HTMLMediaElement.cpp @@ -817,6 +817,7 @@ NS_IMPL_CYCLE_COLLECTION_TRAVERSE_BEGIN_INHERITED(HTMLMediaElement, nsGenericHTM NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mErrorSink->mError) for (uint32_t i = 0; i < tmp->mOutputStreams.Length(); ++i) { NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mOutputStreams[i].mStream); + NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mOutputStreams[i].mPreCreatedTracks); } NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mPlayed); NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mTextTrackManager) @@ -1020,6 +1021,17 @@ void HTMLMediaElement::ShutdownDecoder() RemoveMediaElementFromURITable(); NS_ASSERTION(mDecoder, "Must have decoder to shut down"); mWaitingForKeyListener.DisconnectIfExists(); + for (OutputMediaStream& out : mOutputStreams) { + if (!out.mCapturingDecoder) { + continue; + } + if (!out.mStream) { + continue; + } + out.mNextAvailableTrackID = std::max( + mDecoder->NextAvailableTrackIDFor(out.mStream->GetInputStream()), + out.mNextAvailableTrackID); + } mDecoder->Shutdown(); mDecoder = nullptr; } @@ -2730,6 +2742,7 @@ HTMLMediaElement::CaptureStreamInternal(bool aFinishWhenEnded, if (mDecoder) { out->mCapturingDecoder = true; mDecoder->AddOutputStream(out->mStream->GetInputStream()->AsProcessedStream(), + out->mNextAvailableTrackID, aFinishWhenEnded); } else if (mSrcStream) { out->mCapturingMediaStream = true; @@ -2743,23 +2756,26 @@ HTMLMediaElement::CaptureStreamInternal(bool aFinishWhenEnded, if (mDecoder) { if (HasAudio()) { - TrackID audioTrackId = mMediaInfo.mAudio.mTrackId; + TrackID audioTrackId = out->mNextAvailableTrackID++; RefPtr trackSource = getter->GetMediaStreamTrackSource(audioTrackId); RefPtr track = - out->mStream->CreateDOMTrack(audioTrackId, MediaSegment::AUDIO, + out->mStream->CreateDOMTrack(audioTrackId, + MediaSegment::AUDIO, trackSource); + out->mPreCreatedTracks.AppendElement(track); out->mStream->AddTrackInternal(track); LOG(LogLevel::Debug, ("Created audio track %d for captured decoder", audioTrackId)); } if (IsVideo() && HasVideo() && !out->mCapturingAudioOnly) { - TrackID videoTrackId = mMediaInfo.mVideo.mTrackId; + TrackID videoTrackId = out->mNextAvailableTrackID++; RefPtr trackSource = getter->GetMediaStreamTrackSource(videoTrackId); RefPtr track = out->mStream->CreateDOMTrack(videoTrackId, MediaSegment::VIDEO, trackSource); + out->mPreCreatedTracks.AppendElement(track); out->mStream->AddTrackInternal(track); LOG(LogLevel::Debug, ("Created video track %d for captured decoder", videoTrackId)); @@ -2848,6 +2864,25 @@ NS_IMETHODIMP HTMLMediaElement::GetMozAudioCaptured(bool* aCaptured) return NS_OK; } +void +HTMLMediaElement::EndPreCreatedCapturedDecoderTracks() +{ + MOZ_ASSERT(NS_IsMainThread()); + for (OutputMediaStream& ms : mOutputStreams) { + if (!ms.mCapturingDecoder) { + continue; + } + for (RefPtr& t : ms.mPreCreatedTracks) { + if (t->Ended()) { + continue; + } + NS_DispatchToMainThread(NewRunnableMethod( + t, &MediaStreamTrack::OverrideEnded)); + } + ms.mPreCreatedTracks.Clear(); + } +} + class MediaElementSetForURI : public nsURIHashKey { public: explicit MediaElementSetForURI(const nsIURI* aKey) : nsURIHashKey(aKey) {} @@ -3380,11 +3415,12 @@ HTMLMediaElement::WakeLockRelease() } HTMLMediaElement::OutputMediaStream::OutputMediaStream() - : mFinishWhenEnded(false) + : mNextAvailableTrackID(1) + , mFinishWhenEnded(false) , mCapturingAudioOnly(false) , mCapturingDecoder(false) , mCapturingMediaStream(false) - , mNextAvailableTrackID(1) {} +{} HTMLMediaElement::OutputMediaStream::~OutputMediaStream() { @@ -4008,6 +4044,7 @@ nsresult HTMLMediaElement::FinishDecoderSetup(MediaDecoder* aDecoder, ms.mCapturingDecoder = true; aDecoder->AddOutputStream(ms.mStream->GetInputStream()->AsProcessedStream(), + ms.mNextAvailableTrackID, ms.mFinishWhenEnded); } diff --git a/dom/html/HTMLMediaElement.h b/dom/html/HTMLMediaElement.h index b65049206..af944a318 100644 --- a/dom/html/HTMLMediaElement.h +++ b/dom/html/HTMLMediaElement.h @@ -675,6 +675,11 @@ public: return mAudioCaptured; } + /** + * Ensures any MediaStreamTracks captured from a MediaDecoder are ended. + */ + void EndPreCreatedCapturedDecoderTracks(); + void MozGetMetadata(JSContext* aCx, JS::MutableHandle aResult, ErrorResult& aRv); @@ -815,13 +820,18 @@ protected: ~OutputMediaStream(); RefPtr mStream; + TrackID mNextAvailableTrackID; bool mFinishWhenEnded; bool mCapturingAudioOnly; bool mCapturingDecoder; bool mCapturingMediaStream; + // The following members are keeping state for a captured MediaDecoder. + // Tracks that were created on main thread before MediaDecoder fed them + // to the MediaStreamGraph. + nsTArray> mPreCreatedTracks; + // The following members are keeping state for a captured MediaStream. - TrackID mNextAvailableTrackID; nsTArray>> mTrackPorts; }; diff --git a/dom/media/MediaDecoder.cpp b/dom/media/MediaDecoder.cpp index ab39886ba..9334d1bcb 100644 --- a/dom/media/MediaDecoder.cpp +++ b/dom/media/MediaDecoder.cpp @@ -327,11 +327,13 @@ MediaDecoder::SetVolume(double aVolume) void MediaDecoder::AddOutputStream(ProcessedMediaStream* aStream, + TrackID aNextAvailableTrackID, bool aFinishWhenEnded) { MOZ_ASSERT(NS_IsMainThread()); MOZ_ASSERT(mDecoderStateMachine, "Must be called after Load()."); - mDecoderStateMachine->AddOutputStream(aStream, aFinishWhenEnded); + mDecoderStateMachine->AddOutputStream( + aStream, aNextAvailableTrackID, aFinishWhenEnded); } void @@ -342,6 +344,14 @@ MediaDecoder::RemoveOutputStream(MediaStream* aStream) mDecoderStateMachine->RemoveOutputStream(aStream); } +TrackID +MediaDecoder::NextAvailableTrackIDFor(MediaStream* aOutputStream) const +{ + MOZ_ASSERT(NS_IsMainThread()); + MOZ_ASSERT(mDecoderStateMachine, "Must be called after Load()."); + return mDecoderStateMachine->NextAvailableTrackIDFor(aOutputStream); +} + double MediaDecoder::GetDuration() { @@ -1736,6 +1746,8 @@ MediaDecoder::RemoveMediaTracks() videoList->RemoveTracks(); } + element->EndPreCreatedCapturedDecoderTracks(); + mMediaTracksConstructed = false; } diff --git a/dom/media/MediaDecoder.h b/dom/media/MediaDecoder.h index 825bbbd2c..a4edcbe72 100644 --- a/dom/media/MediaDecoder.h +++ b/dom/media/MediaDecoder.h @@ -210,9 +210,13 @@ public: // Add an output stream. All decoder output will be sent to the stream. // The stream is initially blocked. The decoder is responsible for unblocking // it while it is playing back. - virtual void AddOutputStream(ProcessedMediaStream* aStream, bool aFinishWhenEnded); + virtual void AddOutputStream(ProcessedMediaStream* aStream, + TrackID aNextAvailableTrackID, + bool aFinishWhenEnded); // Remove an output stream added with AddOutputStream. virtual void RemoveOutputStream(MediaStream* aStream); + // The next TrackID that can be used without risk of a collision. + virtual TrackID NextAvailableTrackIDFor(MediaStream* aOutputStream) const; // Return the duration of the video in seconds. virtual double GetDuration(); diff --git a/dom/media/MediaDecoderStateMachine.cpp b/dom/media/MediaDecoderStateMachine.cpp index f13e59b6c..c586139ad 100644 --- a/dom/media/MediaDecoderStateMachine.cpp +++ b/dom/media/MediaDecoderStateMachine.cpp @@ -2835,6 +2835,10 @@ MediaDecoderStateMachine::FinishDecodeFirstFrame() RefPtr MediaDecoderStateMachine::BeginShutdown() { + MOZ_ASSERT(NS_IsMainThread()); + if (mOutputStreamManager) { + mOutputStreamManager->Clear(); + } return InvokeAsync(OwnerThread(), this, __func__, &MediaDecoderStateMachine::Shutdown); } @@ -3250,11 +3254,12 @@ MediaDecoderStateMachine::DumpDebugInfo() } void MediaDecoderStateMachine::AddOutputStream(ProcessedMediaStream* aStream, + TrackID aNextAvailableTrackID, bool aFinishWhenEnded) { MOZ_ASSERT(NS_IsMainThread()); DECODER_LOG("AddOutputStream aStream=%p!", aStream); - mOutputStreamManager->Add(aStream, aFinishWhenEnded); + mOutputStreamManager->Add(aStream, aNextAvailableTrackID, aFinishWhenEnded); nsCOMPtr r = NewRunnableMethod( this, &MediaDecoderStateMachine::SetAudioCaptured, true); OwnerThread()->Dispatch(r.forget()); @@ -3269,9 +3274,17 @@ void MediaDecoderStateMachine::RemoveOutputStream(MediaStream* aStream) nsCOMPtr r = NewRunnableMethod( this, &MediaDecoderStateMachine::SetAudioCaptured, false); OwnerThread()->Dispatch(r.forget()); + } } +TrackID +MediaDecoderStateMachine::NextAvailableTrackIDFor(MediaStream* aOutputStream) const +{ + MOZ_ASSERT(NS_IsMainThread()); + return mOutputStreamManager->NextAvailableTrackIDFor(aOutputStream); +} + size_t MediaDecoderStateMachine::SizeOfVideoQueue() const { diff --git a/dom/media/MediaDecoderStateMachine.h b/dom/media/MediaDecoderStateMachine.h index a61fe13d2..ff3258ff1 100644 --- a/dom/media/MediaDecoderStateMachine.h +++ b/dom/media/MediaDecoderStateMachine.h @@ -168,9 +168,12 @@ public: void DumpDebugInfo(); - void AddOutputStream(ProcessedMediaStream* aStream, bool aFinishWhenEnded); + void AddOutputStream(ProcessedMediaStream* aStream, + TrackID aNextAvailableTrackID, + bool aFinishWhenEnded); // Remove an output stream added with AddOutputStream. void RemoveOutputStream(MediaStream* aStream); + TrackID NextAvailableTrackIDFor(MediaStream* aOutputStream) const; // Seeks to the decoder to aTarget asynchronously. RefPtr InvokeSeek(SeekTarget aTarget); diff --git a/dom/media/mediasink/DecodedStream.cpp b/dom/media/mediasink/DecodedStream.cpp index 9501a6cde..00bc5ea49 100644 --- a/dom/media/mediasink/DecodedStream.cpp +++ b/dom/media/mediasink/DecodedStream.cpp @@ -181,17 +181,22 @@ DecodedStreamData::DecodedStreamData(OutputStreamManager* aOutputStreamManager, , mOutputStreamManager(aOutputStreamManager) { mStream->AddListener(mListener); - mOutputStreamManager->Connect(mStream); + TrackID audioTrack = TRACK_NONE; + TrackID videoTrack = TRACK_NONE; // Initialize tracks. if (aInit.mInfo.HasAudio()) { - mStream->AddAudioTrack(aInit.mInfo.mAudio.mTrackId, + audioTrack = aInit.mInfo.mAudio.mTrackId; + mStream->AddAudioTrack(audioTrack, aInit.mInfo.mAudio.mRate, 0, new AudioSegment()); } if (aInit.mInfo.HasVideo()) { - mStream->AddTrack(aInit.mInfo.mVideo.mTrackId, 0, new VideoSegment()); + videoTrack = aInit.mInfo.mVideo.mTrackId; + mStream->AddTrack(videoTrack, 0, new VideoSegment()); } + + mOutputStreamManager->Connect(mStream, audioTrack, videoTrack); } DecodedStreamData::~DecodedStreamData() diff --git a/dom/media/mediasink/OutputStreamManager.cpp b/dom/media/mediasink/OutputStreamManager.cpp index d5685837a..7ecc203ed 100644 --- a/dom/media/mediasink/OutputStreamManager.cpp +++ b/dom/media/mediasink/OutputStreamManager.cpp @@ -13,29 +13,41 @@ OutputStreamData::~OutputStreamData() { MOZ_ASSERT(NS_IsMainThread()); // Break the connection to the input stream if necessary. - if (mPort) { - mPort->Destroy(); + for (RefPtr& port : mPorts) { + port->Destroy(); } } void -OutputStreamData::Init(OutputStreamManager* aOwner, ProcessedMediaStream* aStream) +OutputStreamData::Init(OutputStreamManager* aOwner, + ProcessedMediaStream* aStream, + TrackID aNextAvailableTrackID) { mOwner = aOwner; mStream = aStream; + mNextAvailableTrackID = aNextAvailableTrackID; } bool -OutputStreamData::Connect(MediaStream* aStream) +OutputStreamData::Connect(MediaStream* aStream, + TrackID aInputAudioTrackID, + TrackID aInputVideoTrackID) { MOZ_ASSERT(NS_IsMainThread()); - MOZ_ASSERT(!mPort, "Already connected?"); + MOZ_ASSERT(mPorts.IsEmpty(), "Already connected?"); if (mStream->IsDestroyed()) { return false; } - mPort = mStream->AllocateInputPort(aStream); + for (TrackID tid : {aInputAudioTrackID, aInputVideoTrackID}) { + if (tid == TRACK_NONE) { + continue; + } + MOZ_ASSERT(IsTrackIDExplicit(tid)); + mPorts.AppendElement(mStream->AllocateInputPort( + aStream, tid, mNextAvailableTrackID++)); + } return true; } @@ -51,11 +63,11 @@ OutputStreamData::Disconnect() return false; } - // Disconnect the existing port if necessary. - if (mPort) { - mPort->Destroy(); - mPort = nullptr; + // Disconnect any existing port. + for (RefPtr& port : mPorts) { + port->Destroy(); } + mPorts.Clear(); return true; } @@ -71,8 +83,16 @@ OutputStreamData::Graph() const return mStream->Graph(); } +TrackID +OutputStreamData::NextAvailableTrackID() const +{ + return mNextAvailableTrackID; +} + void -OutputStreamManager::Add(ProcessedMediaStream* aStream, bool aFinishWhenEnded) +OutputStreamManager::Add(ProcessedMediaStream* aStream, + TrackID aNextAvailableTrackID, + bool aFinishWhenEnded) { MOZ_ASSERT(NS_IsMainThread()); // All streams must belong to the same graph. @@ -84,12 +104,12 @@ OutputStreamManager::Add(ProcessedMediaStream* aStream, bool aFinishWhenEnded) } OutputStreamData* p = mStreams.AppendElement(); - p->Init(this, aStream); + p->Init(this, aStream, aNextAvailableTrackID); // Connect to the input stream if we have one. Otherwise the output stream // will be connected in Connect(). if (mInputStream) { - p->Connect(mInputStream); + p->Connect(mInputStream, mInputAudioTrackID, mInputVideoTrackID); } } @@ -106,12 +126,35 @@ OutputStreamManager::Remove(MediaStream* aStream) } void -OutputStreamManager::Connect(MediaStream* aStream) +OutputStreamManager::Clear() +{ + MOZ_ASSERT(NS_IsMainThread()); + mStreams.Clear(); +} + +TrackID +OutputStreamManager::NextAvailableTrackIDFor(MediaStream* aOutputStream) const +{ + MOZ_ASSERT(NS_IsMainThread()); + for (const OutputStreamData& out : mStreams) { + if (out.Equals(aOutputStream)) { + return out.NextAvailableTrackID(); + } + } + return TRACK_INVALID; +} + +void +OutputStreamManager::Connect(MediaStream* aStream, + TrackID aAudioTrackID, + TrackID aVideoTrackID) { MOZ_ASSERT(NS_IsMainThread()); mInputStream = aStream; + mInputAudioTrackID = aAudioTrackID; + mInputVideoTrackID = aVideoTrackID; for (int32_t i = mStreams.Length() - 1; i >= 0; --i) { - if (!mStreams[i].Connect(aStream)) { + if (!mStreams[i].Connect(aStream, mInputAudioTrackID, mInputVideoTrackID)) { // Probably the DOMMediaStream was GCed. Clean up. mStreams.RemoveElementAt(i); } @@ -123,6 +166,8 @@ OutputStreamManager::Disconnect() { MOZ_ASSERT(NS_IsMainThread()); mInputStream = nullptr; + mInputAudioTrackID = TRACK_INVALID; + mInputVideoTrackID = TRACK_INVALID; for (int32_t i = mStreams.Length() - 1; i >= 0; --i) { if (!mStreams[i].Disconnect()) { // Probably the DOMMediaStream was GCed. Clean up. diff --git a/dom/media/mediasink/OutputStreamManager.h b/dom/media/mediasink/OutputStreamManager.h index 7f91a60c1..941a86cf0 100644 --- a/dom/media/mediasink/OutputStreamManager.h +++ b/dom/media/mediasink/OutputStreamManager.h @@ -9,6 +9,7 @@ #include "mozilla/RefPtr.h" #include "nsTArray.h" +#include "MediaSegment.h" namespace mozilla { @@ -21,11 +22,13 @@ class ProcessedMediaStream; class OutputStreamData { public: ~OutputStreamData(); - void Init(OutputStreamManager* aOwner, ProcessedMediaStream* aStream); + void Init(OutputStreamManager* aOwner, + ProcessedMediaStream* aStream, + TrackID aNextAvailableTrackID); - // Connect mStream to the input stream. + // Connect the given input stream's audio and video tracks to mStream. // Return false is mStream is already destroyed, otherwise true. - bool Connect(MediaStream* aStream); + bool Connect(MediaStream* aStream, TrackID aAudioTrackID, TrackID aVideoTrackID); // Disconnect mStream from its input stream. // Return false is mStream is already destroyed, otherwise true. bool Disconnect(); @@ -34,12 +37,16 @@ public: bool Equals(MediaStream* aStream) const; // Return the graph mStream belongs to. MediaStreamGraph* Graph() const; + // The next TrackID that will not cause a collision in mStream. + TrackID NextAvailableTrackID() const; private: OutputStreamManager* mOwner; RefPtr mStream; - // mPort connects our mStream to an input stream. - RefPtr mPort; + // mPort connects an input stream to our mStream. + nsTArray> mPorts; + // For guaranteeing TrackID uniqueness in our mStream. + TrackID mNextAvailableTrackID = TRACK_INVALID; }; class OutputStreamManager { @@ -47,18 +54,26 @@ class OutputStreamManager { public: // Add the output stream to the collection. - void Add(ProcessedMediaStream* aStream, bool aFinishWhenEnded); + void Add(ProcessedMediaStream* aStream, + TrackID aNextAvailableTrackID, + bool aFinishWhenEnded); // Remove the output stream from the collection. void Remove(MediaStream* aStream); + // Clear all output streams from the collection. + void Clear(); + // The next TrackID that will not cause a collision in aOutputStream. + TrackID NextAvailableTrackIDFor(MediaStream* aOutputStream) const; // Return true if the collection empty. bool IsEmpty() const { MOZ_ASSERT(NS_IsMainThread()); return mStreams.IsEmpty(); } - // Connect all output streams in the collection to the input stream. - void Connect(MediaStream* aStream); - // Disconnect all output streams from the input stream. + // Connect the given input stream's tracks to all output streams. + void Connect(MediaStream* aStream, + TrackID aAudioTrackID, + TrackID aVideoTrackID); + // Disconnect the input stream to all output streams. void Disconnect(); // Return the graph these streams belong to or null if empty. MediaStreamGraph* Graph() const @@ -72,6 +87,8 @@ private: // Keep the input stream so we can connect the output streams that // are added after Connect(). RefPtr mInputStream; + TrackID mInputAudioTrackID = TRACK_INVALID; + TrackID mInputVideoTrackID = TRACK_INVALID; nsTArray mStreams; }; -- cgit v1.2.3