diff options
Diffstat (limited to 'dom/media/MediaStreamGraph.h')
| -rw-r--r-- | dom/media/MediaStreamGraph.h | 1407 |
1 files changed, 1407 insertions, 0 deletions
diff --git a/dom/media/MediaStreamGraph.h b/dom/media/MediaStreamGraph.h new file mode 100644 index 000000000..a6d77880a --- /dev/null +++ b/dom/media/MediaStreamGraph.h @@ -0,0 +1,1407 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*-*/ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef MOZILLA_MEDIASTREAMGRAPH_H_ +#define MOZILLA_MEDIASTREAMGRAPH_H_ + +#include "mozilla/LinkedList.h" +#include "mozilla/Mutex.h" +#include "mozilla/TaskQueue.h" + +#include "mozilla/dom/AudioChannelBinding.h" + +#include "AudioStream.h" +#include "nsTArray.h" +#include "nsIRunnable.h" +#include "VideoSegment.h" +#include "StreamTracks.h" +#include "MainThreadUtils.h" +#include "StreamTracks.h" +#include "nsAutoPtr.h" +#include "nsAutoRef.h" +#include <speex/speex_resampler.h> + +class nsIRunnable; + +template <> +class nsAutoRefTraits<SpeexResamplerState> : public nsPointerRefTraits<SpeexResamplerState> +{ + public: + static void Release(SpeexResamplerState* aState) { speex_resampler_destroy(aState); } +}; + +namespace mozilla { + +extern LazyLogModule gMediaStreamGraphLog; + +namespace dom { + enum class AudioContextOperation; +} + +namespace media { + template<typename V, typename E> class Pledge; +} + +/* + * MediaStreamGraph is a framework for synchronized audio/video processing + * and playback. It is designed to be used by other browser components such as + * HTML media elements, media capture APIs, real-time media streaming APIs, + * multitrack media APIs, and advanced audio APIs. + * + * The MediaStreamGraph uses a dedicated thread to process media --- the media + * graph thread. This ensures that we can process media through the graph + * without blocking on main-thread activity. The media graph is only modified + * on the media graph thread, to ensure graph changes can be processed without + * interfering with media processing. All interaction with the media graph + * thread is done with message passing. + * + * APIs that modify the graph or its properties are described as "control APIs". + * These APIs are asynchronous; they queue graph changes internally and + * those changes are processed all-at-once by the MediaStreamGraph. The + * MediaStreamGraph monitors the main thread event loop via nsIAppShell::RunInStableState + * to ensure that graph changes from a single event loop task are always + * processed all together. Control APIs should only be used on the main thread, + * currently; we may be able to relax that later. + * + * To allow precise synchronization of times in the control API, the + * MediaStreamGraph maintains a "media timeline". Control APIs that take or + * return times use that timeline. Those times never advance during + * an event loop task. This time is returned by MediaStreamGraph::GetCurrentTime(). + * + * Media decoding, audio processing and media playback use thread-safe APIs to + * the media graph to ensure they can continue while the main thread is blocked. + * + * When the graph is changed, we may need to throw out buffered data and + * reprocess it. This is triggered automatically by the MediaStreamGraph. + */ + +class AudioNodeEngine; +class AudioNodeExternalInputStream; +class AudioNodeStream; +class MediaInputPort; +class MediaStream; +class MediaStreamGraph; +class MediaStreamGraphImpl; +class ProcessedMediaStream; +class SourceMediaStream; + +class AudioDataListenerInterface { +protected: + // Protected destructor, to discourage deletion outside of Release(): + virtual ~AudioDataListenerInterface() {} + +public: + /* These are for cubeb audio input & output streams: */ + /** + * Output data to speakers, for use as the "far-end" data for echo + * cancellation. This is not guaranteed to be in any particular size + * chunks. + */ + virtual void NotifyOutputData(MediaStreamGraph* aGraph, + AudioDataValue* aBuffer, size_t aFrames, + TrackRate aRate, uint32_t aChannels) = 0; + /** + * Input data from a microphone (or other audio source. This is not + * guaranteed to be in any particular size chunks. + */ + virtual void NotifyInputData(MediaStreamGraph* aGraph, + const AudioDataValue* aBuffer, size_t aFrames, + TrackRate aRate, uint32_t aChannels) = 0; + + /** + * Called when the underlying audio device has changed. + */ + virtual void DeviceChanged() = 0; +}; + +class AudioDataListener : public AudioDataListenerInterface { +protected: + // Protected destructor, to discourage deletion outside of Release(): + virtual ~AudioDataListener() {} + +public: + NS_INLINE_DECL_THREADSAFE_REFCOUNTING(AudioDataListener) +}; + +/** + * This is a base class for main-thread listener callbacks. + * This callback is invoked on the main thread when the main-thread-visible + * state of a stream has changed. + * + * These methods are called with the media graph monitor held, so + * reentry into general media graph methods is not possible. + * You should do something non-blocking and non-reentrant (e.g. dispatch an + * event) and return. DispatchFromMainThreadAfterNextStreamStateUpdate + * would be a good choice. + * The listener is allowed to synchronously remove itself from the stream, but + * not add or remove any other listeners. + */ +class MainThreadMediaStreamListener { +public: + virtual void NotifyMainThreadStreamFinished() = 0; +}; + +/** + * Helper struct used to keep track of memory usage by AudioNodes. + */ +struct AudioNodeSizes +{ + AudioNodeSizes() : mStream(0), mEngine(0), mNodeType() {} + size_t mStream; + size_t mEngine; + const char* mNodeType; +}; + +class AudioNodeEngine; +class AudioNodeExternalInputStream; +class AudioNodeStream; +class AudioSegment; +class DirectMediaStreamListener; +class DirectMediaStreamTrackListener; +class MediaInputPort; +class MediaStreamGraphImpl; +class MediaStreamListener; +class MediaStreamTrackListener; +class MediaStreamVideoSink; +class ProcessedMediaStream; +class SourceMediaStream; +class TrackUnionStream; + +enum MediaStreamGraphEvent : uint32_t; +enum TrackEventCommand : uint32_t; + +/** + * Helper struct for binding a track listener to a specific TrackID. + */ +template<typename Listener> +struct TrackBound +{ + RefPtr<Listener> mListener; + TrackID mTrackID; +}; + +/** + * Describes how a track should be disabled. + * + * ENABLED Not disabled. + * SILENCE_BLACK Audio data is turned into silence, video frames are made black. + * SILENCE_FREEZE Audio data is turned into silence, video freezes at last frame. + */ +enum class DisabledTrackMode +{ + ENABLED, SILENCE_BLACK, SILENCE_FREEZE +}; +struct DisabledTrack { + DisabledTrack(TrackID aTrackID, DisabledTrackMode aMode) + : mTrackID(aTrackID), mMode(aMode) {} + TrackID mTrackID; + DisabledTrackMode mMode; +}; + +/** + * A stream of synchronized audio and video data. All (not blocked) streams + * progress at the same rate --- "real time". Streams cannot seek. The only + * operation readers can perform on a stream is to read the next data. + * + * Consumers of a stream can be reading from it at different offsets, but that + * should only happen due to the order in which consumers are being run. + * Those offsets must not diverge in the long term, otherwise we would require + * unbounded buffering. + * + * Streams can be in a "blocked" state. While blocked, a stream does not + * produce data. A stream can be explicitly blocked via the control API, + * or implicitly blocked by whatever's generating it (e.g. an underrun in the + * source resource), or implicitly blocked because something consuming it + * blocks, or implicitly because it has finished. + * + * A stream can be in a "finished" state. "Finished" streams are permanently + * blocked. + * + * Transitions into and out of the "blocked" and "finished" states are managed + * by the MediaStreamGraph on the media graph thread. + * + * We buffer media data ahead of the consumers' reading offsets. It is possible + * to have buffered data but still be blocked. + * + * Any stream can have its audio and video playing when requested. The media + * stream graph plays audio by constructing audio output streams as necessary. + * Video is played by setting video frames into an MediaStreamVideoSink at the right + * time. To ensure video plays in sync with audio, make sure that the same + * stream is playing both the audio and video. + * + * The data in a stream is managed by StreamTracks. It consists of a set of + * tracks of various types that can start and end over time. + * + * Streams are explicitly managed. The client creates them via + * MediaStreamGraph::CreateInput/ProcessedMediaStream, and releases them by calling + * Destroy() when no longer needed (actual destruction will be deferred). + * The actual object is owned by the MediaStreamGraph. The basic idea is that + * main thread objects will keep Streams alive as long as necessary (using the + * cycle collector to clean up whenever needed). + * + * We make them refcounted only so that stream-related messages with MediaStream* + * pointers can be sent to the main thread safely. + * + * The lifetimes of MediaStreams are controlled from the main thread. + * For MediaStreams exposed to the DOM, the lifetime is controlled by the DOM + * wrapper; the DOM wrappers own their associated MediaStreams. When a DOM + * wrapper is destroyed, it sends a Destroy message for the associated + * MediaStream and clears its reference (the last main-thread reference to + * the object). When the Destroy message is processed on the graph manager + * thread we immediately release the affected objects (disentangling them + * from other objects as necessary). + * + * This could cause problems for media processing if a MediaStream is + * destroyed while a downstream MediaStream is still using it. Therefore + * the DOM wrappers must keep upstream MediaStreams alive as long as they + * could be being used in the media graph. + * + * At any time, however, a set of MediaStream wrappers could be + * collected via cycle collection. Destroy messages will be sent + * for those objects in arbitrary order and the MediaStreamGraph has to be able + * to handle this. + */ + +// GetCurrentTime is defined in winbase.h as zero argument macro forwarding to +// GetTickCount() and conflicts with MediaStream::GetCurrentTime. +#ifdef GetCurrentTime +#undef GetCurrentTime +#endif + +class MediaStream : public mozilla::LinkedListElement<MediaStream> +{ +public: + NS_INLINE_DECL_THREADSAFE_REFCOUNTING(MediaStream) + + MediaStream(); + +protected: + // Protected destructor, to discourage deletion outside of Release(): + virtual ~MediaStream(); + +public: + /** + * Returns the graph that owns this stream. + */ + MediaStreamGraphImpl* GraphImpl(); + MediaStreamGraph* Graph(); + /** + * Sets the graph that owns this stream. Should only be called once. + */ + void SetGraphImpl(MediaStreamGraphImpl* aGraph); + void SetGraphImpl(MediaStreamGraph* aGraph); + + /** + * Returns sample rate of the graph. + */ + TrackRate GraphRate() { return mTracks.GraphRate(); } + + // Control API. + // Since a stream can be played multiple ways, we need to combine independent + // volume settings. The aKey parameter is used to keep volume settings + // separate. Since the stream is always playing the same contents, only + // a single audio output stream is used; the volumes are combined. + // Currently only the first enabled audio track is played. + // XXX change this so all enabled audio tracks are mixed and played. + virtual void AddAudioOutput(void* aKey); + virtual void SetAudioOutputVolume(void* aKey, float aVolume); + virtual void RemoveAudioOutput(void* aKey); + // Since a stream can be played multiple ways, we need to be able to + // play to multiple MediaStreamVideoSinks. + // Only the first enabled video track is played. + virtual void AddVideoOutput(MediaStreamVideoSink* aSink, + TrackID aID = TRACK_ANY); + virtual void RemoveVideoOutput(MediaStreamVideoSink* aSink, + TrackID aID = TRACK_ANY); + // Explicitly suspend. Useful for example if a media element is pausing + // and we need to stop its stream emitting its buffered data. As soon as the + // Suspend message reaches the graph, the stream stops processing. It + // ignores its inputs and produces silence/no video until Resumed. Its + // current time does not advance. + virtual void Suspend(); + virtual void Resume(); + // Events will be dispatched by calling methods of aListener. + virtual void AddListener(MediaStreamListener* aListener); + virtual void RemoveListener(MediaStreamListener* aListener); + virtual void AddTrackListener(MediaStreamTrackListener* aListener, + TrackID aTrackID); + virtual void RemoveTrackListener(MediaStreamTrackListener* aListener, + TrackID aTrackID); + + /** + * Adds aListener to the source stream of track aTrackID in this stream. + * When the MediaStreamGraph processes the added listener, it will traverse + * the graph and add it to the track's source stream (remapping the TrackID + * along the way). + * Note that the listener will be notified on the MediaStreamGraph thread + * with whether the installation of it at the source was successful or not. + */ + virtual void AddDirectTrackListener(DirectMediaStreamTrackListener* aListener, + TrackID aTrackID); + + /** + * Removes aListener from the source stream of track aTrackID in this stream. + * Note that the listener has already been removed if the link between the + * source of track aTrackID and this stream has been broken (and made track + * aTrackID end). The caller doesn't have to care about this, removing when + * the source cannot be found, or when the listener had already been removed + * does nothing. + */ + virtual void RemoveDirectTrackListener(DirectMediaStreamTrackListener* aListener, + TrackID aTrackID); + + // A disabled track has video replaced by black, and audio replaced by + // silence. + void SetTrackEnabled(TrackID aTrackID, DisabledTrackMode aMode); + + // Finish event will be notified by calling methods of aListener. It is the + // responsibility of the caller to remove aListener before it is destroyed. + void AddMainThreadListener(MainThreadMediaStreamListener* aListener); + // It's safe to call this even if aListener is not currently a listener; + // the call will be ignored. + void RemoveMainThreadListener(MainThreadMediaStreamListener* aListener) + { + MOZ_ASSERT(NS_IsMainThread()); + MOZ_ASSERT(aListener); + mMainThreadListeners.RemoveElement(aListener); + } + + /** + * Ensure a runnable will run on the main thread after running all pending + * updates that were sent from the graph thread or will be sent before the + * graph thread receives the next graph update. + * + * If the graph has been shut down or destroyed, then the runnable will be + * dispatched to the event queue immediately. If the graph is non-realtime + * and has not started, then the runnable will be run + * synchronously/immediately. (There are no pending updates in these + * situations.) + * + * Main thread only. + */ + void RunAfterPendingUpdates(already_AddRefed<nsIRunnable> aRunnable); + + // Signal that the client is done with this MediaStream. It will be deleted + // later. Do not mix usage of Destroy() with RegisterUser()/UnregisterUser(). + // That will cause the MediaStream to be destroyed twice, which will cause + // some assertions to fail. + virtual void Destroy(); + // Signal that a client is using this MediaStream. Useful to not have to + // explicitly manage ownership (responsibility to Destroy()) when there are + // multiple clients using a MediaStream. + void RegisterUser(); + // Signal that a client no longer needs this MediaStream. When the number of + // clients using this MediaStream reaches 0, it will be destroyed. + void UnregisterUser(); + + // Returns the main-thread's view of how much data has been processed by + // this stream. + StreamTime GetCurrentTime() + { + NS_ASSERTION(NS_IsMainThread(), "Call only on main thread"); + return mMainThreadCurrentTime; + } + // Return the main thread's view of whether this stream has finished. + bool IsFinished() + { + NS_ASSERTION(NS_IsMainThread(), "Call only on main thread"); + return mMainThreadFinished; + } + + bool IsDestroyed() + { + NS_ASSERTION(NS_IsMainThread(), "Call only on main thread"); + return mMainThreadDestroyed; + } + + friend class MediaStreamGraphImpl; + friend class MediaInputPort; + friend class AudioNodeExternalInputStream; + + virtual SourceMediaStream* AsSourceStream() { return nullptr; } + virtual ProcessedMediaStream* AsProcessedStream() { return nullptr; } + virtual AudioNodeStream* AsAudioNodeStream() { return nullptr; } + virtual TrackUnionStream* AsTrackUnionStream() { return nullptr; } + + // These Impl methods perform the core functionality of the control methods + // above, on the media graph thread. + /** + * Stop all stream activity and disconnect it from all inputs and outputs. + * This must be idempotent. + */ + virtual void DestroyImpl(); + StreamTime GetTracksEnd() { return mTracks.GetEnd(); } +#ifdef DEBUG + void DumpTrackInfo() { return mTracks.DumpTrackInfo(); } +#endif + void SetAudioOutputVolumeImpl(void* aKey, float aVolume); + void AddAudioOutputImpl(void* aKey); + // Returns true if this stream has an audio output. + bool HasAudioOutput() + { + return !mAudioOutputs.IsEmpty(); + } + void RemoveAudioOutputImpl(void* aKey); + void AddVideoOutputImpl(already_AddRefed<MediaStreamVideoSink> aSink, + TrackID aID); + void RemoveVideoOutputImpl(MediaStreamVideoSink* aSink, TrackID aID); + void AddListenerImpl(already_AddRefed<MediaStreamListener> aListener); + void RemoveListenerImpl(MediaStreamListener* aListener); + void RemoveAllListenersImpl(); + virtual void AddTrackListenerImpl(already_AddRefed<MediaStreamTrackListener> aListener, + TrackID aTrackID); + virtual void RemoveTrackListenerImpl(MediaStreamTrackListener* aListener, + TrackID aTrackID); + virtual void AddDirectTrackListenerImpl(already_AddRefed<DirectMediaStreamTrackListener> aListener, + TrackID aTrackID); + virtual void RemoveDirectTrackListenerImpl(DirectMediaStreamTrackListener* aListener, + TrackID aTrackID); + virtual void SetTrackEnabledImpl(TrackID aTrackID, DisabledTrackMode aMode); + DisabledTrackMode GetDisabledTrackMode(TrackID aTrackID); + + void AddConsumer(MediaInputPort* aPort) + { + mConsumers.AppendElement(aPort); + } + void RemoveConsumer(MediaInputPort* aPort) + { + mConsumers.RemoveElement(aPort); + } + uint32_t ConsumerCount() + { + return mConsumers.Length(); + } + StreamTracks& GetStreamTracks() { return mTracks; } + GraphTime GetStreamTracksStartTime() { return mTracksStartTime; } + + double StreamTimeToSeconds(StreamTime aTime) + { + NS_ASSERTION(0 <= aTime && aTime <= STREAM_TIME_MAX, "Bad time"); + return static_cast<double>(aTime)/mTracks.GraphRate(); + } + int64_t StreamTimeToMicroseconds(StreamTime aTime) + { + NS_ASSERTION(0 <= aTime && aTime <= STREAM_TIME_MAX, "Bad time"); + return (aTime*1000000)/mTracks.GraphRate(); + } + StreamTime SecondsToNearestStreamTime(double aSeconds) + { + NS_ASSERTION(0 <= aSeconds && aSeconds <= TRACK_TICKS_MAX/TRACK_RATE_MAX, + "Bad seconds"); + return mTracks.GraphRate() * aSeconds + 0.5; + } + StreamTime MicrosecondsToStreamTimeRoundDown(int64_t aMicroseconds) { + return (aMicroseconds*mTracks.GraphRate())/1000000; + } + + TrackTicks TimeToTicksRoundUp(TrackRate aRate, StreamTime aTime) + { + return RateConvertTicksRoundUp(aRate, mTracks.GraphRate(), aTime); + } + StreamTime TicksToTimeRoundDown(TrackRate aRate, TrackTicks aTicks) + { + return RateConvertTicksRoundDown(mTracks.GraphRate(), aRate, aTicks); + } + /** + * Convert graph time to stream time. aTime must be <= mStateComputedTime + * to ensure we know exactly how much time this stream will be blocked during + * the interval. + */ + StreamTime GraphTimeToStreamTimeWithBlocking(GraphTime aTime); + /** + * Convert graph time to stream time. This assumes there is no blocking time + * to take account of, which is always true except between a stream + * having its blocking time calculated in UpdateGraph and its blocking time + * taken account of in UpdateCurrentTimeForStreams. + */ + StreamTime GraphTimeToStreamTime(GraphTime aTime); + /** + * Convert stream time to graph time. This assumes there is no blocking time + * to take account of, which is always true except between a stream + * having its blocking time calculated in UpdateGraph and its blocking time + * taken account of in UpdateCurrentTimeForStreams. + */ + GraphTime StreamTimeToGraphTime(StreamTime aTime); + + bool IsFinishedOnGraphThread() { return mFinished; } + void FinishOnGraphThread(); + + bool HasCurrentData() { return mHasCurrentData; } + + /** + * Find track by track id. + */ + StreamTracks::Track* FindTrack(TrackID aID); + + StreamTracks::Track* EnsureTrack(TrackID aTrack); + + virtual void ApplyTrackDisabling(TrackID aTrackID, MediaSegment* aSegment, MediaSegment* aRawSegment = nullptr); + + // Return true if the main thread needs to observe updates from this stream. + virtual bool MainThreadNeedsUpdates() const + { + return true; + } + + virtual size_t SizeOfExcludingThis(MallocSizeOf aMallocSizeOf) const; + virtual size_t SizeOfIncludingThis(MallocSizeOf aMallocSizeOf) const; + + void SetAudioChannelType(dom::AudioChannel aType) { mAudioChannelType = aType; } + dom::AudioChannel AudioChannelType() const { return mAudioChannelType; } + + bool IsSuspended() { return mSuspendedCount > 0; } + void IncrementSuspendCount() { ++mSuspendedCount; } + void DecrementSuspendCount() + { + NS_ASSERTION(mSuspendedCount > 0, "Suspend count underrun"); + --mSuspendedCount; + } + +protected: + // |AdvanceTimeVaryingValuesToCurrentTime| will be override in SourceMediaStream. + virtual void AdvanceTimeVaryingValuesToCurrentTime(GraphTime aCurrentTime, + GraphTime aBlockedTime) + { + mTracksStartTime += aBlockedTime; + mTracks.ForgetUpTo(aCurrentTime - mTracksStartTime); + } + + void NotifyMainThreadListeners() + { + NS_ASSERTION(NS_IsMainThread(), "Call only on main thread"); + + for (int32_t i = mMainThreadListeners.Length() - 1; i >= 0; --i) { + mMainThreadListeners[i]->NotifyMainThreadStreamFinished(); + } + mMainThreadListeners.Clear(); + } + + bool ShouldNotifyStreamFinished() + { + NS_ASSERTION(NS_IsMainThread(), "Call only on main thread"); + if (!mMainThreadFinished || mFinishedNotificationSent) { + return false; + } + + mFinishedNotificationSent = true; + return true; + } + + // This state is all initialized on the main thread but + // otherwise modified only on the media graph thread. + + // Buffered data. The start of the buffer corresponds to mTracksStartTime. + // Conceptually the buffer contains everything this stream has ever played, + // but we forget some prefix of the buffered data to bound the space usage. + StreamTracks mTracks; + // The time when the buffered data could be considered to have started playing. + // This increases over time to account for time the stream was blocked before + // mCurrentTime. + GraphTime mTracksStartTime; + + // Client-set volume of this stream + struct AudioOutput { + explicit AudioOutput(void* aKey) : mKey(aKey), mVolume(1.0f) {} + void* mKey; + float mVolume; + }; + nsTArray<AudioOutput> mAudioOutputs; + nsTArray<TrackBound<MediaStreamVideoSink>> mVideoOutputs; + // We record the last played video frame to avoid playing the frame again + // with a different frame id. + VideoFrame mLastPlayedVideoFrame; + nsTArray<RefPtr<MediaStreamListener> > mListeners; + nsTArray<TrackBound<MediaStreamTrackListener>> mTrackListeners; + nsTArray<MainThreadMediaStreamListener*> mMainThreadListeners; + // List of disabled TrackIDs and their associated disabled mode. + // They can either by disabled by frames being replaced by black, or by + // retaining the previous frame. + nsTArray<DisabledTrack> mDisabledTracks; + + // GraphTime at which this stream starts blocking. + // This is only valid up to mStateComputedTime. The stream is considered to + // have not been blocked before mCurrentTime (its mTracksStartTime is increased + // as necessary to account for that time instead). + GraphTime mStartBlocking; + + // MediaInputPorts to which this is connected + nsTArray<MediaInputPort*> mConsumers; + + // Where audio output is going. There is one AudioOutputStream per + // audio track. + struct AudioOutputStream + { + // When we started audio playback for this track. + // Add mStream->GetPosition() to find the current audio playback position. + GraphTime mAudioPlaybackStartTime; + // Amount of time that we've wanted to play silence because of the stream + // blocking. + MediaTime mBlockedAudioTime; + // Last tick written to the audio output. + StreamTime mLastTickWritten; + TrackID mTrackID; + }; + nsTArray<AudioOutputStream> mAudioOutputStreams; + + /** + * Number of outstanding suspend operations on this stream. Stream is + * suspended when this is > 0. + */ + int32_t mSuspendedCount; + + /** + * When true, this means the stream will be finished once all + * buffered data has been consumed. + */ + bool mFinished; + /** + * When true, mFinished is true and we've played all the data in this stream + * and fired NotifyFinished notifications. + */ + bool mNotifiedFinished; + /** + * When true, the last NotifyBlockingChanged delivered to the listeners + * indicated that the stream is blocked. + */ + bool mNotifiedBlocked; + /** + * True if some data can be present by this stream if/when it's unblocked. + * Set by the stream itself on the MediaStreamGraph thread. Only changes + * from false to true once a stream has data, since we won't + * unblock it until there's more data. + */ + bool mHasCurrentData; + /** + * True if mHasCurrentData is true and we've notified listeners. + */ + bool mNotifiedHasCurrentData; + + // Main-thread views of state + StreamTime mMainThreadCurrentTime; + bool mMainThreadFinished; + bool mFinishedNotificationSent; + bool mMainThreadDestroyed; + int mNrOfMainThreadUsers; + + // Our media stream graph. null if destroyed on the graph thread. + MediaStreamGraphImpl* mGraph; + + dom::AudioChannel mAudioChannelType; +}; + +/** + * This is a stream into which a decoder can write audio and video. + * + * Audio and video can be written on any thread, but you probably want to + * always write from the same thread to avoid unexpected interleavings. + */ +class SourceMediaStream : public MediaStream +{ +public: + explicit SourceMediaStream(); + + SourceMediaStream* AsSourceStream() override { return this; } + + // Media graph thread only + + // Users of audio inputs go through the stream so it can track when the + // last stream referencing an input goes away, so it can close the cubeb + // input. Also note: callable on any thread (though it bounces through + // MainThread to set the command if needed). + nsresult OpenAudioInput(int aID, + AudioDataListener *aListener); + // Note: also implied when Destroy() happens + void CloseAudioInput(); + + void DestroyImpl() override; + + // Call these on any thread. + /** + * Enable or disable pulling. When pulling is enabled, NotifyPull + * gets called on MediaStreamListeners for this stream during the + * MediaStreamGraph control loop. Pulling is initially disabled. + * Due to unavoidable race conditions, after a call to SetPullEnabled(false) + * it is still possible for a NotifyPull to occur. + */ + void SetPullEnabled(bool aEnabled); + + /** + * These add/remove DirectListeners, which allow bypassing the graph and any + * synchronization delays for e.g. PeerConnection, which wants the data ASAP + * and lets the far-end handle sync and playout timing. + */ + void NotifyListenersEventImpl(MediaStreamGraphEvent aEvent); + void NotifyListenersEvent(MediaStreamGraphEvent aEvent); + void AddDirectListener(DirectMediaStreamListener* aListener); + void RemoveDirectListener(DirectMediaStreamListener* aListener); + + enum { + ADDTRACK_QUEUED = 0x01 // Queue track add until FinishAddTracks() + }; + /** + * Add a new track to the stream starting at the given base time (which + * must be greater than or equal to the last time passed to + * AdvanceKnownTracksTime). Takes ownership of aSegment. aSegment should + * contain data starting after aStart. + */ + void AddTrack(TrackID aID, StreamTime aStart, MediaSegment* aSegment, + uint32_t aFlags = 0) + { + AddTrackInternal(aID, GraphRate(), aStart, aSegment, aFlags); + } + + /** + * Like AddTrack, but resamples audio from aRate to the graph rate. + */ + void AddAudioTrack(TrackID aID, TrackRate aRate, StreamTime aStart, + AudioSegment* aSegment, uint32_t aFlags = 0); + + /** + * Call after a series of AddTrack or AddAudioTrack calls to implement + * any pending track adds. + */ + void FinishAddTracks(); + + /** + * Append media data to a track. Ownership of aSegment remains with the caller, + * but aSegment is emptied. + * Returns false if the data was not appended because no such track exists + * or the stream was already finished. + */ + bool AppendToTrack(TrackID aID, MediaSegment* aSegment, MediaSegment *aRawSegment = nullptr); + /** + * Get the stream time of the end of the data that has been appended so far. + * Can be called from any thread but won't be useful if it can race with + * an AppendToTrack call, so should probably just be called from the thread + * that also calls AppendToTrack. + */ + StreamTime GetEndOfAppendedData(TrackID aID); + /** + * Indicate that a track has ended. Do not do any more API calls + * affecting this track. + * Ignored if the track does not exist. + */ + void EndTrack(TrackID aID); + /** + * Indicate that no tracks will be added starting before time aKnownTime. + * aKnownTime must be >= its value at the last call to AdvanceKnownTracksTime. + */ + void AdvanceKnownTracksTime(StreamTime aKnownTime); + /** + * Indicate that this stream should enter the "finished" state. All tracks + * must have been ended via EndTrack. The finish time of the stream is + * when all tracks have ended. + */ + void FinishWithLockHeld(); + void Finish() + { + MutexAutoLock lock(mMutex); + FinishWithLockHeld(); + } + + // Overriding allows us to hold the mMutex lock while changing the track enable status + void SetTrackEnabledImpl(TrackID aTrackID, DisabledTrackMode aMode) override; + + // Overriding allows us to ensure mMutex is locked while changing the track enable status + void + ApplyTrackDisabling(TrackID aTrackID, MediaSegment* aSegment, + MediaSegment* aRawSegment = nullptr) override { + mMutex.AssertCurrentThreadOwns(); + MediaStream::ApplyTrackDisabling(aTrackID, aSegment, aRawSegment); + } + + /** + * End all tracks and Finish() this stream. Used to voluntarily revoke access + * to a LocalMediaStream. + */ + void EndAllTrackAndFinish(); + + void RegisterForAudioMixing(); + + /** + * Returns true if this SourceMediaStream contains at least one audio track + * that is in pending state. + * This is thread safe, and takes the SourceMediaStream mutex. + */ + bool HasPendingAudioTrack(); + + TimeStamp GetStreamTracksStrartTimeStamp() { + MutexAutoLock lock(mMutex); + return mStreamTracksStartTimeStamp; + } + + // XXX need a Reset API + + friend class MediaStreamGraphImpl; + +protected: + enum TrackCommands : uint32_t; + + virtual ~SourceMediaStream(); + + /** + * Data for each track that hasn't ended. + */ + struct TrackData { + TrackID mID; + // Sample rate of the input data. + TrackRate mInputRate; + // Resampler if the rate of the input track does not match the + // MediaStreamGraph's. + nsAutoRef<SpeexResamplerState> mResampler; + int mResamplerChannelCount; + StreamTime mStart; + // End-time of data already flushed to the track (excluding mData) + StreamTime mEndOfFlushedData; + // Each time the track updates are flushed to the media graph thread, + // the segment buffer is emptied. + nsAutoPtr<MediaSegment> mData; + // Each time the track updates are flushed to the media graph thread, + // this is cleared. + uint32_t mCommands; + }; + + bool NeedsMixing(); + + void ResampleAudioToGraphSampleRate(TrackData* aTrackData, MediaSegment* aSegment); + + void AddDirectTrackListenerImpl(already_AddRefed<DirectMediaStreamTrackListener> aListener, + TrackID aTrackID) override; + void RemoveDirectTrackListenerImpl(DirectMediaStreamTrackListener* aListener, + TrackID aTrackID) override; + + void AddTrackInternal(TrackID aID, TrackRate aRate, + StreamTime aStart, MediaSegment* aSegment, + uint32_t aFlags); + + TrackData* FindDataForTrack(TrackID aID) + { + mMutex.AssertCurrentThreadOwns(); + for (uint32_t i = 0; i < mUpdateTracks.Length(); ++i) { + if (mUpdateTracks[i].mID == aID) { + return &mUpdateTracks[i]; + } + } + return nullptr; + } + + /** + * Notify direct consumers of new data to one of the stream tracks. + * The data doesn't have to be resampled (though it may be). This is called + * from AppendToTrack on the thread providing the data, and will call + * the Listeners on this thread. + */ + void NotifyDirectConsumers(TrackData *aTrack, + MediaSegment *aSegment); + + virtual void + AdvanceTimeVaryingValuesToCurrentTime(GraphTime aCurrentTime, + GraphTime aBlockedTime) override; + void SetStreamTracksStartTimeStamp(const TimeStamp& aTimeStamp) + { + MutexAutoLock lock(mMutex); + mStreamTracksStartTimeStamp = aTimeStamp; + } + + // Only accessed on the MSG thread. Used so to ask the MSGImpl to usecount + // users of a specific input. + // XXX Should really be a CubebUtils::AudioDeviceID, but they aren't + // copyable (opaque pointers) + RefPtr<AudioDataListener> mInputListener; + + // This must be acquired *before* MediaStreamGraphImpl's lock, if they are + // held together. + Mutex mMutex; + // protected by mMutex + StreamTime mUpdateKnownTracksTime; + // This time stamp will be updated in adding and blocked SourceMediaStream, + // |AddStreamGraphThread| and |AdvanceTimeVaryingValuesToCurrentTime| in + // particularly. + TimeStamp mStreamTracksStartTimeStamp; + nsTArray<TrackData> mUpdateTracks; + nsTArray<TrackData> mPendingTracks; + nsTArray<RefPtr<DirectMediaStreamListener>> mDirectListeners; + nsTArray<TrackBound<DirectMediaStreamTrackListener>> mDirectTrackListeners; + bool mPullEnabled; + bool mUpdateFinished; + bool mNeedsMixing; +}; + +/** + * The blocking mode decides how a track should be blocked in a MediaInputPort. + */ +enum class BlockingMode +{ + /** + * BlockingMode CREATION blocks the source track from being created + * in the destination. It'll end if it already exists. + */ + CREATION, + /** + * BlockingMode END_EXISTING allows a track to be created in the destination + * but will end it before any data has been passed through. + */ + END_EXISTING, +}; + +/** + * Represents a connection between a ProcessedMediaStream and one of its + * input streams. + * We make these refcounted so that stream-related messages with MediaInputPort* + * pointers can be sent to the main thread safely. + * + * A port can be locked to a specific track in the source stream, in which case + * only this track will be forwarded to the destination stream. TRACK_ANY + * can used to signal that all tracks shall be forwarded. + * + * When a port is locked to a specific track in the source stream, it may also + * indicate a TrackID to map this source track to in the destination stream + * by setting aDestTrack to an explicit ID. When we do this, we must know + * that this TrackID in the destination stream is available. We assert during + * processing that the ID is available and that there are no generic input + * ports already attached to the destination stream. + * Note that this is currently only handled by TrackUnionStreams. + * + * When a port's source or destination stream dies, the stream's DestroyImpl + * calls MediaInputPort::Disconnect to disconnect the port from + * the source and destination streams. + * + * The lifetimes of MediaInputPort are controlled from the main thread. + * The media graph adds a reference to the port. When a MediaInputPort is no + * longer needed, main-thread code sends a Destroy message for the port and + * clears its reference (the last main-thread reference to the object). When + * the Destroy message is processed on the graph manager thread we disconnect + * the port and drop the graph's reference, destroying the object. + */ +class MediaInputPort final +{ +private: + // Do not call this constructor directly. Instead call aDest->AllocateInputPort. + MediaInputPort(MediaStream* aSource, TrackID& aSourceTrack, + ProcessedMediaStream* aDest, TrackID& aDestTrack, + uint16_t aInputNumber, uint16_t aOutputNumber) + : mSource(aSource) + , mSourceTrack(aSourceTrack) + , mDest(aDest) + , mDestTrack(aDestTrack) + , mInputNumber(aInputNumber) + , mOutputNumber(aOutputNumber) + , mGraph(nullptr) + { + MOZ_COUNT_CTOR(MediaInputPort); + } + + // Private destructor, to discourage deletion outside of Release(): + ~MediaInputPort() + { + MOZ_COUNT_DTOR(MediaInputPort); + } + +public: + NS_INLINE_DECL_THREADSAFE_REFCOUNTING(MediaInputPort) + + // Called on graph manager thread + // Do not call these from outside MediaStreamGraph.cpp! + void Init(); + // Called during message processing to trigger removal of this stream. + void Disconnect(); + + // Control API + /** + * Disconnects and destroys the port. The caller must not reference this + * object again. + */ + void Destroy(); + + // Any thread + MediaStream* GetSource() { return mSource; } + TrackID GetSourceTrackId() { return mSourceTrack; } + ProcessedMediaStream* GetDestination() { return mDest; } + TrackID GetDestinationTrackId() { return mDestTrack; } + + /** + * Block aTrackId in the source stream from being passed through the port. + * Consumers will interpret this track as ended. + * Returns a pledge that resolves on the main thread after the track block has + * been applied by the MSG. + */ + already_AddRefed<media::Pledge<bool, nsresult>> BlockSourceTrackId(TrackID aTrackId, + BlockingMode aBlockingMode); +private: + void BlockSourceTrackIdImpl(TrackID aTrackId, BlockingMode aBlockingMode); + +public: + // Returns true if aTrackId has not been blocked for any reason and this port + // has not been locked to another track. + bool PassTrackThrough(TrackID aTrackId) { + bool blocked = false; + for (auto pair : mBlockedTracks) { + if (pair.first() == aTrackId && + (pair.second() == BlockingMode::CREATION || + pair.second() == BlockingMode::END_EXISTING)) { + blocked = true; + break; + } + } + return !blocked && (mSourceTrack == TRACK_ANY || mSourceTrack == aTrackId); + } + + // Returns true if aTrackId has not been blocked for track creation and this + // port has not been locked to another track. + bool AllowCreationOf(TrackID aTrackId) { + bool blocked = false; + for (auto pair : mBlockedTracks) { + if (pair.first() == aTrackId && + pair.second() == BlockingMode::CREATION) { + blocked = true; + break; + } + } + return !blocked && (mSourceTrack == TRACK_ANY || mSourceTrack == aTrackId); + } + + uint16_t InputNumber() const { return mInputNumber; } + uint16_t OutputNumber() const { return mOutputNumber; } + + // Call on graph manager thread + struct InputInterval { + GraphTime mStart; + GraphTime mEnd; + bool mInputIsBlocked; + }; + // Find the next time interval starting at or after aTime during which + // mDest is not blocked and mSource's blocking status does not change. + InputInterval GetNextInputInterval(GraphTime aTime); + + /** + * Returns the graph that owns this port. + */ + MediaStreamGraphImpl* GraphImpl(); + MediaStreamGraph* Graph(); + /** + * Sets the graph that owns this stream. Should only be called once. + */ + void SetGraphImpl(MediaStreamGraphImpl* aGraph); + + size_t SizeOfExcludingThis(MallocSizeOf aMallocSizeOf) const + { + size_t amount = 0; + + // Not owned: + // - mSource + // - mDest + // - mGraph + return amount; + } + + size_t SizeOfIncludingThis(MallocSizeOf aMallocSizeOf) const + { + return aMallocSizeOf(this) + SizeOfExcludingThis(aMallocSizeOf); + } + +private: + friend class MediaStreamGraphImpl; + friend class MediaStream; + friend class ProcessedMediaStream; + // Never modified after Init() + MediaStream* mSource; + TrackID mSourceTrack; + ProcessedMediaStream* mDest; + TrackID mDestTrack; + // The input and output numbers are optional, and are currently only used by + // Web Audio. + const uint16_t mInputNumber; + const uint16_t mOutputNumber; + + typedef Pair<TrackID, BlockingMode> BlockedTrack; + nsTArray<BlockedTrack> mBlockedTracks; + + // Our media stream graph + MediaStreamGraphImpl* mGraph; +}; + +/** + * This stream processes zero or more input streams in parallel to produce + * its output. The details of how the output is produced are handled by + * subclasses overriding the ProcessInput method. + */ +class ProcessedMediaStream : public MediaStream +{ +public: + explicit ProcessedMediaStream() + : MediaStream(), mAutofinish(false), mCycleMarker(0) + {} + + // Control API. + /** + * Allocates a new input port attached to source aStream. + * This stream can be removed by calling MediaInputPort::Remove(). + * + * The input port is tied to aTrackID in the source stream. + * aTrackID can be set to TRACK_ANY to automatically forward all tracks from + * aStream. + * + * If aTrackID is an explicit ID, aDestTrackID can also be made explicit + * to ensure that the track is assigned this ID in the destination stream. + * To avoid intermittent TrackID collisions the destination stream may not + * have any existing generic input ports (with TRACK_ANY source track) when + * you allocate an input port with a destination TrackID. + * + * To end a track in the destination stream forwarded with TRACK_ANY, + * it can be blocked in the input port through MediaInputPort::BlockTrackId(). + * + * Tracks in aBlockedTracks will be blocked in the input port initially. This + * ensures that they don't get created by the MSG-thread before we can + * BlockTrackId() on the main thread. + */ + already_AddRefed<MediaInputPort> + AllocateInputPort(MediaStream* aStream, + TrackID aTrackID = TRACK_ANY, + TrackID aDestTrackID = TRACK_ANY, + uint16_t aInputNumber = 0, + uint16_t aOutputNumber = 0, + nsTArray<TrackID>* aBlockedTracks = nullptr); + /** + * Force this stream into the finished state. + */ + void Finish(); + /** + * Set the autofinish flag on this stream (defaults to false). When this flag + * is set, and all input streams are in the finished state (including if there + * are no input streams), this stream automatically enters the finished state. + */ + void SetAutofinish(bool aAutofinish); + + ProcessedMediaStream* AsProcessedStream() override { return this; } + + friend class MediaStreamGraphImpl; + + // Do not call these from outside MediaStreamGraph.cpp! + virtual void AddInput(MediaInputPort* aPort); + virtual void RemoveInput(MediaInputPort* aPort) + { + mInputs.RemoveElement(aPort); + } + bool HasInputPort(MediaInputPort* aPort) + { + return mInputs.Contains(aPort); + } + uint32_t InputPortCount() + { + return mInputs.Length(); + } + virtual MediaStream* GetInputStreamFor(TrackID aTrackID) { return nullptr; } + virtual TrackID GetInputTrackIDFor(TrackID aTrackID) { return TRACK_NONE; } + void DestroyImpl() override; + /** + * This gets called after we've computed the blocking states for all + * streams (mBlocked is up to date up to mStateComputedTime). + * Also, we've produced output for all streams up to this one. If this stream + * is not in a cycle, then all its source streams have produced data. + * Generate output from aFrom to aTo. + * This will be called on streams that have finished. Most stream types should + * just return immediately if IsFinishedOnGraphThread(), but some may wish to + * update internal state (see AudioNodeStream). + * ProcessInput is allowed to call FinishOnGraphThread only if ALLOW_FINISH + * is in aFlags. (This flag will be set when aTo >= mStateComputedTime, i.e. + * when we've producing the last block of data we need to produce.) Otherwise + * we can get into a situation where we've determined the stream should not + * block before mStateComputedTime, but the stream finishes before + * mStateComputedTime, violating the invariant that finished streams are blocked. + */ + enum { + ALLOW_FINISH = 0x01 + }; + virtual void ProcessInput(GraphTime aFrom, GraphTime aTo, uint32_t aFlags) = 0; + void SetAutofinishImpl(bool aAutofinish) { mAutofinish = aAutofinish; } + + // Only valid after MediaStreamGraphImpl::UpdateStreamOrder() has run. + // A DelayNode is considered to break a cycle and so this will not return + // true for echo loops, only for muted cycles. + bool InMutedCycle() const { return mCycleMarker; } + + size_t SizeOfExcludingThis(MallocSizeOf aMallocSizeOf) const override + { + size_t amount = MediaStream::SizeOfExcludingThis(aMallocSizeOf); + // Not owned: + // - mInputs elements + amount += mInputs.ShallowSizeOfExcludingThis(aMallocSizeOf); + return amount; + } + + size_t SizeOfIncludingThis(MallocSizeOf aMallocSizeOf) const override + { + return aMallocSizeOf(this) + SizeOfExcludingThis(aMallocSizeOf); + } + +protected: + // This state is all accessed only on the media graph thread. + + // The list of all inputs that are currently enabled or waiting to be enabled. + nsTArray<MediaInputPort*> mInputs; + bool mAutofinish; + // After UpdateStreamOrder(), mCycleMarker is either 0 or 1 to indicate + // whether this stream is in a muted cycle. During ordering it can contain + // other marker values - see MediaStreamGraphImpl::UpdateStreamOrder(). + uint32_t mCycleMarker; +}; + +/** + * There can be multiple MediaStreamGraph per process: one per AudioChannel. + * Additionaly, each OfflineAudioContext object creates its own MediaStreamGraph + * object too.. + */ +class MediaStreamGraph +{ +public: + + // We ensure that the graph current time advances in multiples of + // IdealAudioBlockSize()/AudioStream::PreferredSampleRate(). A stream that + // never blocks and has a track with the ideal audio rate will produce audio + // in multiples of the block size. + + // Initializing an graph that outputs audio can be quite long on some + // platforms. Code that want to output audio at some point can express the + // fact that they will need an audio stream at some point by passing + // AUDIO_THREAD_DRIVER when getting an instance of MediaStreamGraph, so that + // the graph starts with the right driver. + enum GraphDriverType { + AUDIO_THREAD_DRIVER, + SYSTEM_THREAD_DRIVER, + OFFLINE_THREAD_DRIVER + }; + static const uint32_t AUDIO_CALLBACK_DRIVER_SHUTDOWN_TIMEOUT = 20*1000; + + // Main thread only + static MediaStreamGraph* GetInstance(GraphDriverType aGraphDriverRequested, + dom::AudioChannel aChannel); + static MediaStreamGraph* CreateNonRealtimeInstance(TrackRate aSampleRate); + // Idempotent + static void DestroyNonRealtimeInstance(MediaStreamGraph* aGraph); + + virtual nsresult OpenAudioInput(int aID, + AudioDataListener *aListener) { + return NS_ERROR_FAILURE; + } + virtual void CloseAudioInput(AudioDataListener *aListener) {} + + // Control API. + /** + * Create a stream that a media decoder (or some other source of + * media data, such as a camera) can write to. + */ + SourceMediaStream* CreateSourceStream(); + /** + * Create a stream that will form the union of the tracks of its input + * streams. + * A TrackUnionStream contains all the tracks of all its input streams. + * Adding a new input stream makes that stream's tracks immediately appear as new + * tracks starting at the time the input stream was added. + * Removing an input stream makes the output tracks corresponding to the + * removed tracks immediately end. + * For each added track, the track ID of the output track is the track ID + * of the input track or one plus the maximum ID of all previously added + * tracks, whichever is greater. + * TODO at some point we will probably need to add API to select + * particular tracks of each input stream. + */ + ProcessedMediaStream* CreateTrackUnionStream(); + /** + * Create a stream that will mix all its audio input. + */ + ProcessedMediaStream* CreateAudioCaptureStream(TrackID aTrackId); + + /** + * Add a new stream to the graph. Main thread. + */ + void AddStream(MediaStream* aStream); + + /* From the main thread, ask the MSG to send back an event when the graph + * thread is running, and audio is being processed. */ + void NotifyWhenGraphStarted(AudioNodeStream* aNodeStream); + /* From the main thread, suspend, resume or close an AudioContext. + * aStreams are the streams of all the AudioNodes of the AudioContext that + * need to be suspended or resumed. This can be empty if this is a second + * consecutive suspend call and all the nodes are already suspended. + * + * This can possibly pause the graph thread, releasing system resources, if + * all streams have been suspended/closed. + * + * When the operation is complete, aPromise is resolved. + */ + void ApplyAudioContextOperation(MediaStream* aDestinationStream, + const nsTArray<MediaStream*>& aStreams, + dom::AudioContextOperation aState, + void* aPromise); + + bool IsNonRealtime() const; + /** + * Start processing non-realtime for a specific number of ticks. + */ + void StartNonRealtimeProcessing(uint32_t aTicksToProcess); + + /** + * Media graph thread only. + * Dispatches a runnable that will run on the main thread after all + * main-thread stream state has been next updated. + * Should only be called during MediaStreamListener callbacks or during + * ProcessedMediaStream::ProcessInput(). + */ + virtual void DispatchToMainThreadAfterStreamStateUpdate(already_AddRefed<nsIRunnable> aRunnable) + { + AssertOnGraphThreadOrNotRunning(); + *mPendingUpdateRunnables.AppendElement() = aRunnable; + } + + /** + * Returns graph sample rate in Hz. + */ + TrackRate GraphRate() const { return mSampleRate; } + + void RegisterCaptureStreamForWindow(uint64_t aWindowId, + ProcessedMediaStream* aCaptureStream); + void UnregisterCaptureStreamForWindow(uint64_t aWindowId); + already_AddRefed<MediaInputPort> ConnectToCaptureStream( + uint64_t aWindowId, MediaStream* aMediaStream); + + /** + * Data going to the speakers from the GraphDriver's DataCallback + * to notify any listeners (for echo cancellation). + */ + void NotifyOutputData(AudioDataValue* aBuffer, size_t aFrames, + TrackRate aRate, uint32_t aChannels); + + void AssertOnGraphThreadOrNotRunning() const; + +protected: + explicit MediaStreamGraph(TrackRate aSampleRate) + : mSampleRate(aSampleRate) + { + MOZ_COUNT_CTOR(MediaStreamGraph); + } + virtual ~MediaStreamGraph() + { + MOZ_COUNT_DTOR(MediaStreamGraph); + } + + // Media graph thread only + nsTArray<nsCOMPtr<nsIRunnable> > mPendingUpdateRunnables; + + /** + * Sample rate at which this graph runs. For real time graphs, this is + * the rate of the audio mixer. For offline graphs, this is the rate specified + * at construction. + */ + TrackRate mSampleRate; + + /** + * CloseAudioInput is async, so hold a reference here. + */ + nsTArray<RefPtr<AudioDataListener>> mAudioInputs; +}; + +} // namespace mozilla + +#endif /* MOZILLA_MEDIASTREAMGRAPH_H_ */ |