summaryrefslogtreecommitdiffstats
path: root/dom/media/MediaStreamGraph.h
diff options
context:
space:
mode:
Diffstat (limited to 'dom/media/MediaStreamGraph.h')
-rw-r--r--dom/media/MediaStreamGraph.h1407
1 files changed, 1407 insertions, 0 deletions
diff --git a/dom/media/MediaStreamGraph.h b/dom/media/MediaStreamGraph.h
new file mode 100644
index 000000000..a6d77880a
--- /dev/null
+++ b/dom/media/MediaStreamGraph.h
@@ -0,0 +1,1407 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*-*/
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef MOZILLA_MEDIASTREAMGRAPH_H_
+#define MOZILLA_MEDIASTREAMGRAPH_H_
+
+#include "mozilla/LinkedList.h"
+#include "mozilla/Mutex.h"
+#include "mozilla/TaskQueue.h"
+
+#include "mozilla/dom/AudioChannelBinding.h"
+
+#include "AudioStream.h"
+#include "nsTArray.h"
+#include "nsIRunnable.h"
+#include "VideoSegment.h"
+#include "StreamTracks.h"
+#include "MainThreadUtils.h"
+#include "StreamTracks.h"
+#include "nsAutoPtr.h"
+#include "nsAutoRef.h"
+#include <speex/speex_resampler.h>
+
+class nsIRunnable;
+
+template <>
+class nsAutoRefTraits<SpeexResamplerState> : public nsPointerRefTraits<SpeexResamplerState>
+{
+ public:
+ static void Release(SpeexResamplerState* aState) { speex_resampler_destroy(aState); }
+};
+
+namespace mozilla {
+
+extern LazyLogModule gMediaStreamGraphLog;
+
+namespace dom {
+ enum class AudioContextOperation;
+}
+
+namespace media {
+ template<typename V, typename E> class Pledge;
+}
+
+/*
+ * MediaStreamGraph is a framework for synchronized audio/video processing
+ * and playback. It is designed to be used by other browser components such as
+ * HTML media elements, media capture APIs, real-time media streaming APIs,
+ * multitrack media APIs, and advanced audio APIs.
+ *
+ * The MediaStreamGraph uses a dedicated thread to process media --- the media
+ * graph thread. This ensures that we can process media through the graph
+ * without blocking on main-thread activity. The media graph is only modified
+ * on the media graph thread, to ensure graph changes can be processed without
+ * interfering with media processing. All interaction with the media graph
+ * thread is done with message passing.
+ *
+ * APIs that modify the graph or its properties are described as "control APIs".
+ * These APIs are asynchronous; they queue graph changes internally and
+ * those changes are processed all-at-once by the MediaStreamGraph. The
+ * MediaStreamGraph monitors the main thread event loop via nsIAppShell::RunInStableState
+ * to ensure that graph changes from a single event loop task are always
+ * processed all together. Control APIs should only be used on the main thread,
+ * currently; we may be able to relax that later.
+ *
+ * To allow precise synchronization of times in the control API, the
+ * MediaStreamGraph maintains a "media timeline". Control APIs that take or
+ * return times use that timeline. Those times never advance during
+ * an event loop task. This time is returned by MediaStreamGraph::GetCurrentTime().
+ *
+ * Media decoding, audio processing and media playback use thread-safe APIs to
+ * the media graph to ensure they can continue while the main thread is blocked.
+ *
+ * When the graph is changed, we may need to throw out buffered data and
+ * reprocess it. This is triggered automatically by the MediaStreamGraph.
+ */
+
+class AudioNodeEngine;
+class AudioNodeExternalInputStream;
+class AudioNodeStream;
+class MediaInputPort;
+class MediaStream;
+class MediaStreamGraph;
+class MediaStreamGraphImpl;
+class ProcessedMediaStream;
+class SourceMediaStream;
+
+class AudioDataListenerInterface {
+protected:
+ // Protected destructor, to discourage deletion outside of Release():
+ virtual ~AudioDataListenerInterface() {}
+
+public:
+ /* These are for cubeb audio input & output streams: */
+ /**
+ * Output data to speakers, for use as the "far-end" data for echo
+ * cancellation. This is not guaranteed to be in any particular size
+ * chunks.
+ */
+ virtual void NotifyOutputData(MediaStreamGraph* aGraph,
+ AudioDataValue* aBuffer, size_t aFrames,
+ TrackRate aRate, uint32_t aChannels) = 0;
+ /**
+ * Input data from a microphone (or other audio source. This is not
+ * guaranteed to be in any particular size chunks.
+ */
+ virtual void NotifyInputData(MediaStreamGraph* aGraph,
+ const AudioDataValue* aBuffer, size_t aFrames,
+ TrackRate aRate, uint32_t aChannels) = 0;
+
+ /**
+ * Called when the underlying audio device has changed.
+ */
+ virtual void DeviceChanged() = 0;
+};
+
+class AudioDataListener : public AudioDataListenerInterface {
+protected:
+ // Protected destructor, to discourage deletion outside of Release():
+ virtual ~AudioDataListener() {}
+
+public:
+ NS_INLINE_DECL_THREADSAFE_REFCOUNTING(AudioDataListener)
+};
+
+/**
+ * This is a base class for main-thread listener callbacks.
+ * This callback is invoked on the main thread when the main-thread-visible
+ * state of a stream has changed.
+ *
+ * These methods are called with the media graph monitor held, so
+ * reentry into general media graph methods is not possible.
+ * You should do something non-blocking and non-reentrant (e.g. dispatch an
+ * event) and return. DispatchFromMainThreadAfterNextStreamStateUpdate
+ * would be a good choice.
+ * The listener is allowed to synchronously remove itself from the stream, but
+ * not add or remove any other listeners.
+ */
+class MainThreadMediaStreamListener {
+public:
+ virtual void NotifyMainThreadStreamFinished() = 0;
+};
+
+/**
+ * Helper struct used to keep track of memory usage by AudioNodes.
+ */
+struct AudioNodeSizes
+{
+ AudioNodeSizes() : mStream(0), mEngine(0), mNodeType() {}
+ size_t mStream;
+ size_t mEngine;
+ const char* mNodeType;
+};
+
+class AudioNodeEngine;
+class AudioNodeExternalInputStream;
+class AudioNodeStream;
+class AudioSegment;
+class DirectMediaStreamListener;
+class DirectMediaStreamTrackListener;
+class MediaInputPort;
+class MediaStreamGraphImpl;
+class MediaStreamListener;
+class MediaStreamTrackListener;
+class MediaStreamVideoSink;
+class ProcessedMediaStream;
+class SourceMediaStream;
+class TrackUnionStream;
+
+enum MediaStreamGraphEvent : uint32_t;
+enum TrackEventCommand : uint32_t;
+
+/**
+ * Helper struct for binding a track listener to a specific TrackID.
+ */
+template<typename Listener>
+struct TrackBound
+{
+ RefPtr<Listener> mListener;
+ TrackID mTrackID;
+};
+
+/**
+ * Describes how a track should be disabled.
+ *
+ * ENABLED Not disabled.
+ * SILENCE_BLACK Audio data is turned into silence, video frames are made black.
+ * SILENCE_FREEZE Audio data is turned into silence, video freezes at last frame.
+ */
+enum class DisabledTrackMode
+{
+ ENABLED, SILENCE_BLACK, SILENCE_FREEZE
+};
+struct DisabledTrack {
+ DisabledTrack(TrackID aTrackID, DisabledTrackMode aMode)
+ : mTrackID(aTrackID), mMode(aMode) {}
+ TrackID mTrackID;
+ DisabledTrackMode mMode;
+};
+
+/**
+ * A stream of synchronized audio and video data. All (not blocked) streams
+ * progress at the same rate --- "real time". Streams cannot seek. The only
+ * operation readers can perform on a stream is to read the next data.
+ *
+ * Consumers of a stream can be reading from it at different offsets, but that
+ * should only happen due to the order in which consumers are being run.
+ * Those offsets must not diverge in the long term, otherwise we would require
+ * unbounded buffering.
+ *
+ * Streams can be in a "blocked" state. While blocked, a stream does not
+ * produce data. A stream can be explicitly blocked via the control API,
+ * or implicitly blocked by whatever's generating it (e.g. an underrun in the
+ * source resource), or implicitly blocked because something consuming it
+ * blocks, or implicitly because it has finished.
+ *
+ * A stream can be in a "finished" state. "Finished" streams are permanently
+ * blocked.
+ *
+ * Transitions into and out of the "blocked" and "finished" states are managed
+ * by the MediaStreamGraph on the media graph thread.
+ *
+ * We buffer media data ahead of the consumers' reading offsets. It is possible
+ * to have buffered data but still be blocked.
+ *
+ * Any stream can have its audio and video playing when requested. The media
+ * stream graph plays audio by constructing audio output streams as necessary.
+ * Video is played by setting video frames into an MediaStreamVideoSink at the right
+ * time. To ensure video plays in sync with audio, make sure that the same
+ * stream is playing both the audio and video.
+ *
+ * The data in a stream is managed by StreamTracks. It consists of a set of
+ * tracks of various types that can start and end over time.
+ *
+ * Streams are explicitly managed. The client creates them via
+ * MediaStreamGraph::CreateInput/ProcessedMediaStream, and releases them by calling
+ * Destroy() when no longer needed (actual destruction will be deferred).
+ * The actual object is owned by the MediaStreamGraph. The basic idea is that
+ * main thread objects will keep Streams alive as long as necessary (using the
+ * cycle collector to clean up whenever needed).
+ *
+ * We make them refcounted only so that stream-related messages with MediaStream*
+ * pointers can be sent to the main thread safely.
+ *
+ * The lifetimes of MediaStreams are controlled from the main thread.
+ * For MediaStreams exposed to the DOM, the lifetime is controlled by the DOM
+ * wrapper; the DOM wrappers own their associated MediaStreams. When a DOM
+ * wrapper is destroyed, it sends a Destroy message for the associated
+ * MediaStream and clears its reference (the last main-thread reference to
+ * the object). When the Destroy message is processed on the graph manager
+ * thread we immediately release the affected objects (disentangling them
+ * from other objects as necessary).
+ *
+ * This could cause problems for media processing if a MediaStream is
+ * destroyed while a downstream MediaStream is still using it. Therefore
+ * the DOM wrappers must keep upstream MediaStreams alive as long as they
+ * could be being used in the media graph.
+ *
+ * At any time, however, a set of MediaStream wrappers could be
+ * collected via cycle collection. Destroy messages will be sent
+ * for those objects in arbitrary order and the MediaStreamGraph has to be able
+ * to handle this.
+ */
+
+// GetCurrentTime is defined in winbase.h as zero argument macro forwarding to
+// GetTickCount() and conflicts with MediaStream::GetCurrentTime.
+#ifdef GetCurrentTime
+#undef GetCurrentTime
+#endif
+
+class MediaStream : public mozilla::LinkedListElement<MediaStream>
+{
+public:
+ NS_INLINE_DECL_THREADSAFE_REFCOUNTING(MediaStream)
+
+ MediaStream();
+
+protected:
+ // Protected destructor, to discourage deletion outside of Release():
+ virtual ~MediaStream();
+
+public:
+ /**
+ * Returns the graph that owns this stream.
+ */
+ MediaStreamGraphImpl* GraphImpl();
+ MediaStreamGraph* Graph();
+ /**
+ * Sets the graph that owns this stream. Should only be called once.
+ */
+ void SetGraphImpl(MediaStreamGraphImpl* aGraph);
+ void SetGraphImpl(MediaStreamGraph* aGraph);
+
+ /**
+ * Returns sample rate of the graph.
+ */
+ TrackRate GraphRate() { return mTracks.GraphRate(); }
+
+ // Control API.
+ // Since a stream can be played multiple ways, we need to combine independent
+ // volume settings. The aKey parameter is used to keep volume settings
+ // separate. Since the stream is always playing the same contents, only
+ // a single audio output stream is used; the volumes are combined.
+ // Currently only the first enabled audio track is played.
+ // XXX change this so all enabled audio tracks are mixed and played.
+ virtual void AddAudioOutput(void* aKey);
+ virtual void SetAudioOutputVolume(void* aKey, float aVolume);
+ virtual void RemoveAudioOutput(void* aKey);
+ // Since a stream can be played multiple ways, we need to be able to
+ // play to multiple MediaStreamVideoSinks.
+ // Only the first enabled video track is played.
+ virtual void AddVideoOutput(MediaStreamVideoSink* aSink,
+ TrackID aID = TRACK_ANY);
+ virtual void RemoveVideoOutput(MediaStreamVideoSink* aSink,
+ TrackID aID = TRACK_ANY);
+ // Explicitly suspend. Useful for example if a media element is pausing
+ // and we need to stop its stream emitting its buffered data. As soon as the
+ // Suspend message reaches the graph, the stream stops processing. It
+ // ignores its inputs and produces silence/no video until Resumed. Its
+ // current time does not advance.
+ virtual void Suspend();
+ virtual void Resume();
+ // Events will be dispatched by calling methods of aListener.
+ virtual void AddListener(MediaStreamListener* aListener);
+ virtual void RemoveListener(MediaStreamListener* aListener);
+ virtual void AddTrackListener(MediaStreamTrackListener* aListener,
+ TrackID aTrackID);
+ virtual void RemoveTrackListener(MediaStreamTrackListener* aListener,
+ TrackID aTrackID);
+
+ /**
+ * Adds aListener to the source stream of track aTrackID in this stream.
+ * When the MediaStreamGraph processes the added listener, it will traverse
+ * the graph and add it to the track's source stream (remapping the TrackID
+ * along the way).
+ * Note that the listener will be notified on the MediaStreamGraph thread
+ * with whether the installation of it at the source was successful or not.
+ */
+ virtual void AddDirectTrackListener(DirectMediaStreamTrackListener* aListener,
+ TrackID aTrackID);
+
+ /**
+ * Removes aListener from the source stream of track aTrackID in this stream.
+ * Note that the listener has already been removed if the link between the
+ * source of track aTrackID and this stream has been broken (and made track
+ * aTrackID end). The caller doesn't have to care about this, removing when
+ * the source cannot be found, or when the listener had already been removed
+ * does nothing.
+ */
+ virtual void RemoveDirectTrackListener(DirectMediaStreamTrackListener* aListener,
+ TrackID aTrackID);
+
+ // A disabled track has video replaced by black, and audio replaced by
+ // silence.
+ void SetTrackEnabled(TrackID aTrackID, DisabledTrackMode aMode);
+
+ // Finish event will be notified by calling methods of aListener. It is the
+ // responsibility of the caller to remove aListener before it is destroyed.
+ void AddMainThreadListener(MainThreadMediaStreamListener* aListener);
+ // It's safe to call this even if aListener is not currently a listener;
+ // the call will be ignored.
+ void RemoveMainThreadListener(MainThreadMediaStreamListener* aListener)
+ {
+ MOZ_ASSERT(NS_IsMainThread());
+ MOZ_ASSERT(aListener);
+ mMainThreadListeners.RemoveElement(aListener);
+ }
+
+ /**
+ * Ensure a runnable will run on the main thread after running all pending
+ * updates that were sent from the graph thread or will be sent before the
+ * graph thread receives the next graph update.
+ *
+ * If the graph has been shut down or destroyed, then the runnable will be
+ * dispatched to the event queue immediately. If the graph is non-realtime
+ * and has not started, then the runnable will be run
+ * synchronously/immediately. (There are no pending updates in these
+ * situations.)
+ *
+ * Main thread only.
+ */
+ void RunAfterPendingUpdates(already_AddRefed<nsIRunnable> aRunnable);
+
+ // Signal that the client is done with this MediaStream. It will be deleted
+ // later. Do not mix usage of Destroy() with RegisterUser()/UnregisterUser().
+ // That will cause the MediaStream to be destroyed twice, which will cause
+ // some assertions to fail.
+ virtual void Destroy();
+ // Signal that a client is using this MediaStream. Useful to not have to
+ // explicitly manage ownership (responsibility to Destroy()) when there are
+ // multiple clients using a MediaStream.
+ void RegisterUser();
+ // Signal that a client no longer needs this MediaStream. When the number of
+ // clients using this MediaStream reaches 0, it will be destroyed.
+ void UnregisterUser();
+
+ // Returns the main-thread's view of how much data has been processed by
+ // this stream.
+ StreamTime GetCurrentTime()
+ {
+ NS_ASSERTION(NS_IsMainThread(), "Call only on main thread");
+ return mMainThreadCurrentTime;
+ }
+ // Return the main thread's view of whether this stream has finished.
+ bool IsFinished()
+ {
+ NS_ASSERTION(NS_IsMainThread(), "Call only on main thread");
+ return mMainThreadFinished;
+ }
+
+ bool IsDestroyed()
+ {
+ NS_ASSERTION(NS_IsMainThread(), "Call only on main thread");
+ return mMainThreadDestroyed;
+ }
+
+ friend class MediaStreamGraphImpl;
+ friend class MediaInputPort;
+ friend class AudioNodeExternalInputStream;
+
+ virtual SourceMediaStream* AsSourceStream() { return nullptr; }
+ virtual ProcessedMediaStream* AsProcessedStream() { return nullptr; }
+ virtual AudioNodeStream* AsAudioNodeStream() { return nullptr; }
+ virtual TrackUnionStream* AsTrackUnionStream() { return nullptr; }
+
+ // These Impl methods perform the core functionality of the control methods
+ // above, on the media graph thread.
+ /**
+ * Stop all stream activity and disconnect it from all inputs and outputs.
+ * This must be idempotent.
+ */
+ virtual void DestroyImpl();
+ StreamTime GetTracksEnd() { return mTracks.GetEnd(); }
+#ifdef DEBUG
+ void DumpTrackInfo() { return mTracks.DumpTrackInfo(); }
+#endif
+ void SetAudioOutputVolumeImpl(void* aKey, float aVolume);
+ void AddAudioOutputImpl(void* aKey);
+ // Returns true if this stream has an audio output.
+ bool HasAudioOutput()
+ {
+ return !mAudioOutputs.IsEmpty();
+ }
+ void RemoveAudioOutputImpl(void* aKey);
+ void AddVideoOutputImpl(already_AddRefed<MediaStreamVideoSink> aSink,
+ TrackID aID);
+ void RemoveVideoOutputImpl(MediaStreamVideoSink* aSink, TrackID aID);
+ void AddListenerImpl(already_AddRefed<MediaStreamListener> aListener);
+ void RemoveListenerImpl(MediaStreamListener* aListener);
+ void RemoveAllListenersImpl();
+ virtual void AddTrackListenerImpl(already_AddRefed<MediaStreamTrackListener> aListener,
+ TrackID aTrackID);
+ virtual void RemoveTrackListenerImpl(MediaStreamTrackListener* aListener,
+ TrackID aTrackID);
+ virtual void AddDirectTrackListenerImpl(already_AddRefed<DirectMediaStreamTrackListener> aListener,
+ TrackID aTrackID);
+ virtual void RemoveDirectTrackListenerImpl(DirectMediaStreamTrackListener* aListener,
+ TrackID aTrackID);
+ virtual void SetTrackEnabledImpl(TrackID aTrackID, DisabledTrackMode aMode);
+ DisabledTrackMode GetDisabledTrackMode(TrackID aTrackID);
+
+ void AddConsumer(MediaInputPort* aPort)
+ {
+ mConsumers.AppendElement(aPort);
+ }
+ void RemoveConsumer(MediaInputPort* aPort)
+ {
+ mConsumers.RemoveElement(aPort);
+ }
+ uint32_t ConsumerCount()
+ {
+ return mConsumers.Length();
+ }
+ StreamTracks& GetStreamTracks() { return mTracks; }
+ GraphTime GetStreamTracksStartTime() { return mTracksStartTime; }
+
+ double StreamTimeToSeconds(StreamTime aTime)
+ {
+ NS_ASSERTION(0 <= aTime && aTime <= STREAM_TIME_MAX, "Bad time");
+ return static_cast<double>(aTime)/mTracks.GraphRate();
+ }
+ int64_t StreamTimeToMicroseconds(StreamTime aTime)
+ {
+ NS_ASSERTION(0 <= aTime && aTime <= STREAM_TIME_MAX, "Bad time");
+ return (aTime*1000000)/mTracks.GraphRate();
+ }
+ StreamTime SecondsToNearestStreamTime(double aSeconds)
+ {
+ NS_ASSERTION(0 <= aSeconds && aSeconds <= TRACK_TICKS_MAX/TRACK_RATE_MAX,
+ "Bad seconds");
+ return mTracks.GraphRate() * aSeconds + 0.5;
+ }
+ StreamTime MicrosecondsToStreamTimeRoundDown(int64_t aMicroseconds) {
+ return (aMicroseconds*mTracks.GraphRate())/1000000;
+ }
+
+ TrackTicks TimeToTicksRoundUp(TrackRate aRate, StreamTime aTime)
+ {
+ return RateConvertTicksRoundUp(aRate, mTracks.GraphRate(), aTime);
+ }
+ StreamTime TicksToTimeRoundDown(TrackRate aRate, TrackTicks aTicks)
+ {
+ return RateConvertTicksRoundDown(mTracks.GraphRate(), aRate, aTicks);
+ }
+ /**
+ * Convert graph time to stream time. aTime must be <= mStateComputedTime
+ * to ensure we know exactly how much time this stream will be blocked during
+ * the interval.
+ */
+ StreamTime GraphTimeToStreamTimeWithBlocking(GraphTime aTime);
+ /**
+ * Convert graph time to stream time. This assumes there is no blocking time
+ * to take account of, which is always true except between a stream
+ * having its blocking time calculated in UpdateGraph and its blocking time
+ * taken account of in UpdateCurrentTimeForStreams.
+ */
+ StreamTime GraphTimeToStreamTime(GraphTime aTime);
+ /**
+ * Convert stream time to graph time. This assumes there is no blocking time
+ * to take account of, which is always true except between a stream
+ * having its blocking time calculated in UpdateGraph and its blocking time
+ * taken account of in UpdateCurrentTimeForStreams.
+ */
+ GraphTime StreamTimeToGraphTime(StreamTime aTime);
+
+ bool IsFinishedOnGraphThread() { return mFinished; }
+ void FinishOnGraphThread();
+
+ bool HasCurrentData() { return mHasCurrentData; }
+
+ /**
+ * Find track by track id.
+ */
+ StreamTracks::Track* FindTrack(TrackID aID);
+
+ StreamTracks::Track* EnsureTrack(TrackID aTrack);
+
+ virtual void ApplyTrackDisabling(TrackID aTrackID, MediaSegment* aSegment, MediaSegment* aRawSegment = nullptr);
+
+ // Return true if the main thread needs to observe updates from this stream.
+ virtual bool MainThreadNeedsUpdates() const
+ {
+ return true;
+ }
+
+ virtual size_t SizeOfExcludingThis(MallocSizeOf aMallocSizeOf) const;
+ virtual size_t SizeOfIncludingThis(MallocSizeOf aMallocSizeOf) const;
+
+ void SetAudioChannelType(dom::AudioChannel aType) { mAudioChannelType = aType; }
+ dom::AudioChannel AudioChannelType() const { return mAudioChannelType; }
+
+ bool IsSuspended() { return mSuspendedCount > 0; }
+ void IncrementSuspendCount() { ++mSuspendedCount; }
+ void DecrementSuspendCount()
+ {
+ NS_ASSERTION(mSuspendedCount > 0, "Suspend count underrun");
+ --mSuspendedCount;
+ }
+
+protected:
+ // |AdvanceTimeVaryingValuesToCurrentTime| will be override in SourceMediaStream.
+ virtual void AdvanceTimeVaryingValuesToCurrentTime(GraphTime aCurrentTime,
+ GraphTime aBlockedTime)
+ {
+ mTracksStartTime += aBlockedTime;
+ mTracks.ForgetUpTo(aCurrentTime - mTracksStartTime);
+ }
+
+ void NotifyMainThreadListeners()
+ {
+ NS_ASSERTION(NS_IsMainThread(), "Call only on main thread");
+
+ for (int32_t i = mMainThreadListeners.Length() - 1; i >= 0; --i) {
+ mMainThreadListeners[i]->NotifyMainThreadStreamFinished();
+ }
+ mMainThreadListeners.Clear();
+ }
+
+ bool ShouldNotifyStreamFinished()
+ {
+ NS_ASSERTION(NS_IsMainThread(), "Call only on main thread");
+ if (!mMainThreadFinished || mFinishedNotificationSent) {
+ return false;
+ }
+
+ mFinishedNotificationSent = true;
+ return true;
+ }
+
+ // This state is all initialized on the main thread but
+ // otherwise modified only on the media graph thread.
+
+ // Buffered data. The start of the buffer corresponds to mTracksStartTime.
+ // Conceptually the buffer contains everything this stream has ever played,
+ // but we forget some prefix of the buffered data to bound the space usage.
+ StreamTracks mTracks;
+ // The time when the buffered data could be considered to have started playing.
+ // This increases over time to account for time the stream was blocked before
+ // mCurrentTime.
+ GraphTime mTracksStartTime;
+
+ // Client-set volume of this stream
+ struct AudioOutput {
+ explicit AudioOutput(void* aKey) : mKey(aKey), mVolume(1.0f) {}
+ void* mKey;
+ float mVolume;
+ };
+ nsTArray<AudioOutput> mAudioOutputs;
+ nsTArray<TrackBound<MediaStreamVideoSink>> mVideoOutputs;
+ // We record the last played video frame to avoid playing the frame again
+ // with a different frame id.
+ VideoFrame mLastPlayedVideoFrame;
+ nsTArray<RefPtr<MediaStreamListener> > mListeners;
+ nsTArray<TrackBound<MediaStreamTrackListener>> mTrackListeners;
+ nsTArray<MainThreadMediaStreamListener*> mMainThreadListeners;
+ // List of disabled TrackIDs and their associated disabled mode.
+ // They can either by disabled by frames being replaced by black, or by
+ // retaining the previous frame.
+ nsTArray<DisabledTrack> mDisabledTracks;
+
+ // GraphTime at which this stream starts blocking.
+ // This is only valid up to mStateComputedTime. The stream is considered to
+ // have not been blocked before mCurrentTime (its mTracksStartTime is increased
+ // as necessary to account for that time instead).
+ GraphTime mStartBlocking;
+
+ // MediaInputPorts to which this is connected
+ nsTArray<MediaInputPort*> mConsumers;
+
+ // Where audio output is going. There is one AudioOutputStream per
+ // audio track.
+ struct AudioOutputStream
+ {
+ // When we started audio playback for this track.
+ // Add mStream->GetPosition() to find the current audio playback position.
+ GraphTime mAudioPlaybackStartTime;
+ // Amount of time that we've wanted to play silence because of the stream
+ // blocking.
+ MediaTime mBlockedAudioTime;
+ // Last tick written to the audio output.
+ StreamTime mLastTickWritten;
+ TrackID mTrackID;
+ };
+ nsTArray<AudioOutputStream> mAudioOutputStreams;
+
+ /**
+ * Number of outstanding suspend operations on this stream. Stream is
+ * suspended when this is > 0.
+ */
+ int32_t mSuspendedCount;
+
+ /**
+ * When true, this means the stream will be finished once all
+ * buffered data has been consumed.
+ */
+ bool mFinished;
+ /**
+ * When true, mFinished is true and we've played all the data in this stream
+ * and fired NotifyFinished notifications.
+ */
+ bool mNotifiedFinished;
+ /**
+ * When true, the last NotifyBlockingChanged delivered to the listeners
+ * indicated that the stream is blocked.
+ */
+ bool mNotifiedBlocked;
+ /**
+ * True if some data can be present by this stream if/when it's unblocked.
+ * Set by the stream itself on the MediaStreamGraph thread. Only changes
+ * from false to true once a stream has data, since we won't
+ * unblock it until there's more data.
+ */
+ bool mHasCurrentData;
+ /**
+ * True if mHasCurrentData is true and we've notified listeners.
+ */
+ bool mNotifiedHasCurrentData;
+
+ // Main-thread views of state
+ StreamTime mMainThreadCurrentTime;
+ bool mMainThreadFinished;
+ bool mFinishedNotificationSent;
+ bool mMainThreadDestroyed;
+ int mNrOfMainThreadUsers;
+
+ // Our media stream graph. null if destroyed on the graph thread.
+ MediaStreamGraphImpl* mGraph;
+
+ dom::AudioChannel mAudioChannelType;
+};
+
+/**
+ * This is a stream into which a decoder can write audio and video.
+ *
+ * Audio and video can be written on any thread, but you probably want to
+ * always write from the same thread to avoid unexpected interleavings.
+ */
+class SourceMediaStream : public MediaStream
+{
+public:
+ explicit SourceMediaStream();
+
+ SourceMediaStream* AsSourceStream() override { return this; }
+
+ // Media graph thread only
+
+ // Users of audio inputs go through the stream so it can track when the
+ // last stream referencing an input goes away, so it can close the cubeb
+ // input. Also note: callable on any thread (though it bounces through
+ // MainThread to set the command if needed).
+ nsresult OpenAudioInput(int aID,
+ AudioDataListener *aListener);
+ // Note: also implied when Destroy() happens
+ void CloseAudioInput();
+
+ void DestroyImpl() override;
+
+ // Call these on any thread.
+ /**
+ * Enable or disable pulling. When pulling is enabled, NotifyPull
+ * gets called on MediaStreamListeners for this stream during the
+ * MediaStreamGraph control loop. Pulling is initially disabled.
+ * Due to unavoidable race conditions, after a call to SetPullEnabled(false)
+ * it is still possible for a NotifyPull to occur.
+ */
+ void SetPullEnabled(bool aEnabled);
+
+ /**
+ * These add/remove DirectListeners, which allow bypassing the graph and any
+ * synchronization delays for e.g. PeerConnection, which wants the data ASAP
+ * and lets the far-end handle sync and playout timing.
+ */
+ void NotifyListenersEventImpl(MediaStreamGraphEvent aEvent);
+ void NotifyListenersEvent(MediaStreamGraphEvent aEvent);
+ void AddDirectListener(DirectMediaStreamListener* aListener);
+ void RemoveDirectListener(DirectMediaStreamListener* aListener);
+
+ enum {
+ ADDTRACK_QUEUED = 0x01 // Queue track add until FinishAddTracks()
+ };
+ /**
+ * Add a new track to the stream starting at the given base time (which
+ * must be greater than or equal to the last time passed to
+ * AdvanceKnownTracksTime). Takes ownership of aSegment. aSegment should
+ * contain data starting after aStart.
+ */
+ void AddTrack(TrackID aID, StreamTime aStart, MediaSegment* aSegment,
+ uint32_t aFlags = 0)
+ {
+ AddTrackInternal(aID, GraphRate(), aStart, aSegment, aFlags);
+ }
+
+ /**
+ * Like AddTrack, but resamples audio from aRate to the graph rate.
+ */
+ void AddAudioTrack(TrackID aID, TrackRate aRate, StreamTime aStart,
+ AudioSegment* aSegment, uint32_t aFlags = 0);
+
+ /**
+ * Call after a series of AddTrack or AddAudioTrack calls to implement
+ * any pending track adds.
+ */
+ void FinishAddTracks();
+
+ /**
+ * Append media data to a track. Ownership of aSegment remains with the caller,
+ * but aSegment is emptied.
+ * Returns false if the data was not appended because no such track exists
+ * or the stream was already finished.
+ */
+ bool AppendToTrack(TrackID aID, MediaSegment* aSegment, MediaSegment *aRawSegment = nullptr);
+ /**
+ * Get the stream time of the end of the data that has been appended so far.
+ * Can be called from any thread but won't be useful if it can race with
+ * an AppendToTrack call, so should probably just be called from the thread
+ * that also calls AppendToTrack.
+ */
+ StreamTime GetEndOfAppendedData(TrackID aID);
+ /**
+ * Indicate that a track has ended. Do not do any more API calls
+ * affecting this track.
+ * Ignored if the track does not exist.
+ */
+ void EndTrack(TrackID aID);
+ /**
+ * Indicate that no tracks will be added starting before time aKnownTime.
+ * aKnownTime must be >= its value at the last call to AdvanceKnownTracksTime.
+ */
+ void AdvanceKnownTracksTime(StreamTime aKnownTime);
+ /**
+ * Indicate that this stream should enter the "finished" state. All tracks
+ * must have been ended via EndTrack. The finish time of the stream is
+ * when all tracks have ended.
+ */
+ void FinishWithLockHeld();
+ void Finish()
+ {
+ MutexAutoLock lock(mMutex);
+ FinishWithLockHeld();
+ }
+
+ // Overriding allows us to hold the mMutex lock while changing the track enable status
+ void SetTrackEnabledImpl(TrackID aTrackID, DisabledTrackMode aMode) override;
+
+ // Overriding allows us to ensure mMutex is locked while changing the track enable status
+ void
+ ApplyTrackDisabling(TrackID aTrackID, MediaSegment* aSegment,
+ MediaSegment* aRawSegment = nullptr) override {
+ mMutex.AssertCurrentThreadOwns();
+ MediaStream::ApplyTrackDisabling(aTrackID, aSegment, aRawSegment);
+ }
+
+ /**
+ * End all tracks and Finish() this stream. Used to voluntarily revoke access
+ * to a LocalMediaStream.
+ */
+ void EndAllTrackAndFinish();
+
+ void RegisterForAudioMixing();
+
+ /**
+ * Returns true if this SourceMediaStream contains at least one audio track
+ * that is in pending state.
+ * This is thread safe, and takes the SourceMediaStream mutex.
+ */
+ bool HasPendingAudioTrack();
+
+ TimeStamp GetStreamTracksStrartTimeStamp() {
+ MutexAutoLock lock(mMutex);
+ return mStreamTracksStartTimeStamp;
+ }
+
+ // XXX need a Reset API
+
+ friend class MediaStreamGraphImpl;
+
+protected:
+ enum TrackCommands : uint32_t;
+
+ virtual ~SourceMediaStream();
+
+ /**
+ * Data for each track that hasn't ended.
+ */
+ struct TrackData {
+ TrackID mID;
+ // Sample rate of the input data.
+ TrackRate mInputRate;
+ // Resampler if the rate of the input track does not match the
+ // MediaStreamGraph's.
+ nsAutoRef<SpeexResamplerState> mResampler;
+ int mResamplerChannelCount;
+ StreamTime mStart;
+ // End-time of data already flushed to the track (excluding mData)
+ StreamTime mEndOfFlushedData;
+ // Each time the track updates are flushed to the media graph thread,
+ // the segment buffer is emptied.
+ nsAutoPtr<MediaSegment> mData;
+ // Each time the track updates are flushed to the media graph thread,
+ // this is cleared.
+ uint32_t mCommands;
+ };
+
+ bool NeedsMixing();
+
+ void ResampleAudioToGraphSampleRate(TrackData* aTrackData, MediaSegment* aSegment);
+
+ void AddDirectTrackListenerImpl(already_AddRefed<DirectMediaStreamTrackListener> aListener,
+ TrackID aTrackID) override;
+ void RemoveDirectTrackListenerImpl(DirectMediaStreamTrackListener* aListener,
+ TrackID aTrackID) override;
+
+ void AddTrackInternal(TrackID aID, TrackRate aRate,
+ StreamTime aStart, MediaSegment* aSegment,
+ uint32_t aFlags);
+
+ TrackData* FindDataForTrack(TrackID aID)
+ {
+ mMutex.AssertCurrentThreadOwns();
+ for (uint32_t i = 0; i < mUpdateTracks.Length(); ++i) {
+ if (mUpdateTracks[i].mID == aID) {
+ return &mUpdateTracks[i];
+ }
+ }
+ return nullptr;
+ }
+
+ /**
+ * Notify direct consumers of new data to one of the stream tracks.
+ * The data doesn't have to be resampled (though it may be). This is called
+ * from AppendToTrack on the thread providing the data, and will call
+ * the Listeners on this thread.
+ */
+ void NotifyDirectConsumers(TrackData *aTrack,
+ MediaSegment *aSegment);
+
+ virtual void
+ AdvanceTimeVaryingValuesToCurrentTime(GraphTime aCurrentTime,
+ GraphTime aBlockedTime) override;
+ void SetStreamTracksStartTimeStamp(const TimeStamp& aTimeStamp)
+ {
+ MutexAutoLock lock(mMutex);
+ mStreamTracksStartTimeStamp = aTimeStamp;
+ }
+
+ // Only accessed on the MSG thread. Used so to ask the MSGImpl to usecount
+ // users of a specific input.
+ // XXX Should really be a CubebUtils::AudioDeviceID, but they aren't
+ // copyable (opaque pointers)
+ RefPtr<AudioDataListener> mInputListener;
+
+ // This must be acquired *before* MediaStreamGraphImpl's lock, if they are
+ // held together.
+ Mutex mMutex;
+ // protected by mMutex
+ StreamTime mUpdateKnownTracksTime;
+ // This time stamp will be updated in adding and blocked SourceMediaStream,
+ // |AddStreamGraphThread| and |AdvanceTimeVaryingValuesToCurrentTime| in
+ // particularly.
+ TimeStamp mStreamTracksStartTimeStamp;
+ nsTArray<TrackData> mUpdateTracks;
+ nsTArray<TrackData> mPendingTracks;
+ nsTArray<RefPtr<DirectMediaStreamListener>> mDirectListeners;
+ nsTArray<TrackBound<DirectMediaStreamTrackListener>> mDirectTrackListeners;
+ bool mPullEnabled;
+ bool mUpdateFinished;
+ bool mNeedsMixing;
+};
+
+/**
+ * The blocking mode decides how a track should be blocked in a MediaInputPort.
+ */
+enum class BlockingMode
+{
+ /**
+ * BlockingMode CREATION blocks the source track from being created
+ * in the destination. It'll end if it already exists.
+ */
+ CREATION,
+ /**
+ * BlockingMode END_EXISTING allows a track to be created in the destination
+ * but will end it before any data has been passed through.
+ */
+ END_EXISTING,
+};
+
+/**
+ * Represents a connection between a ProcessedMediaStream and one of its
+ * input streams.
+ * We make these refcounted so that stream-related messages with MediaInputPort*
+ * pointers can be sent to the main thread safely.
+ *
+ * A port can be locked to a specific track in the source stream, in which case
+ * only this track will be forwarded to the destination stream. TRACK_ANY
+ * can used to signal that all tracks shall be forwarded.
+ *
+ * When a port is locked to a specific track in the source stream, it may also
+ * indicate a TrackID to map this source track to in the destination stream
+ * by setting aDestTrack to an explicit ID. When we do this, we must know
+ * that this TrackID in the destination stream is available. We assert during
+ * processing that the ID is available and that there are no generic input
+ * ports already attached to the destination stream.
+ * Note that this is currently only handled by TrackUnionStreams.
+ *
+ * When a port's source or destination stream dies, the stream's DestroyImpl
+ * calls MediaInputPort::Disconnect to disconnect the port from
+ * the source and destination streams.
+ *
+ * The lifetimes of MediaInputPort are controlled from the main thread.
+ * The media graph adds a reference to the port. When a MediaInputPort is no
+ * longer needed, main-thread code sends a Destroy message for the port and
+ * clears its reference (the last main-thread reference to the object). When
+ * the Destroy message is processed on the graph manager thread we disconnect
+ * the port and drop the graph's reference, destroying the object.
+ */
+class MediaInputPort final
+{
+private:
+ // Do not call this constructor directly. Instead call aDest->AllocateInputPort.
+ MediaInputPort(MediaStream* aSource, TrackID& aSourceTrack,
+ ProcessedMediaStream* aDest, TrackID& aDestTrack,
+ uint16_t aInputNumber, uint16_t aOutputNumber)
+ : mSource(aSource)
+ , mSourceTrack(aSourceTrack)
+ , mDest(aDest)
+ , mDestTrack(aDestTrack)
+ , mInputNumber(aInputNumber)
+ , mOutputNumber(aOutputNumber)
+ , mGraph(nullptr)
+ {
+ MOZ_COUNT_CTOR(MediaInputPort);
+ }
+
+ // Private destructor, to discourage deletion outside of Release():
+ ~MediaInputPort()
+ {
+ MOZ_COUNT_DTOR(MediaInputPort);
+ }
+
+public:
+ NS_INLINE_DECL_THREADSAFE_REFCOUNTING(MediaInputPort)
+
+ // Called on graph manager thread
+ // Do not call these from outside MediaStreamGraph.cpp!
+ void Init();
+ // Called during message processing to trigger removal of this stream.
+ void Disconnect();
+
+ // Control API
+ /**
+ * Disconnects and destroys the port. The caller must not reference this
+ * object again.
+ */
+ void Destroy();
+
+ // Any thread
+ MediaStream* GetSource() { return mSource; }
+ TrackID GetSourceTrackId() { return mSourceTrack; }
+ ProcessedMediaStream* GetDestination() { return mDest; }
+ TrackID GetDestinationTrackId() { return mDestTrack; }
+
+ /**
+ * Block aTrackId in the source stream from being passed through the port.
+ * Consumers will interpret this track as ended.
+ * Returns a pledge that resolves on the main thread after the track block has
+ * been applied by the MSG.
+ */
+ already_AddRefed<media::Pledge<bool, nsresult>> BlockSourceTrackId(TrackID aTrackId,
+ BlockingMode aBlockingMode);
+private:
+ void BlockSourceTrackIdImpl(TrackID aTrackId, BlockingMode aBlockingMode);
+
+public:
+ // Returns true if aTrackId has not been blocked for any reason and this port
+ // has not been locked to another track.
+ bool PassTrackThrough(TrackID aTrackId) {
+ bool blocked = false;
+ for (auto pair : mBlockedTracks) {
+ if (pair.first() == aTrackId &&
+ (pair.second() == BlockingMode::CREATION ||
+ pair.second() == BlockingMode::END_EXISTING)) {
+ blocked = true;
+ break;
+ }
+ }
+ return !blocked && (mSourceTrack == TRACK_ANY || mSourceTrack == aTrackId);
+ }
+
+ // Returns true if aTrackId has not been blocked for track creation and this
+ // port has not been locked to another track.
+ bool AllowCreationOf(TrackID aTrackId) {
+ bool blocked = false;
+ for (auto pair : mBlockedTracks) {
+ if (pair.first() == aTrackId &&
+ pair.second() == BlockingMode::CREATION) {
+ blocked = true;
+ break;
+ }
+ }
+ return !blocked && (mSourceTrack == TRACK_ANY || mSourceTrack == aTrackId);
+ }
+
+ uint16_t InputNumber() const { return mInputNumber; }
+ uint16_t OutputNumber() const { return mOutputNumber; }
+
+ // Call on graph manager thread
+ struct InputInterval {
+ GraphTime mStart;
+ GraphTime mEnd;
+ bool mInputIsBlocked;
+ };
+ // Find the next time interval starting at or after aTime during which
+ // mDest is not blocked and mSource's blocking status does not change.
+ InputInterval GetNextInputInterval(GraphTime aTime);
+
+ /**
+ * Returns the graph that owns this port.
+ */
+ MediaStreamGraphImpl* GraphImpl();
+ MediaStreamGraph* Graph();
+ /**
+ * Sets the graph that owns this stream. Should only be called once.
+ */
+ void SetGraphImpl(MediaStreamGraphImpl* aGraph);
+
+ size_t SizeOfExcludingThis(MallocSizeOf aMallocSizeOf) const
+ {
+ size_t amount = 0;
+
+ // Not owned:
+ // - mSource
+ // - mDest
+ // - mGraph
+ return amount;
+ }
+
+ size_t SizeOfIncludingThis(MallocSizeOf aMallocSizeOf) const
+ {
+ return aMallocSizeOf(this) + SizeOfExcludingThis(aMallocSizeOf);
+ }
+
+private:
+ friend class MediaStreamGraphImpl;
+ friend class MediaStream;
+ friend class ProcessedMediaStream;
+ // Never modified after Init()
+ MediaStream* mSource;
+ TrackID mSourceTrack;
+ ProcessedMediaStream* mDest;
+ TrackID mDestTrack;
+ // The input and output numbers are optional, and are currently only used by
+ // Web Audio.
+ const uint16_t mInputNumber;
+ const uint16_t mOutputNumber;
+
+ typedef Pair<TrackID, BlockingMode> BlockedTrack;
+ nsTArray<BlockedTrack> mBlockedTracks;
+
+ // Our media stream graph
+ MediaStreamGraphImpl* mGraph;
+};
+
+/**
+ * This stream processes zero or more input streams in parallel to produce
+ * its output. The details of how the output is produced are handled by
+ * subclasses overriding the ProcessInput method.
+ */
+class ProcessedMediaStream : public MediaStream
+{
+public:
+ explicit ProcessedMediaStream()
+ : MediaStream(), mAutofinish(false), mCycleMarker(0)
+ {}
+
+ // Control API.
+ /**
+ * Allocates a new input port attached to source aStream.
+ * This stream can be removed by calling MediaInputPort::Remove().
+ *
+ * The input port is tied to aTrackID in the source stream.
+ * aTrackID can be set to TRACK_ANY to automatically forward all tracks from
+ * aStream.
+ *
+ * If aTrackID is an explicit ID, aDestTrackID can also be made explicit
+ * to ensure that the track is assigned this ID in the destination stream.
+ * To avoid intermittent TrackID collisions the destination stream may not
+ * have any existing generic input ports (with TRACK_ANY source track) when
+ * you allocate an input port with a destination TrackID.
+ *
+ * To end a track in the destination stream forwarded with TRACK_ANY,
+ * it can be blocked in the input port through MediaInputPort::BlockTrackId().
+ *
+ * Tracks in aBlockedTracks will be blocked in the input port initially. This
+ * ensures that they don't get created by the MSG-thread before we can
+ * BlockTrackId() on the main thread.
+ */
+ already_AddRefed<MediaInputPort>
+ AllocateInputPort(MediaStream* aStream,
+ TrackID aTrackID = TRACK_ANY,
+ TrackID aDestTrackID = TRACK_ANY,
+ uint16_t aInputNumber = 0,
+ uint16_t aOutputNumber = 0,
+ nsTArray<TrackID>* aBlockedTracks = nullptr);
+ /**
+ * Force this stream into the finished state.
+ */
+ void Finish();
+ /**
+ * Set the autofinish flag on this stream (defaults to false). When this flag
+ * is set, and all input streams are in the finished state (including if there
+ * are no input streams), this stream automatically enters the finished state.
+ */
+ void SetAutofinish(bool aAutofinish);
+
+ ProcessedMediaStream* AsProcessedStream() override { return this; }
+
+ friend class MediaStreamGraphImpl;
+
+ // Do not call these from outside MediaStreamGraph.cpp!
+ virtual void AddInput(MediaInputPort* aPort);
+ virtual void RemoveInput(MediaInputPort* aPort)
+ {
+ mInputs.RemoveElement(aPort);
+ }
+ bool HasInputPort(MediaInputPort* aPort)
+ {
+ return mInputs.Contains(aPort);
+ }
+ uint32_t InputPortCount()
+ {
+ return mInputs.Length();
+ }
+ virtual MediaStream* GetInputStreamFor(TrackID aTrackID) { return nullptr; }
+ virtual TrackID GetInputTrackIDFor(TrackID aTrackID) { return TRACK_NONE; }
+ void DestroyImpl() override;
+ /**
+ * This gets called after we've computed the blocking states for all
+ * streams (mBlocked is up to date up to mStateComputedTime).
+ * Also, we've produced output for all streams up to this one. If this stream
+ * is not in a cycle, then all its source streams have produced data.
+ * Generate output from aFrom to aTo.
+ * This will be called on streams that have finished. Most stream types should
+ * just return immediately if IsFinishedOnGraphThread(), but some may wish to
+ * update internal state (see AudioNodeStream).
+ * ProcessInput is allowed to call FinishOnGraphThread only if ALLOW_FINISH
+ * is in aFlags. (This flag will be set when aTo >= mStateComputedTime, i.e.
+ * when we've producing the last block of data we need to produce.) Otherwise
+ * we can get into a situation where we've determined the stream should not
+ * block before mStateComputedTime, but the stream finishes before
+ * mStateComputedTime, violating the invariant that finished streams are blocked.
+ */
+ enum {
+ ALLOW_FINISH = 0x01
+ };
+ virtual void ProcessInput(GraphTime aFrom, GraphTime aTo, uint32_t aFlags) = 0;
+ void SetAutofinishImpl(bool aAutofinish) { mAutofinish = aAutofinish; }
+
+ // Only valid after MediaStreamGraphImpl::UpdateStreamOrder() has run.
+ // A DelayNode is considered to break a cycle and so this will not return
+ // true for echo loops, only for muted cycles.
+ bool InMutedCycle() const { return mCycleMarker; }
+
+ size_t SizeOfExcludingThis(MallocSizeOf aMallocSizeOf) const override
+ {
+ size_t amount = MediaStream::SizeOfExcludingThis(aMallocSizeOf);
+ // Not owned:
+ // - mInputs elements
+ amount += mInputs.ShallowSizeOfExcludingThis(aMallocSizeOf);
+ return amount;
+ }
+
+ size_t SizeOfIncludingThis(MallocSizeOf aMallocSizeOf) const override
+ {
+ return aMallocSizeOf(this) + SizeOfExcludingThis(aMallocSizeOf);
+ }
+
+protected:
+ // This state is all accessed only on the media graph thread.
+
+ // The list of all inputs that are currently enabled or waiting to be enabled.
+ nsTArray<MediaInputPort*> mInputs;
+ bool mAutofinish;
+ // After UpdateStreamOrder(), mCycleMarker is either 0 or 1 to indicate
+ // whether this stream is in a muted cycle. During ordering it can contain
+ // other marker values - see MediaStreamGraphImpl::UpdateStreamOrder().
+ uint32_t mCycleMarker;
+};
+
+/**
+ * There can be multiple MediaStreamGraph per process: one per AudioChannel.
+ * Additionaly, each OfflineAudioContext object creates its own MediaStreamGraph
+ * object too..
+ */
+class MediaStreamGraph
+{
+public:
+
+ // We ensure that the graph current time advances in multiples of
+ // IdealAudioBlockSize()/AudioStream::PreferredSampleRate(). A stream that
+ // never blocks and has a track with the ideal audio rate will produce audio
+ // in multiples of the block size.
+
+ // Initializing an graph that outputs audio can be quite long on some
+ // platforms. Code that want to output audio at some point can express the
+ // fact that they will need an audio stream at some point by passing
+ // AUDIO_THREAD_DRIVER when getting an instance of MediaStreamGraph, so that
+ // the graph starts with the right driver.
+ enum GraphDriverType {
+ AUDIO_THREAD_DRIVER,
+ SYSTEM_THREAD_DRIVER,
+ OFFLINE_THREAD_DRIVER
+ };
+ static const uint32_t AUDIO_CALLBACK_DRIVER_SHUTDOWN_TIMEOUT = 20*1000;
+
+ // Main thread only
+ static MediaStreamGraph* GetInstance(GraphDriverType aGraphDriverRequested,
+ dom::AudioChannel aChannel);
+ static MediaStreamGraph* CreateNonRealtimeInstance(TrackRate aSampleRate);
+ // Idempotent
+ static void DestroyNonRealtimeInstance(MediaStreamGraph* aGraph);
+
+ virtual nsresult OpenAudioInput(int aID,
+ AudioDataListener *aListener) {
+ return NS_ERROR_FAILURE;
+ }
+ virtual void CloseAudioInput(AudioDataListener *aListener) {}
+
+ // Control API.
+ /**
+ * Create a stream that a media decoder (or some other source of
+ * media data, such as a camera) can write to.
+ */
+ SourceMediaStream* CreateSourceStream();
+ /**
+ * Create a stream that will form the union of the tracks of its input
+ * streams.
+ * A TrackUnionStream contains all the tracks of all its input streams.
+ * Adding a new input stream makes that stream's tracks immediately appear as new
+ * tracks starting at the time the input stream was added.
+ * Removing an input stream makes the output tracks corresponding to the
+ * removed tracks immediately end.
+ * For each added track, the track ID of the output track is the track ID
+ * of the input track or one plus the maximum ID of all previously added
+ * tracks, whichever is greater.
+ * TODO at some point we will probably need to add API to select
+ * particular tracks of each input stream.
+ */
+ ProcessedMediaStream* CreateTrackUnionStream();
+ /**
+ * Create a stream that will mix all its audio input.
+ */
+ ProcessedMediaStream* CreateAudioCaptureStream(TrackID aTrackId);
+
+ /**
+ * Add a new stream to the graph. Main thread.
+ */
+ void AddStream(MediaStream* aStream);
+
+ /* From the main thread, ask the MSG to send back an event when the graph
+ * thread is running, and audio is being processed. */
+ void NotifyWhenGraphStarted(AudioNodeStream* aNodeStream);
+ /* From the main thread, suspend, resume or close an AudioContext.
+ * aStreams are the streams of all the AudioNodes of the AudioContext that
+ * need to be suspended or resumed. This can be empty if this is a second
+ * consecutive suspend call and all the nodes are already suspended.
+ *
+ * This can possibly pause the graph thread, releasing system resources, if
+ * all streams have been suspended/closed.
+ *
+ * When the operation is complete, aPromise is resolved.
+ */
+ void ApplyAudioContextOperation(MediaStream* aDestinationStream,
+ const nsTArray<MediaStream*>& aStreams,
+ dom::AudioContextOperation aState,
+ void* aPromise);
+
+ bool IsNonRealtime() const;
+ /**
+ * Start processing non-realtime for a specific number of ticks.
+ */
+ void StartNonRealtimeProcessing(uint32_t aTicksToProcess);
+
+ /**
+ * Media graph thread only.
+ * Dispatches a runnable that will run on the main thread after all
+ * main-thread stream state has been next updated.
+ * Should only be called during MediaStreamListener callbacks or during
+ * ProcessedMediaStream::ProcessInput().
+ */
+ virtual void DispatchToMainThreadAfterStreamStateUpdate(already_AddRefed<nsIRunnable> aRunnable)
+ {
+ AssertOnGraphThreadOrNotRunning();
+ *mPendingUpdateRunnables.AppendElement() = aRunnable;
+ }
+
+ /**
+ * Returns graph sample rate in Hz.
+ */
+ TrackRate GraphRate() const { return mSampleRate; }
+
+ void RegisterCaptureStreamForWindow(uint64_t aWindowId,
+ ProcessedMediaStream* aCaptureStream);
+ void UnregisterCaptureStreamForWindow(uint64_t aWindowId);
+ already_AddRefed<MediaInputPort> ConnectToCaptureStream(
+ uint64_t aWindowId, MediaStream* aMediaStream);
+
+ /**
+ * Data going to the speakers from the GraphDriver's DataCallback
+ * to notify any listeners (for echo cancellation).
+ */
+ void NotifyOutputData(AudioDataValue* aBuffer, size_t aFrames,
+ TrackRate aRate, uint32_t aChannels);
+
+ void AssertOnGraphThreadOrNotRunning() const;
+
+protected:
+ explicit MediaStreamGraph(TrackRate aSampleRate)
+ : mSampleRate(aSampleRate)
+ {
+ MOZ_COUNT_CTOR(MediaStreamGraph);
+ }
+ virtual ~MediaStreamGraph()
+ {
+ MOZ_COUNT_DTOR(MediaStreamGraph);
+ }
+
+ // Media graph thread only
+ nsTArray<nsCOMPtr<nsIRunnable> > mPendingUpdateRunnables;
+
+ /**
+ * Sample rate at which this graph runs. For real time graphs, this is
+ * the rate of the audio mixer. For offline graphs, this is the rate specified
+ * at construction.
+ */
+ TrackRate mSampleRate;
+
+ /**
+ * CloseAudioInput is async, so hold a reference here.
+ */
+ nsTArray<RefPtr<AudioDataListener>> mAudioInputs;
+};
+
+} // namespace mozilla
+
+#endif /* MOZILLA_MEDIASTREAMGRAPH_H_ */