summaryrefslogtreecommitdiffstats
path: root/dom/media/GraphDriver.h
diff options
context:
space:
mode:
Diffstat (limited to 'dom/media/GraphDriver.h')
-rw-r--r--dom/media/GraphDriver.h583
1 files changed, 583 insertions, 0 deletions
diff --git a/dom/media/GraphDriver.h b/dom/media/GraphDriver.h
new file mode 100644
index 000000000..411e175d3
--- /dev/null
+++ b/dom/media/GraphDriver.h
@@ -0,0 +1,583 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef GRAPHDRIVER_H_
+#define GRAPHDRIVER_H_
+
+#include "nsAutoRef.h"
+#include "AudioBufferUtils.h"
+#include "AudioMixer.h"
+#include "AudioSegment.h"
+#include "SelfRef.h"
+#include "mozilla/Atomics.h"
+#include "mozilla/SharedThreadPool.h"
+#include "mozilla/StaticPtr.h"
+
+struct cubeb_stream;
+
+template <>
+class nsAutoRefTraits<cubeb_stream> : public nsPointerRefTraits<cubeb_stream>
+{
+public:
+ static void Release(cubeb_stream* aStream) { cubeb_stream_destroy(aStream); }
+};
+
+namespace mozilla {
+
+/**
+ * Assume we can run an iteration of the MediaStreamGraph loop in this much time
+ * or less.
+ * We try to run the control loop at this rate.
+ */
+static const int MEDIA_GRAPH_TARGET_PERIOD_MS = 10;
+
+/**
+ * Assume that we might miss our scheduled wakeup of the MediaStreamGraph by
+ * this much.
+ */
+static const int SCHEDULE_SAFETY_MARGIN_MS = 10;
+
+/**
+ * Try have this much audio buffered in streams and queued to the hardware.
+ * The maximum delay to the end of the next control loop
+ * is 2*MEDIA_GRAPH_TARGET_PERIOD_MS + SCHEDULE_SAFETY_MARGIN_MS.
+ * There is no point in buffering more audio than this in a stream at any
+ * given time (until we add processing).
+ * This is not optimal yet.
+ */
+static const int AUDIO_TARGET_MS = 2*MEDIA_GRAPH_TARGET_PERIOD_MS +
+ SCHEDULE_SAFETY_MARGIN_MS;
+
+class MediaStreamGraphImpl;
+
+class AudioCallbackDriver;
+class OfflineClockDriver;
+class SystemClockDriver;
+
+/**
+ * A driver is responsible for the scheduling of the processing, the thread
+ * management, and give the different clocks to a MediaStreamGraph. This is an
+ * abstract base class. A MediaStreamGraph can be driven by an
+ * OfflineClockDriver, if the graph is offline, or a SystemClockDriver, if the
+ * graph is real time.
+ * A MediaStreamGraph holds an owning reference to its driver.
+ *
+ * The lifetime of drivers is a complicated affair. Here are the different
+ * scenarii that can happen:
+ *
+ * Starting a MediaStreamGraph with an AudioCallbackDriver
+ * - A new thread T is created, from the main thread.
+ * - On this thread T, cubeb is initialized if needed, and a cubeb_stream is
+ * created and started
+ * - The thread T posts a message to the main thread to terminate itself.
+ * - The graph runs off the audio thread
+ *
+ * Starting a MediaStreamGraph with a SystemClockDriver:
+ * - A new thread T is created from the main thread.
+ * - The graph runs off this thread.
+ *
+ * Switching from a SystemClockDriver to an AudioCallbackDriver:
+ * - A new AudioCallabackDriver is created and initialized on the graph thread
+ * - At the end of the MSG iteration, the SystemClockDriver transfers its timing
+ * info and a reference to itself to the AudioCallbackDriver. It then starts
+ * the AudioCallbackDriver.
+ * - When the AudioCallbackDriver starts, it checks if it has been switched from
+ * a SystemClockDriver, and if that is the case, sends a message to the main
+ * thread to shut the SystemClockDriver thread down.
+ * - The graph now runs off an audio callback
+ *
+ * Switching from an AudioCallbackDriver to a SystemClockDriver:
+ * - A new SystemClockDriver is created, and set as mNextDriver.
+ * - At the end of the MSG iteration, the AudioCallbackDriver transfers its
+ * timing info and a reference to itself to the SystemClockDriver. A new
+ * SystemClockDriver is started from the current audio thread.
+ * - When starting, the SystemClockDriver checks if it has been switched from an
+ * AudioCallbackDriver. If yes, it creates a new temporary thread to release
+ * the cubeb_streams. This temporary thread closes the cubeb_stream, and
+ * then dispatches a message to the main thread to be terminated.
+ * - The graph now runs off a normal thread.
+ *
+ * Two drivers cannot run at the same time for the same graph. The thread safety
+ * of the different attributes of drivers, and they access pattern is documented
+ * next to the members themselves.
+ *
+ */
+class GraphDriver
+{
+public:
+ explicit GraphDriver(MediaStreamGraphImpl* aGraphImpl);
+
+ NS_INLINE_DECL_THREADSAFE_REFCOUNTING(GraphDriver);
+ /* For real-time graphs, this waits until it's time to process more data. For
+ * offline graphs, this is a no-op. */
+ virtual void WaitForNextIteration() = 0;
+ /* Wakes up the graph if it is waiting. */
+ virtual void WakeUp() = 0;
+ virtual void Destroy() {}
+ /* Start the graph, init the driver, start the thread. */
+ virtual void Start() = 0;
+ /* Stop the graph, shutting down the thread. */
+ virtual void Stop() = 0;
+ /* Resume after a stop */
+ virtual void Resume() = 0;
+ /* Revive this driver, as more messages just arrived. */
+ virtual void Revive() = 0;
+ /* Remove Mixer callbacks when switching */
+ virtual void RemoveCallback() = 0;
+ /* Shutdown GraphDriver (synchronously) */
+ void Shutdown();
+ /* Rate at which the GraphDriver runs, in ms. This can either be user
+ * controlled (because we are using a {System,Offline}ClockDriver, and decide
+ * how often we want to wakeup/how much we want to process per iteration), or
+ * it can be indirectly set by the latency of the audio backend, and the
+ * number of buffers of this audio backend: say we have four buffers, and 40ms
+ * latency, we will get a callback approximately every 10ms. */
+ virtual uint32_t IterationDuration() = 0;
+
+ /* Return whether we are switching or not. */
+ bool Switching();
+
+ // Those are simply or setting the associated pointer, but assert that the
+ // lock is held.
+ GraphDriver* NextDriver();
+ GraphDriver* PreviousDriver();
+ void SetNextDriver(GraphDriver* aNextDriver);
+ void SetPreviousDriver(GraphDriver* aPreviousDriver);
+
+ /**
+ * If we are running a real time graph, get the current time stamp to schedule
+ * video frames. This has to be reimplemented by real time drivers.
+ */
+ virtual TimeStamp GetCurrentTimeStamp() {
+ return mCurrentTimeStamp;
+ }
+
+ GraphTime IterationEnd() {
+ return mIterationEnd;
+ }
+
+ virtual AudioCallbackDriver* AsAudioCallbackDriver() {
+ return nullptr;
+ }
+
+ virtual OfflineClockDriver* AsOfflineClockDriver() {
+ return nullptr;
+ }
+
+ virtual SystemClockDriver* AsSystemClockDriver() {
+ return nullptr;
+ }
+
+ /**
+ * Tell the driver it has to stop and return the current time of the graph, so
+ * another driver can start from the right point in time.
+ */
+ virtual void SwitchAtNextIteration(GraphDriver* aDriver);
+
+ /**
+ * Set the time for a graph, on a driver. This is used so a new driver just
+ * created can start at the right point in time.
+ */
+ void SetGraphTime(GraphDriver* aPreviousDriver,
+ GraphTime aLastSwitchNextIterationStart,
+ GraphTime aLastSwitchNextIterationEnd);
+ /**
+ * Call this to indicate that another iteration of the control loop is
+ * required on its regular schedule. The monitor must not be held.
+ * This function has to be idempotent.
+ */
+ void EnsureNextIteration();
+
+ /**
+ * Same thing, but not locked.
+ */
+ void EnsureNextIterationLocked();
+
+ MediaStreamGraphImpl* GraphImpl() {
+ return mGraphImpl;
+ }
+
+ virtual bool OnThread() = 0;
+
+protected:
+ GraphTime StateComputedTime() const;
+
+ // Time of the start of this graph iteration. This must be accessed while
+ // having the monitor.
+ GraphTime mIterationStart;
+ // Time of the end of this graph iteration. This must be accessed while having
+ // the monitor.
+ GraphTime mIterationEnd;
+ // The MediaStreamGraphImpl that owns this driver. This has a lifetime longer
+ // than the driver, and will never be null. Hence, it can be accesed without
+ // monitor.
+ MediaStreamGraphImpl* mGraphImpl;
+
+ // This enum specifies the wait state of the driver.
+ enum WaitState {
+ // RunThread() is running normally
+ WAITSTATE_RUNNING,
+ // RunThread() is paused waiting for its next iteration, which will
+ // happen soon
+ WAITSTATE_WAITING_FOR_NEXT_ITERATION,
+ // RunThread() is paused indefinitely waiting for something to change
+ WAITSTATE_WAITING_INDEFINITELY,
+ // Something has signaled RunThread() to wake up immediately,
+ // but it hasn't done so yet
+ WAITSTATE_WAKING_UP
+ };
+ // This must be access with the monitor.
+ WaitState mWaitState;
+
+ // This is used on the main thread (during initialization), and the graph
+ // thread. No monitor needed because we know the graph thread does not run
+ // during the initialization.
+ TimeStamp mCurrentTimeStamp;
+ // This is non-null only when this driver has recently switched from an other
+ // driver, and has not cleaned it up yet (for example because the audio stream
+ // is currently calling the callback during initialization).
+ //
+ // This is written to when changing driver, from the previous driver's thread,
+ // or a thread created for the occasion. This is read each time we need to
+ // check whether we're changing driver (in Switching()), from the graph
+ // thread.
+ // This must be accessed using the {Set,Get}PreviousDriver methods.
+ RefPtr<GraphDriver> mPreviousDriver;
+ // This is non-null only when this driver is going to switch to an other
+ // driver at the end of this iteration.
+ // This must be accessed using the {Set,Get}NextDriver methods.
+ RefPtr<GraphDriver> mNextDriver;
+ virtual ~GraphDriver()
+ { }
+};
+
+class MediaStreamGraphInitThreadRunnable;
+
+/**
+ * This class is a driver that manages its own thread.
+ */
+class ThreadedDriver : public GraphDriver
+{
+public:
+ explicit ThreadedDriver(MediaStreamGraphImpl* aGraphImpl);
+ virtual ~ThreadedDriver();
+ void Start() override;
+ void Stop() override;
+ void Resume() override;
+ void Revive() override;
+ void RemoveCallback() override;
+ /**
+ * Runs main control loop on the graph thread. Normally a single invocation
+ * of this runs for the entire lifetime of the graph thread.
+ */
+ void RunThread();
+ friend class MediaStreamGraphInitThreadRunnable;
+ uint32_t IterationDuration() override {
+ return MEDIA_GRAPH_TARGET_PERIOD_MS;
+ }
+
+ bool OnThread() override { return !mThread || NS_GetCurrentThread() == mThread; }
+
+ /* When the graph wakes up to do an iteration, implementations return the
+ * range of time that will be processed. This is called only once per
+ * iteration; it may determine the interval from state in a previous
+ * call. */
+ virtual MediaTime GetIntervalForIteration() = 0;
+protected:
+ nsCOMPtr<nsIThread> mThread;
+};
+
+/**
+ * A SystemClockDriver drives a MediaStreamGraph using a system clock, and waits
+ * using a monitor, between each iteration.
+ */
+class SystemClockDriver : public ThreadedDriver
+{
+public:
+ explicit SystemClockDriver(MediaStreamGraphImpl* aGraphImpl);
+ virtual ~SystemClockDriver();
+ MediaTime GetIntervalForIteration() override;
+ void WaitForNextIteration() override;
+ void WakeUp() override;
+ void MarkAsFallback();
+ bool IsFallback();
+ SystemClockDriver* AsSystemClockDriver() override {
+ return this;
+ }
+
+private:
+ // Those are only modified (after initialization) on the graph thread. The
+ // graph thread does not run during the initialization.
+ TimeStamp mInitialTimeStamp;
+ TimeStamp mLastTimeStamp;
+ // This is true if this SystemClockDriver runs the graph because we could not
+ // open an audio stream.
+ bool mIsFallback;
+};
+
+/**
+ * An OfflineClockDriver runs the graph as fast as possible, without waiting
+ * between iteration.
+ */
+class OfflineClockDriver : public ThreadedDriver
+{
+public:
+ OfflineClockDriver(MediaStreamGraphImpl* aGraphImpl, GraphTime aSlice);
+ virtual ~OfflineClockDriver();
+ MediaTime GetIntervalForIteration() override;
+ void WaitForNextIteration() override;
+ void WakeUp() override;
+ TimeStamp GetCurrentTimeStamp() override;
+ OfflineClockDriver* AsOfflineClockDriver() override {
+ return this;
+ }
+
+private:
+ // Time, in GraphTime, for each iteration
+ GraphTime mSlice;
+};
+
+struct StreamAndPromiseForOperation
+{
+ StreamAndPromiseForOperation(MediaStream* aStream,
+ void* aPromise,
+ dom::AudioContextOperation aOperation);
+ RefPtr<MediaStream> mStream;
+ void* mPromise;
+ dom::AudioContextOperation mOperation;
+};
+
+enum AsyncCubebOperation {
+ INIT,
+ SHUTDOWN
+};
+
+/**
+ * This is a graph driver that is based on callback functions called by the
+ * audio api. This ensures minimal audio latency, because it means there is no
+ * buffering happening: the audio is generated inside the callback.
+ *
+ * This design is less flexible than running our own thread:
+ * - We have no control over the thread:
+ * - It cannot block, and it has to run for a shorter amount of time than the
+ * buffer it is going to fill, or an under-run is going to occur (short burst
+ * of silence in the final audio output).
+ * - We can't know for sure when the callback function is going to be called
+ * (although we compute an estimation so we can schedule video frames)
+ * - Creating and shutting the thread down is a blocking operation, that can
+ * take _seconds_ in some cases (because IPC has to be set up, and
+ * sometimes hardware components are involved and need to be warmed up)
+ * - We have no control on how much audio we generate, we have to return exactly
+ * the number of frames asked for by the callback. Since for the Web Audio
+ * API, we have to do block processing at 128 frames per block, we need to
+ * keep a little spill buffer to store the extra frames.
+ */
+class AudioCallbackDriver : public GraphDriver,
+ public MixerCallbackReceiver
+{
+public:
+ explicit AudioCallbackDriver(MediaStreamGraphImpl* aGraphImpl);
+ virtual ~AudioCallbackDriver();
+
+ void Destroy() override;
+ void Start() override;
+ void Stop() override;
+ void Resume() override;
+ void Revive() override;
+ void RemoveCallback() override;
+ void WaitForNextIteration() override;
+ void WakeUp() override;
+
+ /* Static wrapper function cubeb calls back. */
+ static long DataCallback_s(cubeb_stream * aStream,
+ void * aUser,
+ const void * aInputBuffer,
+ void * aOutputBuffer,
+ long aFrames);
+ static void StateCallback_s(cubeb_stream* aStream, void * aUser,
+ cubeb_state aState);
+ static void DeviceChangedCallback_s(void * aUser);
+ /* This function is called by the underlying audio backend when a refill is
+ * needed. This is what drives the whole graph when it is used to output
+ * audio. If the return value is exactly aFrames, this function will get
+ * called again. If it is less than aFrames, the stream will go in draining
+ * mode, and this function will not be called again. */
+ long DataCallback(const AudioDataValue* aInputBuffer, AudioDataValue* aOutputBuffer, long aFrames);
+ /* This function is called by the underlying audio backend, but is only used
+ * for informational purposes at the moment. */
+ void StateCallback(cubeb_state aState);
+ /* This is an approximation of the number of millisecond there are between two
+ * iterations of the graph. */
+ uint32_t IterationDuration() override;
+
+ /* This function gets called when the graph has produced the audio frames for
+ * this iteration. */
+ void MixerCallback(AudioDataValue* aMixedBuffer,
+ AudioSampleFormat aFormat,
+ uint32_t aChannels,
+ uint32_t aFrames,
+ uint32_t aSampleRate) override;
+
+ // These are invoked on the MSG thread (we don't call this if not LIFECYCLE_RUNNING)
+ virtual void SetInputListener(AudioDataListener *aListener) {
+ MOZ_ASSERT(OnThread());
+ mAudioInput = aListener;
+ }
+ // XXX do we need the param? probably no
+ virtual void RemoveInputListener(AudioDataListener *aListener) {
+ MOZ_ASSERT(OnThread());
+ mAudioInput = nullptr;
+ }
+
+ AudioCallbackDriver* AsAudioCallbackDriver() override {
+ return this;
+ }
+
+ /* Enqueue a promise that is going to be resolved when a specific operation
+ * occurs on the cubeb stream. */
+ void EnqueueStreamAndPromiseForOperation(MediaStream* aStream,
+ void* aPromise,
+ dom::AudioContextOperation aOperation);
+
+ /**
+ * Whether the audio callback is processing. This is for asserting only.
+ */
+ bool InCallback();
+
+ bool OnThread() override { return !mStarted || InCallback(); }
+
+ /* Whether the underlying cubeb stream has been started. See comment for
+ * mStarted for details. */
+ bool IsStarted();
+
+ /* Tell the driver whether this process is using a microphone or not. This is
+ * thread safe. */
+ void SetMicrophoneActive(bool aActive);
+
+ void CompleteAudioContextOperations(AsyncCubebOperation aOperation);
+private:
+ /**
+ * On certain MacBookPro, the microphone is located near the left speaker.
+ * We need to pan the sound output to the right speaker if we are using the
+ * mic and the built-in speaker, or we will have terrible echo. */
+ void PanOutputIfNeeded(bool aMicrophoneActive);
+ /**
+ * This is called when the output device used by the cubeb stream changes. */
+ void DeviceChangedCallback();
+ /* Start the cubeb stream */
+ void StartStream();
+ friend class AsyncCubebTask;
+ void Init();
+ /* MediaStreamGraphs are always down/up mixed to stereo for now. */
+ static const uint32_t ChannelCount = 2;
+ /* The size of this buffer comes from the fact that some audio backends can
+ * call back with a number of frames lower than one block (128 frames), so we
+ * need to keep at most two block in the SpillBuffer, because we always round
+ * up to block boundaries during an iteration.
+ * This is only ever accessed on the audio callback thread. */
+ SpillBuffer<AudioDataValue, WEBAUDIO_BLOCK_SIZE * 2, ChannelCount> mScratchBuffer;
+ /* Wrapper to ensure we write exactly the number of frames we need in the
+ * audio buffer cubeb passes us. This is only ever accessed on the audio
+ * callback thread. */
+ AudioCallbackBufferWrapper<AudioDataValue, ChannelCount> mBuffer;
+ /* cubeb stream for this graph. This is guaranteed to be non-null after Init()
+ * has been called, and is synchronized internaly. */
+ nsAutoRef<cubeb_stream> mAudioStream;
+ /* The sample rate for the aforementionned cubeb stream. This is set on
+ * initialization and can be read safely afterwards. */
+ uint32_t mSampleRate;
+ /* The number of input channels from cubeb. Should be set before opening cubeb
+ * and then be static. */
+ uint32_t mInputChannels;
+ /* Approximation of the time between two callbacks. This is used to schedule
+ * video frames. This is in milliseconds. Only even used (after
+ * inizatialization) on the audio callback thread. */
+ uint32_t mIterationDurationMS;
+ /* cubeb_stream_init calls the audio callback to prefill the buffers. The
+ * previous driver has to be kept alive until the audio stream has been
+ * started, because it is responsible to call cubeb_stream_start, so we delay
+ * the cleanup of the previous driver until it has started the audio stream.
+ * Otherwise, there is a race where we kill the previous driver thread
+ * between cubeb_stream_init and cubeb_stream_start,
+ * and callbacks after the prefill never get called.
+ * This is written on the previous driver's thread (if switching) or main
+ * thread (if this driver is the first one).
+ * This is read on previous driver's thread (during callbacks from
+ * cubeb_stream_init) and the audio thread (when switching away from this
+ * driver back to a SystemClockDriver).
+ * This is synchronized by the Graph's monitor.
+ * */
+ bool mStarted;
+ /* Listener for mic input, if any. */
+ RefPtr<AudioDataListener> mAudioInput;
+
+ struct AutoInCallback
+ {
+ explicit AutoInCallback(AudioCallbackDriver* aDriver);
+ ~AutoInCallback();
+ AudioCallbackDriver* mDriver;
+ };
+
+ /* Thread for off-main-thread initialization and
+ * shutdown of the audio stream. */
+ nsCOMPtr<nsIThread> mInitShutdownThread;
+ /* This must be accessed with the graph monitor held. */
+ AutoTArray<StreamAndPromiseForOperation, 1> mPromisesForOperation;
+ /* This is set during initialization, and can be read safely afterwards. */
+ dom::AudioChannel mAudioChannel;
+ /* Used to queue us to add the mixer callback on first run. */
+ bool mAddedMixer;
+
+ /* This is atomic and is set by the audio callback thread. It can be read by
+ * any thread safely. */
+ Atomic<bool> mInCallback;
+ /**
+ * True if microphone is being used by this process. This is synchronized by
+ * the graph's monitor. */
+ bool mMicrophoneActive;
+ /* Indication of whether a fallback SystemClockDriver should be started if
+ * StateCallback() receives an error. No mutex need be held during access.
+ * The transition to true happens before cubeb_stream_start() is called.
+ * After transitioning to false on the last DataCallback(), the stream is
+ * not accessed from another thread until the graph thread either signals
+ * main thread cleanup or dispatches an event to switch to another
+ * driver. */
+ bool mShouldFallbackIfError;
+ /* True if this driver was created from a driver created because of a previous
+ * AudioCallbackDriver failure. */
+ bool mFromFallback;
+};
+
+class AsyncCubebTask : public Runnable
+{
+public:
+
+ AsyncCubebTask(AudioCallbackDriver* aDriver, AsyncCubebOperation aOperation);
+
+ nsresult Dispatch(uint32_t aFlags = NS_DISPATCH_NORMAL)
+ {
+ nsresult rv = EnsureThread();
+ if (!NS_FAILED(rv)) {
+ rv = sThreadPool->Dispatch(this, aFlags);
+ }
+ return rv;
+ }
+
+protected:
+ virtual ~AsyncCubebTask();
+
+private:
+ static nsresult EnsureThread();
+
+ NS_IMETHOD Run() override final;
+ static StaticRefPtr<nsIThreadPool> sThreadPool;
+ RefPtr<AudioCallbackDriver> mDriver;
+ AsyncCubebOperation mOperation;
+ RefPtr<MediaStreamGraphImpl> mShutdownGrip;
+};
+
+} // namespace mozilla
+
+#endif // GRAPHDRIVER_H_