summaryrefslogtreecommitdiffstats
path: root/dom/media/GraphDriver.cpp
diff options
context:
space:
mode:
Diffstat (limited to 'dom/media/GraphDriver.cpp')
-rw-r--r--dom/media/GraphDriver.cpp1217
1 files changed, 1217 insertions, 0 deletions
diff --git a/dom/media/GraphDriver.cpp b/dom/media/GraphDriver.cpp
new file mode 100644
index 000000000..9b74bd58c
--- /dev/null
+++ b/dom/media/GraphDriver.cpp
@@ -0,0 +1,1217 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include <MediaStreamGraphImpl.h>
+#include "mozilla/dom/AudioContext.h"
+#include "mozilla/SharedThreadPool.h"
+#include "mozilla/ClearOnShutdown.h"
+#include "mozilla/Unused.h"
+#include "CubebUtils.h"
+
+#ifdef MOZ_WEBRTC
+#include "webrtc/MediaEngineWebRTC.h"
+#endif
+
+#ifdef XP_MACOSX
+#include <sys/sysctl.h>
+#endif
+
+extern mozilla::LazyLogModule gMediaStreamGraphLog;
+#define STREAM_LOG(type, msg) MOZ_LOG(gMediaStreamGraphLog, type, msg)
+
+// We don't use NSPR log here because we want this interleaved with adb logcat
+// on Android/B2G
+// #define ENABLE_LIFECYCLE_LOG
+#ifdef ENABLE_LIFECYCLE_LOG
+#ifdef ANDROID
+#include "android/log.h"
+#define LIFECYCLE_LOG(...) __android_log_print(ANDROID_LOG_INFO, "Gecko - MSG" , __VA_ARGS__); printf(__VA_ARGS__);printf("\n");
+#else
+#define LIFECYCLE_LOG(...) printf(__VA_ARGS__);printf("\n");
+#endif
+#else
+#define LIFECYCLE_LOG(...)
+#endif
+
+namespace mozilla {
+
+StaticRefPtr<nsIThreadPool> AsyncCubebTask::sThreadPool;
+
+struct AutoProfilerUnregisterThread
+{
+ // The empty ctor is used to silence a pre-4.8.0 GCC unused variable warning.
+ AutoProfilerUnregisterThread()
+ {
+ }
+
+ ~AutoProfilerUnregisterThread()
+ {
+ profiler_unregister_thread();
+ }
+};
+
+GraphDriver::GraphDriver(MediaStreamGraphImpl* aGraphImpl)
+ : mIterationStart(0),
+ mIterationEnd(0),
+ mGraphImpl(aGraphImpl),
+ mWaitState(WAITSTATE_RUNNING),
+ mCurrentTimeStamp(TimeStamp::Now()),
+ mPreviousDriver(nullptr),
+ mNextDriver(nullptr)
+{ }
+
+void GraphDriver::SetGraphTime(GraphDriver* aPreviousDriver,
+ GraphTime aLastSwitchNextIterationStart,
+ GraphTime aLastSwitchNextIterationEnd)
+{
+ GraphImpl()->GetMonitor().AssertCurrentThreadOwns();
+ // We set mIterationEnd here, because the first thing a driver do when it
+ // does an iteration is to update graph times, so we are in fact setting
+ // mIterationStart of the next iteration by setting the end of the previous
+ // iteration.
+ mIterationStart = aLastSwitchNextIterationStart;
+ mIterationEnd = aLastSwitchNextIterationEnd;
+
+ MOZ_ASSERT(!PreviousDriver());
+ MOZ_ASSERT(aPreviousDriver);
+
+ STREAM_LOG(LogLevel::Debug, ("Setting previous driver: %p (%s)",
+ aPreviousDriver,
+ aPreviousDriver->AsAudioCallbackDriver()
+ ? "AudioCallbackDriver"
+ : "SystemClockDriver"));
+ SetPreviousDriver(aPreviousDriver);
+}
+
+void GraphDriver::SwitchAtNextIteration(GraphDriver* aNextDriver)
+{
+ GraphImpl()->GetMonitor().AssertCurrentThreadOwns();
+ LIFECYCLE_LOG("Switching to new driver: %p (%s)",
+ aNextDriver, aNextDriver->AsAudioCallbackDriver() ?
+ "AudioCallbackDriver" : "SystemClockDriver");
+ if (mNextDriver &&
+ mNextDriver != GraphImpl()->CurrentDriver()) {
+ LIFECYCLE_LOG("Discarding previous next driver: %p (%s)",
+ mNextDriver.get(), mNextDriver->AsAudioCallbackDriver() ?
+ "AudioCallbackDriver" : "SystemClockDriver");
+ }
+ SetNextDriver(aNextDriver);
+}
+
+GraphTime
+GraphDriver::StateComputedTime() const
+{
+ return mGraphImpl->mStateComputedTime;
+}
+
+void GraphDriver::EnsureNextIteration()
+{
+ mGraphImpl->EnsureNextIteration();
+}
+
+void GraphDriver::Shutdown()
+{
+ if (AsAudioCallbackDriver()) {
+ LIFECYCLE_LOG("Releasing audio driver off main thread (GraphDriver::Shutdown).\n");
+ RefPtr<AsyncCubebTask> releaseEvent =
+ new AsyncCubebTask(AsAudioCallbackDriver(), AsyncCubebOperation::SHUTDOWN);
+ releaseEvent->Dispatch(NS_DISPATCH_SYNC);
+ } else {
+ Stop();
+ }
+}
+
+bool GraphDriver::Switching()
+{
+ GraphImpl()->GetMonitor().AssertCurrentThreadOwns();
+ return mNextDriver || mPreviousDriver;
+}
+
+GraphDriver* GraphDriver::NextDriver()
+{
+ GraphImpl()->GetMonitor().AssertCurrentThreadOwns();
+ return mNextDriver;
+}
+
+GraphDriver* GraphDriver::PreviousDriver()
+{
+ GraphImpl()->GetMonitor().AssertCurrentThreadOwns();
+ return mPreviousDriver;
+}
+
+void GraphDriver::SetNextDriver(GraphDriver* aNextDriver)
+{
+ GraphImpl()->GetMonitor().AssertCurrentThreadOwns();
+ mNextDriver = aNextDriver;
+}
+
+void GraphDriver::SetPreviousDriver(GraphDriver* aPreviousDriver)
+{
+ GraphImpl()->GetMonitor().AssertCurrentThreadOwns();
+ mPreviousDriver = aPreviousDriver;
+}
+
+ThreadedDriver::ThreadedDriver(MediaStreamGraphImpl* aGraphImpl)
+ : GraphDriver(aGraphImpl)
+{ }
+
+class MediaStreamGraphShutdownThreadRunnable : public Runnable {
+public:
+ explicit MediaStreamGraphShutdownThreadRunnable(already_AddRefed<nsIThread> aThread)
+ : mThread(aThread)
+ {
+ }
+ NS_IMETHOD Run() override
+ {
+ MOZ_ASSERT(NS_IsMainThread());
+ MOZ_ASSERT(mThread);
+
+ mThread->Shutdown();
+ mThread = nullptr;
+ return NS_OK;
+ }
+private:
+ nsCOMPtr<nsIThread> mThread;
+};
+
+ThreadedDriver::~ThreadedDriver()
+{
+ if (mThread) {
+ if (NS_IsMainThread()) {
+ mThread->Shutdown();
+ } else {
+ nsCOMPtr<nsIRunnable> event =
+ new MediaStreamGraphShutdownThreadRunnable(mThread.forget());
+ NS_DispatchToMainThread(event);
+ }
+ }
+}
+class MediaStreamGraphInitThreadRunnable : public Runnable {
+public:
+ explicit MediaStreamGraphInitThreadRunnable(ThreadedDriver* aDriver)
+ : mDriver(aDriver)
+ {
+ }
+ NS_IMETHOD Run() override
+ {
+ char aLocal;
+ STREAM_LOG(LogLevel::Debug, ("Starting system thread"));
+ profiler_register_thread("MediaStreamGraph", &aLocal);
+ LIFECYCLE_LOG("Starting a new system driver for graph %p\n",
+ mDriver->mGraphImpl);
+
+ RefPtr<GraphDriver> previousDriver;
+ {
+ MonitorAutoLock mon(mDriver->mGraphImpl->GetMonitor());
+ previousDriver = mDriver->PreviousDriver();
+ }
+ if (previousDriver) {
+ LIFECYCLE_LOG("%p releasing an AudioCallbackDriver(%p), for graph %p\n",
+ mDriver,
+ previousDriver,
+ mDriver->GraphImpl());
+ MOZ_ASSERT(!mDriver->AsAudioCallbackDriver());
+ RefPtr<AsyncCubebTask> releaseEvent =
+ new AsyncCubebTask(previousDriver->AsAudioCallbackDriver(), AsyncCubebOperation::SHUTDOWN);
+ releaseEvent->Dispatch();
+
+ MonitorAutoLock mon(mDriver->mGraphImpl->GetMonitor());
+ mDriver->SetPreviousDriver(nullptr);
+ } else {
+ MonitorAutoLock mon(mDriver->mGraphImpl->GetMonitor());
+ MOZ_ASSERT(mDriver->mGraphImpl->MessagesQueued() ||
+ mDriver->mGraphImpl->mForceShutDown, "Don't start a graph without messages queued.");
+ mDriver->mGraphImpl->SwapMessageQueues();
+ }
+
+ mDriver->RunThread();
+ return NS_OK;
+ }
+private:
+ RefPtr<ThreadedDriver> mDriver;
+};
+
+void
+ThreadedDriver::Start()
+{
+ LIFECYCLE_LOG("Starting thread for a SystemClockDriver %p\n", mGraphImpl);
+ Unused << NS_WARN_IF(mThread);
+ if (!mThread) { // Ensure we haven't already started it
+ nsCOMPtr<nsIRunnable> event = new MediaStreamGraphInitThreadRunnable(this);
+ // Note: mThread may be null during event->Run() if we pass to NewNamedThread! See AudioInitTask
+ nsresult rv = NS_NewNamedThread("MediaStreamGrph", getter_AddRefs(mThread));
+ if (NS_SUCCEEDED(rv)) {
+ mThread->Dispatch(event, NS_DISPATCH_NORMAL);
+ }
+ }
+}
+
+void
+ThreadedDriver::Resume()
+{
+ Start();
+}
+
+void
+ThreadedDriver::Revive()
+{
+ // Note: only called on MainThread, without monitor
+ // We know were weren't in a running state
+ STREAM_LOG(LogLevel::Debug, ("AudioCallbackDriver reviving."));
+ // If we were switching, switch now. Otherwise, tell thread to run the main
+ // loop again.
+ MonitorAutoLock mon(mGraphImpl->GetMonitor());
+ if (NextDriver()) {
+ NextDriver()->SetGraphTime(this, mIterationStart, mIterationEnd);
+ mGraphImpl->SetCurrentDriver(NextDriver());
+ NextDriver()->Start();
+ } else {
+ nsCOMPtr<nsIRunnable> event = new MediaStreamGraphInitThreadRunnable(this);
+ mThread->Dispatch(event, NS_DISPATCH_NORMAL);
+ }
+}
+
+void
+ThreadedDriver::RemoveCallback()
+{
+}
+
+void
+ThreadedDriver::Stop()
+{
+ NS_ASSERTION(NS_IsMainThread(), "Must be called on main thread");
+ // mGraph's thread is not running so it's OK to do whatever here
+ STREAM_LOG(LogLevel::Debug, ("Stopping threads for MediaStreamGraph %p", this));
+
+ if (mThread) {
+ mThread->Shutdown();
+ mThread = nullptr;
+ }
+}
+
+SystemClockDriver::SystemClockDriver(MediaStreamGraphImpl* aGraphImpl)
+ : ThreadedDriver(aGraphImpl),
+ mInitialTimeStamp(TimeStamp::Now()),
+ mLastTimeStamp(TimeStamp::Now()),
+ mIsFallback(false)
+{}
+
+SystemClockDriver::~SystemClockDriver()
+{ }
+
+void
+SystemClockDriver::MarkAsFallback()
+{
+ mIsFallback = true;
+}
+
+bool
+SystemClockDriver::IsFallback()
+{
+ return mIsFallback;
+}
+
+void
+ThreadedDriver::RunThread()
+{
+ AutoProfilerUnregisterThread autoUnregister;
+
+ while (true) {
+ mIterationStart = IterationEnd();
+ mIterationEnd += GetIntervalForIteration();
+
+ GraphTime stateComputedTime = StateComputedTime();
+ if (stateComputedTime < mIterationEnd) {
+ STREAM_LOG(LogLevel::Warning, ("Media graph global underrun detected"));
+ mIterationEnd = stateComputedTime;
+ }
+
+ if (mIterationStart >= mIterationEnd) {
+ NS_ASSERTION(mIterationStart == mIterationEnd ,
+ "Time can't go backwards!");
+ // This could happen due to low clock resolution, maybe?
+ STREAM_LOG(LogLevel::Debug, ("Time did not advance"));
+ }
+
+ GraphTime nextStateComputedTime =
+ mGraphImpl->RoundUpToNextAudioBlock(
+ mIterationEnd + mGraphImpl->MillisecondsToMediaTime(AUDIO_TARGET_MS));
+ if (nextStateComputedTime < stateComputedTime) {
+ // A previous driver may have been processing further ahead of
+ // iterationEnd.
+ STREAM_LOG(LogLevel::Warning,
+ ("Prevent state from going backwards. interval[%ld; %ld] state[%ld; %ld]",
+ (long)mIterationStart, (long)mIterationEnd,
+ (long)stateComputedTime, (long)nextStateComputedTime));
+ nextStateComputedTime = stateComputedTime;
+ }
+ STREAM_LOG(LogLevel::Verbose,
+ ("interval[%ld; %ld] state[%ld; %ld]",
+ (long)mIterationStart, (long)mIterationEnd,
+ (long)stateComputedTime, (long)nextStateComputedTime));
+
+ bool stillProcessing = mGraphImpl->OneIteration(nextStateComputedTime);
+
+ if (!stillProcessing) {
+ // Enter shutdown mode. The stable-state handler will detect this
+ // and complete shutdown if the graph does not get restarted.
+ mGraphImpl->SignalMainThreadCleanup();
+ return;
+ }
+ MonitorAutoLock lock(GraphImpl()->GetMonitor());
+ if (NextDriver()) {
+ STREAM_LOG(LogLevel::Debug, ("Switching to AudioCallbackDriver"));
+ RemoveCallback();
+ NextDriver()->SetGraphTime(this, mIterationStart, mIterationEnd);
+ mGraphImpl->SetCurrentDriver(NextDriver());
+ NextDriver()->Start();
+ return;
+ }
+ }
+}
+
+MediaTime
+SystemClockDriver::GetIntervalForIteration()
+{
+ TimeStamp now = TimeStamp::Now();
+ MediaTime interval =
+ mGraphImpl->SecondsToMediaTime((now - mCurrentTimeStamp).ToSeconds());
+ mCurrentTimeStamp = now;
+
+ MOZ_LOG(gMediaStreamGraphLog, LogLevel::Verbose,
+ ("Updating current time to %f (real %f, StateComputedTime() %f)",
+ mGraphImpl->MediaTimeToSeconds(IterationEnd() + interval),
+ (now - mInitialTimeStamp).ToSeconds(),
+ mGraphImpl->MediaTimeToSeconds(StateComputedTime())));
+
+ return interval;
+}
+
+TimeStamp
+OfflineClockDriver::GetCurrentTimeStamp()
+{
+ MOZ_CRASH("This driver does not support getting the current timestamp.");
+ return TimeStamp();
+}
+
+void
+SystemClockDriver::WaitForNextIteration()
+{
+ mGraphImpl->GetMonitor().AssertCurrentThreadOwns();
+
+ PRIntervalTime timeout = PR_INTERVAL_NO_TIMEOUT;
+ TimeStamp now = TimeStamp::Now();
+
+ // This lets us avoid hitting the Atomic twice when we know we won't sleep
+ bool another = mGraphImpl->mNeedAnotherIteration; // atomic
+ if (!another) {
+ mGraphImpl->mGraphDriverAsleep = true; // atomic
+ mWaitState = WAITSTATE_WAITING_INDEFINITELY;
+ }
+ // NOTE: mNeedAnotherIteration while also atomic may have changed before
+ // we could set mGraphDriverAsleep, so we must re-test it.
+ // (EnsureNextIteration sets mNeedAnotherIteration, then tests
+ // mGraphDriverAsleep
+ if (another || mGraphImpl->mNeedAnotherIteration) { // atomic
+ int64_t timeoutMS = MEDIA_GRAPH_TARGET_PERIOD_MS -
+ int64_t((now - mCurrentTimeStamp).ToMilliseconds());
+ // Make sure timeoutMS doesn't overflow 32 bits by waking up at
+ // least once a minute, if we need to wake up at all
+ timeoutMS = std::max<int64_t>(0, std::min<int64_t>(timeoutMS, 60*1000));
+ timeout = PR_MillisecondsToInterval(uint32_t(timeoutMS));
+ STREAM_LOG(LogLevel::Verbose,
+ ("Waiting for next iteration; at %f, timeout=%f",
+ (now - mInitialTimeStamp).ToSeconds(), timeoutMS/1000.0));
+ if (mWaitState == WAITSTATE_WAITING_INDEFINITELY) {
+ mGraphImpl->mGraphDriverAsleep = false; // atomic
+ }
+ mWaitState = WAITSTATE_WAITING_FOR_NEXT_ITERATION;
+ }
+ if (timeout > 0) {
+ mGraphImpl->GetMonitor().Wait(timeout);
+ STREAM_LOG(LogLevel::Verbose, ("Resuming after timeout; at %f, elapsed=%f",
+ (TimeStamp::Now() - mInitialTimeStamp).ToSeconds(),
+ (TimeStamp::Now() - now).ToSeconds()));
+ }
+
+ if (mWaitState == WAITSTATE_WAITING_INDEFINITELY) {
+ mGraphImpl->mGraphDriverAsleep = false; // atomic
+ }
+ // Note: this can race against the EnsureNextIteration setting
+ // WAITSTATE_RUNNING and setting mGraphDriverAsleep to false, so you can
+ // have an iteration with WAITSTATE_WAKING_UP instead of RUNNING.
+ mWaitState = WAITSTATE_RUNNING;
+ mGraphImpl->mNeedAnotherIteration = false; // atomic
+}
+
+void SystemClockDriver::WakeUp()
+{
+ mGraphImpl->GetMonitor().AssertCurrentThreadOwns();
+ // Note: this can race against the thread setting WAITSTATE_RUNNING and
+ // setting mGraphDriverAsleep to false, so you can have an iteration
+ // with WAITSTATE_WAKING_UP instead of RUNNING.
+ mWaitState = WAITSTATE_WAKING_UP;
+ mGraphImpl->mGraphDriverAsleep = false; // atomic
+ mGraphImpl->GetMonitor().Notify();
+}
+
+OfflineClockDriver::OfflineClockDriver(MediaStreamGraphImpl* aGraphImpl, GraphTime aSlice)
+ : ThreadedDriver(aGraphImpl),
+ mSlice(aSlice)
+{
+
+}
+
+OfflineClockDriver::~OfflineClockDriver()
+{
+}
+
+MediaTime
+OfflineClockDriver::GetIntervalForIteration()
+{
+ return mGraphImpl->MillisecondsToMediaTime(mSlice);
+}
+
+void
+OfflineClockDriver::WaitForNextIteration()
+{
+ // No op: we want to go as fast as possible when we are offline
+}
+
+void
+OfflineClockDriver::WakeUp()
+{
+ MOZ_ASSERT(false, "An offline graph should not have to wake up.");
+}
+
+AsyncCubebTask::AsyncCubebTask(AudioCallbackDriver* aDriver, AsyncCubebOperation aOperation)
+ : mDriver(aDriver),
+ mOperation(aOperation),
+ mShutdownGrip(aDriver->GraphImpl())
+{
+ NS_WARNING_ASSERTION(mDriver->mAudioStream || aOperation == INIT,
+ "No audio stream!");
+}
+
+AsyncCubebTask::~AsyncCubebTask()
+{
+}
+
+/* static */
+nsresult
+AsyncCubebTask::EnsureThread()
+{
+ if (!sThreadPool) {
+ nsCOMPtr<nsIThreadPool> threadPool =
+ SharedThreadPool::Get(NS_LITERAL_CSTRING("CubebOperation"), 1);
+ sThreadPool = threadPool;
+ // Need to null this out before xpcom-shutdown-threads Observers run
+ // since we don't know the order that the shutdown-threads observers
+ // will run. ClearOnShutdown guarantees it runs first.
+ if (!NS_IsMainThread()) {
+ NS_DispatchToMainThread(NS_NewRunnableFunction([]() -> void {
+ ClearOnShutdown(&sThreadPool, ShutdownPhase::ShutdownThreads);
+ }));
+ } else {
+ ClearOnShutdown(&sThreadPool, ShutdownPhase::ShutdownThreads);
+ }
+
+ const uint32_t kIdleThreadTimeoutMs = 2000;
+
+ nsresult rv = sThreadPool->SetIdleThreadTimeout(PR_MillisecondsToInterval(kIdleThreadTimeoutMs));
+ if (NS_WARN_IF(NS_FAILED(rv))) {
+ return rv;
+ }
+ }
+
+ return NS_OK;
+}
+
+NS_IMETHODIMP
+AsyncCubebTask::Run()
+{
+ MOZ_ASSERT(mDriver);
+
+ switch(mOperation) {
+ case AsyncCubebOperation::INIT: {
+ LIFECYCLE_LOG("AsyncCubebOperation::INIT driver=%p\n", mDriver.get());
+ mDriver->Init();
+ mDriver->CompleteAudioContextOperations(mOperation);
+ break;
+ }
+ case AsyncCubebOperation::SHUTDOWN: {
+ LIFECYCLE_LOG("AsyncCubebOperation::SHUTDOWN driver=%p\n", mDriver.get());
+ mDriver->Stop();
+
+ mDriver->CompleteAudioContextOperations(mOperation);
+
+ mDriver = nullptr;
+ mShutdownGrip = nullptr;
+ break;
+ }
+ default:
+ MOZ_CRASH("Operation not implemented.");
+ }
+
+ // The thread will kill itself after a bit
+ return NS_OK;
+}
+
+StreamAndPromiseForOperation::StreamAndPromiseForOperation(MediaStream* aStream,
+ void* aPromise,
+ dom::AudioContextOperation aOperation)
+ : mStream(aStream)
+ , mPromise(aPromise)
+ , mOperation(aOperation)
+{
+ // MOZ_ASSERT(aPromise);
+}
+
+AudioCallbackDriver::AudioCallbackDriver(MediaStreamGraphImpl* aGraphImpl)
+ : GraphDriver(aGraphImpl)
+ , mSampleRate(0)
+ , mInputChannels(1)
+ , mIterationDurationMS(MEDIA_GRAPH_TARGET_PERIOD_MS)
+ , mStarted(false)
+ , mAudioInput(nullptr)
+ , mAudioChannel(aGraphImpl->AudioChannel())
+ , mAddedMixer(false)
+ , mInCallback(false)
+ , mMicrophoneActive(false)
+ , mShouldFallbackIfError(false)
+ , mFromFallback(false)
+{
+ STREAM_LOG(LogLevel::Debug, ("AudioCallbackDriver ctor for graph %p", aGraphImpl));
+}
+
+AudioCallbackDriver::~AudioCallbackDriver()
+{
+ MOZ_ASSERT(mPromisesForOperation.IsEmpty());
+}
+
+bool IsMacbookOrMacbookAir()
+{
+#ifdef XP_MACOSX
+ size_t len = 0;
+ sysctlbyname("hw.model", NULL, &len, NULL, 0);
+ if (len) {
+ UniquePtr<char[]> model(new char[len]);
+ // This string can be
+ // MacBook%d,%d for a normal MacBook
+ // MacBookPro%d,%d for a MacBook Pro
+ // MacBookAir%d,%d for a Macbook Air
+ sysctlbyname("hw.model", model.get(), &len, NULL, 0);
+ char* substring = strstr(model.get(), "MacBook");
+ if (substring) {
+ const size_t offset = strlen("MacBook");
+ if (strncmp(model.get() + offset, "Air", len - offset) ||
+ isdigit(model[offset + 1])) {
+ return true;
+ }
+ }
+ return false;
+ }
+#endif
+ return false;
+}
+
+void
+AudioCallbackDriver::Init()
+{
+ cubeb* cubebContext = CubebUtils::GetCubebContext();
+ if (!cubebContext) {
+ NS_WARNING("Could not get cubeb context.");
+ if (!mFromFallback) {
+ CubebUtils::ReportCubebStreamInitFailure(true);
+ }
+ return;
+ }
+
+ cubeb_stream_params output;
+ cubeb_stream_params input;
+ uint32_t latency_frames;
+ bool firstStream = CubebUtils::GetFirstStream();
+
+ MOZ_ASSERT(!NS_IsMainThread(),
+ "This is blocking and should never run on the main thread.");
+
+ mSampleRate = output.rate = CubebUtils::PreferredSampleRate();
+
+#if defined(__ANDROID__)
+#if defined(MOZ_B2G)
+ output.stream_type = CubebUtils::ConvertChannelToCubebType(mAudioChannel);
+#else
+ output.stream_type = CUBEB_STREAM_TYPE_MUSIC;
+#endif
+ if (output.stream_type == CUBEB_STREAM_TYPE_MAX) {
+ NS_WARNING("Bad stream type");
+ return;
+ }
+#else
+ (void)mAudioChannel;
+#endif
+
+ output.channels = mGraphImpl->AudioChannelCount();
+ if (AUDIO_OUTPUT_FORMAT == AUDIO_FORMAT_S16) {
+ output.format = CUBEB_SAMPLE_S16NE;
+ } else {
+ output.format = CUBEB_SAMPLE_FLOAT32NE;
+ }
+
+ Maybe<uint32_t> latencyPref = CubebUtils::GetCubebMSGLatencyInFrames();
+ if (latencyPref) {
+ latency_frames = latencyPref.value();
+ } else {
+ if (cubeb_get_min_latency(cubebContext, output, &latency_frames) != CUBEB_OK) {
+ NS_WARNING("Could not get minimal latency from cubeb.");
+ }
+ }
+
+ // Macbook and MacBook air don't have enough CPU to run very low latency
+ // MediaStreamGraphs, cap the minimal latency to 512 frames int this case.
+ if (IsMacbookOrMacbookAir()) {
+ latency_frames = std::max((uint32_t) 512, latency_frames);
+ }
+
+
+ input = output;
+ input.channels = mInputChannels; // change to support optional stereo capture
+
+ cubeb_stream* stream = nullptr;
+ CubebUtils::AudioDeviceID input_id = nullptr, output_id = nullptr;
+ // We have to translate the deviceID values to cubeb devid's since those can be
+ // freed whenever enumerate is called.
+ {
+#ifdef MOZ_WEBRTC
+ StaticMutexAutoLock lock(AudioInputCubeb::Mutex());
+#endif
+ if ((!mGraphImpl->mInputWanted
+#ifdef MOZ_WEBRTC
+ || AudioInputCubeb::GetDeviceID(mGraphImpl->mInputDeviceID, input_id)
+#endif
+ ) &&
+ (mGraphImpl->mOutputDeviceID == -1 // pass nullptr for ID for default output
+#ifdef MOZ_WEBRTC
+ // XXX we should figure out how we would use a deviceID for output without webrtc.
+ // Currently we don't set this though, so it's ok
+ || AudioInputCubeb::GetDeviceID(mGraphImpl->mOutputDeviceID, output_id)
+#endif
+ ) &&
+ // XXX Only pass input input if we have an input listener. Always
+ // set up output because it's easier, and it will just get silence.
+ // XXX Add support for adding/removing an input listener later.
+ cubeb_stream_init(cubebContext, &stream,
+ "AudioCallbackDriver",
+ input_id,
+ mGraphImpl->mInputWanted ? &input : nullptr,
+ output_id,
+ mGraphImpl->mOutputWanted ? &output : nullptr, latency_frames,
+ DataCallback_s, StateCallback_s, this) == CUBEB_OK) {
+ mAudioStream.own(stream);
+ DebugOnly<int> rv = cubeb_stream_set_volume(mAudioStream, CubebUtils::GetVolumeScale());
+ NS_WARNING_ASSERTION(
+ rv == CUBEB_OK,
+ "Could not set the audio stream volume in GraphDriver.cpp");
+ CubebUtils::ReportCubebBackendUsed();
+ } else {
+#ifdef MOZ_WEBRTC
+ StaticMutexAutoUnlock unlock(AudioInputCubeb::Mutex());
+#endif
+ NS_WARNING("Could not create a cubeb stream for MediaStreamGraph, falling back to a SystemClockDriver");
+ // Only report failures when we're not coming from a driver that was
+ // created itself as a fallback driver because of a previous audio driver
+ // failure.
+ if (!mFromFallback) {
+ CubebUtils::ReportCubebStreamInitFailure(firstStream);
+ }
+ // Fall back to a driver using a normal thread. If needed,
+ // the graph will try to re-open an audio stream later.
+ MonitorAutoLock lock(GraphImpl()->GetMonitor());
+ SystemClockDriver* nextDriver = new SystemClockDriver(GraphImpl());
+ SetNextDriver(nextDriver);
+ nextDriver->MarkAsFallback();
+ nextDriver->SetGraphTime(this, mIterationStart, mIterationEnd);
+ // We're not using SwitchAtNextIteration here, because there
+ // won't be a next iteration if we don't restart things manually:
+ // the audio stream just signaled that it's in error state.
+ mGraphImpl->SetCurrentDriver(nextDriver);
+ nextDriver->Start();
+ return;
+ }
+ }
+ bool aec;
+ Unused << mGraphImpl->AudioTrackPresent(aec);
+ SetMicrophoneActive(aec);
+
+ cubeb_stream_register_device_changed_callback(mAudioStream,
+ AudioCallbackDriver::DeviceChangedCallback_s);
+
+ StartStream();
+
+ STREAM_LOG(LogLevel::Debug, ("AudioCallbackDriver started."));
+}
+
+
+void
+AudioCallbackDriver::Destroy()
+{
+ STREAM_LOG(LogLevel::Debug, ("AudioCallbackDriver destroyed."));
+ mAudioInput = nullptr;
+ mAudioStream.reset();
+}
+
+void
+AudioCallbackDriver::Resume()
+{
+ STREAM_LOG(LogLevel::Debug, ("Resuming audio threads for MediaStreamGraph %p", mGraphImpl));
+ if (cubeb_stream_start(mAudioStream) != CUBEB_OK) {
+ NS_WARNING("Could not start cubeb stream for MSG.");
+ }
+}
+
+void
+AudioCallbackDriver::Start()
+{
+ if (mPreviousDriver) {
+ if (mPreviousDriver->AsAudioCallbackDriver()) {
+ LIFECYCLE_LOG("Releasing audio driver off main thread.");
+ RefPtr<AsyncCubebTask> releaseEvent =
+ new AsyncCubebTask(mPreviousDriver->AsAudioCallbackDriver(),
+ AsyncCubebOperation::SHUTDOWN);
+ releaseEvent->Dispatch();
+ mPreviousDriver = nullptr;
+ } else {
+ LIFECYCLE_LOG("Dropping driver reference for SystemClockDriver.");
+ MOZ_ASSERT(mPreviousDriver->AsSystemClockDriver());
+ mFromFallback = mPreviousDriver->AsSystemClockDriver()->IsFallback();
+ mPreviousDriver = nullptr;
+ }
+ }
+
+ LIFECYCLE_LOG("Starting new audio driver off main thread, "
+ "to ensure it runs after previous shutdown.");
+ RefPtr<AsyncCubebTask> initEvent =
+ new AsyncCubebTask(AsAudioCallbackDriver(), AsyncCubebOperation::INIT);
+ initEvent->Dispatch();
+}
+
+void
+AudioCallbackDriver::StartStream()
+{
+ mShouldFallbackIfError = true;
+ if (cubeb_stream_start(mAudioStream) != CUBEB_OK) {
+ MOZ_CRASH("Could not start cubeb stream for MSG.");
+ }
+
+ {
+ MonitorAutoLock mon(mGraphImpl->GetMonitor());
+ mStarted = true;
+ mWaitState = WAITSTATE_RUNNING;
+ }
+}
+
+void
+AudioCallbackDriver::Stop()
+{
+ if (cubeb_stream_stop(mAudioStream) != CUBEB_OK) {
+ NS_WARNING("Could not stop cubeb stream for MSG.");
+ }
+}
+
+void
+AudioCallbackDriver::Revive()
+{
+ // Note: only called on MainThread, without monitor
+ // We know were weren't in a running state
+ STREAM_LOG(LogLevel::Debug, ("AudioCallbackDriver reviving."));
+ // If we were switching, switch now. Otherwise, start the audio thread again.
+ MonitorAutoLock mon(mGraphImpl->GetMonitor());
+ if (NextDriver()) {
+ RemoveCallback();
+ NextDriver()->SetGraphTime(this, mIterationStart, mIterationEnd);
+ mGraphImpl->SetCurrentDriver(NextDriver());
+ NextDriver()->Start();
+ } else {
+ STREAM_LOG(LogLevel::Debug, ("Starting audio threads for MediaStreamGraph %p from a new thread.", mGraphImpl));
+ RefPtr<AsyncCubebTask> initEvent =
+ new AsyncCubebTask(this, AsyncCubebOperation::INIT);
+ initEvent->Dispatch();
+ }
+}
+
+void
+AudioCallbackDriver::RemoveCallback()
+{
+ if (mAddedMixer) {
+ mGraphImpl->mMixer.RemoveCallback(this);
+ mAddedMixer = false;
+ }
+}
+
+void
+AudioCallbackDriver::WaitForNextIteration()
+{
+}
+
+void
+AudioCallbackDriver::WakeUp()
+{
+ mGraphImpl->GetMonitor().AssertCurrentThreadOwns();
+ mGraphImpl->GetMonitor().Notify();
+}
+
+/* static */ long
+AudioCallbackDriver::DataCallback_s(cubeb_stream* aStream,
+ void* aUser,
+ const void* aInputBuffer,
+ void* aOutputBuffer,
+ long aFrames)
+{
+ AudioCallbackDriver* driver = reinterpret_cast<AudioCallbackDriver*>(aUser);
+ return driver->DataCallback(static_cast<const AudioDataValue*>(aInputBuffer),
+ static_cast<AudioDataValue*>(aOutputBuffer), aFrames);
+}
+
+/* static */ void
+AudioCallbackDriver::StateCallback_s(cubeb_stream* aStream, void * aUser,
+ cubeb_state aState)
+{
+ AudioCallbackDriver* driver = reinterpret_cast<AudioCallbackDriver*>(aUser);
+ driver->StateCallback(aState);
+}
+
+/* static */ void
+AudioCallbackDriver::DeviceChangedCallback_s(void* aUser)
+{
+ AudioCallbackDriver* driver = reinterpret_cast<AudioCallbackDriver*>(aUser);
+ driver->DeviceChangedCallback();
+}
+
+bool AudioCallbackDriver::InCallback() {
+ return mInCallback;
+}
+
+AudioCallbackDriver::AutoInCallback::AutoInCallback(AudioCallbackDriver* aDriver)
+ : mDriver(aDriver)
+{
+ mDriver->mInCallback = true;
+}
+
+AudioCallbackDriver::AutoInCallback::~AutoInCallback() {
+ mDriver->mInCallback = false;
+}
+
+long
+AudioCallbackDriver::DataCallback(const AudioDataValue* aInputBuffer,
+ AudioDataValue* aOutputBuffer, long aFrames)
+{
+ bool stillProcessing;
+
+ // Don't add the callback until we're inited and ready
+ if (!mAddedMixer) {
+ mGraphImpl->mMixer.AddCallback(this);
+ mAddedMixer = true;
+ }
+
+#ifdef DEBUG
+ // DebugOnly<> doesn't work here... it forces an initialization that will cause
+ // mInCallback to be set back to false before we exit the statement. Do it by
+ // hand instead.
+ AutoInCallback aic(this);
+#endif
+
+ GraphTime stateComputedTime = StateComputedTime();
+ if (stateComputedTime == 0) {
+ MonitorAutoLock mon(mGraphImpl->GetMonitor());
+ // Because this function is called during cubeb_stream_init (to prefill the
+ // audio buffers), it can be that we don't have a message here (because this
+ // driver is the first one for this graph), and the graph would exit. Simply
+ // return here until we have messages.
+ if (!mGraphImpl->MessagesQueued()) {
+ PodZero(aOutputBuffer, aFrames * mGraphImpl->AudioChannelCount());
+ return aFrames;
+ }
+ mGraphImpl->SwapMessageQueues();
+ }
+
+ uint32_t durationMS = aFrames * 1000 / mSampleRate;
+
+ // For now, simply average the duration with the previous
+ // duration so there is some damping against sudden changes.
+ if (!mIterationDurationMS) {
+ mIterationDurationMS = durationMS;
+ } else {
+ mIterationDurationMS = (mIterationDurationMS*3) + durationMS;
+ mIterationDurationMS /= 4;
+ }
+
+ // Process mic data if any/needed
+ if (aInputBuffer) {
+ if (mAudioInput) { // for this specific input-only or full-duplex stream
+ mAudioInput->NotifyInputData(mGraphImpl, aInputBuffer,
+ static_cast<size_t>(aFrames),
+ mSampleRate, mInputChannels);
+ }
+ }
+
+ mBuffer.SetBuffer(aOutputBuffer, aFrames);
+ // fill part or all with leftover data from last iteration (since we
+ // align to Audio blocks)
+ mScratchBuffer.Empty(mBuffer);
+ // if we totally filled the buffer (and mScratchBuffer isn't empty),
+ // we don't need to run an iteration and if we do so we may overflow.
+ if (mBuffer.Available()) {
+
+ // State computed time is decided by the audio callback's buffer length. We
+ // compute the iteration start and end from there, trying to keep the amount
+ // of buffering in the graph constant.
+ GraphTime nextStateComputedTime =
+ mGraphImpl->RoundUpToNextAudioBlock(stateComputedTime + mBuffer.Available());
+
+ mIterationStart = mIterationEnd;
+ // inGraph is the number of audio frames there is between the state time and
+ // the current time, i.e. the maximum theoretical length of the interval we
+ // could use as [mIterationStart; mIterationEnd].
+ GraphTime inGraph = stateComputedTime - mIterationStart;
+ // We want the interval [mIterationStart; mIterationEnd] to be before the
+ // interval [stateComputedTime; nextStateComputedTime]. We also want
+ // the distance between these intervals to be roughly equivalent each time, to
+ // ensure there is no clock drift between current time and state time. Since
+ // we can't act on the state time because we have to fill the audio buffer, we
+ // reclock the current time against the state time, here.
+ mIterationEnd = mIterationStart + 0.8 * inGraph;
+
+ STREAM_LOG(LogLevel::Verbose, ("interval[%ld; %ld] state[%ld; %ld] (frames: %ld) (durationMS: %u) (duration ticks: %ld)\n",
+ (long)mIterationStart, (long)mIterationEnd,
+ (long)stateComputedTime, (long)nextStateComputedTime,
+ (long)aFrames, (uint32_t)durationMS,
+ (long)(nextStateComputedTime - stateComputedTime)));
+
+ mCurrentTimeStamp = TimeStamp::Now();
+
+ if (stateComputedTime < mIterationEnd) {
+ STREAM_LOG(LogLevel::Warning, ("Media graph global underrun detected"));
+ mIterationEnd = stateComputedTime;
+ }
+
+ stillProcessing = mGraphImpl->OneIteration(nextStateComputedTime);
+ } else {
+ STREAM_LOG(LogLevel::Verbose, ("DataCallback buffer filled entirely from scratch buffer, skipping iteration."));
+ stillProcessing = true;
+ }
+
+ mBuffer.BufferFilled();
+
+ // Callback any observers for the AEC speaker data. Note that one
+ // (maybe) of these will be full-duplex, the others will get their input
+ // data off separate cubeb callbacks. Take care with how stuff is
+ // removed/added to this list and TSAN issues, but input and output will
+ // use separate callback methods.
+ mGraphImpl->NotifyOutputData(aOutputBuffer, static_cast<size_t>(aFrames),
+ mSampleRate, ChannelCount);
+
+ if (!stillProcessing) {
+ // About to hand over control of the graph. Do not start a new driver if
+ // StateCallback() receives an error for this stream while the main thread
+ // or another driver has control of the graph.
+ mShouldFallbackIfError = false;
+ // Enter shutdown mode. The stable-state handler will detect this
+ // and complete shutdown if the graph does not get restarted.
+ mGraphImpl->SignalMainThreadCleanup();
+ return aFrames - 1;
+ }
+
+ bool switching = false;
+ {
+ MonitorAutoLock mon(mGraphImpl->GetMonitor());
+ switching = !!NextDriver();
+ }
+
+ if (switching) {
+ mShouldFallbackIfError = false;
+ // If the audio stream has not been started by the previous driver or
+ // the graph itself, keep it alive.
+ MonitorAutoLock mon(mGraphImpl->GetMonitor());
+ if (!IsStarted()) {
+ return aFrames;
+ }
+ STREAM_LOG(LogLevel::Debug, ("Switching to system driver."));
+ RemoveCallback();
+ NextDriver()->SetGraphTime(this, mIterationStart, mIterationEnd);
+ mGraphImpl->SetCurrentDriver(NextDriver());
+ NextDriver()->Start();
+ // Returning less than aFrames starts the draining and eventually stops the
+ // audio thread. This function will never get called again.
+ return aFrames - 1;
+ }
+
+ return aFrames;
+}
+
+void
+AudioCallbackDriver::StateCallback(cubeb_state aState)
+{
+ STREAM_LOG(LogLevel::Debug, ("AudioCallbackDriver State: %d", aState));
+ if (aState == CUBEB_STATE_ERROR && mShouldFallbackIfError) {
+ MonitorAutoLock lock(GraphImpl()->GetMonitor());
+ // Fall back to a driver using a normal thread. If needed,
+ // the graph will try to re-open an audio stream later.
+ SystemClockDriver* nextDriver = new SystemClockDriver(GraphImpl());
+ SetNextDriver(nextDriver);
+ RemoveCallback();
+ nextDriver->MarkAsFallback();
+ nextDriver->SetGraphTime(this, mIterationStart, mIterationEnd);
+ // We're not using SwitchAtNextIteration here, because there
+ // won't be a next iteration if we don't restart things manually:
+ // the audio stream just signaled that it's in error state.
+ mGraphImpl->SetCurrentDriver(nextDriver);
+ nextDriver->Start();
+ }
+}
+
+void
+AudioCallbackDriver::MixerCallback(AudioDataValue* aMixedBuffer,
+ AudioSampleFormat aFormat,
+ uint32_t aChannels,
+ uint32_t aFrames,
+ uint32_t aSampleRate)
+{
+ uint32_t toWrite = mBuffer.Available();
+
+ if (!mBuffer.Available()) {
+ NS_WARNING("DataCallback buffer full, expect frame drops.");
+ }
+
+ MOZ_ASSERT(mBuffer.Available() <= aFrames);
+
+ mBuffer.WriteFrames(aMixedBuffer, mBuffer.Available());
+ MOZ_ASSERT(mBuffer.Available() == 0, "Missing frames to fill audio callback's buffer.");
+
+ DebugOnly<uint32_t> written = mScratchBuffer.Fill(aMixedBuffer + toWrite * aChannels, aFrames - toWrite);
+ NS_WARNING_ASSERTION(written == aFrames - toWrite, "Dropping frames.");
+};
+
+void AudioCallbackDriver::PanOutputIfNeeded(bool aMicrophoneActive)
+{
+#ifdef XP_MACOSX
+ cubeb_device* out;
+ int rv;
+ char name[128];
+ size_t length = sizeof(name);
+
+ rv = sysctlbyname("hw.model", name, &length, NULL, 0);
+ if (rv) {
+ return;
+ }
+
+ if (!strncmp(name, "MacBookPro", 10)) {
+ if (cubeb_stream_get_current_device(mAudioStream, &out) == CUBEB_OK) {
+ // Check if we are currently outputing sound on external speakers.
+ if (!strcmp(out->output_name, "ispk")) {
+ // Pan everything to the right speaker.
+ if (aMicrophoneActive) {
+ if (cubeb_stream_set_panning(mAudioStream, 1.0) != CUBEB_OK) {
+ NS_WARNING("Could not pan audio output to the right.");
+ }
+ } else {
+ if (cubeb_stream_set_panning(mAudioStream, 0.0) != CUBEB_OK) {
+ NS_WARNING("Could not pan audio output to the center.");
+ }
+ }
+ } else {
+ if (cubeb_stream_set_panning(mAudioStream, 0.0) != CUBEB_OK) {
+ NS_WARNING("Could not pan audio output to the center.");
+ }
+ }
+ cubeb_stream_device_destroy(mAudioStream, out);
+ }
+ }
+#endif
+}
+
+void
+AudioCallbackDriver::DeviceChangedCallback() {
+ // Tell the audio engine the device has changed, it might want to reset some
+ // state.
+ MonitorAutoLock mon(mGraphImpl->GetMonitor());
+ if (mAudioInput) {
+ mAudioInput->DeviceChanged();
+ }
+#ifdef XP_MACOSX
+ PanOutputIfNeeded(mMicrophoneActive);
+#endif
+}
+
+void
+AudioCallbackDriver::SetMicrophoneActive(bool aActive)
+{
+ MonitorAutoLock mon(mGraphImpl->GetMonitor());
+
+ mMicrophoneActive = aActive;
+
+#ifdef XP_MACOSX
+ PanOutputIfNeeded(mMicrophoneActive);
+#endif
+}
+
+uint32_t
+AudioCallbackDriver::IterationDuration()
+{
+ // The real fix would be to have an API in cubeb to give us the number. Short
+ // of that, we approximate it here. bug 1019507
+ return mIterationDurationMS;
+}
+
+bool
+AudioCallbackDriver::IsStarted() {
+ mGraphImpl->GetMonitor().AssertCurrentThreadOwns();
+ return mStarted;
+}
+
+void
+AudioCallbackDriver::EnqueueStreamAndPromiseForOperation(MediaStream* aStream,
+ void* aPromise,
+ dom::AudioContextOperation aOperation)
+{
+ MonitorAutoLock mon(mGraphImpl->GetMonitor());
+ mPromisesForOperation.AppendElement(StreamAndPromiseForOperation(aStream,
+ aPromise,
+ aOperation));
+}
+
+void AudioCallbackDriver::CompleteAudioContextOperations(AsyncCubebOperation aOperation)
+{
+ AutoTArray<StreamAndPromiseForOperation, 1> array;
+
+ // We can't lock for the whole function because AudioContextOperationCompleted
+ // will grab the monitor
+ {
+ MonitorAutoLock mon(GraphImpl()->GetMonitor());
+ array.SwapElements(mPromisesForOperation);
+ }
+
+ for (uint32_t i = 0; i < array.Length(); i++) {
+ StreamAndPromiseForOperation& s = array[i];
+ if ((aOperation == AsyncCubebOperation::INIT &&
+ s.mOperation == dom::AudioContextOperation::Resume) ||
+ (aOperation == AsyncCubebOperation::SHUTDOWN &&
+ s.mOperation != dom::AudioContextOperation::Resume)) {
+
+ GraphImpl()->AudioContextOperationCompleted(s.mStream,
+ s.mPromise,
+ s.mOperation);
+ array.RemoveElementAt(i);
+ i--;
+ }
+ }
+
+ if (!array.IsEmpty()) {
+ MonitorAutoLock mon(GraphImpl()->GetMonitor());
+ mPromisesForOperation.AppendElements(array);
+ }
+}
+
+
+} // namespace mozilla