summaryrefslogtreecommitdiffstats
path: root/dom/media/webrtc/MediaEngineWebRTC.h
diff options
context:
space:
mode:
Diffstat (limited to 'dom/media/webrtc/MediaEngineWebRTC.h')
-rw-r--r--dom/media/webrtc/MediaEngineWebRTC.h613
1 files changed, 613 insertions, 0 deletions
diff --git a/dom/media/webrtc/MediaEngineWebRTC.h b/dom/media/webrtc/MediaEngineWebRTC.h
new file mode 100644
index 000000000..1834f3bd3
--- /dev/null
+++ b/dom/media/webrtc/MediaEngineWebRTC.h
@@ -0,0 +1,613 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef MEDIAENGINEWEBRTC_H_
+#define MEDIAENGINEWEBRTC_H_
+
+#include "prcvar.h"
+#include "prthread.h"
+#include "prprf.h"
+#include "nsIThread.h"
+#include "nsIRunnable.h"
+
+#include "mozilla/dom/File.h"
+#include "mozilla/Mutex.h"
+#include "mozilla/StaticMutex.h"
+#include "mozilla/Monitor.h"
+#include "mozilla/UniquePtr.h"
+#include "nsAutoPtr.h"
+#include "nsCOMPtr.h"
+#include "nsThreadUtils.h"
+#include "DOMMediaStream.h"
+#include "nsDirectoryServiceDefs.h"
+#include "nsComponentManagerUtils.h"
+#include "nsRefPtrHashtable.h"
+
+#include "VideoUtils.h"
+#include "MediaEngineCameraVideoSource.h"
+#include "VideoSegment.h"
+#include "AudioSegment.h"
+#include "StreamTracks.h"
+#include "MediaStreamGraph.h"
+#include "cubeb/cubeb.h"
+#include "CubebUtils.h"
+#include "AudioPacketizer.h"
+
+#include "MediaEngineWrapper.h"
+#include "mozilla/dom/MediaStreamTrackBinding.h"
+// WebRTC library includes follow
+#include "webrtc/common.h"
+// Audio Engine
+#include "webrtc/voice_engine/include/voe_base.h"
+#include "webrtc/voice_engine/include/voe_codec.h"
+#include "webrtc/voice_engine/include/voe_hardware.h"
+#include "webrtc/voice_engine/include/voe_network.h"
+#include "webrtc/voice_engine/include/voe_audio_processing.h"
+#include "webrtc/voice_engine/include/voe_volume_control.h"
+#include "webrtc/voice_engine/include/voe_external_media.h"
+#include "webrtc/voice_engine/include/voe_audio_processing.h"
+#include "webrtc/modules/audio_processing/include/audio_processing.h"
+
+// Video Engine
+// conflicts with #include of scoped_ptr.h
+#undef FF
+#include "webrtc/video_engine/include/vie_base.h"
+#include "webrtc/video_engine/include/vie_codec.h"
+#include "webrtc/video_engine/include/vie_render.h"
+#include "webrtc/video_engine/include/vie_capture.h"
+#include "CamerasChild.h"
+
+#include "NullTransport.h"
+#include "AudioOutputObserver.h"
+
+namespace mozilla {
+
+class MediaEngineWebRTCAudioCaptureSource : public MediaEngineAudioSource
+{
+public:
+ NS_DECL_THREADSAFE_ISUPPORTS
+
+ explicit MediaEngineWebRTCAudioCaptureSource(const char* aUuid)
+ : MediaEngineAudioSource(kReleased)
+ {
+ }
+ void GetName(nsAString& aName) const override;
+ void GetUUID(nsACString& aUUID) const override;
+ nsresult Allocate(const dom::MediaTrackConstraints& aConstraints,
+ const MediaEnginePrefs& aPrefs,
+ const nsString& aDeviceId,
+ const nsACString& aOrigin,
+ AllocationHandle** aOutHandle,
+ const char** aOutBadConstraint) override
+ {
+ // Nothing to do here, everything is managed in MediaManager.cpp
+ *aOutHandle = nullptr;
+ return NS_OK;
+ }
+ nsresult Deallocate(AllocationHandle* aHandle) override
+ {
+ // Nothing to do here, everything is managed in MediaManager.cpp
+ MOZ_ASSERT(!aHandle);
+ return NS_OK;
+ }
+ nsresult Start(SourceMediaStream* aMediaStream,
+ TrackID aId,
+ const PrincipalHandle& aPrincipalHandle) override;
+ nsresult Stop(SourceMediaStream* aMediaStream, TrackID aId) override;
+ nsresult Restart(AllocationHandle* aHandle,
+ const dom::MediaTrackConstraints& aConstraints,
+ const MediaEnginePrefs &aPrefs,
+ const nsString& aDeviceId,
+ const char** aOutBadConstraint) override;
+ void SetDirectListeners(bool aDirect) override
+ {}
+ void NotifyOutputData(MediaStreamGraph* aGraph,
+ AudioDataValue* aBuffer, size_t aFrames,
+ TrackRate aRate, uint32_t aChannels) override
+ {}
+ void DeviceChanged() override
+ {}
+ void NotifyInputData(MediaStreamGraph* aGraph,
+ const AudioDataValue* aBuffer, size_t aFrames,
+ TrackRate aRate, uint32_t aChannels) override
+ {}
+ void NotifyPull(MediaStreamGraph* aGraph,
+ SourceMediaStream* aSource,
+ TrackID aID,
+ StreamTime aDesiredTime,
+ const PrincipalHandle& aPrincipalHandle) override
+ {}
+ dom::MediaSourceEnum GetMediaSource() const override
+ {
+ return dom::MediaSourceEnum::AudioCapture;
+ }
+ bool IsFake() override
+ {
+ return false;
+ }
+ nsresult TakePhoto(MediaEnginePhotoCallback* aCallback) override
+ {
+ return NS_ERROR_NOT_IMPLEMENTED;
+ }
+ uint32_t GetBestFitnessDistance(
+ const nsTArray<const NormalizedConstraintSet*>& aConstraintSets,
+ const nsString& aDeviceId) const override;
+
+protected:
+ virtual ~MediaEngineWebRTCAudioCaptureSource() {}
+ nsCString mUUID;
+};
+
+// Small subset of VoEHardware
+class AudioInput
+{
+public:
+ explicit AudioInput(webrtc::VoiceEngine* aVoiceEngine) : mVoiceEngine(aVoiceEngine) {};
+ // Threadsafe because it's referenced from an MicrophoneSource, which can
+ // had references to it on other threads.
+ NS_INLINE_DECL_THREADSAFE_REFCOUNTING(AudioInput)
+
+ virtual int GetNumOfRecordingDevices(int& aDevices) = 0;
+ virtual int GetRecordingDeviceName(int aIndex, char aStrNameUTF8[128],
+ char aStrGuidUTF8[128]) = 0;
+ virtual int GetRecordingDeviceStatus(bool& aIsAvailable) = 0;
+ virtual void StartRecording(SourceMediaStream *aStream, AudioDataListener *aListener) = 0;
+ virtual void StopRecording(SourceMediaStream *aStream) = 0;
+ virtual int SetRecordingDevice(int aIndex) = 0;
+
+protected:
+ // Protected destructor, to discourage deletion outside of Release():
+ virtual ~AudioInput() {}
+
+ webrtc::VoiceEngine* mVoiceEngine;
+};
+
+class AudioInputCubeb final : public AudioInput
+{
+public:
+ explicit AudioInputCubeb(webrtc::VoiceEngine* aVoiceEngine, int aIndex = 0) :
+ AudioInput(aVoiceEngine), mSelectedDevice(aIndex), mInUseCount(0)
+ {
+ if (!mDeviceIndexes) {
+ mDeviceIndexes = new nsTArray<int>;
+ mDeviceNames = new nsTArray<nsCString>;
+ mDefaultDevice = -1;
+ }
+ }
+
+ static void CleanupGlobalData()
+ {
+ if (mDevices) {
+ // This doesn't require anything more than support for free()
+ cubeb_device_collection_destroy(mDevices);
+ mDevices = nullptr;
+ }
+ delete mDeviceIndexes;
+ mDeviceIndexes = nullptr;
+ delete mDeviceNames;
+ mDeviceNames = nullptr;
+ }
+
+ int GetNumOfRecordingDevices(int& aDevices)
+ {
+ UpdateDeviceList();
+ aDevices = mDeviceIndexes->Length();
+ return 0;
+ }
+
+ static int32_t DeviceIndex(int aIndex)
+ {
+ // -1 = system default if any
+ if (aIndex == -1) {
+ if (mDefaultDevice == -1) {
+ aIndex = 0;
+ } else {
+ aIndex = mDefaultDevice;
+ }
+ }
+ if (aIndex < 0 || aIndex >= (int) mDeviceIndexes->Length()) {
+ return -1;
+ }
+ // Note: if the device is gone, this will be -1
+ return (*mDeviceIndexes)[aIndex]; // translate to mDevices index
+ }
+
+ static StaticMutex& Mutex()
+ {
+ return sMutex;
+ }
+
+ static bool GetDeviceID(int aDeviceIndex, CubebUtils::AudioDeviceID &aID)
+ {
+ // Assert sMutex is held
+ sMutex.AssertCurrentThreadOwns();
+ int dev_index = DeviceIndex(aDeviceIndex);
+ if (dev_index != -1) {
+ aID = mDevices->device[dev_index]->devid;
+ return true;
+ }
+ return false;
+ }
+
+ int GetRecordingDeviceName(int aIndex, char aStrNameUTF8[128],
+ char aStrGuidUTF8[128])
+ {
+ int32_t devindex = DeviceIndex(aIndex);
+ if (!mDevices || devindex < 0) {
+ return 1;
+ }
+ PR_snprintf(aStrNameUTF8, 128, "%s%s", aIndex == -1 ? "default: " : "",
+ mDevices->device[devindex]->friendly_name);
+ aStrGuidUTF8[0] = '\0';
+ return 0;
+ }
+
+ int GetRecordingDeviceStatus(bool& aIsAvailable)
+ {
+ // With cubeb, we only expose devices of type CUBEB_DEVICE_TYPE_INPUT,
+ // so unless it was removed, say it's available
+ aIsAvailable = true;
+ return 0;
+ }
+
+ void StartRecording(SourceMediaStream *aStream, AudioDataListener *aListener)
+ {
+ MOZ_ASSERT(mDevices);
+
+ if (mInUseCount == 0) {
+ ScopedCustomReleasePtr<webrtc::VoEExternalMedia> ptrVoERender;
+ ptrVoERender = webrtc::VoEExternalMedia::GetInterface(mVoiceEngine);
+ if (ptrVoERender) {
+ ptrVoERender->SetExternalRecordingStatus(true);
+ }
+ mAnyInUse = true;
+ }
+ mInUseCount++;
+ // Always tell the stream we're using it for input
+ aStream->OpenAudioInput(mSelectedDevice, aListener);
+ }
+
+ void StopRecording(SourceMediaStream *aStream)
+ {
+ aStream->CloseAudioInput();
+ if (--mInUseCount == 0) {
+ mAnyInUse = false;
+ }
+ }
+
+ int SetRecordingDevice(int aIndex)
+ {
+ mSelectedDevice = aIndex;
+ return 0;
+ }
+
+protected:
+ ~AudioInputCubeb() {
+ MOZ_RELEASE_ASSERT(mInUseCount == 0);
+ }
+
+private:
+ // It would be better to watch for device-change notifications
+ void UpdateDeviceList();
+
+ // We have an array, which consists of indexes to the current mDevices
+ // list. This is updated on mDevices updates. Many devices in mDevices
+ // won't be included in the array (wrong type, etc), or if a device is
+ // removed it will map to -1 (and opens of this device will need to check
+ // for this - and be careful of threading access. The mappings need to
+ // updated on each re-enumeration.
+ int mSelectedDevice;
+ uint32_t mInUseCount;
+
+ // pointers to avoid static constructors
+ static nsTArray<int>* mDeviceIndexes;
+ static int mDefaultDevice; // -1 == not set
+ static nsTArray<nsCString>* mDeviceNames;
+ static cubeb_device_collection *mDevices;
+ static bool mAnyInUse;
+ static StaticMutex sMutex;
+};
+
+class AudioInputWebRTC final : public AudioInput
+{
+public:
+ explicit AudioInputWebRTC(webrtc::VoiceEngine* aVoiceEngine) : AudioInput(aVoiceEngine) {}
+
+ int GetNumOfRecordingDevices(int& aDevices)
+ {
+ ScopedCustomReleasePtr<webrtc::VoEHardware> ptrVoEHw;
+ ptrVoEHw = webrtc::VoEHardware::GetInterface(mVoiceEngine);
+ if (!ptrVoEHw) {
+ return 1;
+ }
+ return ptrVoEHw->GetNumOfRecordingDevices(aDevices);
+ }
+
+ int GetRecordingDeviceName(int aIndex, char aStrNameUTF8[128],
+ char aStrGuidUTF8[128])
+ {
+ ScopedCustomReleasePtr<webrtc::VoEHardware> ptrVoEHw;
+ ptrVoEHw = webrtc::VoEHardware::GetInterface(mVoiceEngine);
+ if (!ptrVoEHw) {
+ return 1;
+ }
+ return ptrVoEHw->GetRecordingDeviceName(aIndex, aStrNameUTF8,
+ aStrGuidUTF8);
+ }
+
+ int GetRecordingDeviceStatus(bool& aIsAvailable)
+ {
+ ScopedCustomReleasePtr<webrtc::VoEHardware> ptrVoEHw;
+ ptrVoEHw = webrtc::VoEHardware::GetInterface(mVoiceEngine);
+ if (!ptrVoEHw) {
+ return 1;
+ }
+ ptrVoEHw->GetRecordingDeviceStatus(aIsAvailable);
+ return 0;
+ }
+
+ void StartRecording(SourceMediaStream *aStream, AudioDataListener *aListener) {}
+ void StopRecording(SourceMediaStream *aStream) {}
+
+ int SetRecordingDevice(int aIndex)
+ {
+ ScopedCustomReleasePtr<webrtc::VoEHardware> ptrVoEHw;
+ ptrVoEHw = webrtc::VoEHardware::GetInterface(mVoiceEngine);
+ if (!ptrVoEHw) {
+ return 1;
+ }
+ return ptrVoEHw->SetRecordingDevice(aIndex);
+ }
+
+protected:
+ // Protected destructor, to discourage deletion outside of Release():
+ ~AudioInputWebRTC() {}
+};
+
+class WebRTCAudioDataListener : public AudioDataListener
+{
+protected:
+ // Protected destructor, to discourage deletion outside of Release():
+ virtual ~WebRTCAudioDataListener() {}
+
+public:
+ explicit WebRTCAudioDataListener(MediaEngineAudioSource* aAudioSource)
+ : mMutex("WebRTCAudioDataListener")
+ , mAudioSource(aAudioSource)
+ {}
+
+ // AudioDataListenerInterface methods
+ virtual void NotifyOutputData(MediaStreamGraph* aGraph,
+ AudioDataValue* aBuffer, size_t aFrames,
+ TrackRate aRate, uint32_t aChannels) override
+ {
+ MutexAutoLock lock(mMutex);
+ if (mAudioSource) {
+ mAudioSource->NotifyOutputData(aGraph, aBuffer, aFrames, aRate, aChannels);
+ }
+ }
+ virtual void NotifyInputData(MediaStreamGraph* aGraph,
+ const AudioDataValue* aBuffer, size_t aFrames,
+ TrackRate aRate, uint32_t aChannels) override
+ {
+ MutexAutoLock lock(mMutex);
+ if (mAudioSource) {
+ mAudioSource->NotifyInputData(aGraph, aBuffer, aFrames, aRate, aChannels);
+ }
+ }
+ virtual void DeviceChanged() override
+ {
+ MutexAutoLock lock(mMutex);
+ if (mAudioSource) {
+ mAudioSource->DeviceChanged();
+ }
+ }
+
+ void Shutdown()
+ {
+ MutexAutoLock lock(mMutex);
+ mAudioSource = nullptr;
+ }
+
+private:
+ Mutex mMutex;
+ RefPtr<MediaEngineAudioSource> mAudioSource;
+};
+
+class MediaEngineWebRTCMicrophoneSource : public MediaEngineAudioSource,
+ public webrtc::VoEMediaProcess
+{
+ typedef MediaEngineAudioSource Super;
+public:
+ MediaEngineWebRTCMicrophoneSource(webrtc::VoiceEngine* aVoiceEnginePtr,
+ mozilla::AudioInput* aAudioInput,
+ int aIndex,
+ const char* name,
+ const char* uuid);
+
+ void GetName(nsAString& aName) const override;
+ void GetUUID(nsACString& aUUID) const override;
+
+ nsresult Deallocate(AllocationHandle* aHandle) override;
+ nsresult Start(SourceMediaStream* aStream,
+ TrackID aID,
+ const PrincipalHandle& aPrincipalHandle) override;
+ nsresult Stop(SourceMediaStream* aSource, TrackID aID) override;
+ nsresult Restart(AllocationHandle* aHandle,
+ const dom::MediaTrackConstraints& aConstraints,
+ const MediaEnginePrefs &aPrefs,
+ const nsString& aDeviceId,
+ const char** aOutBadConstraint) override;
+ void SetDirectListeners(bool aHasDirectListeners) override {};
+
+ void NotifyPull(MediaStreamGraph* aGraph,
+ SourceMediaStream* aSource,
+ TrackID aId,
+ StreamTime aDesiredTime,
+ const PrincipalHandle& aPrincipalHandle) override;
+
+ // AudioDataListenerInterface methods
+ void NotifyOutputData(MediaStreamGraph* aGraph,
+ AudioDataValue* aBuffer, size_t aFrames,
+ TrackRate aRate, uint32_t aChannels) override;
+ void NotifyInputData(MediaStreamGraph* aGraph,
+ const AudioDataValue* aBuffer, size_t aFrames,
+ TrackRate aRate, uint32_t aChannels) override;
+
+ void DeviceChanged() override;
+
+ bool IsFake() override {
+ return false;
+ }
+
+ dom::MediaSourceEnum GetMediaSource() const override {
+ return dom::MediaSourceEnum::Microphone;
+ }
+
+ nsresult TakePhoto(MediaEnginePhotoCallback* aCallback) override
+ {
+ return NS_ERROR_NOT_IMPLEMENTED;
+ }
+
+ uint32_t GetBestFitnessDistance(
+ const nsTArray<const NormalizedConstraintSet*>& aConstraintSets,
+ const nsString& aDeviceId) const override;
+
+ // VoEMediaProcess.
+ void Process(int channel, webrtc::ProcessingTypes type,
+ int16_t audio10ms[], int length,
+ int samplingFreq, bool isStereo) override;
+
+ void Shutdown() override;
+
+ NS_DECL_THREADSAFE_ISUPPORTS
+
+protected:
+ ~MediaEngineWebRTCMicrophoneSource() {}
+
+private:
+ nsresult
+ UpdateSingleSource(const AllocationHandle* aHandle,
+ const NormalizedConstraints& aNetConstraints,
+ const MediaEnginePrefs& aPrefs,
+ const nsString& aDeviceId,
+ const char** aOutBadConstraint) override;
+
+ void SetLastPrefs(const MediaEnginePrefs& aPrefs);
+
+ // These allocate/configure and release the channel
+ bool AllocChannel();
+ void FreeChannel();
+ // These start/stop VoEBase and associated interfaces
+ bool InitEngine();
+ void DeInitEngine();
+
+ // This is true when all processing is disabled, we can skip
+ // packetization, resampling and other processing passes.
+ bool PassThrough() {
+ return mSkipProcessing;
+ }
+ template<typename T>
+ void InsertInGraph(const T* aBuffer,
+ size_t aFrames,
+ uint32_t aChannels);
+
+ void PacketizeAndProcess(MediaStreamGraph* aGraph,
+ const AudioDataValue* aBuffer,
+ size_t aFrames,
+ TrackRate aRate,
+ uint32_t aChannels);
+
+ webrtc::VoiceEngine* mVoiceEngine;
+ RefPtr<mozilla::AudioInput> mAudioInput;
+ RefPtr<WebRTCAudioDataListener> mListener;
+
+ // Note: shared across all microphone sources - we don't want to Terminate()
+ // the VoEBase until there are no active captures
+ static int sChannelsOpen;
+ static ScopedCustomReleasePtr<webrtc::VoEBase> mVoEBase;
+ static ScopedCustomReleasePtr<webrtc::VoEExternalMedia> mVoERender;
+ static ScopedCustomReleasePtr<webrtc::VoENetwork> mVoENetwork;
+ static ScopedCustomReleasePtr<webrtc::VoEAudioProcessing> mVoEProcessing;
+
+ // accessed from the GraphDriver thread except for deletion
+ nsAutoPtr<AudioPacketizer<AudioDataValue, int16_t>> mPacketizer;
+ ScopedCustomReleasePtr<webrtc::VoEExternalMedia> mVoERenderListener;
+
+ // mMonitor protects mSources[] and mPrinicpalIds[] access/changes, and
+ // transitions of mState from kStarted to kStopped (which are combined with
+ // EndTrack()). mSources[] and mPrincipalHandles[] are accessed from webrtc
+ // threads.
+ Monitor mMonitor;
+ nsTArray<RefPtr<SourceMediaStream>> mSources;
+ nsTArray<PrincipalHandle> mPrincipalHandles; // Maps to mSources.
+
+ int mCapIndex;
+ int mChannel;
+ MOZ_INIT_OUTSIDE_CTOR TrackID mTrackID;
+ bool mStarted;
+
+ nsString mDeviceName;
+ nsCString mDeviceUUID;
+
+ int32_t mSampleFrequency;
+ int32_t mPlayoutDelay;
+
+ NullTransport *mNullTransport;
+
+ nsTArray<int16_t> mInputBuffer;
+ // mSkipProcessing is true if none of the processing passes are enabled,
+ // because of prefs or constraints. This allows simply copying the audio into
+ // the MSG, skipping resampling and the whole webrtc.org code.
+ bool mSkipProcessing;
+
+ // To only update microphone when needed, we keep track of previous settings.
+ MediaEnginePrefs mLastPrefs;
+};
+
+class MediaEngineWebRTC : public MediaEngine
+{
+ typedef MediaEngine Super;
+public:
+ explicit MediaEngineWebRTC(MediaEnginePrefs& aPrefs);
+
+ virtual void SetFakeDeviceChangeEvents() override;
+
+ // Clients should ensure to clean-up sources video/audio sources
+ // before invoking Shutdown on this class.
+ void Shutdown() override;
+
+ // Returns whether the host supports duplex audio stream.
+ bool SupportsDuplex();
+
+ void EnumerateVideoDevices(dom::MediaSourceEnum,
+ nsTArray<RefPtr<MediaEngineVideoSource>>*) override;
+ void EnumerateAudioDevices(dom::MediaSourceEnum,
+ nsTArray<RefPtr<MediaEngineAudioSource>>*) override;
+private:
+ ~MediaEngineWebRTC() {
+ gFarendObserver = nullptr;
+ }
+
+ nsCOMPtr<nsIThread> mThread;
+
+ // gUM runnables can e.g. Enumerate from multiple threads
+ Mutex mMutex;
+ webrtc::VoiceEngine* mVoiceEngine;
+ webrtc::Config mConfig;
+ RefPtr<mozilla::AudioInput> mAudioInput;
+ bool mFullDuplex;
+ bool mExtendedFilter;
+ bool mDelayAgnostic;
+ bool mHasTabVideoSource;
+
+ // Store devices we've already seen in a hashtable for quick return.
+ // Maps UUID to MediaEngineSource (one set for audio, one for video).
+ nsRefPtrHashtable<nsStringHashKey, MediaEngineVideoSource> mVideoSources;
+ nsRefPtrHashtable<nsStringHashKey, MediaEngineAudioSource> mAudioSources;
+};
+
+}
+
+#endif /* NSMEDIAENGINEWEBRTC_H_ */