summaryrefslogtreecommitdiffstats
path: root/dom/media/encoder/MediaEncoder.cpp
diff options
context:
space:
mode:
authorMatt A. Tobin <mattatobin@localhost.localdomain>2018-02-02 04:16:08 -0500
committerMatt A. Tobin <mattatobin@localhost.localdomain>2018-02-02 04:16:08 -0500
commit5f8de423f190bbb79a62f804151bc24824fa32d8 (patch)
tree10027f336435511475e392454359edea8e25895d /dom/media/encoder/MediaEncoder.cpp
parent49ee0794b5d912db1f95dce6eb52d781dc210db5 (diff)
downloadUXP-5f8de423f190bbb79a62f804151bc24824fa32d8.tar
UXP-5f8de423f190bbb79a62f804151bc24824fa32d8.tar.gz
UXP-5f8de423f190bbb79a62f804151bc24824fa32d8.tar.lz
UXP-5f8de423f190bbb79a62f804151bc24824fa32d8.tar.xz
UXP-5f8de423f190bbb79a62f804151bc24824fa32d8.zip
Add m-esr52 at 52.6.0
Diffstat (limited to 'dom/media/encoder/MediaEncoder.cpp')
-rw-r--r--dom/media/encoder/MediaEncoder.cpp404
1 files changed, 404 insertions, 0 deletions
diff --git a/dom/media/encoder/MediaEncoder.cpp b/dom/media/encoder/MediaEncoder.cpp
new file mode 100644
index 000000000..864b486e4
--- /dev/null
+++ b/dom/media/encoder/MediaEncoder.cpp
@@ -0,0 +1,404 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*-*/
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+#include "MediaEncoder.h"
+#include "MediaDecoder.h"
+#include "nsIPrincipal.h"
+#include "nsMimeTypes.h"
+#include "mozilla/Logging.h"
+#include "mozilla/Preferences.h"
+#include "mozilla/StaticPtr.h"
+#include "mozilla/gfx/Point.h" // IntSize
+
+#include"GeckoProfiler.h"
+#include "OggWriter.h"
+#include "OpusTrackEncoder.h"
+
+#ifdef MOZ_WEBM_ENCODER
+#include "VP8TrackEncoder.h"
+#include "WebMWriter.h"
+#endif
+
+#ifdef LOG
+#undef LOG
+#endif
+
+mozilla::LazyLogModule gMediaEncoderLog("MediaEncoder");
+#define LOG(type, msg) MOZ_LOG(gMediaEncoderLog, type, msg)
+
+namespace mozilla {
+
+void
+MediaStreamVideoRecorderSink::SetCurrentFrames(const VideoSegment& aSegment)
+{
+ MOZ_ASSERT(mVideoEncoder);
+ mVideoEncoder->SetCurrentFrames(aSegment);
+}
+
+void
+MediaEncoder::SetDirectConnect(bool aConnected)
+{
+ mDirectConnected = aConnected;
+}
+
+void
+MediaEncoder::NotifyRealtimeData(MediaStreamGraph* aGraph,
+ TrackID aID,
+ StreamTime aTrackOffset,
+ uint32_t aTrackEvents,
+ const MediaSegment& aRealtimeMedia)
+{
+ if (mSuspended == RECORD_NOT_SUSPENDED) {
+ // Process the incoming raw track data from MediaStreamGraph, called on the
+ // thread of MediaStreamGraph.
+ if (mAudioEncoder && aRealtimeMedia.GetType() == MediaSegment::AUDIO) {
+ mAudioEncoder->NotifyQueuedTrackChanges(aGraph, aID,
+ aTrackOffset, aTrackEvents,
+ aRealtimeMedia);
+ } else if (mVideoEncoder &&
+ aRealtimeMedia.GetType() == MediaSegment::VIDEO &&
+ aTrackEvents != TrackEventCommand::TRACK_EVENT_NONE) {
+ mVideoEncoder->NotifyQueuedTrackChanges(aGraph, aID,
+ aTrackOffset, aTrackEvents,
+ aRealtimeMedia);
+ }
+ }
+}
+
+void
+MediaEncoder::NotifyQueuedTrackChanges(MediaStreamGraph* aGraph,
+ TrackID aID,
+ StreamTime aTrackOffset,
+ TrackEventCommand aTrackEvents,
+ const MediaSegment& aQueuedMedia,
+ MediaStream* aInputStream,
+ TrackID aInputTrackID)
+{
+ if (!mDirectConnected) {
+ NotifyRealtimeData(aGraph, aID, aTrackOffset, aTrackEvents, aQueuedMedia);
+ } else {
+ if (aTrackEvents != TrackEventCommand::TRACK_EVENT_NONE) {
+ // forward events (TRACK_EVENT_ENDED) but not the media
+ if (aQueuedMedia.GetType() == MediaSegment::VIDEO) {
+ VideoSegment segment;
+ NotifyRealtimeData(aGraph, aID, aTrackOffset, aTrackEvents, segment);
+ } else {
+ AudioSegment segment;
+ NotifyRealtimeData(aGraph, aID, aTrackOffset, aTrackEvents, segment);
+ }
+ }
+ if (mSuspended == RECORD_RESUMED) {
+ if (mVideoEncoder) {
+ if (aQueuedMedia.GetType() == MediaSegment::VIDEO) {
+ // insert a null frame of duration equal to the first segment passed
+ // after Resume(), so it'll get added to one of the DirectListener frames
+ VideoSegment segment;
+ gfx::IntSize size(0,0);
+ segment.AppendFrame(nullptr, aQueuedMedia.GetDuration(), size,
+ PRINCIPAL_HANDLE_NONE);
+ mVideoEncoder->NotifyQueuedTrackChanges(aGraph, aID,
+ aTrackOffset, aTrackEvents,
+ segment);
+ mSuspended = RECORD_NOT_SUSPENDED;
+ }
+ } else {
+ mSuspended = RECORD_NOT_SUSPENDED; // no video
+ }
+ }
+ }
+}
+
+void
+MediaEncoder::NotifyQueuedAudioData(MediaStreamGraph* aGraph, TrackID aID,
+ StreamTime aTrackOffset,
+ const AudioSegment& aQueuedMedia,
+ MediaStream* aInputStream,
+ TrackID aInputTrackID)
+{
+ if (!mDirectConnected) {
+ NotifyRealtimeData(aGraph, aID, aTrackOffset, 0, aQueuedMedia);
+ } else {
+ if (mSuspended == RECORD_RESUMED) {
+ if (!mVideoEncoder) {
+ mSuspended = RECORD_NOT_SUSPENDED; // no video
+ }
+ }
+ }
+}
+
+void
+MediaEncoder::NotifyEvent(MediaStreamGraph* aGraph,
+ MediaStreamGraphEvent event)
+{
+ // In case that MediaEncoder does not receive a TRACK_EVENT_ENDED event.
+ LOG(LogLevel::Debug, ("NotifyRemoved in [MediaEncoder]."));
+ if (mAudioEncoder) {
+ mAudioEncoder->NotifyEvent(aGraph, event);
+ }
+ if (mVideoEncoder) {
+ mVideoEncoder->NotifyEvent(aGraph, event);
+ }
+}
+
+/* static */
+already_AddRefed<MediaEncoder>
+MediaEncoder::CreateEncoder(const nsAString& aMIMEType, uint32_t aAudioBitrate,
+ uint32_t aVideoBitrate, uint32_t aBitrate,
+ uint8_t aTrackTypes,
+ TrackRate aTrackRate)
+{
+ PROFILER_LABEL("MediaEncoder", "CreateEncoder",
+ js::ProfileEntry::Category::OTHER);
+
+ nsAutoPtr<ContainerWriter> writer;
+ nsAutoPtr<AudioTrackEncoder> audioEncoder;
+ nsAutoPtr<VideoTrackEncoder> videoEncoder;
+ RefPtr<MediaEncoder> encoder;
+ nsString mimeType;
+ if (!aTrackTypes) {
+ LOG(LogLevel::Error, ("NO TrackTypes!!!"));
+ return nullptr;
+ }
+#ifdef MOZ_WEBM_ENCODER
+ else if (MediaEncoder::IsWebMEncoderEnabled() &&
+ (aMIMEType.EqualsLiteral(VIDEO_WEBM) ||
+ (aTrackTypes & ContainerWriter::CREATE_VIDEO_TRACK))) {
+ if (aTrackTypes & ContainerWriter::CREATE_AUDIO_TRACK
+ && MediaDecoder::IsOpusEnabled()) {
+ audioEncoder = new OpusTrackEncoder();
+ NS_ENSURE_TRUE(audioEncoder, nullptr);
+ }
+ videoEncoder = new VP8TrackEncoder(aTrackRate);
+ writer = new WebMWriter(aTrackTypes);
+ NS_ENSURE_TRUE(writer, nullptr);
+ NS_ENSURE_TRUE(videoEncoder, nullptr);
+ mimeType = NS_LITERAL_STRING(VIDEO_WEBM);
+ }
+#endif //MOZ_WEBM_ENCODER
+ else if (MediaDecoder::IsOggEnabled() && MediaDecoder::IsOpusEnabled() &&
+ (aMIMEType.EqualsLiteral(AUDIO_OGG) ||
+ (aTrackTypes & ContainerWriter::CREATE_AUDIO_TRACK))) {
+ writer = new OggWriter();
+ audioEncoder = new OpusTrackEncoder();
+ NS_ENSURE_TRUE(writer, nullptr);
+ NS_ENSURE_TRUE(audioEncoder, nullptr);
+ mimeType = NS_LITERAL_STRING(AUDIO_OGG);
+ }
+ else {
+ LOG(LogLevel::Error, ("Can not find any encoder to record this media stream"));
+ return nullptr;
+ }
+ LOG(LogLevel::Debug, ("Create encoder result:a[%d] v[%d] w[%d] mimeType = %s.",
+ audioEncoder != nullptr, videoEncoder != nullptr,
+ writer != nullptr, mimeType.get()));
+ if (videoEncoder && aVideoBitrate != 0) {
+ videoEncoder->SetBitrate(aVideoBitrate);
+ }
+ if (audioEncoder && aAudioBitrate != 0) {
+ audioEncoder->SetBitrate(aAudioBitrate);
+ }
+ encoder = new MediaEncoder(writer.forget(), audioEncoder.forget(),
+ videoEncoder.forget(), mimeType, aAudioBitrate,
+ aVideoBitrate, aBitrate);
+ return encoder.forget();
+}
+
+/**
+ * GetEncodedData() runs as a state machine, starting with mState set to
+ * GET_METADDATA, the procedure should be as follow:
+ *
+ * While non-stop
+ * If mState is GET_METADDATA
+ * Get the meta data from audio/video encoder
+ * If a meta data is generated
+ * Get meta data from audio/video encoder
+ * Set mState to ENCODE_TRACK
+ * Return the final container data
+ *
+ * If mState is ENCODE_TRACK
+ * Get encoded track data from audio/video encoder
+ * If a packet of track data is generated
+ * Insert encoded track data into the container stream of writer
+ * If the final container data is copied to aOutput
+ * Return the copy of final container data
+ * If this is the last packet of input stream
+ * Set mState to ENCODE_DONE
+ *
+ * If mState is ENCODE_DONE or ENCODE_ERROR
+ * Stop the loop
+ */
+void
+MediaEncoder::GetEncodedData(nsTArray<nsTArray<uint8_t> >* aOutputBufs,
+ nsAString& aMIMEType)
+{
+ MOZ_ASSERT(!NS_IsMainThread());
+
+ aMIMEType = mMIMEType;
+ PROFILER_LABEL("MediaEncoder", "GetEncodedData",
+ js::ProfileEntry::Category::OTHER);
+
+ bool reloop = true;
+ while (reloop) {
+ switch (mState) {
+ case ENCODE_METADDATA: {
+ LOG(LogLevel::Debug, ("ENCODE_METADDATA TimeStamp = %f", GetEncodeTimeStamp()));
+ nsresult rv = CopyMetadataToMuxer(mAudioEncoder.get());
+ if (NS_FAILED(rv)) {
+ LOG(LogLevel::Error, ("Error! Fail to Set Audio Metadata"));
+ break;
+ }
+ rv = CopyMetadataToMuxer(mVideoEncoder.get());
+ if (NS_FAILED(rv)) {
+ LOG(LogLevel::Error, ("Error! Fail to Set Video Metadata"));
+ break;
+ }
+
+ rv = mWriter->GetContainerData(aOutputBufs,
+ ContainerWriter::GET_HEADER);
+ if (aOutputBufs != nullptr) {
+ mSizeOfBuffer = aOutputBufs->ShallowSizeOfExcludingThis(MallocSizeOf);
+ }
+ if (NS_FAILED(rv)) {
+ LOG(LogLevel::Error,("Error! writer fail to generate header!"));
+ mState = ENCODE_ERROR;
+ break;
+ }
+ LOG(LogLevel::Debug, ("Finish ENCODE_METADDATA TimeStamp = %f", GetEncodeTimeStamp()));
+ mState = ENCODE_TRACK;
+ break;
+ }
+
+ case ENCODE_TRACK: {
+ LOG(LogLevel::Debug, ("ENCODE_TRACK TimeStamp = %f", GetEncodeTimeStamp()));
+ EncodedFrameContainer encodedData;
+ nsresult rv = NS_OK;
+ // We're most likely to actually wait for a video frame, so do that first to minimize
+ // capture offset/lipsync issues
+ rv = WriteEncodedDataToMuxer(mVideoEncoder.get());
+ if (NS_FAILED(rv)) {
+ LOG(LogLevel::Error, ("Fail to write video encoder data to muxer"));
+ break;
+ }
+ rv = WriteEncodedDataToMuxer(mAudioEncoder.get());
+ if (NS_FAILED(rv)) {
+ LOG(LogLevel::Error, ("Error! Fail to write audio encoder data to muxer"));
+ break;
+ }
+ LOG(LogLevel::Debug, ("Audio encoded TimeStamp = %f", GetEncodeTimeStamp()));
+ LOG(LogLevel::Debug, ("Video encoded TimeStamp = %f", GetEncodeTimeStamp()));
+ // In audio only or video only case, let unavailable track's flag to be true.
+ bool isAudioCompleted = (mAudioEncoder && mAudioEncoder->IsEncodingComplete()) || !mAudioEncoder;
+ bool isVideoCompleted = (mVideoEncoder && mVideoEncoder->IsEncodingComplete()) || !mVideoEncoder;
+ rv = mWriter->GetContainerData(aOutputBufs,
+ isAudioCompleted && isVideoCompleted ?
+ ContainerWriter::FLUSH_NEEDED : 0);
+ if (aOutputBufs != nullptr) {
+ mSizeOfBuffer = aOutputBufs->ShallowSizeOfExcludingThis(MallocSizeOf);
+ }
+ if (NS_SUCCEEDED(rv)) {
+ // Successfully get the copy of final container data from writer.
+ reloop = false;
+ }
+ mState = (mWriter->IsWritingComplete()) ? ENCODE_DONE : ENCODE_TRACK;
+ LOG(LogLevel::Debug, ("END ENCODE_TRACK TimeStamp = %f "
+ "mState = %d aComplete %d vComplete %d",
+ GetEncodeTimeStamp(), mState, isAudioCompleted, isVideoCompleted));
+ break;
+ }
+
+ case ENCODE_DONE:
+ case ENCODE_ERROR:
+ LOG(LogLevel::Debug, ("MediaEncoder has been shutdown."));
+ mSizeOfBuffer = 0;
+ mShutdown = true;
+ reloop = false;
+ break;
+ default:
+ MOZ_CRASH("Invalid encode state");
+ }
+ }
+}
+
+nsresult
+MediaEncoder::WriteEncodedDataToMuxer(TrackEncoder *aTrackEncoder)
+{
+ if (aTrackEncoder == nullptr) {
+ return NS_OK;
+ }
+ if (aTrackEncoder->IsEncodingComplete()) {
+ return NS_OK;
+ }
+
+ PROFILER_LABEL("MediaEncoder", "WriteEncodedDataToMuxer",
+ js::ProfileEntry::Category::OTHER);
+
+ EncodedFrameContainer encodedVideoData;
+ nsresult rv = aTrackEncoder->GetEncodedTrack(encodedVideoData);
+ if (NS_FAILED(rv)) {
+ // Encoding might be canceled.
+ LOG(LogLevel::Error, ("Error! Fail to get encoded data from video encoder."));
+ mState = ENCODE_ERROR;
+ return rv;
+ }
+ rv = mWriter->WriteEncodedTrack(encodedVideoData,
+ aTrackEncoder->IsEncodingComplete() ?
+ ContainerWriter::END_OF_STREAM : 0);
+ if (NS_FAILED(rv)) {
+ LOG(LogLevel::Error, ("Error! Fail to write encoded video track to the media container."));
+ mState = ENCODE_ERROR;
+ }
+ return rv;
+}
+
+nsresult
+MediaEncoder::CopyMetadataToMuxer(TrackEncoder *aTrackEncoder)
+{
+ if (aTrackEncoder == nullptr) {
+ return NS_OK;
+ }
+
+ PROFILER_LABEL("MediaEncoder", "CopyMetadataToMuxer",
+ js::ProfileEntry::Category::OTHER);
+
+ RefPtr<TrackMetadataBase> meta = aTrackEncoder->GetMetadata();
+ if (meta == nullptr) {
+ LOG(LogLevel::Error, ("Error! metadata = null"));
+ mState = ENCODE_ERROR;
+ return NS_ERROR_ABORT;
+ }
+
+ nsresult rv = mWriter->SetMetadata(meta);
+ if (NS_FAILED(rv)) {
+ LOG(LogLevel::Error, ("Error! SetMetadata fail"));
+ mState = ENCODE_ERROR;
+ }
+ return rv;
+}
+
+#ifdef MOZ_WEBM_ENCODER
+bool
+MediaEncoder::IsWebMEncoderEnabled()
+{
+ return Preferences::GetBool("media.encoder.webm.enabled");
+}
+#endif
+
+/*
+ * SizeOfExcludingThis measures memory being used by the Media Encoder.
+ * Currently it measures the size of the Encoder buffer and memory occupied
+ * by mAudioEncoder and mVideoEncoder.
+ */
+size_t
+MediaEncoder::SizeOfExcludingThis(mozilla::MallocSizeOf aMallocSizeOf) const
+{
+ size_t amount = 0;
+ if (mState == ENCODE_TRACK) {
+ amount = mSizeOfBuffer +
+ (mAudioEncoder != nullptr ? mAudioEncoder->SizeOfExcludingThis(aMallocSizeOf) : 0) +
+ (mVideoEncoder != nullptr ? mVideoEncoder->SizeOfExcludingThis(aMallocSizeOf) : 0);
+ }
+ return amount;
+}
+
+} // namespace mozilla