diff options
Diffstat (limited to 'dom/media/platforms')
21 files changed, 566 insertions, 152 deletions
diff --git a/dom/media/platforms/MediaTelemetryConstants.h b/dom/media/platforms/MediaTelemetryConstants.h deleted file mode 100644 index 5024949a8..000000000 --- a/dom/media/platforms/MediaTelemetryConstants.h +++ /dev/null @@ -1,22 +0,0 @@ -/* -*- Mode: IDL; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- - * - * This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ -#ifndef dom_media_platforms_MediaTelemetryConstants_h___ -#define dom_media_platforms_MediaTelemetryConstants_h___ - -namespace mozilla { -namespace media { - -enum class MediaDecoderBackend : uint32_t -{ - WMFSoftware = 0, - WMFDXVA2D3D9 = 1, - WMFDXVA2D3D11 = 2 -}; - -} // namespace media -} // namespace mozilla - -#endif // dom_media_platforms_MediaTelemetryConstants_h___ diff --git a/dom/media/platforms/PDMFactory.cpp b/dom/media/platforms/PDMFactory.cpp index c1e58fdc2..5bfdcffb7 100644 --- a/dom/media/platforms/PDMFactory.cpp +++ b/dom/media/platforms/PDMFactory.cpp @@ -19,9 +19,6 @@ #ifdef MOZ_APPLEMEDIA #include "AppleDecoderModule.h" #endif -#ifdef MOZ_GONK_MEDIACODEC -#include "GonkDecoderModule.h" -#endif #ifdef MOZ_WIDGET_ANDROID #include "AndroidDecoderModule.h" #endif @@ -390,12 +387,6 @@ PDMFactory::CreatePDMs() m = new AppleDecoderModule(); StartupPDM(m); #endif -#ifdef MOZ_GONK_MEDIACODEC - if (MediaPrefs::PDMGonkDecoderEnabled()) { - m = new GonkDecoderModule(); - StartupPDM(m); - } -#endif #ifdef MOZ_WIDGET_ANDROID if(MediaPrefs::PDMAndroidMediaCodecEnabled()){ m = new AndroidDecoderModule(); diff --git a/dom/media/platforms/agnostic/AOMDecoder.cpp b/dom/media/platforms/agnostic/AOMDecoder.cpp new file mode 100644 index 000000000..b5d21375e --- /dev/null +++ b/dom/media/platforms/agnostic/AOMDecoder.cpp @@ -0,0 +1,332 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "AOMDecoder.h" +#include "MediaResult.h" +#include "TimeUnits.h" +#include "aom/aomdx.h" +#include "aom/aom_image.h" +#include "gfx2DGlue.h" +#include "mozilla/PodOperations.h" +#include "mozilla/SyncRunnable.h" +#include "nsError.h" +#include "prsystem.h" + +#include <algorithm> + +#undef LOG +#define LOG(arg, ...) MOZ_LOG(sPDMLog, mozilla::LogLevel::Debug, ("AOMDecoder(%p)::%s: " arg, this, __func__, ##__VA_ARGS__)) + +namespace mozilla { + +using namespace gfx; +using namespace layers; + +AOMDecoder::AOMDecoder(const CreateDecoderParams& aParams) + : mImageContainer(aParams.mImageContainer) + , mTaskQueue(aParams.mTaskQueue) + , mCallback(aParams.mCallback) + , mIsFlushing(false) + , mInfo(aParams.VideoConfig()) +{ + PodZero(&mCodec); +} + +AOMDecoder::~AOMDecoder() +{ +} + +void +AOMDecoder::Shutdown() +{ + aom_codec_destroy(&mCodec); +} + +RefPtr<MediaDataDecoder::InitPromise> +AOMDecoder::Init() +{ + int decode_threads = 2; + + aom_codec_iface_t* dx = aom_codec_av1_dx(); + if (mInfo.mDisplay.width >= 2048) { + decode_threads = 8; + } + else if (mInfo.mDisplay.width >= 1024) { + decode_threads = 4; + } + decode_threads = std::min(decode_threads, PR_GetNumberOfProcessors()); + + aom_codec_dec_cfg_t config; + PodZero(&config); + config.threads = decode_threads; + config.w = config.h = 0; // set after decode + config.allow_lowbitdepth = true; + + aom_codec_flags_t flags = 0; + + if (!dx || aom_codec_dec_init(&mCodec, dx, &config, flags)) { + return InitPromise::CreateAndReject(NS_ERROR_DOM_MEDIA_FATAL_ERR, __func__); + } + return InitPromise::CreateAndResolve(TrackInfo::kVideoTrack, __func__); +} + +void +AOMDecoder::Flush() +{ + MOZ_ASSERT(mCallback->OnReaderTaskQueue()); + mIsFlushing = true; + nsCOMPtr<nsIRunnable> r = NS_NewRunnableFunction([this] () { + // nothing to do for now. + }); + SyncRunnable::DispatchToThread(mTaskQueue, r); + mIsFlushing = false; +} + +// Ported from third_party/aom/tools_common.c. +static aom_codec_err_t +highbd_img_downshift(aom_image_t *dst, aom_image_t *src, int down_shift) { + int plane; + if (dst->d_w != src->d_w || dst->d_h != src->d_h) + return AOM_CODEC_INVALID_PARAM; + if (dst->x_chroma_shift != src->x_chroma_shift) + return AOM_CODEC_INVALID_PARAM; + if (dst->y_chroma_shift != src->y_chroma_shift) + return AOM_CODEC_INVALID_PARAM; + if (dst->fmt != (src->fmt & ~AOM_IMG_FMT_HIGHBITDEPTH)) + return AOM_CODEC_INVALID_PARAM; + if (down_shift < 0) + return AOM_CODEC_INVALID_PARAM; + switch (dst->fmt) { + case AOM_IMG_FMT_I420: + case AOM_IMG_FMT_I422: + case AOM_IMG_FMT_I444: + break; + default: + return AOM_CODEC_INVALID_PARAM; + } + switch (src->fmt) { + case AOM_IMG_FMT_I42016: + case AOM_IMG_FMT_I42216: + case AOM_IMG_FMT_I44416: + break; + default: + // We don't support anything that's not 16 bit + return AOM_CODEC_UNSUP_BITSTREAM; + } + for (plane = 0; plane < 3; plane++) { + int w = src->d_w; + int h = src->d_h; + int x, y; + if (plane) { + w = (w + src->x_chroma_shift) >> src->x_chroma_shift; + h = (h + src->y_chroma_shift) >> src->y_chroma_shift; + } + for (y = 0; y < h; y++) { + uint16_t *p_src = + (uint16_t *)(src->planes[plane] + y * src->stride[plane]); + uint8_t *p_dst = + dst->planes[plane] + y * dst->stride[plane]; + for (x = 0; x < w; x++) *p_dst++ = (*p_src++ >> down_shift) & 0xFF; + } + } + return AOM_CODEC_OK; +} + +// UniquePtr dtor wrapper for aom_image_t. +struct AomImageFree { + void operator()(aom_image_t* img) { aom_img_free(img); } +}; + +MediaResult +AOMDecoder::DoDecode(MediaRawData* aSample) +{ + MOZ_ASSERT(mTaskQueue->IsCurrentThreadIn()); + +#if defined(DEBUG) + NS_ASSERTION(IsKeyframe(*aSample) == aSample->mKeyframe, + "AOM Decode Keyframe error sample->mKeyframe and si.si_kf out of sync"); +#endif + + if (aom_codec_err_t r = aom_codec_decode(&mCodec, aSample->Data(), aSample->Size(), nullptr)) { + LOG("AOM Decode error: %s", aom_codec_err_to_string(r)); + return MediaResult( + NS_ERROR_DOM_MEDIA_DECODE_ERR, + RESULT_DETAIL("AOM error decoding AV1 sample: %s", aom_codec_err_to_string(r))); + } + + aom_codec_iter_t iter = nullptr; + aom_image_t *img; + UniquePtr<aom_image_t, AomImageFree> img8; + + while ((img = aom_codec_get_frame(&mCodec, &iter))) { + // Track whether the underlying buffer is 8 or 16 bits per channel. + bool highbd = bool(img->fmt & AOM_IMG_FMT_HIGHBITDEPTH); + if (highbd) { + // Downsample images with more than 8 bits per channel. + aom_img_fmt_t fmt8 = static_cast<aom_img_fmt_t>(img->fmt ^ AOM_IMG_FMT_HIGHBITDEPTH); + img8.reset(aom_img_alloc(NULL, fmt8, img->d_w, img->d_h, 16)); + if (img8 == nullptr) { + LOG("Couldn't allocate bitdepth reduction target!"); + return MediaResult( + NS_ERROR_OUT_OF_MEMORY, + RESULT_DETAIL("Couldn't allocate conversion buffer for AV1 frame")); + } + if (aom_codec_err_t r = highbd_img_downshift(img8.get(), img, img->bit_depth - 8)) { + return MediaResult( + NS_ERROR_DOM_MEDIA_DECODE_ERR, + RESULT_DETAIL("Error converting AV1 frame to 8 bits: %s", + aom_codec_err_to_string(r))); + } + // img normally points to storage owned by mCodec, so it is not freed. + // To copy out the contents of img8 we can overwrite img with an alias. + // Since img is assigned at the start of the while loop and img8 is held + // outside that loop, the alias won't outlive the storage it points to. + img = img8.get(); + highbd = false; + } + + NS_ASSERTION(img->fmt == AOM_IMG_FMT_I420 || + img->fmt == AOM_IMG_FMT_I42016 || + img->fmt == AOM_IMG_FMT_I444 || + img->fmt == AOM_IMG_FMT_I44416, + "AV1 image format not I420 or I444"); + + // Chroma shifts are rounded down as per the decoding examples in the SDK + VideoData::YCbCrBuffer b; + b.mPlanes[0].mData = img->planes[0]; + b.mPlanes[0].mStride = img->stride[0]; + b.mPlanes[0].mHeight = img->d_h; + b.mPlanes[0].mWidth = img->d_w; + b.mPlanes[0].mOffset = 0; + b.mPlanes[0].mSkip = highbd ? 1 : 0; + + b.mPlanes[1].mData = img->planes[1]; + b.mPlanes[1].mStride = img->stride[1]; + b.mPlanes[1].mOffset = 0; + b.mPlanes[1].mSkip = highbd ? 1 : 0; + + b.mPlanes[2].mData = img->planes[2]; + b.mPlanes[2].mStride = img->stride[2]; + b.mPlanes[2].mOffset = 0; + b.mPlanes[2].mSkip = highbd ? 1 : 0; + + if (img->fmt == AOM_IMG_FMT_I420 || + img->fmt == AOM_IMG_FMT_I42016) { + b.mPlanes[1].mHeight = (img->d_h + 1) >> img->y_chroma_shift; + b.mPlanes[1].mWidth = (img->d_w + 1) >> img->x_chroma_shift; + + b.mPlanes[2].mHeight = (img->d_h + 1) >> img->y_chroma_shift; + b.mPlanes[2].mWidth = (img->d_w + 1) >> img->x_chroma_shift; + } else if (img->fmt == AOM_IMG_FMT_I444) { + b.mPlanes[1].mHeight = img->d_h; + b.mPlanes[1].mWidth = img->d_w; + + b.mPlanes[2].mHeight = img->d_h; + b.mPlanes[2].mWidth = img->d_w; + } else { + LOG("AOM Unknown image format"); + return MediaResult(NS_ERROR_DOM_MEDIA_DECODE_ERR, + RESULT_DETAIL("AOM Unknown image format")); + } + + RefPtr<VideoData> v = + VideoData::CreateAndCopyData(mInfo, + mImageContainer, + aSample->mOffset, + aSample->mTime, + aSample->mDuration, + b, + aSample->mKeyframe, + aSample->mTimecode, + mInfo.ScaledImageRect(img->d_w, + img->d_h)); + + if (!v) { + LOG("Image allocation error source %ux%u display %ux%u picture %ux%u", + img->d_w, img->d_h, mInfo.mDisplay.width, mInfo.mDisplay.height, + mInfo.mImage.width, mInfo.mImage.height); + return MediaResult(NS_ERROR_OUT_OF_MEMORY, __func__); + } + mCallback->Output(v); + } + return NS_OK; +} + +void +AOMDecoder::ProcessDecode(MediaRawData* aSample) +{ + MOZ_ASSERT(mTaskQueue->IsCurrentThreadIn()); + if (mIsFlushing) { + return; + } + MediaResult rv = DoDecode(aSample); + if (NS_FAILED(rv)) { + mCallback->Error(rv); + } else { + mCallback->InputExhausted(); + } +} + +void +AOMDecoder::Input(MediaRawData* aSample) +{ + MOZ_ASSERT(mCallback->OnReaderTaskQueue()); + mTaskQueue->Dispatch(NewRunnableMethod<RefPtr<MediaRawData>>( + this, &AOMDecoder::ProcessDecode, aSample)); +} + +void +AOMDecoder::ProcessDrain() +{ + MOZ_ASSERT(mTaskQueue->IsCurrentThreadIn()); + mCallback->DrainComplete(); +} + +void +AOMDecoder::Drain() +{ + MOZ_ASSERT(mCallback->OnReaderTaskQueue()); + mTaskQueue->Dispatch(NewRunnableMethod(this, &AOMDecoder::ProcessDrain)); +} + +/* static */ +bool +AOMDecoder::IsAV1(const nsACString& aMimeType) +{ + return aMimeType.EqualsLiteral("video/webm; codecs=av1") || + aMimeType.EqualsLiteral("video/av1"); +} + +/* static */ +bool +AOMDecoder::IsKeyframe(Span<const uint8_t> aBuffer) { + aom_codec_stream_info_t info; + PodZero(&info); + + aom_codec_peek_stream_info(aom_codec_av1_dx(), + aBuffer.Elements(), + aBuffer.Length(), + &info); + + return bool(info.is_kf); +} + +/* static */ +nsIntSize +AOMDecoder::GetFrameSize(Span<const uint8_t> aBuffer) { + aom_codec_stream_info_t info; + PodZero(&info); + + aom_codec_peek_stream_info(aom_codec_av1_dx(), + aBuffer.Elements(), + aBuffer.Length(), + &info); + + return nsIntSize(info.w, info.h); +} + +} // namespace mozilla +#undef LOG diff --git a/dom/media/platforms/agnostic/AOMDecoder.h b/dom/media/platforms/agnostic/AOMDecoder.h new file mode 100644 index 000000000..ec6b1f95a --- /dev/null +++ b/dom/media/platforms/agnostic/AOMDecoder.h @@ -0,0 +1,62 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#if !defined(AOMDecoder_h_) +#define AOMDecoder_h_ + +#include "PlatformDecoderModule.h" +#include "mozilla/Span.h" + +#include <stdint.h> +#include "aom/aom_decoder.h" + +namespace mozilla { + +class AOMDecoder : public MediaDataDecoder +{ +public: + explicit AOMDecoder(const CreateDecoderParams& aParams); + + RefPtr<InitPromise> Init() override; + void Input(MediaRawData* aSample) override; + void Flush() override; + void Drain() override; + void Shutdown() override; + const char* GetDescriptionName() const override + { + return "libaom (AV1) video decoder"; + } + + // Return true if aMimeType is a one of the strings used + // by our demuxers to identify AV1 streams. + static bool IsAV1(const nsACString& aMimeType); + + // Return true if a sample is a keyframe. + static bool IsKeyframe(Span<const uint8_t> aBuffer); + + // Return the frame dimensions for a sample. + static nsIntSize GetFrameSize(Span<const uint8_t> aBuffer); + +private: + ~AOMDecoder(); + void ProcessDecode(MediaRawData* aSample); + MediaResult DoDecode(MediaRawData* aSample); + void ProcessDrain(); + + const RefPtr<layers::ImageContainer> mImageContainer; + const RefPtr<TaskQueue> mTaskQueue; + MediaDataDecoderCallback* mCallback; + Atomic<bool> mIsFlushing; + + // AOM decoder state + aom_codec_ctx_t mCodec; + + const VideoInfo& mInfo; +}; + +} // namespace mozilla + +#endif // AOMDecoder_h_ diff --git a/dom/media/platforms/agnostic/AgnosticDecoderModule.cpp b/dom/media/platforms/agnostic/AgnosticDecoderModule.cpp index 7bd75b7fe..cf5ed3d22 100644 --- a/dom/media/platforms/agnostic/AgnosticDecoderModule.cpp +++ b/dom/media/platforms/agnostic/AgnosticDecoderModule.cpp @@ -5,6 +5,7 @@ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ #include "AgnosticDecoderModule.h" +#include "MediaPrefs.h" #include "mozilla/Logging.h" #include "OpusDecoder.h" #include "VorbisDecoder.h" @@ -12,6 +13,10 @@ #include "WAVDecoder.h" #include "TheoraDecoder.h" +#ifdef MOZ_AV1 +#include "AOMDecoder.h" +#endif + namespace mozilla { bool @@ -24,6 +29,11 @@ AgnosticDecoderModule::SupportsMimeType(const nsACString& aMimeType, VorbisDataDecoder::IsVorbis(aMimeType) || WaveDataDecoder::IsWave(aMimeType) || TheoraDecoder::IsTheora(aMimeType); +#ifdef MOZ_AV1 + if (MediaPrefs::AV1Enabled()) { + supports |= AOMDecoder::IsAV1(aMimeType); + } +#endif MOZ_LOG(sPDMLog, LogLevel::Debug, ("Agnostic decoder %s requested type", supports ? "supports" : "rejects")); return supports; @@ -36,7 +46,14 @@ AgnosticDecoderModule::CreateVideoDecoder(const CreateDecoderParams& aParams) if (VPXDecoder::IsVPX(aParams.mConfig.mMimeType)) { m = new VPXDecoder(aParams); - } else if (TheoraDecoder::IsTheora(aParams.mConfig.mMimeType)) { + } +#ifdef MOZ_AV1 + else if (AOMDecoder::IsAV1(aParams.mConfig.mMimeType) && + MediaPrefs::AV1Enabled()) { + m = new AOMDecoder(aParams); + } +#endif + else if (TheoraDecoder::IsTheora(aParams.mConfig.mMimeType)) { m = new TheoraDecoder(aParams); } diff --git a/dom/media/platforms/agnostic/VPXDecoder.cpp b/dom/media/platforms/agnostic/VPXDecoder.cpp index 77c81b51b..f2f84487f 100644 --- a/dom/media/platforms/agnostic/VPXDecoder.cpp +++ b/dom/media/platforms/agnostic/VPXDecoder.cpp @@ -22,7 +22,7 @@ namespace mozilla { using namespace gfx; using namespace layers; -static int MimeTypeToCodec(const nsACString& aMimeType) +static VPXDecoder::Codec MimeTypeToCodec(const nsACString& aMimeType) { if (aMimeType.EqualsLiteral("video/webm; codecs=vp8")) { return VPXDecoder::Codec::VP8; @@ -31,7 +31,7 @@ static int MimeTypeToCodec(const nsACString& aMimeType) } else if (aMimeType.EqualsLiteral("video/vp9")) { return VPXDecoder::Codec::VP9; } - return -1; + return VPXDecoder::Codec::Unknown; } VPXDecoder::VPXDecoder(const CreateDecoderParams& aParams) @@ -101,17 +101,10 @@ MediaResult VPXDecoder::DoDecode(MediaRawData* aSample) { MOZ_ASSERT(mTaskQueue->IsCurrentThreadIn()); + #if defined(DEBUG) - vpx_codec_stream_info_t si; - PodZero(&si); - si.sz = sizeof(si); - if (mCodec == Codec::VP8) { - vpx_codec_peek_stream_info(vpx_codec_vp8_dx(), aSample->Data(), aSample->Size(), &si); - } else if (mCodec == Codec::VP9) { - vpx_codec_peek_stream_info(vpx_codec_vp9_dx(), aSample->Data(), aSample->Size(), &si); - } - NS_ASSERTION(bool(si.is_kf) == aSample->mKeyframe, - "VPX Decode Keyframe error sample->mKeyframe and si.si_kf out of sync"); + NS_ASSERTION(IsKeyframe(*aSample, mCodec) == aSample->mKeyframe, + "VPX Decode Keyframe error sample->mKeyframe and sample data out of sync"); #endif if (vpx_codec_err_t r = vpx_codec_decode(&mVPX, aSample->Data(), aSample->Size(), nullptr, 0)) { @@ -249,5 +242,41 @@ VPXDecoder::IsVP9(const nsACString& aMimeType) return IsVPX(aMimeType, VPXDecoder::VP9); } +/* static */ +bool +VPXDecoder::IsKeyframe(Span<const uint8_t> aBuffer, Codec aCodec) +{ + vpx_codec_stream_info_t si; + PodZero(&si); + si.sz = sizeof(si); + + if (aCodec == Codec::VP8) { + vpx_codec_peek_stream_info(vpx_codec_vp8_dx(), aBuffer.Elements(), aBuffer.Length(), &si); + return bool(si.is_kf); + } else if (aCodec == Codec::VP9) { + vpx_codec_peek_stream_info(vpx_codec_vp9_dx(), aBuffer.Elements(), aBuffer.Length(), &si); + return bool(si.is_kf); + } + + return false; +} + +/* static */ +nsIntSize +VPXDecoder::GetFrameSize(Span<const uint8_t> aBuffer, Codec aCodec) +{ + vpx_codec_stream_info_t si; + PodZero(&si); + si.sz = sizeof(si); + + if (aCodec == Codec::VP8) { + vpx_codec_peek_stream_info(vpx_codec_vp8_dx(), aBuffer.Elements(), aBuffer.Length(), &si); + } else if (aCodec == Codec::VP9) { + vpx_codec_peek_stream_info(vpx_codec_vp9_dx(), aBuffer.Elements(), aBuffer.Length(), &si); + } + + return nsIntSize(si.w, si.h); +} + } // namespace mozilla #undef LOG diff --git a/dom/media/platforms/agnostic/VPXDecoder.h b/dom/media/platforms/agnostic/VPXDecoder.h index d420ec069..4e8d83617 100644 --- a/dom/media/platforms/agnostic/VPXDecoder.h +++ b/dom/media/platforms/agnostic/VPXDecoder.h @@ -7,6 +7,7 @@ #define VPXDecoder_h_ #include "PlatformDecoderModule.h" +#include "mozilla/Span.h" #include <stdint.h> #define VPX_DONT_DEFINE_STDINT_TYPES @@ -36,7 +37,8 @@ public: enum Codec: uint8_t { VP8 = 1 << 0, - VP9 = 1 << 1 + VP9 = 1 << 1, + Unknown = 1 << 7, }; // Return true if aMimeType is a one of the strings used by our demuxers to @@ -46,6 +48,12 @@ public: static bool IsVP8(const nsACString& aMimeType); static bool IsVP9(const nsACString& aMimeType); + // Return true if a sample is a keyframe for the specified codec. + static bool IsKeyframe(Span<const uint8_t> aBuffer, Codec aCodec); + + // Return the frame dimensions for a sample for the specified codec. + static nsIntSize GetFrameSize(Span<const uint8_t> aBuffer, Codec aCodec); + private: void ProcessDecode(MediaRawData* aSample); MediaResult DoDecode(MediaRawData* aSample); @@ -61,7 +69,7 @@ private: const VideoInfo& mInfo; - const int mCodec; + const Codec mCodec; }; } // namespace mozilla diff --git a/dom/media/platforms/ffmpeg/FFmpegDataDecoder.cpp b/dom/media/platforms/ffmpeg/FFmpegDataDecoder.cpp index 8cb5c8578..8655ce25f 100644 --- a/dom/media/platforms/ffmpeg/FFmpegDataDecoder.cpp +++ b/dom/media/platforms/ffmpeg/FFmpegDataDecoder.cpp @@ -69,12 +69,21 @@ FFmpegDataDecoder<LIBAV_VER>::InitDecoder() mCodecContext->extradata_size = mExtraData->Length(); // FFmpeg may use SIMD instructions to access the data which reads the // data in 32 bytes block. Must ensure we have enough data to read. + uint32_t padding_size = #if LIBAVCODEC_VERSION_MAJOR >= 58 - mExtraData->AppendElements(AV_INPUT_BUFFER_PADDING_SIZE); + AV_INPUT_BUFFER_PADDING_SIZE; #else - mExtraData->AppendElements(FF_INPUT_BUFFER_PADDING_SIZE); + FF_INPUT_BUFFER_PADDING_SIZE; #endif - mCodecContext->extradata = mExtraData->Elements(); + mCodecContext->extradata = static_cast<uint8_t*>( + mLib->av_malloc(mExtraData->Length() + padding_size)); + if (!mCodecContext->extradata) { + return MediaResult(NS_ERROR_OUT_OF_MEMORY, + RESULT_DETAIL("Couldn't init ffmpeg extradata")); + } + memcpy(mCodecContext->extradata, + mExtraData->Elements(), + mExtraData->Length()); } else { mCodecContext->extradata_size = 0; } @@ -165,6 +174,9 @@ FFmpegDataDecoder<LIBAV_VER>::ProcessShutdown() StaticMutexAutoLock mon(sMonitor); if (mCodecContext) { + if (mCodecContext->extradata) { + mLib->av_freep(&mCodecContext->extradata); + } mLib->avcodec_close(mCodecContext); mLib->av_freep(&mCodecContext); #if LIBAVCODEC_VERSION_MAJOR >= 55 diff --git a/dom/media/platforms/ffmpeg/FFmpegLibWrapper.cpp b/dom/media/platforms/ffmpeg/FFmpegLibWrapper.cpp index 6302882a6..426e9f74b 100644 --- a/dom/media/platforms/ffmpeg/FFmpegLibWrapper.cpp +++ b/dom/media/platforms/ffmpeg/FFmpegLibWrapper.cpp @@ -7,6 +7,7 @@ #include "MediaPrefs.h" #include "mozilla/PodOperations.h" #include "mozilla/Types.h" +#include "PlatformDecoderModule.h" #include "prlink.h" #define AV_LOG_DEBUG 48 @@ -144,6 +145,8 @@ FFmpegLibWrapper::Link() AV_FUNC(avcodec_alloc_frame, (AV_FUNC_53 | AV_FUNC_54)) AV_FUNC(avcodec_get_frame_defaults, (AV_FUNC_53 | AV_FUNC_54)) AV_FUNC(avcodec_free_frame, AV_FUNC_54) + AV_FUNC(avcodec_send_packet, AV_FUNC_58) + AV_FUNC(avcodec_receive_frame, AV_FUNC_58) AV_FUNC(av_log_set_level, AV_FUNC_AVUTIL_ALL) AV_FUNC(av_malloc, AV_FUNC_AVUTIL_ALL) AV_FUNC(av_freep, AV_FUNC_AVUTIL_ALL) diff --git a/dom/media/platforms/ffmpeg/FFmpegLibWrapper.h b/dom/media/platforms/ffmpeg/FFmpegLibWrapper.h index d6944a1d8..b968edd32 100644 --- a/dom/media/platforms/ffmpeg/FFmpegLibWrapper.h +++ b/dom/media/platforms/ffmpeg/FFmpegLibWrapper.h @@ -5,6 +5,7 @@ #ifndef __FFmpegLibWrapper_h__ #define __FFmpegLibWrapper_h__ +#include "mozilla/Attributes.h" #include "mozilla/Types.h" struct AVCodec; @@ -70,6 +71,10 @@ struct FFmpegLibWrapper // libavcodec v54 only void (*avcodec_free_frame)(AVFrame** frame); + // libavcodec v58 and later only + int (*avcodec_send_packet)(AVCodecContext* avctx, const AVPacket* avpkt); + int (*avcodec_receive_frame)(AVCodecContext* avctx, AVFrame* frame); + // libavutil void (*av_log_set_level)(int level); void* (*av_malloc)(size_t size); @@ -91,4 +96,4 @@ private: } // namespace mozilla -#endif // FFmpegLibWrapper
\ No newline at end of file +#endif // FFmpegLibWrapper diff --git a/dom/media/platforms/ffmpeg/FFmpegVideoDecoder.cpp b/dom/media/platforms/ffmpeg/FFmpegVideoDecoder.cpp index aec1e9136..f3101e44c 100644 --- a/dom/media/platforms/ffmpeg/FFmpegVideoDecoder.cpp +++ b/dom/media/platforms/ffmpeg/FFmpegVideoDecoder.cpp @@ -174,12 +174,15 @@ FFmpegVideoDecoder<LIBAV_VER>::DoDecode(MediaRawData* aSample, bool* aGotFrame) uint8_t* inputData = const_cast<uint8_t*>(aSample->Data()); size_t inputSize = aSample->Size(); -#if LIBAVCODEC_VERSION_MAJOR >= 54 +#if LIBAVCODEC_VERSION_MAJOR >= 54 && LIBAVCODEC_VERSION_MAJOR < 58 if (inputSize && mCodecParser && (mCodecID == AV_CODEC_ID_VP8 -#if LIBAVCODEC_VERSION_MAJOR >= 55 +#if LIBAVCODEC_VERSION_MAJOR >= 55 && LIBAVCODEC_VERSION_MAJOR < 58 || mCodecID == AV_CODEC_ID_VP9 #endif - )) { + )) +#endif +#if LIBAVCODEC_VERSION_MAJOR >= 54 + { while (inputSize) { uint8_t* data = inputData; int size = inputSize; @@ -224,6 +227,48 @@ FFmpegVideoDecoder<LIBAV_VER>::DoDecode(MediaRawData* aSample, packet.flags = aSample->mKeyframe ? AV_PKT_FLAG_KEY : 0; packet.pos = aSample->mOffset; +#if LIBAVCODEC_VERSION_MAJOR >= 58 + packet.duration = aSample->mDuration; + int res = mLib->avcodec_send_packet(mCodecContext, &packet); + if (res < 0) { + // In theory, avcodec_send_packet could sent -EAGAIN should its internal + // buffers be full. In practice this can't happen as we only feed one frame + // at a time, and we immediately call avcodec_receive_frame right after. + FFMPEG_LOG("avcodec_send_packet error: %d", res); + return MediaResult(NS_ERROR_DOM_MEDIA_DECODE_ERR, + RESULT_DETAIL("avcodec_send_packet error: %d", res)); + } + + if (aGotFrame) { + *aGotFrame = false; + } + do { + if (!PrepareFrame()) { + NS_WARNING("FFmpeg h264 decoder failed to allocate frame."); + return MediaResult(NS_ERROR_OUT_OF_MEMORY, __func__); + } + res = mLib->avcodec_receive_frame(mCodecContext, mFrame); + if (res == int(AVERROR_EOF)) { + return NS_ERROR_DOM_MEDIA_END_OF_STREAM; + } + if (res == AVERROR(EAGAIN)) { + return NS_OK; + } + if (res < 0) { + FFMPEG_LOG("avcodec_receive_frame error: %d", res); + return MediaResult(NS_ERROR_DOM_MEDIA_DECODE_ERR, + RESULT_DETAIL("avcodec_receive_frame error: %d", res)); + } + MediaResult rv = CreateImage(mFrame->pkt_pos, mFrame->pkt_pts, + mFrame->pkt_duration); + if (NS_FAILED(rv)) { + return rv; + } + if (aGotFrame) { + *aGotFrame = true; + } + } while (true); +#else // LibAV provides no API to retrieve the decoded sample's duration. // (FFmpeg >= 1.0 provides av_frame_get_pkt_duration) // As such we instead use a map using the dts as key that we will retrieve @@ -276,8 +321,21 @@ FFmpegVideoDecoder<LIBAV_VER>::DoDecode(MediaRawData* aSample, // against the map becoming extremely big. mDurationMap.Clear(); } + + MediaResult rv = CreateImage(aSample->mOffset, pts, duration); + if (NS_SUCCEEDED(rv) && aGotFrame) { + *aGotFrame = true; + } + return rv; +#endif +} + +MediaResult +FFmpegVideoDecoder<LIBAV_VER>::CreateImage(int64_t aOffset, int64_t aPts, + int64_t aDuration) +{ FFMPEG_LOG("Got one frame output with pts=%lld dts=%lld duration=%lld opaque=%lld", - pts, mFrame->pkt_dts, duration, mCodecContext->reordered_opaque); + aPts, mFrame->pkt_dts, aDuration, mCodecContext->reordered_opaque); VideoData::YCbCrBuffer b; b.mPlanes[0].mData = mFrame->data[0]; @@ -317,9 +375,9 @@ FFmpegVideoDecoder<LIBAV_VER>::DoDecode(MediaRawData* aSample, RefPtr<VideoData> v = VideoData::CreateAndCopyData(mInfo, mImageContainer, - aSample->mOffset, - pts, - duration, + aOffset, + aPts, + aDuration, b, !!mFrame->key_frame, -1, @@ -331,9 +389,6 @@ FFmpegVideoDecoder<LIBAV_VER>::DoDecode(MediaRawData* aSample, RESULT_DETAIL("image allocation error")); } mCallback->Output(v); - if (aGotFrame) { - *aGotFrame = true; - } return NS_OK; } diff --git a/dom/media/platforms/ffmpeg/FFmpegVideoDecoder.h b/dom/media/platforms/ffmpeg/FFmpegVideoDecoder.h index 786df0da1..49a55e8a6 100644 --- a/dom/media/platforms/ffmpeg/FFmpegVideoDecoder.h +++ b/dom/media/platforms/ffmpeg/FFmpegVideoDecoder.h @@ -49,6 +49,7 @@ private: MediaResult DoDecode(MediaRawData* aSample) override; MediaResult DoDecode(MediaRawData* aSample, bool* aGotFrame); MediaResult DoDecode(MediaRawData* aSample, uint8_t* aData, int aSize, bool* aGotFrame); + MediaResult CreateImage(int64_t aOffset, int64_t aPts, int64_t aDuration); void ProcessDrain() override; void ProcessFlush() override; void OutputDelayedFrames(); diff --git a/dom/media/platforms/ffmpeg/ffvpx/moz.build b/dom/media/platforms/ffmpeg/ffvpx/moz.build index aee58b5b0..c0041a4d4 100644 --- a/dom/media/platforms/ffmpeg/ffvpx/moz.build +++ b/dom/media/platforms/ffmpeg/ffvpx/moz.build @@ -20,7 +20,7 @@ SOURCES += [ ] LOCAL_INCLUDES += [ '..', - '../ffmpeg57/include', + '../ffmpeg58/include', ] if CONFIG['OS_ARCH'] == 'WINNT': diff --git a/dom/media/platforms/moz.build b/dom/media/platforms/moz.build index 3fb0cc842..f5fb72c5d 100644 --- a/dom/media/platforms/moz.build +++ b/dom/media/platforms/moz.build @@ -10,7 +10,6 @@ EXPORTS += [ 'agnostic/TheoraDecoder.h', 'agnostic/VorbisDecoder.h', 'agnostic/VPXDecoder.h', - 'MediaTelemetryConstants.h', 'PDMFactory.h', 'PlatformDecoderModule.h', 'wrappers/FuzzingWrapper.h', @@ -55,6 +54,14 @@ if CONFIG['MOZ_FFMPEG']: 'ffmpeg', ] +if CONFIG['MOZ_AV1']: + EXPORTS += [ + 'agnostic/AOMDecoder.h', + ] + UNIFIED_SOURCES += [ + 'agnostic/AOMDecoder.cpp', + ] + if CONFIG['MOZ_APPLEMEDIA']: EXPORTS += [ 'apple/AppleDecoderModule.h', diff --git a/dom/media/platforms/omx/OmxPlatformLayer.cpp b/dom/media/platforms/omx/OmxPlatformLayer.cpp index 039b4a22f..15b3062a4 100644 --- a/dom/media/platforms/omx/OmxPlatformLayer.cpp +++ b/dom/media/platforms/omx/OmxPlatformLayer.cpp @@ -282,26 +282,7 @@ OmxPlatformLayer::CompressionFormat() } } -// Implementations for different platforms will be defined in their own files. -#ifdef OMX_PLATFORM_GONK - -bool -OmxPlatformLayer::SupportsMimeType(const nsACString& aMimeType) -{ - return GonkOmxPlatformLayer::FindComponents(aMimeType); -} - -OmxPlatformLayer* -OmxPlatformLayer::Create(OmxDataDecoder* aDataDecoder, - OmxPromiseLayer* aPromiseLayer, - TaskQueue* aTaskQueue, - layers::ImageContainer* aImageContainer) -{ - return new GonkOmxPlatformLayer(aDataDecoder, aPromiseLayer, aTaskQueue, aImageContainer); -} - -#else // For platforms without OMX IL support. - +// For platforms without OMX IL support. bool OmxPlatformLayer::SupportsMimeType(const nsACString& aMimeType) { @@ -317,6 +298,4 @@ OmxPlatformLayer::Create(OmxDataDecoder* aDataDecoder, return nullptr; } -#endif - } diff --git a/dom/media/platforms/wmf/DXVA2Manager.cpp b/dom/media/platforms/wmf/DXVA2Manager.cpp index 0c1734c54..69e002f7f 100644 --- a/dom/media/platforms/wmf/DXVA2Manager.cpp +++ b/dom/media/platforms/wmf/DXVA2Manager.cpp @@ -14,8 +14,6 @@ #include "mozilla/layers/D3D11ShareHandleImage.h" #include "mozilla/layers/ImageBridgeChild.h" #include "mozilla/layers/TextureForwarder.h" -#include "mozilla/Telemetry.h" -#include "MediaTelemetryConstants.h" #include "mfapi.h" #include "gfxPrefs.h" #include "MFTDecoder.h" @@ -442,9 +440,6 @@ D3D9DXVA2Manager::Init(layers::KnowsCompositor* aKnowsCompositor, } mTextureClientAllocator->SetMaxPoolSize(5); - Telemetry::Accumulate(Telemetry::MEDIA_DECODER_BACKEND_USED, - uint32_t(media::MediaDecoderBackend::WMFDXVA2D3D9)); - return S_OK; } @@ -775,9 +770,6 @@ D3D11DXVA2Manager::Init(layers::KnowsCompositor* aKnowsCompositor, } mTextureClientAllocator->SetMaxPoolSize(5); - Telemetry::Accumulate(Telemetry::MEDIA_DECODER_BACKEND_USED, - uint32_t(media::MediaDecoderBackend::WMFDXVA2D3D11)); - return S_OK; } diff --git a/dom/media/platforms/wmf/WMFAudioMFTManager.cpp b/dom/media/platforms/wmf/WMFAudioMFTManager.cpp index 69b62da51..3dacdf0aa 100644 --- a/dom/media/platforms/wmf/WMFAudioMFTManager.cpp +++ b/dom/media/platforms/wmf/WMFAudioMFTManager.cpp @@ -252,7 +252,6 @@ WMFAudioMFTManager::Output(int64_t aStreamOffset, LOG("Audio MFTDecoder returned success but null output."); nsCOMPtr<nsIRunnable> task = NS_NewRunnableFunction([]() -> void { LOG("Reporting telemetry AUDIO_MFT_OUTPUT_NULL_SAMPLES"); - Telemetry::Accumulate(Telemetry::ID::AUDIO_MFT_OUTPUT_NULL_SAMPLES, 1); }); AbstractThread::MainThread()->Dispatch(task.forget()); return E_FAIL; diff --git a/dom/media/platforms/wmf/WMFDecoderModule.h b/dom/media/platforms/wmf/WMFDecoderModule.h index cd7b8c660..6582f8056 100644 --- a/dom/media/platforms/wmf/WMFDecoderModule.h +++ b/dom/media/platforms/wmf/WMFDecoderModule.h @@ -40,10 +40,8 @@ public: static int GetNumDecoderThreads(); // Accessors that report whether we have the required MFTs available - // on the system to play various codecs. Windows Vista doesn't have the - // H.264/AAC decoders if the "Platform Update Supplement for Windows Vista" - // is not installed, and Window N and KN variants also require a "Media - // Feature Pack" to be installed. Windows XP doesn't have WMF. + // on the system to play various codecs. Windows N and KN variants + // require a "Media Feature Pack" to be installed. static bool HasAAC(); static bool HasH264(); diff --git a/dom/media/platforms/wmf/WMFMediaDataDecoder.cpp b/dom/media/platforms/wmf/WMFMediaDataDecoder.cpp index d2c13eac7..e6dd29c6d 100644 --- a/dom/media/platforms/wmf/WMFMediaDataDecoder.cpp +++ b/dom/media/platforms/wmf/WMFMediaDataDecoder.cpp @@ -8,7 +8,6 @@ #include "VideoUtils.h" #include "WMFUtils.h" #include "nsTArray.h" -#include "mozilla/Telemetry.h" #include "mozilla/Logging.h" #include "mozilla/SyncRunnable.h" @@ -39,39 +38,6 @@ WMFMediaDataDecoder::Init() return InitPromise::CreateAndResolve(mMFTManager->GetType(), __func__); } -// A single telemetry sample is reported for each MediaDataDecoder object -// that has detected error or produced output successfully. -static void -SendTelemetry(unsigned long hr) -{ - // Collapse the error codes into a range of 0-0xff that can be viewed in - // telemetry histograms. For most MF_E_* errors, unique samples are used, - // retaining the least significant 7 or 8 bits. Other error codes are - // bucketed. - uint32_t sample; - if (SUCCEEDED(hr)) { - sample = 0; - } else if (hr < 0xc00d36b0) { - sample = 1; // low bucket - } else if (hr < 0xc00d3700) { - sample = hr & 0xffU; // MF_E_* - } else if (hr <= 0xc00d3705) { - sample = 0x80 + (hr & 0xfU); // more MF_E_* - } else if (hr < 0xc00d6d60) { - sample = 2; // mid bucket - } else if (hr <= 0xc00d6d78) { - sample = hr & 0xffU; // MF_E_TRANSFORM_* - } else { - sample = 3; // high bucket - } - - nsCOMPtr<nsIRunnable> runnable = NS_NewRunnableFunction( - [sample] { - Telemetry::Accumulate(Telemetry::MEDIA_WMF_DECODE_ERROR, sample); - }); - NS_DispatchToMainThread(runnable); -} - void WMFMediaDataDecoder::Shutdown() { @@ -91,9 +57,6 @@ WMFMediaDataDecoder::ProcessShutdown() if (mMFTManager) { mMFTManager->Shutdown(); mMFTManager = nullptr; - if (!mRecordedError && mHasSuccessfulOutput) { - SendTelemetry(S_OK); - } } } @@ -125,10 +88,6 @@ WMFMediaDataDecoder::ProcessDecode(MediaRawData* aSample) NS_WARNING("MFTManager rejected sample"); mCallback->Error(MediaResult(NS_ERROR_DOM_MEDIA_DECODE_ERR, RESULT_DETAIL("MFTManager::Input:%x", hr))); - if (!mRecordedError) { - SendTelemetry(hr); - mRecordedError = true; - } return; } @@ -144,7 +103,6 @@ WMFMediaDataDecoder::ProcessOutput() HRESULT hr = S_OK; while (SUCCEEDED(hr = mMFTManager->Output(mLastStreamOffset, output)) && output) { - mHasSuccessfulOutput = true; mCallback->Output(output); } if (hr == MF_E_TRANSFORM_NEED_MORE_INPUT) { @@ -153,10 +111,6 @@ WMFMediaDataDecoder::ProcessOutput() NS_WARNING("WMFMediaDataDecoder failed to output data"); mCallback->Error(MediaResult(NS_ERROR_DOM_MEDIA_DECODE_ERR, RESULT_DETAIL("MFTManager::Output:%x", hr))); - if (!mRecordedError) { - SendTelemetry(hr); - mRecordedError = true; - } } } diff --git a/dom/media/platforms/wmf/WMFMediaDataDecoder.h b/dom/media/platforms/wmf/WMFMediaDataDecoder.h index a4dd49f56..f869012e7 100644 --- a/dom/media/platforms/wmf/WMFMediaDataDecoder.h +++ b/dom/media/platforms/wmf/WMFMediaDataDecoder.h @@ -33,7 +33,7 @@ public: // Returns S_OK on success, or MF_E_TRANSFORM_NEED_MORE_INPUT if there's not // enough data to produce more output. If this returns a failure code other // than MF_E_TRANSFORM_NEED_MORE_INPUT, an error will be reported to the - // MP4Reader. + // MP4Demuxer. virtual HRESULT Output(int64_t aStreamOffset, RefPtr<MediaData>& aOutput) = 0; @@ -136,10 +136,6 @@ private: Atomic<bool> mIsFlushing; bool mIsShutDown; - - // For telemetry - bool mHasSuccessfulOutput = false; - bool mRecordedError = false; }; } // namespace mozilla diff --git a/dom/media/platforms/wmf/WMFVideoMFTManager.cpp b/dom/media/platforms/wmf/WMFVideoMFTManager.cpp index 291bc5b74..a7633a7de 100644 --- a/dom/media/platforms/wmf/WMFVideoMFTManager.cpp +++ b/dom/media/platforms/wmf/WMFVideoMFTManager.cpp @@ -27,7 +27,6 @@ #include "mozilla/WindowsVersion.h" #include "mozilla/Telemetry.h" #include "nsPrintfCString.h" -#include "MediaTelemetryConstants.h" #include "GMPUtils.h" // For SplitAt. TODO: Move SplitAt to a central place. #include "MP4Decoder.h" #include "VPXDecoder.h" @@ -128,7 +127,6 @@ WMFVideoMFTManager::~WMFVideoMFTManager() nsCOMPtr<nsIRunnable> task = NS_NewRunnableFunction([=]() -> void { LOG(nsPrintfCString("Reporting telemetry VIDEO_MFT_OUTPUT_NULL_SAMPLES=%d", telemetry).get()); - Telemetry::Accumulate(Telemetry::ID::VIDEO_MFT_OUTPUT_NULL_SAMPLES, telemetry); }); AbstractThread::MainThread()->Dispatch(task.forget()); } @@ -511,8 +509,6 @@ WMFVideoMFTManager::InitInternal(bool aForceD3D9) if (mStreamType == VP9 || mStreamType == VP8) { return false; } - Telemetry::Accumulate(Telemetry::MEDIA_DECODER_BACKEND_USED, - uint32_t(media::MediaDecoderBackend::WMFSoftware)); } mDecoder = decoder; |