diff options
Diffstat (limited to 'dom/media/platforms/android/RemoteDataDecoder.cpp')
| -rw-r--r-- | dom/media/platforms/android/RemoteDataDecoder.cpp | 489 |
1 files changed, 489 insertions, 0 deletions
diff --git a/dom/media/platforms/android/RemoteDataDecoder.cpp b/dom/media/platforms/android/RemoteDataDecoder.cpp new file mode 100644 index 000000000..56af2601f --- /dev/null +++ b/dom/media/platforms/android/RemoteDataDecoder.cpp @@ -0,0 +1,489 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "AndroidDecoderModule.h" +#include "AndroidBridge.h" +#include "AndroidSurfaceTexture.h" +#include "FennecJNINatives.h" +#include "GLImages.h" + +#include "MediaData.h" +#include "MediaInfo.h" +#include "VideoUtils.h" +#include "VPXDecoder.h" + +#include "nsThreadUtils.h" +#include "nsPromiseFlatString.h" +#include "nsIGfxInfo.h" + +#include "prlog.h" + +#include <jni.h> + +#include <deque> + +#undef LOG +#define LOG(arg, ...) MOZ_LOG(sAndroidDecoderModuleLog, \ + mozilla::LogLevel::Debug, ("RemoteDataDecoder(%p)::%s: " arg, \ + this, __func__, ##__VA_ARGS__)) + +using namespace mozilla; +using namespace mozilla::gl; +using namespace mozilla::java; +using namespace mozilla::java::sdk; +using media::TimeUnit; + +namespace mozilla { + +class JavaCallbacksSupport + : public CodecProxy::NativeCallbacks::Natives<JavaCallbacksSupport> +{ +public: + typedef CodecProxy::NativeCallbacks::Natives<JavaCallbacksSupport> Base; + using Base::AttachNative; + + JavaCallbacksSupport(MediaDataDecoderCallback* aDecoderCallback) + : mDecoderCallback(aDecoderCallback) + { + MOZ_ASSERT(aDecoderCallback); + } + + virtual ~JavaCallbacksSupport() {} + + void OnInputExhausted() + { + if (mDecoderCallback) { + mDecoderCallback->InputExhausted(); + } + } + + virtual void HandleOutput(Sample::Param aSample) = 0; + + void OnOutput(jni::Object::Param aSample) + { + if (mDecoderCallback) { + HandleOutput(Sample::Ref::From(aSample)); + } + } + + virtual void HandleOutputFormatChanged(MediaFormat::Param aFormat) {}; + + void OnOutputFormatChanged(jni::Object::Param aFormat) + { + if (mDecoderCallback) { + HandleOutputFormatChanged(MediaFormat::Ref::From(aFormat)); + } + } + + void OnError(bool aIsFatal) + { + if (mDecoderCallback) { + mDecoderCallback->Error(aIsFatal ? + MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR, __func__) : + MediaResult(NS_ERROR_DOM_MEDIA_DECODE_ERR, __func__)); + } + } + + void DisposeNative() + { + // TODO + } + + void Cancel() + { + mDecoderCallback = nullptr; + } + +protected: + MediaDataDecoderCallback* mDecoderCallback; +}; + +struct SampleTime final +{ + SampleTime(int64_t aStart, int64_t aDuration) + : mStart(aStart) + , mDuration(aDuration) + {} + + int64_t mStart; + int64_t mDuration; +}; + + +class RemoteVideoDecoder final : public RemoteDataDecoder +{ +public: + class CallbacksSupport final : public JavaCallbacksSupport + { + public: + CallbacksSupport(RemoteVideoDecoder* aDecoder, MediaDataDecoderCallback* aCallback) + : JavaCallbacksSupport(aCallback) + , mDecoder(aDecoder) + {} + + virtual ~CallbacksSupport() {} + + void HandleOutput(Sample::Param aSample) override + { + Maybe<int64_t> durationUs = mDecoder->mInputDurations.Get(); + if (!durationUs) { + return; + } + + BufferInfo::LocalRef info = aSample->Info(); + + int32_t flags; + bool ok = NS_SUCCEEDED(info->Flags(&flags)); + MOZ_ASSERT(ok); + + int32_t offset; + ok |= NS_SUCCEEDED(info->Offset(&offset)); + MOZ_ASSERT(ok); + + int64_t presentationTimeUs; + ok |= NS_SUCCEEDED(info->PresentationTimeUs(&presentationTimeUs)); + MOZ_ASSERT(ok); + + int32_t size; + ok |= NS_SUCCEEDED(info->Size(&size)); + MOZ_ASSERT(ok); + + NS_ENSURE_TRUE_VOID(ok); + + if (size > 0) { + RefPtr<layers::Image> img = + new SurfaceTextureImage(mDecoder->mSurfaceTexture.get(), mDecoder->mConfig.mDisplay, + gl::OriginPos::BottomLeft); + + RefPtr<VideoData> v = + VideoData::CreateFromImage(mDecoder->mConfig, + offset, + presentationTimeUs, + durationUs.value(), + img, + !!(flags & MediaCodec::BUFFER_FLAG_SYNC_FRAME), + presentationTimeUs, + gfx::IntRect(0, 0, + mDecoder->mConfig.mDisplay.width, + mDecoder->mConfig.mDisplay.height)); + + mDecoderCallback->Output(v); + } + + if ((flags & MediaCodec::BUFFER_FLAG_END_OF_STREAM) != 0) { + mDecoderCallback->DrainComplete(); + } + } + + friend class RemoteDataDecoder; + + private: + RemoteVideoDecoder* mDecoder; + }; + + RemoteVideoDecoder(const VideoInfo& aConfig, + MediaFormat::Param aFormat, + MediaDataDecoderCallback* aCallback, + layers::ImageContainer* aImageContainer) + : RemoteDataDecoder(MediaData::Type::VIDEO_DATA, aConfig.mMimeType, + aFormat, aCallback) + , mImageContainer(aImageContainer) + , mConfig(aConfig) + { + } + + RefPtr<InitPromise> Init() override + { + mSurfaceTexture = AndroidSurfaceTexture::Create(); + if (!mSurfaceTexture) { + NS_WARNING("Failed to create SurfaceTexture for video decode\n"); + return InitPromise::CreateAndReject(NS_ERROR_DOM_MEDIA_FATAL_ERR, __func__); + } + + if (!jni::IsFennec()) { + NS_WARNING("Remote decoding not supported in non-Fennec environment\n"); + return InitPromise::CreateAndReject(NS_ERROR_DOM_MEDIA_FATAL_ERR, __func__); + } + + // Register native methods. + JavaCallbacksSupport::Init(); + + mJavaCallbacks = CodecProxy::NativeCallbacks::New(); + JavaCallbacksSupport::AttachNative(mJavaCallbacks, + mozilla::MakeUnique<CallbacksSupport>(this, mCallback)); + + mJavaDecoder = CodecProxy::Create(mFormat, mSurfaceTexture->JavaSurface(), mJavaCallbacks); + if (mJavaDecoder == nullptr) { + return InitPromise::CreateAndReject(NS_ERROR_DOM_MEDIA_FATAL_ERR, __func__); + } + + mInputDurations.Clear(); + + return InitPromise::CreateAndResolve(TrackInfo::kVideoTrack, __func__); + } + + void Flush() override + { + mInputDurations.Clear(); + RemoteDataDecoder::Flush(); + } + + void Drain() override + { + RemoteDataDecoder::Drain(); + mInputDurations.Put(0); + } + + void Input(MediaRawData* aSample) override + { + RemoteDataDecoder::Input(aSample); + mInputDurations.Put(aSample->mDuration); + } + +private: + class DurationQueue { + public: + + void Clear() + { + mValues.clear(); + } + + void Put(int64_t aDurationUs) + { + mValues.emplace_back(aDurationUs); + } + + Maybe<int64_t> Get() + { + if (mValues.empty()) { + return Nothing(); + } + + auto value = Some(mValues.front()); + mValues.pop_front(); + + return value; + } + + private: + std::deque<int64_t> mValues; + }; + + layers::ImageContainer* mImageContainer; + const VideoInfo& mConfig; + RefPtr<AndroidSurfaceTexture> mSurfaceTexture; + DurationQueue mInputDurations; +}; + +class RemoteAudioDecoder final : public RemoteDataDecoder +{ +public: + RemoteAudioDecoder(const AudioInfo& aConfig, + MediaFormat::Param aFormat, + MediaDataDecoderCallback* aCallback) + : RemoteDataDecoder(MediaData::Type::AUDIO_DATA, aConfig.mMimeType, + aFormat, aCallback) + , mConfig(aConfig) + { + JNIEnv* const env = jni::GetEnvForThread(); + + bool formatHasCSD = false; + NS_ENSURE_SUCCESS_VOID(aFormat->ContainsKey(NS_LITERAL_STRING("csd-0"), &formatHasCSD)); + + if (!formatHasCSD && aConfig.mCodecSpecificConfig->Length() >= 2) { + jni::ByteBuffer::LocalRef buffer(env); + buffer = jni::ByteBuffer::New( + aConfig.mCodecSpecificConfig->Elements(), + aConfig.mCodecSpecificConfig->Length()); + NS_ENSURE_SUCCESS_VOID(aFormat->SetByteBuffer(NS_LITERAL_STRING("csd-0"), + buffer)); + } + } + + RefPtr<InitPromise> Init() override + { + // Register native methods. + JavaCallbacksSupport::Init(); + + mJavaCallbacks = CodecProxy::NativeCallbacks::New(); + JavaCallbacksSupport::AttachNative(mJavaCallbacks, + mozilla::MakeUnique<CallbacksSupport>(this, mCallback)); + + mJavaDecoder = CodecProxy::Create(mFormat, nullptr, mJavaCallbacks); + if (mJavaDecoder == nullptr) { + return InitPromise::CreateAndReject(NS_ERROR_DOM_MEDIA_FATAL_ERR, __func__); + } + + return InitPromise::CreateAndResolve(TrackInfo::kAudioTrack, __func__); + } + +private: + class CallbacksSupport final : public JavaCallbacksSupport + { + public: + CallbacksSupport(RemoteAudioDecoder* aDecoder, MediaDataDecoderCallback* aCallback) + : JavaCallbacksSupport(aCallback) + , mDecoder(aDecoder) + {} + + virtual ~CallbacksSupport() {} + + void HandleOutput(Sample::Param aSample) override + { + BufferInfo::LocalRef info = aSample->Info(); + + int32_t flags; + bool ok = NS_SUCCEEDED(info->Flags(&flags)); + MOZ_ASSERT(ok); + + int32_t offset; + ok |= NS_SUCCEEDED(info->Offset(&offset)); + MOZ_ASSERT(ok); + + int64_t presentationTimeUs; + ok |= NS_SUCCEEDED(info->PresentationTimeUs(&presentationTimeUs)); + MOZ_ASSERT(ok); + + int32_t size; + ok |= NS_SUCCEEDED(info->Size(&size)); + MOZ_ASSERT(ok); + + NS_ENSURE_TRUE_VOID(ok); + + if (size > 0) { +#ifdef MOZ_SAMPLE_TYPE_S16 + const int32_t numSamples = size / 2; +#else +#error We only support 16-bit integer PCM +#endif + + const int32_t numFrames = numSamples / mOutputChannels; + AlignedAudioBuffer audio(numSamples); + if (!audio) { + return; + } + + jni::ByteBuffer::LocalRef dest = jni::ByteBuffer::New(audio.get(), size); + aSample->WriteToByteBuffer(dest); + + RefPtr<AudioData> data = new AudioData(0, presentationTimeUs, + FramesToUsecs(numFrames, mOutputSampleRate).value(), + numFrames, + Move(audio), + mOutputChannels, + mOutputSampleRate); + + mDecoderCallback->Output(data); + } + + if ((flags & MediaCodec::BUFFER_FLAG_END_OF_STREAM) != 0) { + mDecoderCallback->DrainComplete(); + return; + } + } + + void HandleOutputFormatChanged(MediaFormat::Param aFormat) override + { + aFormat->GetInteger(NS_LITERAL_STRING("channel-count"), &mOutputChannels); + AudioConfig::ChannelLayout layout(mOutputChannels); + if (!layout.IsValid()) { + mDecoderCallback->Error(MediaResult( + NS_ERROR_DOM_MEDIA_FATAL_ERR, + RESULT_DETAIL("Invalid channel layout:%d", mOutputChannels))); + return; + } + aFormat->GetInteger(NS_LITERAL_STRING("sample-rate"), &mOutputSampleRate); + LOG("Audio output format changed: channels:%d sample rate:%d", mOutputChannels, mOutputSampleRate); + } + + private: + RemoteAudioDecoder* mDecoder; + int32_t mOutputChannels; + int32_t mOutputSampleRate; + }; + + const AudioInfo& mConfig; +}; + +MediaDataDecoder* +RemoteDataDecoder::CreateAudioDecoder(const AudioInfo& aConfig, + MediaFormat::Param aFormat, + MediaDataDecoderCallback* aCallback) +{ + return new RemoteAudioDecoder(aConfig, aFormat, aCallback); +} + +MediaDataDecoder* +RemoteDataDecoder::CreateVideoDecoder(const VideoInfo& aConfig, + MediaFormat::Param aFormat, + MediaDataDecoderCallback* aCallback, + layers::ImageContainer* aImageContainer) +{ + return new RemoteVideoDecoder(aConfig, aFormat, aCallback, aImageContainer); +} + +RemoteDataDecoder::RemoteDataDecoder(MediaData::Type aType, + const nsACString& aMimeType, + MediaFormat::Param aFormat, + MediaDataDecoderCallback* aCallback) + : mType(aType) + , mMimeType(aMimeType) + , mFormat(aFormat) + , mCallback(aCallback) +{ +} + +void +RemoteDataDecoder::Flush() +{ + mJavaDecoder->Flush(); +} + +void +RemoteDataDecoder::Drain() +{ + BufferInfo::LocalRef bufferInfo; + nsresult rv = BufferInfo::New(&bufferInfo); + NS_ENSURE_SUCCESS_VOID(rv); + bufferInfo->Set(0, 0, -1, MediaCodec::BUFFER_FLAG_END_OF_STREAM); + + mJavaDecoder->Input(nullptr, bufferInfo, nullptr); +} + +void +RemoteDataDecoder::Shutdown() +{ + LOG(""); + MOZ_ASSERT(mJavaDecoder && mJavaCallbacks); + + mJavaDecoder->Release(); + mJavaDecoder = nullptr; + + JavaCallbacksSupport::GetNative(mJavaCallbacks)->Cancel(); + mJavaCallbacks = nullptr; + + mFormat = nullptr; +} + +void +RemoteDataDecoder::Input(MediaRawData* aSample) +{ + MOZ_ASSERT(aSample != nullptr); + + jni::ByteBuffer::LocalRef bytes = jni::ByteBuffer::New(const_cast<uint8_t*>(aSample->Data()), + aSample->Size()); + + BufferInfo::LocalRef bufferInfo; + nsresult rv = BufferInfo::New(&bufferInfo); + if (NS_FAILED(rv)) { + mCallback->Error(MediaResult(NS_ERROR_OUT_OF_MEMORY, __func__)); + return; + } + bufferInfo->Set(0, aSample->Size(), aSample->mTime, 0); + + mJavaDecoder->Input(bytes, bufferInfo, GetCryptoInfoFromSample(aSample)); +} + +} // mozilla |