diff options
author | Matt A. Tobin <mattatobin@localhost.localdomain> | 2018-02-02 04:16:08 -0500 |
---|---|---|
committer | Matt A. Tobin <mattatobin@localhost.localdomain> | 2018-02-02 04:16:08 -0500 |
commit | 5f8de423f190bbb79a62f804151bc24824fa32d8 (patch) | |
tree | 10027f336435511475e392454359edea8e25895d /dom/media/platforms/gonk | |
parent | 49ee0794b5d912db1f95dce6eb52d781dc210db5 (diff) | |
download | UXP-5f8de423f190bbb79a62f804151bc24824fa32d8.tar UXP-5f8de423f190bbb79a62f804151bc24824fa32d8.tar.gz UXP-5f8de423f190bbb79a62f804151bc24824fa32d8.tar.lz UXP-5f8de423f190bbb79a62f804151bc24824fa32d8.tar.xz UXP-5f8de423f190bbb79a62f804151bc24824fa32d8.zip |
Add m-esr52 at 52.6.0
Diffstat (limited to 'dom/media/platforms/gonk')
-rw-r--r-- | dom/media/platforms/gonk/GonkAudioDecoderManager.cpp | 268 | ||||
-rw-r--r-- | dom/media/platforms/gonk/GonkAudioDecoderManager.h | 59 | ||||
-rw-r--r-- | dom/media/platforms/gonk/GonkDecoderModule.cpp | 63 | ||||
-rw-r--r-- | dom/media/platforms/gonk/GonkDecoderModule.h | 37 | ||||
-rw-r--r-- | dom/media/platforms/gonk/GonkMediaDataDecoder.cpp | 385 | ||||
-rw-r--r-- | dom/media/platforms/gonk/GonkMediaDataDecoder.h | 214 | ||||
-rw-r--r-- | dom/media/platforms/gonk/GonkVideoDecoderManager.cpp | 772 | ||||
-rw-r--r-- | dom/media/platforms/gonk/GonkVideoDecoderManager.h | 149 | ||||
-rw-r--r-- | dom/media/platforms/gonk/moz.build | 39 |
9 files changed, 1986 insertions, 0 deletions
diff --git a/dom/media/platforms/gonk/GonkAudioDecoderManager.cpp b/dom/media/platforms/gonk/GonkAudioDecoderManager.cpp new file mode 100644 index 000000000..0bc3fbea9 --- /dev/null +++ b/dom/media/platforms/gonk/GonkAudioDecoderManager.cpp @@ -0,0 +1,268 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +#include "MediaCodecProxy.h" +#include <OMX_IVCommon.h> +#include <gui/Surface.h> +#include <ICrypto.h> +#include "GonkAudioDecoderManager.h" +#include "MediaDecoderReader.h" +#include "VideoUtils.h" +#include "nsTArray.h" +#include "mozilla/Logging.h" +#include "stagefright/MediaBuffer.h" +#include "stagefright/MetaData.h" +#include "stagefright/MediaErrors.h" +#include <stagefright/foundation/AMessage.h> +#include <stagefright/foundation/ALooper.h> +#include "media/openmax/OMX_Audio.h" +#include "MediaData.h" +#include "MediaInfo.h" + +#define CODECCONFIG_TIMEOUT_US 10000LL +#define READ_OUTPUT_BUFFER_TIMEOUT_US 0LL + +#include <android/log.h> +#define GADM_LOG(...) __android_log_print(ANDROID_LOG_DEBUG, "GonkAudioDecoderManager", __VA_ARGS__) + +#define LOG(...) MOZ_LOG(sPDMLog, mozilla::LogLevel::Debug, (__VA_ARGS__)) + +using namespace android; +typedef android::MediaCodecProxy MediaCodecProxy; + +namespace mozilla { + +GonkAudioDecoderManager::GonkAudioDecoderManager(const AudioInfo& aConfig) + : mAudioChannels(aConfig.mChannels) + , mAudioRate(aConfig.mRate) + , mAudioProfile(aConfig.mProfile) + , mAudioCompactor(mAudioQueue) +{ + MOZ_COUNT_CTOR(GonkAudioDecoderManager); + MOZ_ASSERT(mAudioChannels); + mCodecSpecificData = aConfig.mCodecSpecificConfig; + mMimeType = aConfig.mMimeType; +} + +GonkAudioDecoderManager::~GonkAudioDecoderManager() +{ + MOZ_COUNT_DTOR(GonkAudioDecoderManager); +} + +RefPtr<MediaDataDecoder::InitPromise> +GonkAudioDecoderManager::Init() +{ + if (InitMediaCodecProxy()) { + return InitPromise::CreateAndResolve(TrackType::kAudioTrack, __func__); + } else { + return InitPromise::CreateAndReject(NS_ERROR_DOM_MEDIA_FATAL_ERR, __func__); + } +} + +bool +GonkAudioDecoderManager::InitMediaCodecProxy() +{ + status_t rv = OK; + if (!InitLoopers(MediaData::AUDIO_DATA)) { + return false; + } + + mDecoder = MediaCodecProxy::CreateByType(mDecodeLooper, mMimeType.get(), false); + if (!mDecoder.get()) { + return false; + } + if (!mDecoder->AllocateAudioMediaCodec()) + { + mDecoder = nullptr; + return false; + } + sp<AMessage> format = new AMessage; + // Fixed values + GADM_LOG("Configure audio mime type:%s, chan no:%d, sample-rate:%d, profile:%d", + mMimeType.get(), mAudioChannels, mAudioRate, mAudioProfile); + format->setString("mime", mMimeType.get()); + format->setInt32("channel-count", mAudioChannels); + format->setInt32("sample-rate", mAudioRate); + format->setInt32("aac-profile", mAudioProfile); + status_t err = mDecoder->configure(format, nullptr, nullptr, 0); + if (err != OK || !mDecoder->Prepare()) { + return false; + } + + if (mMimeType.EqualsLiteral("audio/mp4a-latm")) { + rv = mDecoder->Input(mCodecSpecificData->Elements(), mCodecSpecificData->Length(), 0, + android::MediaCodec::BUFFER_FLAG_CODECCONFIG, + CODECCONFIG_TIMEOUT_US); + } + + if (rv == OK) { + return true; + } else { + GADM_LOG("Failed to input codec specific data!"); + return false; + } +} + +nsresult +GonkAudioDecoderManager::CreateAudioData(MediaBuffer* aBuffer, int64_t aStreamOffset) +{ + if (!(aBuffer != nullptr && aBuffer->data() != nullptr)) { + GADM_LOG("Audio Buffer is not valid!"); + return NS_ERROR_UNEXPECTED; + } + + int64_t timeUs; + if (!aBuffer->meta_data()->findInt64(kKeyTime, &timeUs)) { + return NS_ERROR_UNEXPECTED; + } + + if (aBuffer->range_length() == 0) { + // Some decoders may return spurious empty buffers that we just want to ignore + // quoted from Android's AwesomePlayer.cpp + return NS_ERROR_NOT_AVAILABLE; + } + + if (mLastTime > timeUs) { + GADM_LOG("Output decoded sample time is revert. time=%lld", timeUs); + MOZ_ASSERT(false); + return NS_ERROR_NOT_AVAILABLE; + } + mLastTime = timeUs; + + const uint8_t *data = static_cast<const uint8_t*>(aBuffer->data()); + size_t dataOffset = aBuffer->range_offset(); + size_t size = aBuffer->range_length(); + + uint32_t frames = size / (2 * mAudioChannels); + + CheckedInt64 duration = FramesToUsecs(frames, mAudioRate); + if (!duration.isValid()) { + return NS_ERROR_UNEXPECTED; + } + + typedef AudioCompactor::NativeCopy OmxCopy; + mAudioCompactor.Push(aStreamOffset, + timeUs, + mAudioRate, + frames, + mAudioChannels, + OmxCopy(data+dataOffset, + size, + mAudioChannels)); + return NS_OK; +} + +nsresult +GonkAudioDecoderManager::Output(int64_t aStreamOffset, + RefPtr<MediaData>& aOutData) +{ + aOutData = nullptr; + if (mAudioQueue.GetSize() > 0) { + aOutData = mAudioQueue.PopFront(); + return mAudioQueue.AtEndOfStream() ? NS_ERROR_ABORT : NS_OK; + } + + status_t err; + MediaBuffer* audioBuffer = nullptr; + err = mDecoder->Output(&audioBuffer, READ_OUTPUT_BUFFER_TIMEOUT_US); + AutoReleaseMediaBuffer a(audioBuffer, mDecoder.get()); + + switch (err) { + case OK: + { + nsresult rv = CreateAudioData(audioBuffer, aStreamOffset); + NS_ENSURE_SUCCESS(rv, rv); + break; + } + case android::INFO_FORMAT_CHANGED: + { + // If the format changed, update our cached info. + GADM_LOG("Decoder format changed"); + sp<AMessage> audioCodecFormat; + + if (mDecoder->getOutputFormat(&audioCodecFormat) != OK || + audioCodecFormat == nullptr) { + return NS_ERROR_UNEXPECTED; + } + + int32_t codec_channel_count = 0; + int32_t codec_sample_rate = 0; + + if (!audioCodecFormat->findInt32("channel-count", &codec_channel_count) || + !audioCodecFormat->findInt32("sample-rate", &codec_sample_rate)) { + return NS_ERROR_UNEXPECTED; + } + + // Update AudioInfo + AudioConfig::ChannelLayout layout(codec_channel_count); + if (!layout.IsValid()) { + return NS_ERROR_FAILURE; + } + mAudioChannels = codec_channel_count; + mAudioRate = codec_sample_rate; + + return Output(aStreamOffset, aOutData); + } + case android::INFO_OUTPUT_BUFFERS_CHANGED: + { + GADM_LOG("Info Output Buffers Changed"); + if (mDecoder->UpdateOutputBuffers()) { + return Output(aStreamOffset, aOutData); + } + return NS_ERROR_FAILURE; + } + case -EAGAIN: + { + return NS_ERROR_NOT_AVAILABLE; + } + case android::ERROR_END_OF_STREAM: + { + GADM_LOG("Got EOS frame!"); + nsresult rv = CreateAudioData(audioBuffer, aStreamOffset); + NS_ENSURE_SUCCESS(rv, NS_ERROR_ABORT); + MOZ_ASSERT(mAudioQueue.GetSize() > 0); + mAudioQueue.Finish(); + break; + } + case -ETIMEDOUT: + { + GADM_LOG("Timeout. can try again next time"); + return NS_ERROR_UNEXPECTED; + } + default: + { + GADM_LOG("Decoder failed, err=%d", err); + return NS_ERROR_UNEXPECTED; + } + } + + if (mAudioQueue.GetSize() > 0) { + aOutData = mAudioQueue.PopFront(); + // Return NS_ERROR_ABORT at the last sample. + return mAudioQueue.AtEndOfStream() ? NS_ERROR_ABORT : NS_OK; + } + + return NS_ERROR_NOT_AVAILABLE; +} + +void +GonkAudioDecoderManager::ProcessFlush() +{ + GADM_LOG("FLUSH<<<"); + mAudioQueue.Reset(); + GADM_LOG(">>>FLUSH"); + GonkDecoderManager::ProcessFlush(); +} + +void +GonkAudioDecoderManager::ResetEOS() +{ + GADM_LOG("ResetEOS(<<<"); + mAudioQueue.Reset(); + GADM_LOG(">>>ResetEOS("); + GonkDecoderManager::ResetEOS(); +} + +} // namespace mozilla diff --git a/dom/media/platforms/gonk/GonkAudioDecoderManager.h b/dom/media/platforms/gonk/GonkAudioDecoderManager.h new file mode 100644 index 000000000..aa35d620e --- /dev/null +++ b/dom/media/platforms/gonk/GonkAudioDecoderManager.h @@ -0,0 +1,59 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#if !defined(GonkAudioDecoderManager_h_) +#define GonkAudioDecoderManager_h_ + +#include "AudioCompactor.h" +#include "mozilla/RefPtr.h" +#include "GonkMediaDataDecoder.h" + +using namespace android; + +namespace android { +class MOZ_EXPORT MediaBuffer; +} // namespace android + +namespace mozilla { + +class GonkAudioDecoderManager : public GonkDecoderManager { +typedef android::MediaCodecProxy MediaCodecProxy; +public: + GonkAudioDecoderManager(const AudioInfo& aConfig); + + virtual ~GonkAudioDecoderManager(); + + RefPtr<InitPromise> Init() override; + + nsresult Output(int64_t aStreamOffset, + RefPtr<MediaData>& aOutput) override; + + void ProcessFlush() override; + virtual void ResetEOS() override; + + const char* GetDescriptionName() const override + { + return "gonk audio decoder"; + } + +private: + bool InitMediaCodecProxy(); + + nsresult CreateAudioData(MediaBuffer* aBuffer, int64_t aStreamOffset); + + uint32_t mAudioChannels; + uint32_t mAudioRate; + const uint32_t mAudioProfile; + + MediaQueue<AudioData> mAudioQueue; + + AudioCompactor mAudioCompactor; + +}; + +} // namespace mozilla + +#endif // GonkAudioDecoderManager_h_ diff --git a/dom/media/platforms/gonk/GonkDecoderModule.cpp b/dom/media/platforms/gonk/GonkDecoderModule.cpp new file mode 100644 index 000000000..537bc299c --- /dev/null +++ b/dom/media/platforms/gonk/GonkDecoderModule.cpp @@ -0,0 +1,63 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +#include "GonkDecoderModule.h" +#include "GonkVideoDecoderManager.h" +#include "GonkAudioDecoderManager.h" +#include "mozilla/DebugOnly.h" +#include "GonkMediaDataDecoder.h" + +namespace mozilla { +GonkDecoderModule::GonkDecoderModule() +{ +} + +GonkDecoderModule::~GonkDecoderModule() +{ +} + +already_AddRefed<MediaDataDecoder> +GonkDecoderModule::CreateVideoDecoder(const CreateDecoderParams& aParams) +{ + RefPtr<MediaDataDecoder> decoder = + new GonkMediaDataDecoder(new GonkVideoDecoderManager(aParams.mImageContainer, aParams.VideoConfig()), + aParams.mCallback); + return decoder.forget(); +} + +already_AddRefed<MediaDataDecoder> +GonkDecoderModule::CreateAudioDecoder(const CreateDecoderParams& aParams) +{ + RefPtr<MediaDataDecoder> decoder = + new GonkMediaDataDecoder(new GonkAudioDecoderManager(aParams.AudioConfig()), + aParams.mCallback); + return decoder.forget(); +} + +PlatformDecoderModule::ConversionRequired +GonkDecoderModule::DecoderNeedsConversion(const TrackInfo& aConfig) const +{ + if (aConfig.IsVideo()) { + return ConversionRequired::kNeedAnnexB; + } else { + return ConversionRequired::kNeedNone; + } +} + +bool +GonkDecoderModule::SupportsMimeType(const nsACString& aMimeType, + DecoderDoctorDiagnostics* aDiagnostics) const +{ + return aMimeType.EqualsLiteral("audio/mp4a-latm") || + aMimeType.EqualsLiteral("audio/3gpp") || + aMimeType.EqualsLiteral("audio/amr-wb") || + aMimeType.EqualsLiteral("audio/mpeg") || + aMimeType.EqualsLiteral("video/mp4") || + aMimeType.EqualsLiteral("video/mp4v-es") || + aMimeType.EqualsLiteral("video/avc") || + aMimeType.EqualsLiteral("video/3gpp"); +} + +} // namespace mozilla diff --git a/dom/media/platforms/gonk/GonkDecoderModule.h b/dom/media/platforms/gonk/GonkDecoderModule.h new file mode 100644 index 000000000..4f29f0e75 --- /dev/null +++ b/dom/media/platforms/gonk/GonkDecoderModule.h @@ -0,0 +1,37 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#if !defined(GonkPlatformDecoderModule_h_) +#define GonkPlatformDecoderModule_h_ + +#include "PlatformDecoderModule.h" + +namespace mozilla { + +class GonkDecoderModule : public PlatformDecoderModule { +public: + GonkDecoderModule(); + virtual ~GonkDecoderModule(); + + // Decode thread. + already_AddRefed<MediaDataDecoder> + CreateVideoDecoder(const CreateDecoderParams& aParams) override; + + // Decode thread. + already_AddRefed<MediaDataDecoder> + CreateAudioDecoder(const CreateDecoderParams& aParams) override; + + ConversionRequired + DecoderNeedsConversion(const TrackInfo& aConfig) const override; + + bool SupportsMimeType(const nsACString& aMimeType, + DecoderDoctorDiagnostics* aDiagnostics) const override; + +}; + +} // namespace mozilla + +#endif diff --git a/dom/media/platforms/gonk/GonkMediaDataDecoder.cpp b/dom/media/platforms/gonk/GonkMediaDataDecoder.cpp new file mode 100644 index 000000000..6d59d72e1 --- /dev/null +++ b/dom/media/platforms/gonk/GonkMediaDataDecoder.cpp @@ -0,0 +1,385 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +#include "GonkMediaDataDecoder.h" +#include "VideoUtils.h" +#include "nsTArray.h" +#include "MediaCodecProxy.h" + +#include <stagefright/foundation/ADebug.h> + +#include "mozilla/Logging.h" +#include <android/log.h> +#define GMDD_LOG(...) __android_log_print(ANDROID_LOG_DEBUG, "GonkMediaDataDecoder", __VA_ARGS__) +#define INPUT_TIMEOUT_US 0LL // Don't wait for buffer if none is available. +#define MIN_QUEUED_SAMPLES 2 + +#ifdef DEBUG +#include <utils/AndroidThreads.h> +#endif + +#define LOG(...) MOZ_LOG(sPDMLog, mozilla::LogLevel::Debug, (__VA_ARGS__)) + +using namespace android; + +namespace mozilla { + +bool +GonkDecoderManager::InitLoopers(MediaData::Type aType) +{ + MOZ_ASSERT(mDecodeLooper.get() == nullptr && mTaskLooper.get() == nullptr); + MOZ_ASSERT(aType == MediaData::VIDEO_DATA || aType == MediaData::AUDIO_DATA); + + const char* suffix = (aType == MediaData::VIDEO_DATA ? "video" : "audio"); + mDecodeLooper = new ALooper; + android::AString name("MediaCodecProxy/"); + name.append(suffix); + mDecodeLooper->setName(name.c_str()); + + mTaskLooper = new ALooper; + name.setTo("GonkDecoderManager/"); + name.append(suffix); + mTaskLooper->setName(name.c_str()); + mTaskLooper->registerHandler(this); + +#ifdef DEBUG + sp<AMessage> findThreadId(new AMessage(kNotifyFindLooperId, id())); + findThreadId->post(); +#endif + + return mDecodeLooper->start() == OK && mTaskLooper->start() == OK; +} + +nsresult +GonkDecoderManager::Input(MediaRawData* aSample) +{ + RefPtr<MediaRawData> sample; + + if (aSample) { + sample = aSample; + } else { + // It means EOS with empty sample. + sample = new MediaRawData(); + } + { + MutexAutoLock lock(mMutex); + mQueuedSamples.AppendElement(sample); + } + + sp<AMessage> input = new AMessage(kNotifyProcessInput, id()); + if (!aSample) { + input->setInt32("input-eos", 1); + } + input->post(); + return NS_OK; +} + +int32_t +GonkDecoderManager::ProcessQueuedSamples() +{ + MOZ_ASSERT(OnTaskLooper()); + + MutexAutoLock lock(mMutex); + status_t rv; + while (mQueuedSamples.Length()) { + RefPtr<MediaRawData> data = mQueuedSamples.ElementAt(0); + rv = mDecoder->Input(reinterpret_cast<const uint8_t*>(data->Data()), + data->Size(), + data->mTime, + 0, + INPUT_TIMEOUT_US); + if (rv == OK) { + mQueuedSamples.RemoveElementAt(0); + mWaitOutput.AppendElement(WaitOutputInfo(data->mOffset, data->mTime, + /* eos */ data->Data() == nullptr)); + } else if (rv == -EAGAIN || rv == -ETIMEDOUT) { + // In most cases, EAGAIN or ETIMEOUT are safe because OMX can't fill + // buffer on time. + break; + } else { + return rv; + } + } + return mQueuedSamples.Length(); +} + +nsresult +GonkDecoderManager::Flush() +{ + if (mDecoder == nullptr) { + GMDD_LOG("Decoder is not initialized"); + return NS_ERROR_UNEXPECTED; + } + + if (!mInitPromise.IsEmpty()) { + return NS_OK; + } + + { + MutexAutoLock lock(mMutex); + mQueuedSamples.Clear(); + } + + MonitorAutoLock lock(mFlushMonitor); + mIsFlushing = true; + sp<AMessage> flush = new AMessage(kNotifyProcessFlush, id()); + flush->post(); + while (mIsFlushing) { + lock.Wait(); + } + return NS_OK; +} + +nsresult +GonkDecoderManager::Shutdown() +{ + if (mDecoder.get()) { + mDecoder->stop(); + mDecoder->ReleaseMediaResources(); + mDecoder = nullptr; + } + + mInitPromise.RejectIfExists(NS_ERROR_DOM_MEDIA_CANCELED, __func__); + + return NS_OK; +} + +size_t +GonkDecoderManager::NumQueuedSamples() +{ + MutexAutoLock lock(mMutex); + return mQueuedSamples.Length(); +} + +void +GonkDecoderManager::ProcessInput(bool aEndOfStream) +{ + MOZ_ASSERT(OnTaskLooper()); + + status_t rv = ProcessQueuedSamples(); + if (rv >= 0) { + if (!aEndOfStream && rv <= MIN_QUEUED_SAMPLES) { + mDecodeCallback->InputExhausted(); + } + + if (mToDo.get() == nullptr) { + mToDo = new AMessage(kNotifyDecoderActivity, id()); + if (aEndOfStream) { + mToDo->setInt32("input-eos", 1); + } + mDecoder->requestActivityNotification(mToDo); + } else if (aEndOfStream) { + mToDo->setInt32("input-eos", 1); + } + } else { + GMDD_LOG("input processed: error#%d", rv); + mDecodeCallback->Error(MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR, + __func__)); + } +} + +void +GonkDecoderManager::ProcessFlush() +{ + MOZ_ASSERT(OnTaskLooper()); + + mLastTime = INT64_MIN; + MonitorAutoLock lock(mFlushMonitor); + mWaitOutput.Clear(); + if (mDecoder->flush() != OK) { + GMDD_LOG("flush error"); + mDecodeCallback->Error(MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR, + __func__)); + } + mIsFlushing = false; + lock.NotifyAll(); +} + +// Use output timestamp to determine which output buffer is already returned +// and remove corresponding info, except for EOS, from the waiting list. +// This method handles the cases that audio decoder sends multiple output +// buffers for one input. +void +GonkDecoderManager::UpdateWaitingList(int64_t aForgetUpTo) +{ + MOZ_ASSERT(OnTaskLooper()); + + size_t i; + for (i = 0; i < mWaitOutput.Length(); i++) { + const auto& item = mWaitOutput.ElementAt(i); + if (item.mEOS || item.mTimestamp > aForgetUpTo) { + break; + } + } + if (i > 0) { + mWaitOutput.RemoveElementsAt(0, i); + } +} + +void +GonkDecoderManager::ProcessToDo(bool aEndOfStream) +{ + MOZ_ASSERT(OnTaskLooper()); + + MOZ_ASSERT(mToDo.get() != nullptr); + mToDo.clear(); + + if (NumQueuedSamples() > 0 && ProcessQueuedSamples() < 0) { + mDecodeCallback->Error(MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR, + __func__)); + return; + } + + while (mWaitOutput.Length() > 0) { + RefPtr<MediaData> output; + WaitOutputInfo wait = mWaitOutput.ElementAt(0); + nsresult rv = Output(wait.mOffset, output); + if (rv == NS_OK) { + MOZ_ASSERT(output); + mDecodeCallback->Output(output); + UpdateWaitingList(output->mTime); + } else if (rv == NS_ERROR_ABORT) { + // EOS + MOZ_ASSERT(mQueuedSamples.IsEmpty()); + if (output) { + mDecodeCallback->Output(output); + UpdateWaitingList(output->mTime); + } + MOZ_ASSERT(mWaitOutput.Length() == 1); + mWaitOutput.RemoveElementAt(0); + mDecodeCallback->DrainComplete(); + ResetEOS(); + return; + } else if (rv == NS_ERROR_NOT_AVAILABLE) { + break; + } else { + mDecodeCallback->Error(MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR, + __func__)); + return; + } + } + + if (!aEndOfStream && NumQueuedSamples() <= MIN_QUEUED_SAMPLES) { + mDecodeCallback->InputExhausted(); + // No need to shedule todo task this time because InputExhausted() will + // cause Input() to be invoked and do it for us. + return; + } + + if (NumQueuedSamples() || mWaitOutput.Length() > 0) { + mToDo = new AMessage(kNotifyDecoderActivity, id()); + if (aEndOfStream) { + mToDo->setInt32("input-eos", 1); + } + mDecoder->requestActivityNotification(mToDo); + } +} + +void +GonkDecoderManager::ResetEOS() +{ + // After eos, android::MediaCodec needs to be flushed to receive next input + mWaitOutput.Clear(); + if (mDecoder->flush() != OK) { + GMDD_LOG("flush error"); + mDecodeCallback->Error(MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR, + __func__)); + } +} + +void +GonkDecoderManager::onMessageReceived(const sp<AMessage> &aMessage) +{ + switch (aMessage->what()) { + case kNotifyProcessInput: + { + int32_t eos = 0; + ProcessInput(aMessage->findInt32("input-eos", &eos) && eos); + break; + } + case kNotifyProcessFlush: + { + ProcessFlush(); + break; + } + case kNotifyDecoderActivity: + { + int32_t eos = 0; + ProcessToDo(aMessage->findInt32("input-eos", &eos) && eos); + break; + } +#ifdef DEBUG + case kNotifyFindLooperId: + { + mTaskLooperId = androidGetThreadId(); + MOZ_ASSERT(mTaskLooperId); + break; + } +#endif + default: + { + TRESPASS(); + break; + } + } +} + +#ifdef DEBUG +bool +GonkDecoderManager::OnTaskLooper() +{ + return androidGetThreadId() == mTaskLooperId; +} +#endif + +GonkMediaDataDecoder::GonkMediaDataDecoder(GonkDecoderManager* aManager, + MediaDataDecoderCallback* aCallback) + : mManager(aManager) +{ + MOZ_COUNT_CTOR(GonkMediaDataDecoder); + mManager->SetDecodeCallback(aCallback); +} + +GonkMediaDataDecoder::~GonkMediaDataDecoder() +{ + MOZ_COUNT_DTOR(GonkMediaDataDecoder); +} + +RefPtr<MediaDataDecoder::InitPromise> +GonkMediaDataDecoder::Init() +{ + return mManager->Init(); +} + +void +GonkMediaDataDecoder::Shutdown() +{ + mManager->Shutdown(); + + // Because codec allocated runnable and init promise is at reader TaskQueue, + // so manager needs to be destroyed at reader TaskQueue to prevent racing. + mManager = nullptr; +} + +// Inserts data into the decoder's pipeline. +void +GonkMediaDataDecoder::Input(MediaRawData* aSample) +{ + mManager->Input(aSample); +} + +void +GonkMediaDataDecoder::Flush() +{ + mManager->Flush(); +} + +void +GonkMediaDataDecoder::Drain() +{ + mManager->Input(nullptr); +} + +} // namespace mozilla diff --git a/dom/media/platforms/gonk/GonkMediaDataDecoder.h b/dom/media/platforms/gonk/GonkMediaDataDecoder.h new file mode 100644 index 000000000..bba2a8645 --- /dev/null +++ b/dom/media/platforms/gonk/GonkMediaDataDecoder.h @@ -0,0 +1,214 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#if !defined(GonkMediaDataDecoder_h_) +#define GonkMediaDataDecoder_h_ +#include "PlatformDecoderModule.h" +#include <stagefright/foundation/AHandler.h> + +namespace android { +struct ALooper; +class MediaBuffer; +class MediaCodecProxy; +} // namespace android + +namespace mozilla { +class MediaRawData; + +// Manage the data flow from inputting encoded data and outputting decode data. +class GonkDecoderManager : public android::AHandler { +public: + typedef TrackInfo::TrackType TrackType; + typedef MediaDataDecoder::InitPromise InitPromise; + + virtual ~GonkDecoderManager() {} + + virtual RefPtr<InitPromise> Init() = 0; + virtual const char* GetDescriptionName() const = 0; + + // Asynchronously send sample into mDecoder. If out of input buffer, aSample + // will be queued for later re-send. + nsresult Input(MediaRawData* aSample); + + // Flush the queued samples and signal decoder to throw all pending input/output away. + nsresult Flush(); + + // Shutdown decoder and rejects the init promise. + virtual nsresult Shutdown(); + + // How many samples are waiting for processing. + size_t NumQueuedSamples(); + + // Set callback for decoder events, such as requesting more input, + // returning output, or reporting error. + void SetDecodeCallback(MediaDataDecoderCallback* aCallback) + { + mDecodeCallback = aCallback; + } + +protected: + GonkDecoderManager() + : mMutex("GonkDecoderManager") + , mLastTime(INT64_MIN) + , mFlushMonitor("GonkDecoderManager::Flush") + , mIsFlushing(false) + , mDecodeCallback(nullptr) + {} + + bool InitLoopers(MediaData::Type aType); + + void onMessageReceived(const android::sp<android::AMessage> &aMessage) override; + + // Produces decoded output. It returns NS_OK on success, or NS_ERROR_NOT_AVAILABLE + // when output is not produced yet. + // If this returns a failure code other than NS_ERROR_NOT_AVAILABLE, an error + // will be reported through mDecodeCallback. + virtual nsresult Output(int64_t aStreamOffset, + RefPtr<MediaData>& aOutput) = 0; + + // Send queued samples to OMX. It returns how many samples are still in + // queue after processing, or negative error code if failed. + int32_t ProcessQueuedSamples(); + + void ProcessInput(bool aEndOfStream); + virtual void ProcessFlush(); + void ProcessToDo(bool aEndOfStream); + virtual void ResetEOS(); + + RefPtr<MediaByteBuffer> mCodecSpecificData; + + nsAutoCString mMimeType; + + // MediaCodedc's wrapper that performs the decoding. + android::sp<android::MediaCodecProxy> mDecoder; + // Looper for mDecoder to run on. + android::sp<android::ALooper> mDecodeLooper; + // Looper to run decode tasks such as processing input, output, flush, and + // recycling output buffers. + android::sp<android::ALooper> mTaskLooper; + // Message codes for tasks running on mTaskLooper. + enum { + // Decoder will send this to indicate internal state change such as input or + // output buffers availability. Used to run pending input & output tasks. + kNotifyDecoderActivity = 'nda ', + // Signal the decoder to flush. + kNotifyProcessFlush = 'npf ', + // Used to process queued samples when there is new input. + kNotifyProcessInput = 'npi ', +#ifdef DEBUG + kNotifyFindLooperId = 'nfli', +#endif + }; + + MozPromiseHolder<InitPromise> mInitPromise; + + Mutex mMutex; // Protects mQueuedSamples. + // A queue that stores the samples waiting to be sent to mDecoder. + // Empty element means EOS and there shouldn't be any sample be queued after it. + // Samples are queued in caller's thread and dequeued in mTaskLooper. + nsTArray<RefPtr<MediaRawData>> mQueuedSamples; + + // The last decoded frame presentation time. Only accessed on mTaskLooper. + int64_t mLastTime; + + Monitor mFlushMonitor; // Waits for flushing to complete. + bool mIsFlushing; // Protected by mFlushMonitor. + + // Remembers the notification that is currently waiting for the decoder event + // to avoid requesting more than one notification at the time, which is + // forbidden by mDecoder. + android::sp<android::AMessage> mToDo; + + // Stores sample info for output buffer processing later. + struct WaitOutputInfo { + WaitOutputInfo(int64_t aOffset, int64_t aTimestamp, bool aEOS) + : mOffset(aOffset) + , mTimestamp(aTimestamp) + , mEOS(aEOS) + {} + const int64_t mOffset; + const int64_t mTimestamp; + const bool mEOS; + }; + + nsTArray<WaitOutputInfo> mWaitOutput; + + MediaDataDecoderCallback* mDecodeCallback; // Reports decoder output or error. + +private: + void UpdateWaitingList(int64_t aForgetUpTo); + +#ifdef DEBUG + typedef void* LooperId; + + bool OnTaskLooper(); + LooperId mTaskLooperId; +#endif +}; + +class AutoReleaseMediaBuffer +{ +public: + AutoReleaseMediaBuffer(android::MediaBuffer* aBuffer, android::MediaCodecProxy* aCodec) + : mBuffer(aBuffer) + , mCodec(aCodec) + {} + + ~AutoReleaseMediaBuffer() + { + MOZ_ASSERT(mCodec.get()); + if (mBuffer) { + mCodec->ReleaseMediaBuffer(mBuffer); + } + } + + android::MediaBuffer* forget() + { + android::MediaBuffer* tmp = mBuffer; + mBuffer = nullptr; + return tmp; + } + +private: + android::MediaBuffer* mBuffer; + android::sp<android::MediaCodecProxy> mCodec; +}; + +// Samples are decoded using the GonkDecoder (MediaCodec) +// created by the GonkDecoderManager. This class implements +// the higher-level logic that drives mapping the Gonk to the async +// MediaDataDecoder interface. The specifics of decoding the exact stream +// type are handled by GonkDecoderManager and the GonkDecoder it creates. +class GonkMediaDataDecoder : public MediaDataDecoder { +public: + GonkMediaDataDecoder(GonkDecoderManager* aDecoderManager, + MediaDataDecoderCallback* aCallback); + + ~GonkMediaDataDecoder(); + + RefPtr<InitPromise> Init() override; + + void Input(MediaRawData* aSample) override; + + void Flush() override; + + void Drain() override; + + void Shutdown() override; + + const char* GetDescriptionName() const override + { + return "gonk decoder"; + } + +private: + + android::sp<GonkDecoderManager> mManager; +}; + +} // namespace mozilla + +#endif // GonkMediaDataDecoder_h_ diff --git a/dom/media/platforms/gonk/GonkVideoDecoderManager.cpp b/dom/media/platforms/gonk/GonkVideoDecoderManager.cpp new file mode 100644 index 000000000..0c7b3b6af --- /dev/null +++ b/dom/media/platforms/gonk/GonkVideoDecoderManager.cpp @@ -0,0 +1,772 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +#include "MediaCodecProxy.h" +#include <OMX_IVCommon.h> +#include <gui/Surface.h> +#include <ICrypto.h> +#include "GonkVideoDecoderManager.h" +#include "GrallocImages.h" +#include "MediaDecoderReader.h" +#include "ImageContainer.h" +#include "VideoUtils.h" +#include "nsThreadUtils.h" +#include "Layers.h" +#include "mozilla/Logging.h" +#include <stagefright/MediaBuffer.h> +#include <stagefright/MetaData.h> +#include <stagefright/MediaErrors.h> +#include <stagefright/foundation/AString.h> +#include "GonkNativeWindow.h" +#include "mozilla/layers/GrallocTextureClient.h" +#include "mozilla/layers/ImageBridgeChild.h" +#include "mozilla/layers/TextureClient.h" +#include "mozilla/layers/TextureClientRecycleAllocator.h" +#include <cutils/properties.h> + +#define CODECCONFIG_TIMEOUT_US 10000LL +#define READ_OUTPUT_BUFFER_TIMEOUT_US 0LL + +#include <android/log.h> +#define GVDM_LOG(...) __android_log_print(ANDROID_LOG_DEBUG, "GonkVideoDecoderManager", __VA_ARGS__) + +#define LOG(...) MOZ_LOG(sPDMLog, mozilla::LogLevel::Debug, (__VA_ARGS__)) +using namespace mozilla::layers; +using namespace android; +typedef android::MediaCodecProxy MediaCodecProxy; + +namespace mozilla { + +class GonkTextureClientAllocationHelper : public layers::ITextureClientAllocationHelper +{ +public: + GonkTextureClientAllocationHelper(uint32_t aGrallocFormat, + gfx::IntSize aSize) + : ITextureClientAllocationHelper(gfx::SurfaceFormat::UNKNOWN, + aSize, + BackendSelector::Content, + TextureFlags::DEALLOCATE_CLIENT, + ALLOC_DISALLOW_BUFFERTEXTURECLIENT) + , mGrallocFormat(aGrallocFormat) + {} + + already_AddRefed<TextureClient> Allocate(KnowsCompositor* aAllocator) override + { + uint32_t usage = android::GraphicBuffer::USAGE_SW_READ_OFTEN | + android::GraphicBuffer::USAGE_SW_WRITE_OFTEN | + android::GraphicBuffer::USAGE_HW_TEXTURE; + + GrallocTextureData* texData = GrallocTextureData::Create(mSize, mGrallocFormat, + gfx::BackendType::NONE, + usage, aAllocator->GetTextureForwarder()); + if (!texData) { + return nullptr; + } + sp<GraphicBuffer> graphicBuffer = texData->GetGraphicBuffer(); + if (!graphicBuffer.get()) { + return nullptr; + } + RefPtr<TextureClient> textureClient = + TextureClient::CreateWithData(texData, TextureFlags::DEALLOCATE_CLIENT, aAllocator->GetTextureForwarder()); + return textureClient.forget(); + } + + bool IsCompatible(TextureClient* aTextureClient) override + { + if (!aTextureClient) { + return false; + } + sp<GraphicBuffer> graphicBuffer = + static_cast<GrallocTextureData*>(aTextureClient->GetInternalData())->GetGraphicBuffer(); + if (!graphicBuffer.get() || + static_cast<uint32_t>(graphicBuffer->getPixelFormat()) != mGrallocFormat || + aTextureClient->GetSize() != mSize) { + return false; + } + return true; + } + +private: + uint32_t mGrallocFormat; +}; + +GonkVideoDecoderManager::GonkVideoDecoderManager( + mozilla::layers::ImageContainer* aImageContainer, + const VideoInfo& aConfig) + : mConfig(aConfig) + , mImageContainer(aImageContainer) + , mColorConverterBufferSize(0) + , mPendingReleaseItemsLock("GonkVideoDecoderManager::mPendingReleaseItemsLock") + , mNeedsCopyBuffer(false) +{ + MOZ_COUNT_CTOR(GonkVideoDecoderManager); +} + +GonkVideoDecoderManager::~GonkVideoDecoderManager() +{ + MOZ_COUNT_DTOR(GonkVideoDecoderManager); +} + +nsresult +GonkVideoDecoderManager::Shutdown() +{ + mVideoCodecRequest.DisconnectIfExists(); + return GonkDecoderManager::Shutdown(); +} + +RefPtr<MediaDataDecoder::InitPromise> +GonkVideoDecoderManager::Init() +{ + mNeedsCopyBuffer = false; + + uint32_t maxWidth, maxHeight; + char propValue[PROPERTY_VALUE_MAX]; + property_get("ro.moz.omx.hw.max_width", propValue, "-1"); + maxWidth = -1 == atoi(propValue) ? MAX_VIDEO_WIDTH : atoi(propValue); + property_get("ro.moz.omx.hw.max_height", propValue, "-1"); + maxHeight = -1 == atoi(propValue) ? MAX_VIDEO_HEIGHT : atoi(propValue) ; + + if (uint32_t(mConfig.mImage.width * mConfig.mImage.height) > maxWidth * maxHeight) { + GVDM_LOG("Video resolution exceeds hw codec capability"); + return InitPromise::CreateAndReject(NS_ERROR_DOM_MEDIA_FATAL_ERR, __func__); + } + + // Validate the container-reported frame and pictureRect sizes. This ensures + // that our video frame creation code doesn't overflow. + if (!IsValidVideoRegion(mConfig.mImage, mConfig.ImageRect(), mConfig.mDisplay)) { + GVDM_LOG("It is not a valid region"); + return InitPromise::CreateAndReject(NS_ERROR_DOM_MEDIA_FATAL_ERR, __func__); + } + + mReaderTaskQueue = AbstractThread::GetCurrent()->AsTaskQueue(); + MOZ_ASSERT(mReaderTaskQueue); + + if (mDecodeLooper.get() != nullptr) { + return InitPromise::CreateAndReject(NS_ERROR_DOM_MEDIA_FATAL_ERR, __func__); + } + + if (!InitLoopers(MediaData::VIDEO_DATA)) { + return InitPromise::CreateAndReject(NS_ERROR_DOM_MEDIA_FATAL_ERR, __func__); + } + + RefPtr<InitPromise> p = mInitPromise.Ensure(__func__); + android::sp<GonkVideoDecoderManager> self = this; + mDecoder = MediaCodecProxy::CreateByType(mDecodeLooper, + mConfig.mMimeType.get(), + false); + + uint32_t capability = MediaCodecProxy::kEmptyCapability; + if (mDecoder->getCapability(&capability) == OK && (capability & + MediaCodecProxy::kCanExposeGraphicBuffer)) { +#if ANDROID_VERSION >= 21 + sp<IGonkGraphicBufferConsumer> consumer; + GonkBufferQueue::createBufferQueue(&mGraphicBufferProducer, &consumer); + mNativeWindow = new GonkNativeWindow(consumer); +#else + mNativeWindow = new GonkNativeWindow(); +#endif + } + + mVideoCodecRequest.Begin(mDecoder->AsyncAllocateVideoMediaCodec() + ->Then(mReaderTaskQueue, __func__, + [self] (bool) -> void { + self->mVideoCodecRequest.Complete(); + self->codecReserved(); + }, [self] (bool) -> void { + self->mVideoCodecRequest.Complete(); + self->codecCanceled(); + })); + + return p; +} + +nsresult +GonkVideoDecoderManager::CreateVideoData(MediaBuffer* aBuffer, + int64_t aStreamOffset, + VideoData **v) +{ + *v = nullptr; + RefPtr<VideoData> data; + int64_t timeUs; + int32_t keyFrame; + + if (aBuffer == nullptr) { + GVDM_LOG("Video Buffer is not valid!"); + return NS_ERROR_UNEXPECTED; + } + + AutoReleaseMediaBuffer autoRelease(aBuffer, mDecoder.get()); + + if (!aBuffer->meta_data()->findInt64(kKeyTime, &timeUs)) { + GVDM_LOG("Decoder did not return frame time"); + return NS_ERROR_UNEXPECTED; + } + + if (mLastTime > timeUs) { + GVDM_LOG("Output decoded sample time is revert. time=%lld", timeUs); + return NS_ERROR_NOT_AVAILABLE; + } + mLastTime = timeUs; + + if (aBuffer->range_length() == 0) { + // Some decoders may return spurious empty buffers that we just want to ignore + // quoted from Android's AwesomePlayer.cpp + return NS_ERROR_NOT_AVAILABLE; + } + + if (!aBuffer->meta_data()->findInt32(kKeyIsSyncFrame, &keyFrame)) { + keyFrame = 0; + } + + gfx::IntRect picture = + mConfig.ScaledImageRect(mFrameInfo.mWidth, mFrameInfo.mHeight); + if (aBuffer->graphicBuffer().get()) { + data = CreateVideoDataFromGraphicBuffer(aBuffer, picture); + if (data && !mNeedsCopyBuffer) { + // RecycleCallback() will be responsible for release the buffer. + autoRelease.forget(); + } + mNeedsCopyBuffer = false; + } else { + data = CreateVideoDataFromDataBuffer(aBuffer, picture); + } + + if (!data) { + return NS_ERROR_UNEXPECTED; + } + // Fill necessary info. + data->mOffset = aStreamOffset; + data->mTime = timeUs; + data->mKeyframe = keyFrame; + + data.forget(v); + return NS_OK; +} + +// Copy pixels from one planar YUV to another. +static void +CopyYUV(PlanarYCbCrData& aSource, PlanarYCbCrData& aDestination) +{ + // Fill Y plane. + uint8_t* srcY = aSource.mYChannel; + gfx::IntSize ySize = aSource.mYSize; + uint8_t* destY = aDestination.mYChannel; + // Y plane. + for (int i = 0; i < ySize.height; i++) { + memcpy(destY, srcY, ySize.width); + srcY += aSource.mYStride; + destY += aDestination.mYStride; + } + + // Fill UV plane. + // Line start + uint8_t* srcU = aSource.mCbChannel; + uint8_t* srcV = aSource.mCrChannel; + uint8_t* destU = aDestination.mCbChannel; + uint8_t* destV = aDestination.mCrChannel; + + gfx::IntSize uvSize = aSource.mCbCrSize; + for (int i = 0; i < uvSize.height; i++) { + uint8_t* su = srcU; + uint8_t* sv = srcV; + uint8_t* du = destU; + uint8_t* dv =destV; + for (int j = 0; j < uvSize.width; j++) { + *du++ = *su++; + *dv++ = *sv++; + // Move to next pixel. + su += aSource.mCbSkip; + sv += aSource.mCrSkip; + du += aDestination.mCbSkip; + dv += aDestination.mCrSkip; + } + // Move to next line. + srcU += aSource.mCbCrStride; + srcV += aSource.mCbCrStride; + destU += aDestination.mCbCrStride; + destV += aDestination.mCbCrStride; + } +} + +inline static int +Align(int aX, int aAlign) +{ + return (aX + aAlign - 1) & ~(aAlign - 1); +} + +// Venus formats are doucmented in kernel/include/media/msm_media_info.h: +// * Y_Stride : Width aligned to 128 +// * UV_Stride : Width aligned to 128 +// * Y_Scanlines: Height aligned to 32 +// * UV_Scanlines: Height/2 aligned to 16 +// * Total size = align((Y_Stride * Y_Scanlines +// * + UV_Stride * UV_Scanlines + 4096), 4096) +static void +CopyVenus(uint8_t* aSrc, uint8_t* aDest, uint32_t aWidth, uint32_t aHeight) +{ + size_t yStride = Align(aWidth, 128); + uint8_t* s = aSrc; + uint8_t* d = aDest; + for (size_t i = 0; i < aHeight; i++) { + memcpy(d, s, aWidth); + s += yStride; + d += yStride; + } + size_t uvStride = yStride; + size_t uvLines = (aHeight + 1) / 2; + size_t ySize = yStride * Align(aHeight, 32); + s = aSrc + ySize; + d = aDest + ySize; + for (size_t i = 0; i < uvLines; i++) { + memcpy(d, s, aWidth); + s += uvStride; + d += uvStride; + } +} + +static void +CopyGraphicBuffer(sp<GraphicBuffer>& aSource, sp<GraphicBuffer>& aDestination) +{ + void* srcPtr = nullptr; + aSource->lock(GraphicBuffer::USAGE_SW_READ_OFTEN, &srcPtr); + void* destPtr = nullptr; + aDestination->lock(GraphicBuffer::USAGE_SW_WRITE_OFTEN, &destPtr); + MOZ_ASSERT(srcPtr && destPtr); + + // Build PlanarYCbCrData for source buffer. + PlanarYCbCrData srcData; + switch (aSource->getPixelFormat()) { + case HAL_PIXEL_FORMAT_YV12: { + // Android YV12 format is defined in system/core/include/system/graphics.h + srcData.mYChannel = static_cast<uint8_t*>(srcPtr); + srcData.mYSkip = 0; + srcData.mYSize.width = aSource->getWidth(); + srcData.mYSize.height = aSource->getHeight(); + srcData.mYStride = aSource->getStride(); + // 4:2:0. + srcData.mCbCrSize.width = srcData.mYSize.width / 2; + srcData.mCbCrSize.height = srcData.mYSize.height / 2; + srcData.mCrChannel = srcData.mYChannel + (srcData.mYStride * srcData.mYSize.height); + // Aligned to 16 bytes boundary. + srcData.mCbCrStride = Align(srcData.mYStride / 2, 16); + srcData.mCrSkip = 0; + srcData.mCbChannel = srcData.mCrChannel + (srcData.mCbCrStride * srcData.mCbCrSize.height); + srcData.mCbSkip = 0; + + // Build PlanarYCbCrData for destination buffer. + PlanarYCbCrData destData; + destData.mYChannel = static_cast<uint8_t*>(destPtr); + destData.mYSkip = 0; + destData.mYSize.width = aDestination->getWidth(); + destData.mYSize.height = aDestination->getHeight(); + destData.mYStride = aDestination->getStride(); + // 4:2:0. + destData.mCbCrSize.width = destData.mYSize.width / 2; + destData.mCbCrSize.height = destData.mYSize.height / 2; + destData.mCrChannel = destData.mYChannel + (destData.mYStride * destData.mYSize.height); + // Aligned to 16 bytes boundary. + destData.mCbCrStride = Align(destData.mYStride / 2, 16); + destData.mCrSkip = 0; + destData.mCbChannel = destData.mCrChannel + (destData.mCbCrStride * destData.mCbCrSize.height); + destData.mCbSkip = 0; + + CopyYUV(srcData, destData); + break; + } + case GrallocImage::HAL_PIXEL_FORMAT_YCbCr_420_SP_VENUS: + CopyVenus(static_cast<uint8_t*>(srcPtr), + static_cast<uint8_t*>(destPtr), + aSource->getWidth(), + aSource->getHeight()); + break; + default: + NS_ERROR("Unsupported input gralloc image type. Should never be here."); + } + + + aSource->unlock(); + aDestination->unlock(); +} + +already_AddRefed<VideoData> +GonkVideoDecoderManager::CreateVideoDataFromGraphicBuffer(MediaBuffer* aSource, + gfx::IntRect& aPicture) +{ + sp<GraphicBuffer> srcBuffer(aSource->graphicBuffer()); + RefPtr<TextureClient> textureClient; + + if (mNeedsCopyBuffer) { + // Copy buffer contents for bug 1199809. + if (!mCopyAllocator) { + RefPtr<layers::ImageBridgeChild> bridge = layers::ImageBridgeChild::GetSingleton(); + mCopyAllocator = new TextureClientRecycleAllocator(bridge); + } + if (!mCopyAllocator) { + GVDM_LOG("Create buffer allocator failed!"); + return nullptr; + } + + gfx::IntSize size(srcBuffer->getWidth(), srcBuffer->getHeight()); + GonkTextureClientAllocationHelper helper(srcBuffer->getPixelFormat(), size); + textureClient = mCopyAllocator->CreateOrRecycle(helper); + if (!textureClient) { + GVDM_LOG("Copy buffer allocation failed!"); + return nullptr; + } + + sp<GraphicBuffer> destBuffer = + static_cast<GrallocTextureData*>(textureClient->GetInternalData())->GetGraphicBuffer(); + + CopyGraphicBuffer(srcBuffer, destBuffer); + } else { + textureClient = mNativeWindow->getTextureClientFromBuffer(srcBuffer.get()); + textureClient->SetRecycleCallback(GonkVideoDecoderManager::RecycleCallback, this); + static_cast<GrallocTextureData*>(textureClient->GetInternalData())->SetMediaBuffer(aSource); + } + + RefPtr<VideoData> data = + VideoData::CreateAndCopyIntoTextureClient(mConfig, + 0, // Filled later by caller. + 0, // Filled later by caller. + 1, // No way to pass sample duration from muxer to + // OMX codec, so we hardcode the duration here. + textureClient, + false, // Filled later by caller. + -1, + aPicture); + return data.forget(); +} + +already_AddRefed<VideoData> +GonkVideoDecoderManager::CreateVideoDataFromDataBuffer(MediaBuffer* aSource, gfx::IntRect& aPicture) +{ + if (!aSource->data()) { + GVDM_LOG("No data in Video Buffer!"); + return nullptr; + } + uint8_t *yuv420p_buffer = (uint8_t *)aSource->data(); + int32_t stride = mFrameInfo.mStride; + int32_t slice_height = mFrameInfo.mSliceHeight; + + // Converts to OMX_COLOR_FormatYUV420Planar + if (mFrameInfo.mColorFormat != OMX_COLOR_FormatYUV420Planar) { + ARect crop; + crop.top = 0; + crop.bottom = mFrameInfo.mHeight; + crop.left = 0; + crop.right = mFrameInfo.mWidth; + yuv420p_buffer = GetColorConverterBuffer(mFrameInfo.mWidth, mFrameInfo.mHeight); + if (mColorConverter.convertDecoderOutputToI420(aSource->data(), + mFrameInfo.mWidth, mFrameInfo.mHeight, crop, yuv420p_buffer) != OK) { + GVDM_LOG("Color conversion failed!"); + return nullptr; + } + stride = mFrameInfo.mWidth; + slice_height = mFrameInfo.mHeight; + } + + size_t yuv420p_y_size = stride * slice_height; + size_t yuv420p_u_size = ((stride + 1) / 2) * ((slice_height + 1) / 2); + uint8_t *yuv420p_y = yuv420p_buffer; + uint8_t *yuv420p_u = yuv420p_y + yuv420p_y_size; + uint8_t *yuv420p_v = yuv420p_u + yuv420p_u_size; + + VideoData::YCbCrBuffer b; + b.mPlanes[0].mData = yuv420p_y; + b.mPlanes[0].mWidth = mFrameInfo.mWidth; + b.mPlanes[0].mHeight = mFrameInfo.mHeight; + b.mPlanes[0].mStride = stride; + b.mPlanes[0].mOffset = 0; + b.mPlanes[0].mSkip = 0; + + b.mPlanes[1].mData = yuv420p_u; + b.mPlanes[1].mWidth = (mFrameInfo.mWidth + 1) / 2; + b.mPlanes[1].mHeight = (mFrameInfo.mHeight + 1) / 2; + b.mPlanes[1].mStride = (stride + 1) / 2; + b.mPlanes[1].mOffset = 0; + b.mPlanes[1].mSkip = 0; + + b.mPlanes[2].mData = yuv420p_v; + b.mPlanes[2].mWidth =(mFrameInfo.mWidth + 1) / 2; + b.mPlanes[2].mHeight = (mFrameInfo.mHeight + 1) / 2; + b.mPlanes[2].mStride = (stride + 1) / 2; + b.mPlanes[2].mOffset = 0; + b.mPlanes[2].mSkip = 0; + + RefPtr<VideoData> data = + VideoData::CreateAndCopyData(mConfig, + mImageContainer, + 0, // Filled later by caller. + 0, // Filled later by caller. + 1, // We don't know the duration. + b, + 0, // Filled later by caller. + -1, + aPicture); + + return data.forget(); +} + +bool +GonkVideoDecoderManager::SetVideoFormat() +{ + // read video metadata from MediaCodec + sp<AMessage> codecFormat; + if (mDecoder->getOutputFormat(&codecFormat) == OK) { + AString mime; + int32_t width = 0; + int32_t height = 0; + int32_t stride = 0; + int32_t slice_height = 0; + int32_t color_format = 0; + int32_t crop_left = 0; + int32_t crop_top = 0; + int32_t crop_right = 0; + int32_t crop_bottom = 0; + if (!codecFormat->findString("mime", &mime) || + !codecFormat->findInt32("width", &width) || + !codecFormat->findInt32("height", &height) || + !codecFormat->findInt32("stride", &stride) || + !codecFormat->findInt32("slice-height", &slice_height) || + !codecFormat->findInt32("color-format", &color_format) || + !codecFormat->findRect("crop", &crop_left, &crop_top, &crop_right, &crop_bottom)) { + GVDM_LOG("Failed to find values"); + return false; + } + mFrameInfo.mWidth = width; + mFrameInfo.mHeight = height; + mFrameInfo.mStride = stride; + mFrameInfo.mSliceHeight = slice_height; + mFrameInfo.mColorFormat = color_format; + + nsIntSize displaySize(width, height); + if (!IsValidVideoRegion(mConfig.mDisplay, + mConfig.ScaledImageRect(width, height), + displaySize)) { + GVDM_LOG("It is not a valid region"); + return false; + } + return true; + } + GVDM_LOG("Fail to get output format"); + return false; +} + +// Blocks until decoded sample is produced by the deoder. +nsresult +GonkVideoDecoderManager::Output(int64_t aStreamOffset, + RefPtr<MediaData>& aOutData) +{ + aOutData = nullptr; + status_t err; + if (mDecoder == nullptr) { + GVDM_LOG("Decoder is not inited"); + return NS_ERROR_UNEXPECTED; + } + MediaBuffer* outputBuffer = nullptr; + err = mDecoder->Output(&outputBuffer, READ_OUTPUT_BUFFER_TIMEOUT_US); + + switch (err) { + case OK: + { + RefPtr<VideoData> data; + nsresult rv = CreateVideoData(outputBuffer, aStreamOffset, getter_AddRefs(data)); + if (rv == NS_ERROR_NOT_AVAILABLE) { + // Decoder outputs a empty video buffer, try again + return NS_ERROR_NOT_AVAILABLE; + } else if (rv != NS_OK || data == nullptr) { + GVDM_LOG("Failed to create VideoData"); + return NS_ERROR_UNEXPECTED; + } + aOutData = data; + return NS_OK; + } + case android::INFO_FORMAT_CHANGED: + { + // If the format changed, update our cached info. + GVDM_LOG("Decoder format changed"); + if (!SetVideoFormat()) { + return NS_ERROR_UNEXPECTED; + } + return Output(aStreamOffset, aOutData); + } + case android::INFO_OUTPUT_BUFFERS_CHANGED: + { + if (mDecoder->UpdateOutputBuffers()) { + return Output(aStreamOffset, aOutData); + } + GVDM_LOG("Fails to update output buffers!"); + return NS_ERROR_FAILURE; + } + case -EAGAIN: + { +// GVDM_LOG("Need to try again!"); + return NS_ERROR_NOT_AVAILABLE; + } + case android::ERROR_END_OF_STREAM: + { + GVDM_LOG("Got the EOS frame!"); + RefPtr<VideoData> data; + nsresult rv = CreateVideoData(outputBuffer, aStreamOffset, getter_AddRefs(data)); + if (rv == NS_ERROR_NOT_AVAILABLE) { + // For EOS, no need to do any thing. + return NS_ERROR_ABORT; + } + if (rv != NS_OK || data == nullptr) { + GVDM_LOG("Failed to create video data"); + return NS_ERROR_UNEXPECTED; + } + aOutData = data; + return NS_ERROR_ABORT; + } + case -ETIMEDOUT: + { + GVDM_LOG("Timeout. can try again next time"); + return NS_ERROR_UNEXPECTED; + } + default: + { + GVDM_LOG("Decoder failed, err=%d", err); + return NS_ERROR_UNEXPECTED; + } + } + + return NS_OK; +} + +void +GonkVideoDecoderManager::codecReserved() +{ + if (mInitPromise.IsEmpty()) { + return; + } + GVDM_LOG("codecReserved"); + sp<AMessage> format = new AMessage; + sp<Surface> surface; + status_t rv = OK; + // Fixed values + GVDM_LOG("Configure video mime type: %s, width:%d, height:%d", mConfig.mMimeType.get(), mConfig.mImage.width, mConfig.mImage.height); + format->setString("mime", mConfig.mMimeType.get()); + format->setInt32("width", mConfig.mImage.width); + format->setInt32("height", mConfig.mImage.height); + // Set the "moz-use-undequeued-bufs" to use the undeque buffers to accelerate + // the video decoding. + format->setInt32("moz-use-undequeued-bufs", 1); + if (mNativeWindow != nullptr) { +#if ANDROID_VERSION >= 21 + surface = new Surface(mGraphicBufferProducer); +#else + surface = new Surface(mNativeWindow->getBufferQueue()); +#endif + } + mDecoder->configure(format, surface, nullptr, 0); + mDecoder->Prepare(); + + if (mConfig.mMimeType.EqualsLiteral("video/mp4v-es")) { + rv = mDecoder->Input(mConfig.mCodecSpecificConfig->Elements(), + mConfig.mCodecSpecificConfig->Length(), 0, + android::MediaCodec::BUFFER_FLAG_CODECCONFIG, + CODECCONFIG_TIMEOUT_US); + } + + if (rv != OK) { + GVDM_LOG("Failed to configure codec!!!!"); + mInitPromise.Reject(NS_ERROR_DOM_MEDIA_FATAL_ERR, __func__); + return; + } + + mInitPromise.Resolve(TrackType::kVideoTrack, __func__); +} + +void +GonkVideoDecoderManager::codecCanceled() +{ + GVDM_LOG("codecCanceled"); + mInitPromise.RejectIfExists(NS_ERROR_DOM_MEDIA_CANCELED, __func__); +} + +// Called on GonkDecoderManager::mTaskLooper thread. +void +GonkVideoDecoderManager::onMessageReceived(const sp<AMessage> &aMessage) +{ + switch (aMessage->what()) { + case kNotifyPostReleaseBuffer: + { + ReleaseAllPendingVideoBuffers(); + break; + } + + default: + { + GonkDecoderManager::onMessageReceived(aMessage); + break; + } + } +} + +uint8_t * +GonkVideoDecoderManager::GetColorConverterBuffer(int32_t aWidth, int32_t aHeight) +{ + // Allocate a temporary YUV420Planer buffer. + size_t yuv420p_y_size = aWidth * aHeight; + size_t yuv420p_u_size = ((aWidth + 1) / 2) * ((aHeight + 1) / 2); + size_t yuv420p_v_size = yuv420p_u_size; + size_t yuv420p_size = yuv420p_y_size + yuv420p_u_size + yuv420p_v_size; + if (mColorConverterBufferSize != yuv420p_size) { + mColorConverterBuffer = MakeUnique<uint8_t[]>(yuv420p_size); + mColorConverterBufferSize = yuv420p_size; + } + return mColorConverterBuffer.get(); +} + +/* static */ +void +GonkVideoDecoderManager::RecycleCallback(TextureClient* aClient, void* aClosure) +{ + MOZ_ASSERT(aClient && !aClient->IsDead()); + GonkVideoDecoderManager* videoManager = static_cast<GonkVideoDecoderManager*>(aClosure); + GrallocTextureData* client = static_cast<GrallocTextureData*>(aClient->GetInternalData()); + aClient->ClearRecycleCallback(); + FenceHandle handle = aClient->GetAndResetReleaseFenceHandle(); + videoManager->PostReleaseVideoBuffer(client->GetMediaBuffer(), handle); +} + +void GonkVideoDecoderManager::PostReleaseVideoBuffer( + android::MediaBuffer *aBuffer, + FenceHandle aReleaseFence) +{ + { + MutexAutoLock autoLock(mPendingReleaseItemsLock); + if (aBuffer) { + mPendingReleaseItems.AppendElement(ReleaseItem(aBuffer, aReleaseFence)); + } + } + sp<AMessage> notify = + new AMessage(kNotifyPostReleaseBuffer, id()); + notify->post(); + +} + +void GonkVideoDecoderManager::ReleaseAllPendingVideoBuffers() +{ + nsTArray<ReleaseItem> releasingItems; + { + MutexAutoLock autoLock(mPendingReleaseItemsLock); + releasingItems.AppendElements(mPendingReleaseItems); + mPendingReleaseItems.Clear(); + } + + // Free all pending video buffers without holding mPendingReleaseItemsLock. + size_t size = releasingItems.Length(); + for (size_t i = 0; i < size; i++) { + RefPtr<FenceHandle::FdObj> fdObj = releasingItems[i].mReleaseFence.GetAndResetFdObj(); + sp<Fence> fence = new Fence(fdObj->GetAndResetFd()); + fence->waitForever("GonkVideoDecoderManager"); + mDecoder->ReleaseMediaBuffer(releasingItems[i].mBuffer); + } + releasingItems.Clear(); +} + +} // namespace mozilla diff --git a/dom/media/platforms/gonk/GonkVideoDecoderManager.h b/dom/media/platforms/gonk/GonkVideoDecoderManager.h new file mode 100644 index 000000000..343bb2a5c --- /dev/null +++ b/dom/media/platforms/gonk/GonkVideoDecoderManager.h @@ -0,0 +1,149 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#if !defined(GonkVideoDecoderManager_h_) +#define GonkVideoDecoderManager_h_ + +#include "nsRect.h" +#include "GonkMediaDataDecoder.h" +#include "mozilla/RefPtr.h" +#include "I420ColorConverterHelper.h" +#include "MediaCodecProxy.h" +#include "GonkNativeWindow.h" +#include "mozilla/layers/FenceUtils.h" +#include "mozilla/UniquePtr.h" +#include <ui/Fence.h> + +using namespace android; + +namespace android { +class MediaBuffer; +struct MOZ_EXPORT AString; +class GonkNativeWindow; +} // namespace android + +namespace mozilla { + +namespace layers { +class TextureClient; +class TextureClientRecycleAllocator; +} // namespace mozilla::layers + +class GonkVideoDecoderManager : public GonkDecoderManager { +typedef android::MediaCodecProxy MediaCodecProxy; +typedef mozilla::layers::TextureClient TextureClient; + +public: + GonkVideoDecoderManager(mozilla::layers::ImageContainer* aImageContainer, + const VideoInfo& aConfig); + + virtual ~GonkVideoDecoderManager(); + + RefPtr<InitPromise> Init() override; + + nsresult Output(int64_t aStreamOffset, + RefPtr<MediaData>& aOutput) override; + + nsresult Shutdown() override; + + const char* GetDescriptionName() const override + { + return "gonk video decoder"; + } + + static void RecycleCallback(TextureClient* aClient, void* aClosure); + +protected: + // Bug 1199809: workaround to avoid sending the graphic buffer by making a + // copy of output buffer after calling flush(). Bug 1203859 was created to + // reimplementing Gonk PDM on top of OpenMax IL directly. Its buffer + // management will work better with Gecko and solve problems like this. + void ProcessFlush() override + { + mNeedsCopyBuffer = true; + GonkDecoderManager::ProcessFlush(); + } + +private: + struct FrameInfo + { + int32_t mWidth = 0; + int32_t mHeight = 0; + int32_t mStride = 0; + int32_t mSliceHeight = 0; + int32_t mColorFormat = 0; + int32_t mCropLeft = 0; + int32_t mCropTop = 0; + int32_t mCropRight = 0; + int32_t mCropBottom = 0; + }; + + void onMessageReceived(const android::sp<android::AMessage> &aMessage) override; + + bool SetVideoFormat(); + + nsresult CreateVideoData(MediaBuffer* aBuffer, int64_t aStreamOffset, VideoData** aOutData); + already_AddRefed<VideoData> CreateVideoDataFromGraphicBuffer(android::MediaBuffer* aSource, + gfx::IntRect& aPicture); + already_AddRefed<VideoData> CreateVideoDataFromDataBuffer(android::MediaBuffer* aSource, + gfx::IntRect& aPicture); + + uint8_t* GetColorConverterBuffer(int32_t aWidth, int32_t aHeight); + + // For codec resource management + void codecReserved(); + void codecCanceled(); + + void ReleaseAllPendingVideoBuffers(); + void PostReleaseVideoBuffer(android::MediaBuffer *aBuffer, + layers::FenceHandle mReleaseFence); + + VideoInfo mConfig; + + RefPtr<layers::ImageContainer> mImageContainer; + RefPtr<layers::TextureClientRecycleAllocator> mCopyAllocator; + + MozPromiseRequestHolder<android::MediaCodecProxy::CodecPromise> mVideoCodecRequest; + FrameInfo mFrameInfo; + + // color converter + android::I420ColorConverterHelper mColorConverter; + UniquePtr<uint8_t[]> mColorConverterBuffer; + size_t mColorConverterBufferSize; + + android::sp<android::GonkNativeWindow> mNativeWindow; +#if ANDROID_VERSION >= 21 + android::sp<android::IGraphicBufferProducer> mGraphicBufferProducer; +#endif + + enum { + kNotifyPostReleaseBuffer = 'nprb', + }; + + struct ReleaseItem { + ReleaseItem(android::MediaBuffer* aBuffer, layers::FenceHandle& aFence) + : mBuffer(aBuffer) + , mReleaseFence(aFence) {} + android::MediaBuffer* mBuffer; + layers::FenceHandle mReleaseFence; + }; + nsTArray<ReleaseItem> mPendingReleaseItems; + + // The lock protects mPendingReleaseItems. + Mutex mPendingReleaseItemsLock; + + // This TaskQueue should be the same one in mDecodeCallback->OnReaderTaskQueue(). + // It is for codec resource mangement, decoding task should not dispatch to it. + RefPtr<TaskQueue> mReaderTaskQueue; + + // Bug 1199809: do we need to make a copy of output buffer? Used only when + // the decoder outputs graphic buffers. + bool mNeedsCopyBuffer; +}; + +} // namespace mozilla + +#endif // GonkVideoDecoderManager_h_ diff --git a/dom/media/platforms/gonk/moz.build b/dom/media/platforms/gonk/moz.build new file mode 100644 index 000000000..014594977 --- /dev/null +++ b/dom/media/platforms/gonk/moz.build @@ -0,0 +1,39 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +EXPORTS += [ + 'GonkAudioDecoderManager.h', + 'GonkDecoderModule.h', + 'GonkMediaDataDecoder.h', + 'GonkVideoDecoderManager.h', +] +UNIFIED_SOURCES += [ + 'GonkAudioDecoderManager.cpp', + 'GonkDecoderModule.cpp', + 'GonkMediaDataDecoder.cpp', + 'GonkVideoDecoderManager.cpp', +] +LOCAL_INCLUDES += [ + '/dom/media/omx/', +] +include('/ipc/chromium/chromium-config.mozbuild') + +# Suppress some GCC/clang warnings being treated as errors: +# - about attributes on forward declarations for types that are already +# defined, which complains about an important MOZ_EXPORT for android::AString +# - about multi-character constants which are used in codec-related code +if CONFIG['GNU_CC'] or CONFIG['CLANG_CL']: + CXXFLAGS += [ + '-Wno-error=attributes', + '-Wno-error=multichar' + ] + +FINAL_LIBRARY = 'xul' + +LOCAL_INCLUDES += [ + '%' + '%s/%s' % (CONFIG['ANDROID_SOURCE'], d) for d in [ + 'frameworks/native/opengl/include',] +] |