diff options
Diffstat (limited to 'dom/media/platforms/omx')
| -rw-r--r-- | dom/media/platforms/omx/GonkOmxPlatformLayer.cpp | 667 | ||||
| -rw-r--r-- | dom/media/platforms/omx/GonkOmxPlatformLayer.h | 205 | ||||
| -rw-r--r-- | dom/media/platforms/omx/OmxDataDecoder.cpp | 1038 | ||||
| -rw-r--r-- | dom/media/platforms/omx/OmxDataDecoder.h | 214 | ||||
| -rw-r--r-- | dom/media/platforms/omx/OmxDecoderModule.cpp | 45 | ||||
| -rw-r--r-- | dom/media/platforms/omx/OmxDecoderModule.h | 30 | ||||
| -rw-r--r-- | dom/media/platforms/omx/OmxPlatformLayer.cpp | 327 | ||||
| -rw-r--r-- | dom/media/platforms/omx/OmxPlatformLayer.h | 100 | ||||
| -rw-r--r-- | dom/media/platforms/omx/OmxPromiseLayer.cpp | 382 | ||||
| -rw-r--r-- | dom/media/platforms/omx/OmxPromiseLayer.h | 252 | ||||
| -rw-r--r-- | dom/media/platforms/omx/moz.build | 57 |
11 files changed, 3317 insertions, 0 deletions
diff --git a/dom/media/platforms/omx/GonkOmxPlatformLayer.cpp b/dom/media/platforms/omx/GonkOmxPlatformLayer.cpp new file mode 100644 index 000000000..870566cf5 --- /dev/null +++ b/dom/media/platforms/omx/GonkOmxPlatformLayer.cpp @@ -0,0 +1,667 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "GonkOmxPlatformLayer.h" + +#include <binder/MemoryDealer.h> +#include <cutils/properties.h> +#include <media/IOMX.h> +#include <media/stagefright/MediaCodecList.h> +#include <utils/List.h> + +#include "mozilla/Monitor.h" +#include "mozilla/layers/TextureClient.h" +#include "mozilla/layers/GrallocTextureClient.h" +#include "mozilla/layers/ImageBridgeChild.h" +#include "mozilla/layers/TextureClientRecycleAllocator.h" + +#include "ImageContainer.h" +#include "MediaInfo.h" +#include "OmxDataDecoder.h" + + +#ifdef LOG +#undef LOG +#endif + +#define LOG(arg, ...) MOZ_LOG(sPDMLog, mozilla::LogLevel::Debug, ("GonkOmxPlatformLayer(%p)::%s: " arg, this, __func__, ##__VA_ARGS__)) + +#define CHECK_ERR(err) \ + if (err != OK) { \ + LOG("error %d at %s", err, __func__); \ + return NS_ERROR_FAILURE; \ + } \ + +// Android proprietary value. +#define ANDROID_OMX_VIDEO_CodingVP8 (static_cast<OMX_VIDEO_CODINGTYPE>(9)) + +using namespace android; + +namespace mozilla { + +// In Gonk, the software component name has prefix "OMX.google". It needs to +// have a way to use hardware codec first. +bool IsSoftwareCodec(const char* aComponentName) +{ + nsAutoCString str(aComponentName); + return (str.Find(NS_LITERAL_CSTRING("OMX.google.")) == -1 ? false : true); +} + +bool IsInEmulator() +{ + char propQemu[PROPERTY_VALUE_MAX]; + property_get("ro.kernel.qemu", propQemu, ""); + return !strncmp(propQemu, "1", 1); +} + +class GonkOmxObserver : public BnOMXObserver { +public: + void onMessage(const omx_message& aMsg) + { + switch (aMsg.type) { + case omx_message::EVENT: + { + sp<GonkOmxObserver> self = this; + nsCOMPtr<nsIRunnable> r = NS_NewRunnableFunction([self, aMsg] () { + if (self->mClient && self->mClient->Event(aMsg.u.event_data.event, + aMsg.u.event_data.data1, + aMsg.u.event_data.data2)) + { + return; + } + }); + mTaskQueue->Dispatch(r.forget()); + break; + } + case omx_message::EMPTY_BUFFER_DONE: + { + sp<GonkOmxObserver> self = this; + nsCOMPtr<nsIRunnable> r = NS_NewRunnableFunction([self, aMsg] () { + if (!self->mPromiseLayer) { + return; + } + BufferData::BufferID id = (BufferData::BufferID)aMsg.u.buffer_data.buffer; + self->mPromiseLayer->EmptyFillBufferDone(OMX_DirInput, id); + }); + mTaskQueue->Dispatch(r.forget()); + break; + } + case omx_message::FILL_BUFFER_DONE: + { + sp<GonkOmxObserver> self = this; + nsCOMPtr<nsIRunnable> r = NS_NewRunnableFunction([self, aMsg] () { + if (!self->mPromiseLayer) { + return; + } + + // TODO: these codes look a little ugly, it'd be better to improve them. + RefPtr<BufferData> buf; + BufferData::BufferID id = (BufferData::BufferID)aMsg.u.extended_buffer_data.buffer; + buf = self->mPromiseLayer->FindAndRemoveBufferHolder(OMX_DirOutput, id); + MOZ_RELEASE_ASSERT(buf); + GonkBufferData* gonkBuffer = static_cast<GonkBufferData*>(buf.get()); + + // Copy the critical information to local buffer. + if (gonkBuffer->IsLocalBuffer()) { + gonkBuffer->mBuffer->nOffset = aMsg.u.extended_buffer_data.range_offset; + gonkBuffer->mBuffer->nFilledLen = aMsg.u.extended_buffer_data.range_length; + gonkBuffer->mBuffer->nFlags = aMsg.u.extended_buffer_data.flags; + gonkBuffer->mBuffer->nTimeStamp = aMsg.u.extended_buffer_data.timestamp; + } + self->mPromiseLayer->EmptyFillBufferDone(OMX_DirOutput, buf); + }); + mTaskQueue->Dispatch(r.forget()); + break; + } + default: + { + LOG("Unhandle event %d", aMsg.type); + } + } + } + + void Shutdown() + { + MOZ_ASSERT(mTaskQueue->IsCurrentThreadIn()); + mPromiseLayer = nullptr; + mClient = nullptr; + } + + GonkOmxObserver(TaskQueue* aTaskQueue, OmxPromiseLayer* aPromiseLayer, OmxDataDecoder* aDataDecoder) + : mTaskQueue(aTaskQueue) + , mPromiseLayer(aPromiseLayer) + , mClient(aDataDecoder) + {} + +protected: + RefPtr<TaskQueue> mTaskQueue; + // TODO: + // we should combine both event handlers into one. And we should provide + // an unified way for event handling in OmxPlatformLayer class. + RefPtr<OmxPromiseLayer> mPromiseLayer; + RefPtr<OmxDataDecoder> mClient; +}; + +// This class allocates Gralloc buffer and manages TextureClient's recycle. +class GonkTextureClientRecycleHandler : public layers::ITextureClientRecycleAllocator +{ + typedef MozPromise<layers::TextureClient*, nsresult, /* IsExclusive = */ true> TextureClientRecyclePromise; + +public: + GonkTextureClientRecycleHandler(OMX_VIDEO_PORTDEFINITIONTYPE& aDef) + : ITextureClientRecycleAllocator() + , mMonitor("GonkTextureClientRecycleHandler") + { + RefPtr<layers::ImageBridgeChild> bridge = layers::ImageBridgeChild::GetSingleton(); + + // Allocate Gralloc texture memory. + layers::GrallocTextureData* textureData = + layers::GrallocTextureData::Create(gfx::IntSize(aDef.nFrameWidth, aDef.nFrameHeight), + aDef.eColorFormat, + gfx::BackendType::NONE, + GraphicBuffer::USAGE_HW_TEXTURE | GraphicBuffer::USAGE_SW_READ_OFTEN, + bridge); + + mGraphBuffer = textureData->GetGraphicBuffer(); + MOZ_ASSERT(mGraphBuffer.get()); + + mTextureClient = + layers::TextureClient::CreateWithData(textureData, + layers::TextureFlags::DEALLOCATE_CLIENT | layers::TextureFlags::RECYCLE, + bridge); + MOZ_ASSERT(mTextureClient); + + mPromise.SetMonitor(&mMonitor); + } + + RefPtr<TextureClientRecyclePromise> WaitforRecycle() + { + MonitorAutoLock lock(mMonitor); + MOZ_ASSERT(!!mGraphBuffer.get()); + + mTextureClient->SetRecycleAllocator(this); + return mPromise.Ensure(__func__); + } + + // DO NOT use smart pointer to receive TextureClient; otherwise it will + // distrupt the reference count. + layers::TextureClient* GetTextureClient() + { + return mTextureClient; + } + + GraphicBuffer* GetGraphicBuffer() + { + MonitorAutoLock lock(mMonitor); + return mGraphBuffer.get(); + } + + // This function is called from layers thread. + void RecycleTextureClient(layers::TextureClient* aClient) override + { + MOZ_ASSERT(mTextureClient == aClient); + + // Clearing the recycle allocator drops a reference, so make sure we stay alive + // for the duration of this function. + RefPtr<GonkTextureClientRecycleHandler> kungFuDeathGrip(this); + aClient->SetRecycleAllocator(nullptr); + + { + MonitorAutoLock lock(mMonitor); + mPromise.ResolveIfExists(mTextureClient, __func__); + } + } + + void Shutdown() + { + MonitorAutoLock lock(mMonitor); + + mPromise.RejectIfExists(NS_ERROR_FAILURE, __func__); + + // DO NOT clear TextureClient here. + // The ref count could be 1 and RecycleCallback will be called if we clear + // the ref count here. That breaks the whole mechanism. (RecycleCallback + // should be called from layers) + mGraphBuffer = nullptr; + } + +private: + // Because TextureClient calls RecycleCallbackl when ref count is 1, so we + // should hold only one reference here and use raw pointer when out of this + // class. + RefPtr<layers::TextureClient> mTextureClient; + + // It is protected by mMonitor. + sp<android::GraphicBuffer> mGraphBuffer; + + // It is protected by mMonitor. + MozPromiseHolder<TextureClientRecyclePromise> mPromise; + + Monitor mMonitor; +}; + +GonkBufferData::GonkBufferData(bool aLiveInLocal, + GonkOmxPlatformLayer* aGonkPlatformLayer) + : BufferData(nullptr) + , mId(0) + , mGonkPlatformLayer(aGonkPlatformLayer) +{ + if (!aLiveInLocal) { + mMirrorBuffer = new OMX_BUFFERHEADERTYPE; + PodZero(mMirrorBuffer.get()); + mBuffer = mMirrorBuffer.get(); + } +} + +void +GonkBufferData::ReleaseBuffer() +{ + if (mTextureClientRecycleHandler) { + mTextureClientRecycleHandler->Shutdown(); + mTextureClientRecycleHandler = nullptr; + } +} + +nsresult +GonkBufferData::InitSharedMemory(android::IMemory* aMemory) +{ + MOZ_RELEASE_ASSERT(mMirrorBuffer.get()); + + // aMemory is a IPC memory, it is safe to use it here. + mBuffer->pBuffer = (OMX_U8*)aMemory->pointer(); + mBuffer->nAllocLen = aMemory->size(); + return NS_OK; +} + +nsresult +GonkBufferData::InitLocalBuffer(IOMX::buffer_id aId) +{ + MOZ_RELEASE_ASSERT(!mMirrorBuffer.get()); + + mBuffer = (OMX_BUFFERHEADERTYPE*)aId; + return NS_OK; +} + +nsresult +GonkBufferData::InitGraphicBuffer(OMX_VIDEO_PORTDEFINITIONTYPE& aDef) +{ + mTextureClientRecycleHandler = new GonkTextureClientRecycleHandler(aDef); + + if (!mTextureClientRecycleHandler->GetGraphicBuffer()) { + return NS_ERROR_FAILURE; + } + + return NS_OK; +} + +already_AddRefed<MediaData> +GonkBufferData::GetPlatformMediaData() +{ + if (mGonkPlatformLayer->GetTrackInfo()->GetAsAudioInfo()) { + // This is audio decoding. + return nullptr; + } + + if (!mTextureClientRecycleHandler) { + // There is no GraphicBuffer, it should fallback to normal YUV420 VideoData. + return nullptr; + } + + VideoInfo info(*mGonkPlatformLayer->GetTrackInfo()->GetAsVideoInfo()); + RefPtr<VideoData> data = + VideoData::CreateAndCopyIntoTextureClient(info, + 0, + mBuffer->nTimeStamp, + 1, + mTextureClientRecycleHandler->GetTextureClient(), + false, + 0, + info.ImageRect()); + LOG("%p, disp width %d, height %d, pic width %d, height %d, time %ld", + this, info.mDisplay.width, info.mDisplay.height, + info.mImage.width, info.mImage.height, mBuffer->nTimeStamp); + + // Get TextureClient Promise here to wait for resolved. + RefPtr<GonkBufferData> self(this); + mTextureClientRecycleHandler->WaitforRecycle() + ->Then(mGonkPlatformLayer->GetTaskQueue(), __func__, + [self] () { + self->mPromise.ResolveIfExists(self, __func__); + }, + [self] () { + OmxBufferFailureHolder failure(OMX_ErrorUndefined, self); + self->mPromise.RejectIfExists(failure, __func__); + }); + + return data.forget(); +} + +GonkOmxPlatformLayer::GonkOmxPlatformLayer(OmxDataDecoder* aDataDecoder, + OmxPromiseLayer* aPromiseLayer, + TaskQueue* aTaskQueue, + layers::ImageContainer* aImageContainer) + : mTaskQueue(aTaskQueue) + , mImageContainer(aImageContainer) + , mNode(0) +{ + mOmxObserver = new GonkOmxObserver(mTaskQueue, aPromiseLayer, aDataDecoder); +} + +nsresult +GonkOmxPlatformLayer::AllocateOmxBuffer(OMX_DIRTYPE aType, + BUFFERLIST* aBufferList) +{ + MOZ_ASSERT(!mMemoryDealer[aType].get()); + + // Get port definition. + OMX_PARAM_PORTDEFINITIONTYPE def; + nsTArray<uint32_t> portindex; + GetPortIndices(portindex); + for (auto idx : portindex) { + InitOmxParameter(&def); + def.nPortIndex = idx; + + OMX_ERRORTYPE err = GetParameter(OMX_IndexParamPortDefinition, + &def, + sizeof(OMX_PARAM_PORTDEFINITIONTYPE)); + if (err != OMX_ErrorNone) { + return NS_ERROR_FAILURE; + } else if (def.eDir == aType) { + LOG("Get OMX_IndexParamPortDefinition: port: %d, type: %d", def.nPortIndex, def.eDir); + break; + } + } + + size_t t = 0; + + // Configure video output GraphicBuffer for video decoding acceleration. + bool useGralloc = false; + if (aType == OMX_DirOutput && mQuirks.test(kRequiresAllocateBufferOnOutputPorts) && + (def.eDomain == OMX_PortDomainVideo)) { + if (NS_FAILED(EnableOmxGraphicBufferPort(def))) { + return NS_ERROR_FAILURE; + } + + LOG("Enable OMX GraphicBuffer port, number %d, width %d, height %d", def.nBufferCountActual, + def.format.video.nFrameWidth, def.format.video.nFrameHeight); + + useGralloc = true; + + t = 1024; // MemoryDealer doesn't like 0, it's just for MemoryDealer happy. + } else { + t = def.nBufferCountActual * def.nBufferSize; + LOG("Buffer count %d, buffer size %d", def.nBufferCountActual, def.nBufferSize); + } + + bool liveinlocal = mOmx->livesLocally(mNode, getpid()); + + // MemoryDealer is a IPC buffer allocator in Gonk because IOMX is actually + // lives in mediaserver. + mMemoryDealer[aType] = new MemoryDealer(t, "Gecko-OMX"); + for (OMX_U32 i = 0; i < def.nBufferCountActual; ++i) { + RefPtr<GonkBufferData> buffer; + IOMX::buffer_id bufferID; + status_t st; + nsresult rv; + + buffer = new GonkBufferData(liveinlocal, this); + if (useGralloc) { + // Buffer is lived remotely. Use GraphicBuffer for decoded video frame display. + rv = buffer->InitGraphicBuffer(def.format.video); + NS_ENSURE_SUCCESS(rv, rv); + st = mOmx->useGraphicBuffer(mNode, + def.nPortIndex, + buffer->mTextureClientRecycleHandler->GetGraphicBuffer(), + &bufferID); + CHECK_ERR(st); + } else { + sp<IMemory> mem = mMemoryDealer[aType]->allocate(def.nBufferSize); + MOZ_ASSERT(mem.get()); + + if ((mQuirks.test(kRequiresAllocateBufferOnInputPorts) && aType == OMX_DirInput) || + (mQuirks.test(kRequiresAllocateBufferOnOutputPorts) && aType == OMX_DirOutput)) { + // Buffer is lived remotely. We allocate a local OMX_BUFFERHEADERTYPE + // as the mirror of the remote OMX_BUFFERHEADERTYPE. + st = mOmx->allocateBufferWithBackup(mNode, aType, mem, &bufferID); + CHECK_ERR(st); + rv = buffer->InitSharedMemory(mem.get()); + NS_ENSURE_SUCCESS(rv, rv); + } else { + // Buffer is lived locally, bufferID is the actually OMX_BUFFERHEADERTYPE + // pointer. + st = mOmx->useBuffer(mNode, aType, mem, &bufferID); + CHECK_ERR(st); + rv = buffer->InitLocalBuffer(bufferID); + NS_ENSURE_SUCCESS(rv, rv); + } + } + + rv = buffer->SetBufferId(bufferID); + NS_ENSURE_SUCCESS(rv, rv); + + aBufferList->AppendElement(buffer); + } + + return NS_OK; +} + +nsresult +GonkOmxPlatformLayer::ReleaseOmxBuffer(OMX_DIRTYPE aType, + BUFFERLIST* aBufferList) +{ + status_t st; + uint32_t len = aBufferList->Length(); + for (uint32_t i = 0; i < len; i++) { + GonkBufferData* buffer = static_cast<GonkBufferData*>(aBufferList->ElementAt(i).get()); + IOMX::buffer_id id = (OMX_BUFFERHEADERTYPE*) buffer->ID(); + st = mOmx->freeBuffer(mNode, aType, id); + if (st != OK) { + return NS_ERROR_FAILURE; + } + buffer->ReleaseBuffer(); + } + aBufferList->Clear(); + mMemoryDealer[aType].clear(); + + return NS_OK; +} + +nsresult +GonkOmxPlatformLayer::EnableOmxGraphicBufferPort(OMX_PARAM_PORTDEFINITIONTYPE& aDef) +{ + status_t st; + + st = mOmx->enableGraphicBuffers(mNode, aDef.nPortIndex, OMX_TRUE); + CHECK_ERR(st); + + return NS_OK; +} + +OMX_ERRORTYPE +GonkOmxPlatformLayer::GetState(OMX_STATETYPE* aType) +{ + return (OMX_ERRORTYPE)mOmx->getState(mNode, aType); +} + +OMX_ERRORTYPE +GonkOmxPlatformLayer::GetParameter(OMX_INDEXTYPE aParamIndex, + OMX_PTR aComponentParameterStructure, + OMX_U32 aComponentParameterSize) +{ + return (OMX_ERRORTYPE)mOmx->getParameter(mNode, + aParamIndex, + aComponentParameterStructure, + aComponentParameterSize); +} + +OMX_ERRORTYPE +GonkOmxPlatformLayer::SetParameter(OMX_INDEXTYPE aParamIndex, + OMX_PTR aComponentParameterStructure, + OMX_U32 aComponentParameterSize) +{ + return (OMX_ERRORTYPE)mOmx->setParameter(mNode, + aParamIndex, + aComponentParameterStructure, + aComponentParameterSize); +} + +nsresult +GonkOmxPlatformLayer::Shutdown() +{ + mOmx->freeNode(mNode); + mOmxObserver->Shutdown(); + mOmxObserver = nullptr; + mOmxClient.disconnect(); + + return NS_OK; +} + +OMX_ERRORTYPE +GonkOmxPlatformLayer::InitOmxToStateLoaded(const TrackInfo* aInfo) +{ + mInfo = aInfo; + status_t err = mOmxClient.connect(); + if (err != OK) { + return OMX_ErrorUndefined; + } + mOmx = mOmxClient.interface(); + if (!mOmx.get()) { + return OMX_ErrorUndefined; + } + + LOG("find componenet for mime type %s", mInfo->mMimeType.Data()); + + nsTArray<ComponentInfo> components; + if (FindComponents(mInfo->mMimeType, &components)) { + for (auto comp : components) { + if (LoadComponent(comp)) { + return OMX_ErrorNone; + } + } + } + + LOG("no component is loaded"); + return OMX_ErrorUndefined; +} + +OMX_ERRORTYPE +GonkOmxPlatformLayer::EmptyThisBuffer(BufferData* aData) +{ + return (OMX_ERRORTYPE)mOmx->emptyBuffer(mNode, + (IOMX::buffer_id)aData->ID(), + aData->mBuffer->nOffset, + aData->mBuffer->nFilledLen, + aData->mBuffer->nFlags, + aData->mBuffer->nTimeStamp); +} + +OMX_ERRORTYPE +GonkOmxPlatformLayer::FillThisBuffer(BufferData* aData) +{ + return (OMX_ERRORTYPE)mOmx->fillBuffer(mNode, (IOMX::buffer_id)aData->ID()); +} + +OMX_ERRORTYPE +GonkOmxPlatformLayer::SendCommand(OMX_COMMANDTYPE aCmd, + OMX_U32 aParam1, + OMX_PTR aCmdData) +{ + return (OMX_ERRORTYPE)mOmx->sendCommand(mNode, aCmd, aParam1); +} + +bool +GonkOmxPlatformLayer::LoadComponent(const ComponentInfo& aComponent) +{ + status_t err = mOmx->allocateNode(aComponent.mName, mOmxObserver, &mNode); + if (err == OK) { + mQuirks = aComponent.mQuirks; + LOG("Load OpenMax component %s, alloc input %d, alloc output %d, live locally %d", + aComponent.mName, mQuirks.test(kRequiresAllocateBufferOnInputPorts), + mQuirks.test(kRequiresAllocateBufferOnOutputPorts), + mOmx->livesLocally(mNode, getpid())); + return true; + } + return false; +} + +layers::ImageContainer* +GonkOmxPlatformLayer::GetImageContainer() +{ + return mImageContainer; +} + +const TrackInfo* +GonkOmxPlatformLayer::GetTrackInfo() +{ + return mInfo; +} + +bool +GonkOmxPlatformLayer::FindComponents(const nsACString& aMimeType, + nsTArray<ComponentInfo>* aComponents) +{ + static const MediaCodecList* codecs = MediaCodecList::getInstance(); + + bool useHardwareCodecOnly = false; + + // H264 and H263 has different profiles, software codec doesn't support high profile. + // So we use hardware codec only. + if (!IsInEmulator() && + (aMimeType.EqualsLiteral("video/avc") || + aMimeType.EqualsLiteral("video/mp4") || + aMimeType.EqualsLiteral("video/mp4v-es") || + aMimeType.EqualsLiteral("video/3gp"))) { + useHardwareCodecOnly = true; + } + + const char* mime = aMimeType.Data(); + // Translate VP8 MIME type to Android format. + if (aMimeType.EqualsLiteral("video/webm; codecs=vp8")) { + mime = "video/x-vnd.on2.vp8"; + } + + size_t start = 0; + bool found = false; + while (true) { + ssize_t index = codecs->findCodecByType(mime, false /* encoder */, start); + if (index < 0) { + break; + } + start = index + 1; + + const char* name = codecs->getCodecName(index); + if (IsSoftwareCodec(name) && useHardwareCodecOnly) { + continue; + } + + found = true; + + if (!aComponents) { + continue; + } + ComponentInfo* comp = aComponents->AppendElement(); + comp->mName = name; + if (codecs->codecHasQuirk(index, "requires-allocate-on-input-ports")) { + comp->mQuirks.set(kRequiresAllocateBufferOnInputPorts); + } + if (codecs->codecHasQuirk(index, "requires-allocate-on-output-ports")) { + comp->mQuirks.set(kRequiresAllocateBufferOnOutputPorts); + } + } + + return found; +} + +OMX_VIDEO_CODINGTYPE +GonkOmxPlatformLayer::CompressionFormat() +{ + MOZ_ASSERT(mInfo); + + return mInfo->mMimeType.EqualsLiteral("video/webm; codecs=vp8") ? + ANDROID_OMX_VIDEO_CodingVP8 : OmxPlatformLayer::CompressionFormat(); +} + +} // mozilla diff --git a/dom/media/platforms/omx/GonkOmxPlatformLayer.h b/dom/media/platforms/omx/GonkOmxPlatformLayer.h new file mode 100644 index 000000000..aaa8c654d --- /dev/null +++ b/dom/media/platforms/omx/GonkOmxPlatformLayer.h @@ -0,0 +1,205 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#if !defined(GonkOmxPlatformLayer_h_) +#define GonkOmxPlatformLayer_h_ + +#pragma GCC visibility push(default) + +#include <bitset> + +#include <utils/RefBase.h> +#include <media/stagefright/OMXClient.h> +#include "nsAutoPtr.h" + +#include "OMX_Component.h" + +#include "OmxPlatformLayer.h" + +class nsACString; + +namespace android { +class IMemory; +class MemoryDealer; +} + +namespace mozilla { + +class GonkOmxObserver; +class GonkOmxPlatformLayer; +class GonkTextureClientRecycleHandler; + +/* + * Due to Android's omx node could live in local process (client) or remote + * process (mediaserver). And there are 3 kinds of buffer in Android OMX. + * + * 1. + * When buffer is in local process, the IOMX::buffer_id is OMX_BUFFERHEADERTYPE + * pointer actually, it is safe to use it directly. + * + * 2. + * When buffer is in remote process, the OMX_BUFFERHEADERTYPE pointer is 'IN' the + * remote process. It can't be used in local process, so here it allocates a + * local OMX_BUFFERHEADERTYPE. The raw/decoded data is in the android shared + * memory, IMemory. + * + * 3. + * When buffer is in remote process for the display output port. It uses + * GraphicBuffer to accelerate the decoding and display. + * + */ +class GonkBufferData : public OmxPromiseLayer::BufferData { +protected: + virtual ~GonkBufferData() {} + +public: + GonkBufferData(bool aLiveInLocal, + GonkOmxPlatformLayer* aLayer); + + BufferID ID() override + { + return mId; + } + + already_AddRefed<MediaData> GetPlatformMediaData() override; + + bool IsLocalBuffer() + { + return !!mMirrorBuffer.get(); + } + + void ReleaseBuffer(); + + nsresult SetBufferId(android::IOMX::buffer_id aId) + { + mId = aId; + return NS_OK; + } + + // The mBuffer is in local process. And aId is actually the OMX_BUFFERHEADERTYPE + // pointer. It doesn't need a mirror buffer. + nsresult InitLocalBuffer(android::IOMX::buffer_id aId); + + // aMemory is an IPC based memory which will be used as the pBuffer in + // mBuffer. And the mBuffer will be the mirror OMX_BUFFERHEADERTYPE + // of the one in the remote process. + nsresult InitSharedMemory(android::IMemory* aMemory); + + // GraphicBuffer is for video decoding acceleration on output port. + // Then mBuffer is the mirror OMX_BUFFERHEADERTYPE of the one in the remote + // process. + nsresult InitGraphicBuffer(OMX_VIDEO_PORTDEFINITIONTYPE& aDef); + + // Android OMX uses this id to pass the buffer between OMX component and + // client. + android::IOMX::buffer_id mId; + + // mMirrorBuffer are used only when the omx node is in mediaserver. + // Due to IPC problem, the mId is the OMX_BUFFERHEADERTYPE address in mediaserver. + // It can't mapping to client process, so we need a local OMX_BUFFERHEADERTYPE + // here to mirror the remote OMX_BUFFERHEADERTYPE in mediaserver. + nsAutoPtr<OMX_BUFFERHEADERTYPE> mMirrorBuffer; + + // It creates GraphicBuffer and manages TextureClient. + RefPtr<GonkTextureClientRecycleHandler> mTextureClientRecycleHandler; + + GonkOmxPlatformLayer* mGonkPlatformLayer; +}; + +class GonkOmxPlatformLayer : public OmxPlatformLayer { +public: + enum { + kRequiresAllocateBufferOnInputPorts = 0, + kRequiresAllocateBufferOnOutputPorts, + QUIRKS, + }; + typedef std::bitset<QUIRKS> Quirks; + + struct ComponentInfo { + const char* mName; + Quirks mQuirks; + }; + + GonkOmxPlatformLayer(OmxDataDecoder* aDataDecoder, + OmxPromiseLayer* aPromiseLayer, + TaskQueue* aTaskQueue, + layers::ImageContainer* aImageContainer); + + nsresult AllocateOmxBuffer(OMX_DIRTYPE aType, BUFFERLIST* aBufferList) override; + + nsresult ReleaseOmxBuffer(OMX_DIRTYPE aType, BUFFERLIST* aBufferList) override; + + OMX_ERRORTYPE GetState(OMX_STATETYPE* aType) override; + + OMX_ERRORTYPE GetParameter(OMX_INDEXTYPE aParamIndex, + OMX_PTR aComponentParameterStructure, + OMX_U32 aComponentParameterSize) override; + + OMX_ERRORTYPE SetParameter(OMX_INDEXTYPE nIndex, + OMX_PTR aComponentParameterStructure, + OMX_U32 aComponentParameterSize) override; + + OMX_ERRORTYPE InitOmxToStateLoaded(const TrackInfo* aInfo) override; + + OMX_ERRORTYPE EmptyThisBuffer(BufferData* aData) override; + + OMX_ERRORTYPE FillThisBuffer(BufferData* aData) override; + + OMX_ERRORTYPE SendCommand(OMX_COMMANDTYPE aCmd, + OMX_U32 aParam1, + OMX_PTR aCmdData) override; + + nsresult Shutdown() override; + + static bool FindComponents(const nsACString& aMimeType, + nsTArray<ComponentInfo>* aComponents = nullptr); + + // Android/QCOM decoder uses its own OMX_VIDEO_CodingVP8 definition in + // frameworks/native/media/include/openmax/OMX_Video.h, not the one defined + // in OpenMAX v1.1.2 OMX_VideoExt.h + OMX_VIDEO_CODINGTYPE CompressionFormat() override; + +protected: + friend GonkBufferData; + + layers::ImageContainer* GetImageContainer(); + + const TrackInfo* GetTrackInfo(); + + TaskQueue* GetTaskQueue() + { + return mTaskQueue; + } + + nsresult EnableOmxGraphicBufferPort(OMX_PARAM_PORTDEFINITIONTYPE& aDef); + + bool LoadComponent(const ComponentInfo& aComponent); + + friend class GonkOmxObserver; + + RefPtr<TaskQueue> mTaskQueue; + + RefPtr<layers::ImageContainer> mImageContainer; + + // OMX_DirInput is 0, OMX_DirOutput is 1. + android::sp<android::MemoryDealer> mMemoryDealer[2]; + + android::sp<GonkOmxObserver> mOmxObserver; + + android::sp<android::IOMX> mOmx; + + android::IOMX::node_id mNode; + + android::OMXClient mOmxClient; + + Quirks mQuirks; +}; + +} + +#pragma GCC visibility pop + +#endif // GonkOmxPlatformLayer_h_ diff --git a/dom/media/platforms/omx/OmxDataDecoder.cpp b/dom/media/platforms/omx/OmxDataDecoder.cpp new file mode 100644 index 000000000..33f9dad30 --- /dev/null +++ b/dom/media/platforms/omx/OmxDataDecoder.cpp @@ -0,0 +1,1038 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "OmxDataDecoder.h" + +#include "OMX_Audio.h" +#include "OMX_Component.h" +#include "OMX_Types.h" + +#include "OmxPlatformLayer.h" + + +#ifdef LOG +#undef LOG +#undef LOGL +#endif + +#define LOG(arg, ...) MOZ_LOG(sPDMLog, mozilla::LogLevel::Debug, ("OmxDataDecoder(%p)::%s: " arg, this, __func__, ##__VA_ARGS__)) + +#define LOGL(arg, ...) \ + { \ + void* p = self; \ + MOZ_LOG(sPDMLog, mozilla::LogLevel::Debug, \ + ("OmxDataDecoder(%p)::%s: " arg, p, __func__, ##__VA_ARGS__)); \ + } + +#define CHECK_OMX_ERR(err) \ + if (err != OMX_ErrorNone) { \ + NotifyError(err, __func__);\ + return; \ + } \ + +namespace mozilla { + +static const char* +StateTypeToStr(OMX_STATETYPE aType) +{ + MOZ_ASSERT(aType == OMX_StateLoaded || + aType == OMX_StateIdle || + aType == OMX_StateExecuting || + aType == OMX_StatePause || + aType == OMX_StateWaitForResources || + aType == OMX_StateInvalid); + + switch (aType) { + case OMX_StateLoaded: + return "OMX_StateLoaded"; + case OMX_StateIdle: + return "OMX_StateIdle"; + case OMX_StateExecuting: + return "OMX_StateExecuting"; + case OMX_StatePause: + return "OMX_StatePause"; + case OMX_StateWaitForResources: + return "OMX_StateWaitForResources"; + case OMX_StateInvalid: + return "OMX_StateInvalid"; + default: + return "Unknown"; + } +} + +// A helper class to retrieve AudioData or VideoData. +class MediaDataHelper { +protected: + virtual ~MediaDataHelper() {} + +public: + NS_INLINE_DECL_THREADSAFE_REFCOUNTING(MediaDataHelper) + + MediaDataHelper(const TrackInfo* aTrackInfo, + layers::ImageContainer* aImageContainer, + OmxPromiseLayer* aOmxLayer); + + already_AddRefed<MediaData> GetMediaData(BufferData* aBufferData, bool& aPlatformDepenentData); + +protected: + already_AddRefed<AudioData> CreateAudioData(BufferData* aBufferData); + + already_AddRefed<VideoData> CreateYUV420VideoData(BufferData* aBufferData); + + const TrackInfo* mTrackInfo; + + OMX_PARAM_PORTDEFINITIONTYPE mOutputPortDef; + + // audio output + MediaQueue<AudioData> mAudioQueue; + + AudioCompactor mAudioCompactor; + + // video output + RefPtr<layers::ImageContainer> mImageContainer; +}; + +OmxDataDecoder::OmxDataDecoder(const TrackInfo& aTrackInfo, + MediaDataDecoderCallback* aCallback, + layers::ImageContainer* aImageContainer) + : mMonitor("OmxDataDecoder") + , mOmxTaskQueue(CreateMediaDecodeTaskQueue()) + , mImageContainer(aImageContainer) + , mWatchManager(this, mOmxTaskQueue) + , mOmxState(OMX_STATETYPE::OMX_StateInvalid, "OmxDataDecoder::mOmxState") + , mTrackInfo(aTrackInfo.Clone()) + , mFlushing(false) + , mShuttingDown(false) + , mCheckingInputExhausted(false) + , mPortSettingsChanged(-1, "OmxDataDecoder::mPortSettingsChanged") + , mCallback(aCallback) +{ + LOG(""); + mOmxLayer = new OmxPromiseLayer(mOmxTaskQueue, this, aImageContainer); + + mOmxTaskQueue->Dispatch(NewRunnableMethod(this, &OmxDataDecoder::InitializationTask)); +} + +OmxDataDecoder::~OmxDataDecoder() +{ + LOG(""); +} + +void +OmxDataDecoder::InitializationTask() +{ + mWatchManager.Watch(mOmxState, &OmxDataDecoder::OmxStateRunner); + mWatchManager.Watch(mPortSettingsChanged, &OmxDataDecoder::PortSettingsChanged); +} + +void +OmxDataDecoder::EndOfStream() +{ + LOG(""); + MOZ_ASSERT(mOmxTaskQueue->IsCurrentThreadIn()); + + mFlushing = true; + RefPtr<OmxDataDecoder> self = this; + mOmxLayer->SendCommand(OMX_CommandFlush, OMX_ALL, nullptr) + ->Then(mReaderTaskQueue, __func__, + [self] () { + self->mFlushing = false; + self->mCallback->DrainComplete(); + }, + [self] () { + self->mFlushing = false; + self->mCallback->DrainComplete(); + }); +} + +RefPtr<MediaDataDecoder::InitPromise> +OmxDataDecoder::Init() +{ + LOG(""); + mReaderTaskQueue = AbstractThread::GetCurrent()->AsTaskQueue(); + MOZ_ASSERT(mReaderTaskQueue); + + RefPtr<InitPromise> p = mInitPromise.Ensure(__func__); + RefPtr<OmxDataDecoder> self = this; + + // TODO: it needs to get permission from resource manager before allocating + // Omx component. + InvokeAsync(mOmxTaskQueue, mOmxLayer.get(), __func__, &OmxPromiseLayer::Init, + mTrackInfo.get()) + ->Then(mOmxTaskQueue, __func__, + [self] () { + // Omx state should be OMX_StateIdle. + self->mOmxState = self->mOmxLayer->GetState(); + MOZ_ASSERT(self->mOmxState != OMX_StateIdle); + }, + [self] () { + self->RejectInitPromise(NS_ERROR_DOM_MEDIA_FATAL_ERR, __func__); + }); + + return p; +} + +void +OmxDataDecoder::Input(MediaRawData* aSample) +{ + LOG("sample %p", aSample); + MOZ_ASSERT(mInitPromise.IsEmpty()); + + RefPtr<OmxDataDecoder> self = this; + RefPtr<MediaRawData> sample = aSample; + + nsCOMPtr<nsIRunnable> r = + NS_NewRunnableFunction([self, sample] () { + self->mMediaRawDatas.AppendElement(sample); + + // Start to fill/empty buffers. + if (self->mOmxState == OMX_StateIdle || + self->mOmxState == OMX_StateExecuting) { + self->FillAndEmptyBuffers(); + } + }); + mOmxTaskQueue->Dispatch(r.forget()); +} + +void +OmxDataDecoder::Flush() +{ + LOG(""); + + mFlushing = true; + + mOmxTaskQueue->Dispatch(NewRunnableMethod(this, &OmxDataDecoder::DoFlush)); + + // According to the definition of Flush() in PDM: + // "the decoder must be ready to accept new input for decoding". + // So it needs to wait for the Omx to complete the flush command. + MonitorAutoLock lock(mMonitor); + while (mFlushing) { + lock.Wait(); + } +} + +void +OmxDataDecoder::Drain() +{ + LOG(""); + + mOmxTaskQueue->Dispatch(NewRunnableMethod(this, &OmxDataDecoder::SendEosBuffer)); +} + +void +OmxDataDecoder::Shutdown() +{ + LOG(""); + + mShuttingDown = true; + + mOmxTaskQueue->Dispatch(NewRunnableMethod(this, &OmxDataDecoder::DoAsyncShutdown)); + + { + // DoAsyncShutdown() will be running for a while, it could be still running + // when reader releasing the decoder and then it causes problem. To avoid it, + // Shutdown() must block until DoAsyncShutdown() is completed. + MonitorAutoLock lock(mMonitor); + while (mShuttingDown) { + lock.Wait(); + } + } + + mOmxTaskQueue->BeginShutdown(); + mOmxTaskQueue->AwaitShutdownAndIdle(); +} + +void +OmxDataDecoder::DoAsyncShutdown() +{ + LOG(""); + MOZ_ASSERT(mOmxTaskQueue->IsCurrentThreadIn()); + MOZ_ASSERT(!mFlushing); + + mWatchManager.Unwatch(mOmxState, &OmxDataDecoder::OmxStateRunner); + mWatchManager.Unwatch(mPortSettingsChanged, &OmxDataDecoder::PortSettingsChanged); + + // Flush to all ports, so all buffers can be returned from component. + RefPtr<OmxDataDecoder> self = this; + mOmxLayer->SendCommand(OMX_CommandFlush, OMX_ALL, nullptr) + ->Then(mOmxTaskQueue, __func__, + [self] () -> RefPtr<OmxCommandPromise> { + LOGL("DoAsyncShutdown: flush complete"); + return self->mOmxLayer->SendCommand(OMX_CommandStateSet, OMX_StateIdle, nullptr); + }, + [self] () { + self->mOmxLayer->Shutdown(); + }) + ->CompletionPromise() + ->Then(mOmxTaskQueue, __func__, + [self] () -> RefPtr<OmxCommandPromise> { + RefPtr<OmxCommandPromise> p = + self->mOmxLayer->SendCommand(OMX_CommandStateSet, OMX_StateLoaded, nullptr); + + // According to spec 3.1.1.2.2.1: + // OMX_StateLoaded needs to be sent before releasing buffers. + // And state transition from OMX_StateIdle to OMX_StateLoaded + // is completed when all of the buffers have been removed + // from the component. + // Here the buffer promises are not resolved due to displaying + // in layer, it needs to wait before the layer returns the + // buffers. + LOGL("DoAsyncShutdown: releasing buffers..."); + self->ReleaseBuffers(OMX_DirInput); + self->ReleaseBuffers(OMX_DirOutput); + + return p; + }, + [self] () { + self->mOmxLayer->Shutdown(); + }) + ->CompletionPromise() + ->Then(mOmxTaskQueue, __func__, + [self] () { + LOGL("DoAsyncShutdown: OMX_StateLoaded, it is safe to shutdown omx"); + self->mOmxLayer->Shutdown(); + self->mWatchManager.Shutdown(); + self->mOmxLayer = nullptr; + self->mMediaDataHelper = nullptr; + + MonitorAutoLock lock(self->mMonitor); + self->mShuttingDown = false; + self->mMonitor.Notify(); + }, + [self] () { + self->mOmxLayer->Shutdown(); + self->mWatchManager.Shutdown(); + self->mOmxLayer = nullptr; + self->mMediaDataHelper = nullptr; + + MonitorAutoLock lock(self->mMonitor); + self->mShuttingDown = false; + self->mMonitor.Notify(); + }); +} + +void +OmxDataDecoder::FillBufferDone(BufferData* aData) +{ + MOZ_ASSERT(!aData || aData->mStatus == BufferData::BufferStatus::OMX_CLIENT); + + // Don't output sample when flush or shutting down, especially for video + // decoded frame. Because video decoded frame has a promise in BufferData + // waiting for layer to resolve it via recycle callback on Gonk, if other + // module doesn't send it to layer, it will cause a unresolved promise and + // waiting for resolve infinitely. + if (mFlushing || mShuttingDown) { + LOG("mFlush or mShuttingDown, drop data"); + aData->mStatus = BufferData::BufferStatus::FREE; + return; + } + + if (aData->mBuffer->nFlags & OMX_BUFFERFLAG_EOS) { + // Reach eos, it's an empty data so it doesn't need to output. + EndOfStream(); + aData->mStatus = BufferData::BufferStatus::FREE; + } else { + Output(aData); + FillAndEmptyBuffers(); + } +} + +void +OmxDataDecoder::Output(BufferData* aData) +{ + if (!mMediaDataHelper) { + mMediaDataHelper = new MediaDataHelper(mTrackInfo.get(), mImageContainer, mOmxLayer); + } + + bool isPlatformData = false; + RefPtr<MediaData> data = mMediaDataHelper->GetMediaData(aData, isPlatformData); + if (!data) { + aData->mStatus = BufferData::BufferStatus::FREE; + return; + } + + if (isPlatformData) { + // If the MediaData is platform dependnet data, it's mostly a kind of + // limited resource, for example, GraphicBuffer on Gonk. So we use promise + // to notify when the resource is free. + aData->mStatus = BufferData::BufferStatus::OMX_CLIENT_OUTPUT; + + MOZ_RELEASE_ASSERT(aData->mPromise.IsEmpty()); + RefPtr<OmxBufferPromise> p = aData->mPromise.Ensure(__func__); + + RefPtr<OmxDataDecoder> self = this; + RefPtr<BufferData> buffer = aData; + p->Then(mOmxTaskQueue, __func__, + [self, buffer] () { + MOZ_RELEASE_ASSERT(buffer->mStatus == BufferData::BufferStatus::OMX_CLIENT_OUTPUT); + buffer->mStatus = BufferData::BufferStatus::FREE; + self->FillAndEmptyBuffers(); + }, + [buffer] () { + MOZ_RELEASE_ASSERT(buffer->mStatus == BufferData::BufferStatus::OMX_CLIENT_OUTPUT); + buffer->mStatus = BufferData::BufferStatus::FREE; + }); + } else { + aData->mStatus = BufferData::BufferStatus::FREE; + } + + mCallback->Output(data); +} + +void +OmxDataDecoder::FillBufferFailure(OmxBufferFailureHolder aFailureHolder) +{ + NotifyError(aFailureHolder.mError, __func__); +} + +void +OmxDataDecoder::EmptyBufferDone(BufferData* aData) +{ + MOZ_ASSERT(!aData || aData->mStatus == BufferData::BufferStatus::OMX_CLIENT); + + // Nothing to do when status of input buffer is OMX_CLIENT. + aData->mStatus = BufferData::BufferStatus::FREE; + FillAndEmptyBuffers(); + + // There is no way to know if component gets enough raw samples to generate + // output, especially for video decoding. So here it needs to request raw + // samples aggressively. + if (!mCheckingInputExhausted && !mMediaRawDatas.Length()) { + mCheckingInputExhausted = true; + + RefPtr<OmxDataDecoder> self = this; + nsCOMPtr<nsIRunnable> r = + NS_NewRunnableFunction([self] () { + MOZ_ASSERT(self->mOmxTaskQueue->IsCurrentThreadIn()); + + self->mCheckingInputExhausted = false; + + if (self->mMediaRawDatas.Length()) { + return; + } + + LOGL("Call InputExhausted()"); + self->mCallback->InputExhausted(); + }); + + mOmxTaskQueue->Dispatch(r.forget()); + } +} + +void +OmxDataDecoder::EmptyBufferFailure(OmxBufferFailureHolder aFailureHolder) +{ + NotifyError(aFailureHolder.mError, __func__); +} + +void +OmxDataDecoder::NotifyError(OMX_ERRORTYPE aOmxError, const char* aLine, const MediaResult& aError) +{ + LOG("NotifyError %d (%d) at %s", aOmxError, aError.Code(), aLine); + mCallback->Error(aError); +} + +void +OmxDataDecoder::FillAndEmptyBuffers() +{ + MOZ_ASSERT(mOmxTaskQueue->IsCurrentThreadIn()); + MOZ_ASSERT(mOmxState == OMX_StateExecuting); + + // During the port setting changed, it is forbidden to do any buffer operation. + if (mPortSettingsChanged != -1 || mShuttingDown || mFlushing) { + return; + } + + // Trigger input port. + while (!!mMediaRawDatas.Length()) { + // input buffer must be used by component if there is data available. + RefPtr<BufferData> inbuf = FindAvailableBuffer(OMX_DirInput); + if (!inbuf) { + LOG("no input buffer!"); + break; + } + + RefPtr<MediaRawData> data = mMediaRawDatas[0]; + // Buffer size should large enough for raw data. + MOZ_RELEASE_ASSERT(inbuf->mBuffer->nAllocLen >= data->Size()); + + memcpy(inbuf->mBuffer->pBuffer, data->Data(), data->Size()); + inbuf->mBuffer->nFilledLen = data->Size(); + inbuf->mBuffer->nOffset = 0; + inbuf->mBuffer->nFlags = inbuf->mBuffer->nAllocLen > data->Size() ? + OMX_BUFFERFLAG_ENDOFFRAME : 0; + inbuf->mBuffer->nTimeStamp = data->mTime; + if (data->Size()) { + inbuf->mRawData = mMediaRawDatas[0]; + } else { + LOG("send EOS buffer"); + inbuf->mBuffer->nFlags |= OMX_BUFFERFLAG_EOS; + } + + LOG("feed sample %p to omx component, len %d, flag %X", data.get(), + inbuf->mBuffer->nFilledLen, inbuf->mBuffer->nFlags); + mOmxLayer->EmptyBuffer(inbuf)->Then(mOmxTaskQueue, __func__, this, + &OmxDataDecoder::EmptyBufferDone, + &OmxDataDecoder::EmptyBufferFailure); + mMediaRawDatas.RemoveElementAt(0); + } + + // Trigger output port. + while (true) { + RefPtr<BufferData> outbuf = FindAvailableBuffer(OMX_DirOutput); + if (!outbuf) { + break; + } + + mOmxLayer->FillBuffer(outbuf)->Then(mOmxTaskQueue, __func__, this, + &OmxDataDecoder::FillBufferDone, + &OmxDataDecoder::FillBufferFailure); + } +} + +OmxPromiseLayer::BufferData* +OmxDataDecoder::FindAvailableBuffer(OMX_DIRTYPE aType) +{ + BUFFERLIST* buffers = GetBuffers(aType); + + for (uint32_t i = 0; i < buffers->Length(); i++) { + BufferData* buf = buffers->ElementAt(i); + if (buf->mStatus == BufferData::BufferStatus::FREE) { + return buf; + } + } + + return nullptr; +} + +nsresult +OmxDataDecoder::AllocateBuffers(OMX_DIRTYPE aType) +{ + MOZ_ASSERT(mOmxTaskQueue->IsCurrentThreadIn()); + + return mOmxLayer->AllocateOmxBuffer(aType, GetBuffers(aType)); +} + +nsresult +OmxDataDecoder::ReleaseBuffers(OMX_DIRTYPE aType) +{ + MOZ_ASSERT(mOmxTaskQueue->IsCurrentThreadIn()); + + return mOmxLayer->ReleaseOmxBuffer(aType, GetBuffers(aType)); +} + +nsTArray<RefPtr<OmxPromiseLayer::BufferData>>* +OmxDataDecoder::GetBuffers(OMX_DIRTYPE aType) +{ + MOZ_ASSERT(aType == OMX_DIRTYPE::OMX_DirInput || + aType == OMX_DIRTYPE::OMX_DirOutput); + + if (aType == OMX_DIRTYPE::OMX_DirInput) { + return &mInPortBuffers; + } + return &mOutPortBuffers; +} + +void +OmxDataDecoder::ResolveInitPromise(const char* aMethodName) +{ + LOG("called from %s", aMethodName); + RefPtr<OmxDataDecoder> self = this; + nsCOMPtr<nsIRunnable> r = + NS_NewRunnableFunction([self, aMethodName] () { + MOZ_ASSERT(self->mReaderTaskQueue->IsCurrentThreadIn()); + self->mInitPromise.ResolveIfExists(self->mTrackInfo->GetType(), aMethodName); + }); + mReaderTaskQueue->Dispatch(r.forget()); +} + +void +OmxDataDecoder::RejectInitPromise(MediaResult aError, const char* aMethodName) +{ + RefPtr<OmxDataDecoder> self = this; + nsCOMPtr<nsIRunnable> r = + NS_NewRunnableFunction([self, aError, aMethodName] () { + MOZ_ASSERT(self->mReaderTaskQueue->IsCurrentThreadIn()); + self->mInitPromise.RejectIfExists(aError, aMethodName); + }); + mReaderTaskQueue->Dispatch(r.forget()); +} + +void +OmxDataDecoder::OmxStateRunner() +{ + MOZ_ASSERT(mOmxTaskQueue->IsCurrentThreadIn()); + LOG("OMX state: %s", StateTypeToStr(mOmxState)); + + // TODO: maybe it'd be better to use promise CompletionPromise() to replace + // this state machine. + if (mOmxState == OMX_StateLoaded) { + ConfigCodec(); + + // Send OpenMax state command to OMX_StateIdle. + RefPtr<OmxDataDecoder> self = this; + mOmxLayer->SendCommand(OMX_CommandStateSet, OMX_StateIdle, nullptr) + ->Then(mOmxTaskQueue, __func__, + [self] () { + // Current state should be OMX_StateIdle. + self->mOmxState = self->mOmxLayer->GetState(); + MOZ_ASSERT(self->mOmxState == OMX_StateIdle); + }, + [self] () { + self->RejectInitPromise(NS_ERROR_DOM_MEDIA_FATAL_ERR, __func__); + }); + + // Allocate input and output buffers. + OMX_DIRTYPE types[] = {OMX_DIRTYPE::OMX_DirInput, OMX_DIRTYPE::OMX_DirOutput}; + for(const auto id : types) { + if (NS_FAILED(AllocateBuffers(id))) { + LOG("Failed to allocate buffer on port %d", id); + RejectInitPromise(NS_ERROR_DOM_MEDIA_FATAL_ERR, __func__); + break; + } + } + } else if (mOmxState == OMX_StateIdle) { + RefPtr<OmxDataDecoder> self = this; + mOmxLayer->SendCommand(OMX_CommandStateSet, OMX_StateExecuting, nullptr) + ->Then(mOmxTaskQueue, __func__, + [self] () { + self->mOmxState = self->mOmxLayer->GetState(); + MOZ_ASSERT(self->mOmxState == OMX_StateExecuting); + + self->ResolveInitPromise(__func__); + }, + [self] () { + self->RejectInitPromise(NS_ERROR_DOM_MEDIA_FATAL_ERR, __func__); + }); + } else if (mOmxState == OMX_StateExecuting) { + // Configure codec once it gets OMX_StateExecuting state. + FillCodecConfigDataToOmx(); + } else { + MOZ_ASSERT(0); + } +} + +void +OmxDataDecoder::ConfigCodec() +{ + OMX_ERRORTYPE err = mOmxLayer->Config(); + CHECK_OMX_ERR(err); +} + +void +OmxDataDecoder::FillCodecConfigDataToOmx() +{ + // Codec configure data should be the first sample running on Omx TaskQueue. + MOZ_ASSERT(mOmxTaskQueue->IsCurrentThreadIn()); + MOZ_ASSERT(!mMediaRawDatas.Length()); + MOZ_ASSERT(mOmxState == OMX_StateIdle || mOmxState == OMX_StateExecuting); + + + RefPtr<BufferData> inbuf = FindAvailableBuffer(OMX_DirInput); + RefPtr<MediaByteBuffer> csc; + if (mTrackInfo->IsAudio()) { + csc = mTrackInfo->GetAsAudioInfo()->mCodecSpecificConfig; + } else if (mTrackInfo->IsVideo()) { + csc = mTrackInfo->GetAsVideoInfo()->mCodecSpecificConfig; + } + + MOZ_RELEASE_ASSERT(csc); + + // Some codecs like h264, its codec specific data is at the first packet, not in container. + if (csc->Length()) { + memcpy(inbuf->mBuffer->pBuffer, + csc->Elements(), + csc->Length()); + inbuf->mBuffer->nFilledLen = csc->Length(); + inbuf->mBuffer->nOffset = 0; + inbuf->mBuffer->nFlags = (OMX_BUFFERFLAG_ENDOFFRAME | OMX_BUFFERFLAG_CODECCONFIG); + + LOG("Feed codec configure data to OMX component"); + mOmxLayer->EmptyBuffer(inbuf)->Then(mOmxTaskQueue, __func__, this, + &OmxDataDecoder::EmptyBufferDone, + &OmxDataDecoder::EmptyBufferFailure); + } +} + +bool +OmxDataDecoder::Event(OMX_EVENTTYPE aEvent, OMX_U32 aData1, OMX_U32 aData2) +{ + MOZ_ASSERT(mOmxTaskQueue->IsCurrentThreadIn()); + + if (mOmxLayer->Event(aEvent, aData1, aData2)) { + return true; + } + + switch (aEvent) { + case OMX_EventPortSettingsChanged: + { + // Don't always disable port. See bug 1235340. + if (aData2 == 0 || + aData2 == OMX_IndexParamPortDefinition) { + // According to spec: "To prevent the loss of any input data, the + // component issuing the OMX_EventPortSettingsChanged event on its input + // port should buffer all input port data that arrives between the + // emission of the OMX_EventPortSettingsChanged event and the arrival of + // the command to disable the input port." + // + // So client needs to disable port and reallocate buffers. + MOZ_ASSERT(mPortSettingsChanged == -1); + mPortSettingsChanged = aData1; + } + LOG("Got OMX_EventPortSettingsChanged event"); + break; + } + default: + { + // Got error during decoding, send msg to MFR skipping to next key frame. + if (aEvent == OMX_EventError && mOmxState == OMX_StateExecuting) { + NotifyError((OMX_ERRORTYPE)aData1, __func__, + MediaResult(NS_ERROR_DOM_MEDIA_DECODE_ERR, __func__)); + return true; + } + LOG("WARNING: got none handle event: %d, aData1: %d, aData2: %d", + aEvent, aData1, aData2); + return false; + } + } + + return true; +} + +bool +OmxDataDecoder::BuffersCanBeReleased(OMX_DIRTYPE aType) +{ + BUFFERLIST* buffers = GetBuffers(aType); + uint32_t len = buffers->Length(); + for (uint32_t i = 0; i < len; i++) { + BufferData::BufferStatus buf_status = buffers->ElementAt(i)->mStatus; + if (buf_status == BufferData::BufferStatus::OMX_COMPONENT || + buf_status == BufferData::BufferStatus::OMX_CLIENT_OUTPUT) { + return false; + } + } + return true; +} + +OMX_DIRTYPE +OmxDataDecoder::GetPortDirection(uint32_t aPortIndex) +{ + OMX_PARAM_PORTDEFINITIONTYPE def; + InitOmxParameter(&def); + def.nPortIndex = mPortSettingsChanged; + + OMX_ERRORTYPE err = mOmxLayer->GetParameter(OMX_IndexParamPortDefinition, + &def, + sizeof(def)); + if (err != OMX_ErrorNone) { + return OMX_DirMax; + } + return def.eDir; +} + +RefPtr<OmxPromiseLayer::OmxBufferPromise::AllPromiseType> +OmxDataDecoder::CollectBufferPromises(OMX_DIRTYPE aType) +{ + MOZ_ASSERT(mOmxTaskQueue->IsCurrentThreadIn()); + + nsTArray<RefPtr<OmxBufferPromise>> promises; + OMX_DIRTYPE types[] = {OMX_DIRTYPE::OMX_DirInput, OMX_DIRTYPE::OMX_DirOutput}; + for (const auto type : types) { + if ((aType == type) || (aType == OMX_DirMax)) { + // find the buffer which has promise. + BUFFERLIST* buffers = GetBuffers(type); + + for (uint32_t i = 0; i < buffers->Length(); i++) { + BufferData* buf = buffers->ElementAt(i); + if (!buf->mPromise.IsEmpty()) { + // OmxBufferPromise is not exclusive, it can be multiple "Then"s, so it + // is safe to call "Ensure" here. + promises.AppendElement(buf->mPromise.Ensure(__func__)); + } + } + } + } + + LOG("CollectBufferPromises: type %d, total %d promiese", aType, promises.Length()); + if (promises.Length()) { + return OmxBufferPromise::All(mOmxTaskQueue, promises); + } + + nsTArray<BufferData*> headers; + return OmxBufferPromise::AllPromiseType::CreateAndResolve(headers, __func__); +} + +void +OmxDataDecoder::PortSettingsChanged() +{ + MOZ_ASSERT(mOmxTaskQueue->IsCurrentThreadIn()); + + if (mPortSettingsChanged == -1 || mOmxState == OMX_STATETYPE::OMX_StateInvalid) { + return; + } + + // The PortSettingsChanged algorithm: + // + // 1. disable port. + // 2. wait for port buffers return to client and then release these buffers. + // 3. enable port. + // 4. allocate port buffers. + // + + // Disable port. Get port definition if the target port is enable. + OMX_PARAM_PORTDEFINITIONTYPE def; + InitOmxParameter(&def); + def.nPortIndex = mPortSettingsChanged; + + OMX_ERRORTYPE err = mOmxLayer->GetParameter(OMX_IndexParamPortDefinition, + &def, + sizeof(def)); + CHECK_OMX_ERR(err); + + RefPtr<OmxDataDecoder> self = this; + if (def.bEnabled) { + // 1. disable port. + LOG("PortSettingsChanged: disable port %d", def.nPortIndex); + mOmxLayer->SendCommand(OMX_CommandPortDisable, mPortSettingsChanged, nullptr) + ->Then(mOmxTaskQueue, __func__, + [self, def] () -> RefPtr<OmxCommandPromise> { + // 3. enable port. + // Send enable port command. + RefPtr<OmxCommandPromise> p = + self->mOmxLayer->SendCommand(OMX_CommandPortEnable, + self->mPortSettingsChanged, + nullptr); + + // 4. allocate port buffers. + // Allocate new port buffers. + nsresult rv = self->AllocateBuffers(def.eDir); + if (NS_FAILED(rv)) { + self->NotifyError(OMX_ErrorUndefined, __func__); + } + + return p; + }, + [self] () { + self->NotifyError(OMX_ErrorUndefined, __func__); + }) + ->CompletionPromise() + ->Then(mOmxTaskQueue, __func__, + [self] () { + LOGL("PortSettingsChanged: port settings changed complete"); + // finish port setting changed. + self->mPortSettingsChanged = -1; + self->FillAndEmptyBuffers(); + }, + [self] () { + self->NotifyError(OMX_ErrorUndefined, __func__); + }); + + // 2. wait for port buffers return to client and then release these buffers. + // + // Port buffers will be returned to client soon once OMX_CommandPortDisable + // command is sent. Then releasing these buffers. + CollectBufferPromises(def.eDir) + ->Then(mOmxTaskQueue, __func__, + [self, def] () { + MOZ_ASSERT(self->BuffersCanBeReleased(def.eDir)); + nsresult rv = self->ReleaseBuffers(def.eDir); + if (NS_FAILED(rv)) { + MOZ_RELEASE_ASSERT(0); + self->NotifyError(OMX_ErrorUndefined, __func__); + } + }, + [self] () { + self->NotifyError(OMX_ErrorUndefined, __func__); + }); + } +} + +void +OmxDataDecoder::SendEosBuffer() +{ + MOZ_ASSERT(mOmxTaskQueue->IsCurrentThreadIn()); + + // There is no 'Drain' API in OpenMax, so it needs to wait for output sample + // with EOS flag. However, MediaRawData doesn't provide EOS information, + // so here it generates an empty BufferData with eos OMX_BUFFERFLAG_EOS in queue. + // This behaviour should be compliant with spec, I think... + RefPtr<MediaRawData> eos_data = new MediaRawData(); + mMediaRawDatas.AppendElement(eos_data); + FillAndEmptyBuffers(); +} + +void +OmxDataDecoder::DoFlush() +{ + MOZ_ASSERT(mOmxTaskQueue->IsCurrentThreadIn()); + + // 1. Call OMX command OMX_CommandFlush in Omx TaskQueue. + // 2. Remove all elements in mMediaRawDatas when flush is completed. + mOmxLayer->SendCommand(OMX_CommandFlush, OMX_ALL, nullptr) + ->Then(mOmxTaskQueue, __func__, this, + &OmxDataDecoder::FlushComplete, + &OmxDataDecoder::FlushFailure); +} + +void +OmxDataDecoder::FlushComplete(OMX_COMMANDTYPE aCommandType) +{ + mMediaRawDatas.Clear(); + mFlushing = false; + + MonitorAutoLock lock(mMonitor); + mMonitor.Notify(); + LOG("Flush complete"); +} + +void OmxDataDecoder::FlushFailure(OmxCommandFailureHolder aFailureHolder) +{ + NotifyError(OMX_ErrorUndefined, __func__); + mFlushing = false; + + MonitorAutoLock lock(mMonitor); + mMonitor.Notify(); +} + +MediaDataHelper::MediaDataHelper(const TrackInfo* aTrackInfo, + layers::ImageContainer* aImageContainer, + OmxPromiseLayer* aOmxLayer) + : mTrackInfo(aTrackInfo) + , mAudioCompactor(mAudioQueue) + , mImageContainer(aImageContainer) +{ + InitOmxParameter(&mOutputPortDef); + mOutputPortDef.nPortIndex = aOmxLayer->OutputPortIndex(); + aOmxLayer->GetParameter(OMX_IndexParamPortDefinition, &mOutputPortDef, sizeof(mOutputPortDef)); +} + +already_AddRefed<MediaData> +MediaDataHelper::GetMediaData(BufferData* aBufferData, bool& aPlatformDepenentData) +{ + aPlatformDepenentData = false; + RefPtr<MediaData> data; + + if (mTrackInfo->IsAudio()) { + if (!aBufferData->mBuffer->nFilledLen) { + return nullptr; + } + data = CreateAudioData(aBufferData); + } else if (mTrackInfo->IsVideo()) { + data = aBufferData->GetPlatformMediaData(); + if (data) { + aPlatformDepenentData = true; + } else { + if (!aBufferData->mBuffer->nFilledLen) { + return nullptr; + } + // Get YUV VideoData, it uses more CPU, in most cases, on software codec. + data = CreateYUV420VideoData(aBufferData); + } + + // Update video time code, duration... from the raw data. + VideoData* video(data->As<VideoData>()); + if (aBufferData->mRawData) { + video->mTime = aBufferData->mRawData->mTime; + video->mTimecode = aBufferData->mRawData->mTimecode; + video->mOffset = aBufferData->mRawData->mOffset; + video->mDuration = aBufferData->mRawData->mDuration; + video->mKeyframe = aBufferData->mRawData->mKeyframe; + } + } + + return data.forget(); +} + +already_AddRefed<AudioData> +MediaDataHelper::CreateAudioData(BufferData* aBufferData) +{ + RefPtr<AudioData> audio; + OMX_BUFFERHEADERTYPE* buf = aBufferData->mBuffer; + const AudioInfo* info = mTrackInfo->GetAsAudioInfo(); + if (buf->nFilledLen) { + uint64_t offset = 0; + uint32_t frames = buf->nFilledLen / (2 * info->mChannels); + if (aBufferData->mRawData) { + offset = aBufferData->mRawData->mOffset; + } + typedef AudioCompactor::NativeCopy OmxCopy; + mAudioCompactor.Push(offset, + buf->nTimeStamp, + info->mRate, + frames, + info->mChannels, + OmxCopy(buf->pBuffer + buf->nOffset, + buf->nFilledLen, + info->mChannels)); + audio = mAudioQueue.PopFront(); + } + + return audio.forget(); +} + +already_AddRefed<VideoData> +MediaDataHelper::CreateYUV420VideoData(BufferData* aBufferData) +{ + uint8_t *yuv420p_buffer = (uint8_t *)aBufferData->mBuffer->pBuffer; + int32_t stride = mOutputPortDef.format.video.nStride; + int32_t slice_height = mOutputPortDef.format.video.nSliceHeight; + int32_t width = mTrackInfo->GetAsVideoInfo()->mImage.width; + int32_t height = mTrackInfo->GetAsVideoInfo()->mImage.height; + + // TODO: convert other formats to YUV420. + if (mOutputPortDef.format.video.eColorFormat != OMX_COLOR_FormatYUV420Planar) { + return nullptr; + } + + size_t yuv420p_y_size = stride * slice_height; + size_t yuv420p_u_size = ((stride + 1) / 2) * ((slice_height + 1) / 2); + uint8_t *yuv420p_y = yuv420p_buffer; + uint8_t *yuv420p_u = yuv420p_y + yuv420p_y_size; + uint8_t *yuv420p_v = yuv420p_u + yuv420p_u_size; + + VideoData::YCbCrBuffer b; + b.mPlanes[0].mData = yuv420p_y; + b.mPlanes[0].mWidth = width; + b.mPlanes[0].mHeight = height; + b.mPlanes[0].mStride = stride; + b.mPlanes[0].mOffset = 0; + b.mPlanes[0].mSkip = 0; + + b.mPlanes[1].mData = yuv420p_u; + b.mPlanes[1].mWidth = (width + 1) / 2; + b.mPlanes[1].mHeight = (height + 1) / 2; + b.mPlanes[1].mStride = (stride + 1) / 2; + b.mPlanes[1].mOffset = 0; + b.mPlanes[1].mSkip = 0; + + b.mPlanes[2].mData = yuv420p_v; + b.mPlanes[2].mWidth =(width + 1) / 2; + b.mPlanes[2].mHeight = (height + 1) / 2; + b.mPlanes[2].mStride = (stride + 1) / 2; + b.mPlanes[2].mOffset = 0; + b.mPlanes[2].mSkip = 0; + + VideoInfo info(*mTrackInfo->GetAsVideoInfo()); + RefPtr<VideoData> data = + VideoData::CreateAndCopyData(info, + mImageContainer, + 0, // Filled later by caller. + 0, // Filled later by caller. + 1, // We don't know the duration. + b, + 0, // Filled later by caller. + -1, + info.ImageRect()); + + LOG("YUV420 VideoData: disp width %d, height %d, pic width %d, height %d, time %ld", + info.mDisplay.width, info.mDisplay.height, info.mImage.width, + info.mImage.height, aBufferData->mBuffer->nTimeStamp); + + return data.forget(); +} + +} diff --git a/dom/media/platforms/omx/OmxDataDecoder.h b/dom/media/platforms/omx/OmxDataDecoder.h new file mode 100644 index 000000000..ea75b2a2a --- /dev/null +++ b/dom/media/platforms/omx/OmxDataDecoder.h @@ -0,0 +1,214 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#if !defined(OmxDataDecoder_h_) +#define OmxDataDecoder_h_ + +#include "mozilla/Monitor.h" + +#include "AudioCompactor.h" +#include "ImageContainer.h" +#include "MediaInfo.h" +#include "PlatformDecoderModule.h" + +#include "OMX_Component.h" + +#include "OmxPromiseLayer.h" + +namespace mozilla { + +class MediaDataHelper; + +typedef OmxPromiseLayer::OmxCommandPromise OmxCommandPromise; +typedef OmxPromiseLayer::OmxBufferPromise OmxBufferPromise; +typedef OmxPromiseLayer::OmxBufferFailureHolder OmxBufferFailureHolder; +typedef OmxPromiseLayer::OmxCommandFailureHolder OmxCommandFailureHolder; +typedef OmxPromiseLayer::BufferData BufferData; +typedef OmxPromiseLayer::BUFFERLIST BUFFERLIST; + +/* OmxDataDecoder is the major class which performs followings: + * 1. Translate PDM function into OMX commands. + * 2. Keeping the buffers between client and component. + * 3. Manage the OMX state. + * + * From the definition in OpenMax spec. "2.2.1", there are 3 major roles in + * OpenMax IL. + * + * IL client: + * "The IL client may be a layer below the GUI application, such as GStreamer, + * or may be several layers below the GUI layer." + * + * OmxDataDecoder acts as the IL client. + * + * OpenMAX IL component: + * "A component that is intended to wrap functionality that is required in the + * target system." + * + * OmxPromiseLayer acts as the OpenMAX IL component. + * + * OpenMAX IL core: + * "Platform-specific code that has the functionality necessary to locate and + * then load an OpenMAX IL component into main memory." + * + * OmxPlatformLayer acts as the OpenMAX IL core. + */ +class OmxDataDecoder : public MediaDataDecoder { +protected: + virtual ~OmxDataDecoder(); + +public: + OmxDataDecoder(const TrackInfo& aTrackInfo, + MediaDataDecoderCallback* aCallback, + layers::ImageContainer* aImageContainer); + + RefPtr<InitPromise> Init() override; + + void Input(MediaRawData* aSample) override; + + void Flush() override; + + void Drain() override; + + void Shutdown() override; + + const char* GetDescriptionName() const override + { + return "omx decoder"; + } + + // Return true if event is handled. + bool Event(OMX_EVENTTYPE aEvent, OMX_U32 aData1, OMX_U32 aData2); + +protected: + void InitializationTask(); + + void ResolveInitPromise(const char* aMethodName); + + void RejectInitPromise(MediaResult aError, const char* aMethodName); + + void OmxStateRunner(); + + void FillAndEmptyBuffers(); + + void FillBufferDone(BufferData* aData); + + void FillBufferFailure(OmxBufferFailureHolder aFailureHolder); + + void EmptyBufferDone(BufferData* aData); + + void EmptyBufferFailure(OmxBufferFailureHolder aFailureHolder); + + void NotifyError(OMX_ERRORTYPE aOmxError, + const char* aLine, + const MediaResult& aError = MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR)); + + // Configure audio/video codec. + // Some codec may just ignore this and rely on codec specific data in + // FillCodecConfigDataToOmx(). + void ConfigCodec(); + + // Sending codec specific data to OMX component. OMX component could send a + // OMX_EventPortSettingsChanged back to client. And then client needs to + // disable port and reallocate buffer. + void FillCodecConfigDataToOmx(); + + void SendEosBuffer(); + + void EndOfStream(); + + // It could be called after codec specific data is sent and component found + // the port format is changed due to different codec specific. + void PortSettingsChanged(); + + void Output(BufferData* aData); + + // Buffer can be released if its status is not OMX_COMPONENT or + // OMX_CLIENT_OUTPUT. + bool BuffersCanBeReleased(OMX_DIRTYPE aType); + + OMX_DIRTYPE GetPortDirection(uint32_t aPortIndex); + + void DoAsyncShutdown(); + + void DoFlush(); + + void FlushComplete(OMX_COMMANDTYPE aCommandType); + + void FlushFailure(OmxCommandFailureHolder aFailureHolder); + + BUFFERLIST* GetBuffers(OMX_DIRTYPE aType); + + nsresult AllocateBuffers(OMX_DIRTYPE aType); + + nsresult ReleaseBuffers(OMX_DIRTYPE aType); + + BufferData* FindAvailableBuffer(OMX_DIRTYPE aType); + + // aType could be OMX_DirMax for all types. + RefPtr<OmxPromiseLayer::OmxBufferPromise::AllPromiseType> + CollectBufferPromises(OMX_DIRTYPE aType); + + Monitor mMonitor; + + // The Omx TaskQueue. + RefPtr<TaskQueue> mOmxTaskQueue; + + RefPtr<TaskQueue> mReaderTaskQueue; + + RefPtr<layers::ImageContainer> mImageContainer; + + WatchManager<OmxDataDecoder> mWatchManager; + + // It is accessed in omx TaskQueue. + Watchable<OMX_STATETYPE> mOmxState; + + RefPtr<OmxPromiseLayer> mOmxLayer; + + UniquePtr<TrackInfo> mTrackInfo; + + // It is accessed in both omx and reader TaskQueue. + Atomic<bool> mFlushing; + + // It is accessed in Omx/reader TaskQeueu. + Atomic<bool> mShuttingDown; + + // It is accessed in Omx TaskQeueu. + bool mCheckingInputExhausted; + + // It is accessed in reader TaskQueue. + MozPromiseHolder<InitPromise> mInitPromise; + + // It is written in Omx TaskQeueu. Read in Omx TaskQueue. + // It value means the port index which port settings is changed. + // -1 means no port setting changed. + // + // Note: when port setting changed, there should be no buffer operations + // via EmptyBuffer or FillBuffer. + Watchable<int32_t> mPortSettingsChanged; + + // It is access in Omx TaskQueue. + nsTArray<RefPtr<MediaRawData>> mMediaRawDatas; + + BUFFERLIST mInPortBuffers; + + BUFFERLIST mOutPortBuffers; + + RefPtr<MediaDataHelper> mMediaDataHelper; + + MediaDataDecoderCallback* mCallback; +}; + +template<class T> +void InitOmxParameter(T* aParam) +{ + PodZero(aParam); + aParam->nSize = sizeof(T); + aParam->nVersion.s.nVersionMajor = 1; +} + +} + +#endif /* OmxDataDecoder_h_ */ diff --git a/dom/media/platforms/omx/OmxDecoderModule.cpp b/dom/media/platforms/omx/OmxDecoderModule.cpp new file mode 100644 index 000000000..9a3c2f929 --- /dev/null +++ b/dom/media/platforms/omx/OmxDecoderModule.cpp @@ -0,0 +1,45 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "OmxDecoderModule.h" + +#include "OmxDataDecoder.h" +#include "OmxPlatformLayer.h" + +namespace mozilla { + +already_AddRefed<MediaDataDecoder> +OmxDecoderModule::CreateVideoDecoder(const CreateDecoderParams& aParams) +{ + RefPtr<OmxDataDecoder> decoder = new OmxDataDecoder(aParams.mConfig, + aParams.mCallback, + aParams.mImageContainer); + return decoder.forget(); +} + +already_AddRefed<MediaDataDecoder> +OmxDecoderModule::CreateAudioDecoder(const CreateDecoderParams& aParams) +{ + RefPtr<OmxDataDecoder> decoder = new OmxDataDecoder(aParams.mConfig, + aParams.mCallback, + nullptr); + return decoder.forget(); +} + +PlatformDecoderModule::ConversionRequired +OmxDecoderModule::DecoderNeedsConversion(const TrackInfo& aConfig) const +{ + return ConversionRequired::kNeedNone; +} + +bool +OmxDecoderModule::SupportsMimeType(const nsACString& aMimeType, + DecoderDoctorDiagnostics* aDiagnostics) const +{ + return OmxPlatformLayer::SupportsMimeType(aMimeType); +} + +} diff --git a/dom/media/platforms/omx/OmxDecoderModule.h b/dom/media/platforms/omx/OmxDecoderModule.h new file mode 100644 index 000000000..432368a69 --- /dev/null +++ b/dom/media/platforms/omx/OmxDecoderModule.h @@ -0,0 +1,30 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#if !defined(OmxDecoderModule_h_) +#define OmxDecoderModule_h_ + +#include "PlatformDecoderModule.h" + +namespace mozilla { + +class OmxDecoderModule : public PlatformDecoderModule { +public: + already_AddRefed<MediaDataDecoder> + CreateVideoDecoder(const CreateDecoderParams& aParams) override; + + already_AddRefed<MediaDataDecoder> + CreateAudioDecoder(const CreateDecoderParams& aParams) override; + + bool SupportsMimeType(const nsACString& aMimeType, + DecoderDoctorDiagnostics* aDiagnostics) const override; + + ConversionRequired DecoderNeedsConversion(const TrackInfo& aConfig) const override; +}; + +} // namespace mozilla + +#endif // OmxDecoderModule_h_ diff --git a/dom/media/platforms/omx/OmxPlatformLayer.cpp b/dom/media/platforms/omx/OmxPlatformLayer.cpp new file mode 100644 index 000000000..d1f43144d --- /dev/null +++ b/dom/media/platforms/omx/OmxPlatformLayer.cpp @@ -0,0 +1,327 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "OmxPlatformLayer.h" + +#include "OMX_VideoExt.h" // For VP8. + +#if defined(MOZ_WIDGET_GONK) && (ANDROID_VERSION == 20 || ANDROID_VERSION == 19) +#define OMX_PLATFORM_GONK +#include "GonkOmxPlatformLayer.h" +#endif + +#include "VPXDecoder.h" + +#ifdef LOG +#undef LOG +#endif + +#define LOG(arg, ...) MOZ_LOG(sPDMLog, mozilla::LogLevel::Debug, ("OmxPlatformLayer -- %s: " arg, __func__, ##__VA_ARGS__)) + +#define RETURN_IF_ERR(err) \ + if (err != OMX_ErrorNone) { \ + LOG("error: 0x%08x", err); \ + return err; \ + } \ + +// Common OMX decoder configuration code. +namespace mozilla { + +// This helper class encapsulates the details of component parameters setting +// for different OMX audio & video codecs. +template<typename ParamType> +class OmxConfig +{ +public: + virtual ~OmxConfig() {} + // Subclasses should implement this method to configure the codec. + virtual OMX_ERRORTYPE Apply(OmxPlatformLayer& aOmx, const ParamType& aParam) = 0; +}; + +typedef OmxConfig<AudioInfo> OmxAudioConfig; +typedef OmxConfig<VideoInfo> OmxVideoConfig; + +template<typename ConfigType> +UniquePtr<ConfigType> ConfigForMime(const nsACString&); + +static OMX_ERRORTYPE +ConfigAudioOutputPort(OmxPlatformLayer& aOmx, const AudioInfo& aInfo) +{ + OMX_ERRORTYPE err; + + OMX_PARAM_PORTDEFINITIONTYPE def; + InitOmxParameter(&def); + def.nPortIndex = aOmx.OutputPortIndex(); + err = aOmx.GetParameter(OMX_IndexParamPortDefinition, &def, sizeof(def)); + RETURN_IF_ERR(err); + + def.format.audio.eEncoding = OMX_AUDIO_CodingPCM; + err = aOmx.SetParameter(OMX_IndexParamPortDefinition, &def, sizeof(def)); + RETURN_IF_ERR(err); + + OMX_AUDIO_PARAM_PCMMODETYPE pcmParams; + InitOmxParameter(&pcmParams); + pcmParams.nPortIndex = def.nPortIndex; + err = aOmx.GetParameter(OMX_IndexParamAudioPcm, &pcmParams, sizeof(pcmParams)); + RETURN_IF_ERR(err); + + pcmParams.nChannels = aInfo.mChannels; + pcmParams.eNumData = OMX_NumericalDataSigned; + pcmParams.bInterleaved = OMX_TRUE; + pcmParams.nBitPerSample = 16; + pcmParams.nSamplingRate = aInfo.mRate; + pcmParams.ePCMMode = OMX_AUDIO_PCMModeLinear; + err = aOmx.SetParameter(OMX_IndexParamAudioPcm, &pcmParams, sizeof(pcmParams)); + RETURN_IF_ERR(err); + + LOG("Config OMX_IndexParamAudioPcm, channel %d, sample rate %d", + pcmParams.nChannels, pcmParams.nSamplingRate); + + return OMX_ErrorNone; +} + +class OmxAacConfig : public OmxAudioConfig +{ +public: + OMX_ERRORTYPE Apply(OmxPlatformLayer& aOmx, const AudioInfo& aInfo) override + { + OMX_ERRORTYPE err; + + OMX_AUDIO_PARAM_AACPROFILETYPE aacProfile; + InitOmxParameter(&aacProfile); + aacProfile.nPortIndex = aOmx.InputPortIndex(); + err = aOmx.GetParameter(OMX_IndexParamAudioAac, &aacProfile, sizeof(aacProfile)); + RETURN_IF_ERR(err); + + aacProfile.nChannels = aInfo.mChannels; + aacProfile.nSampleRate = aInfo.mRate; + aacProfile.eAACProfile = static_cast<OMX_AUDIO_AACPROFILETYPE>(aInfo.mProfile); + err = aOmx.SetParameter(OMX_IndexParamAudioAac, &aacProfile, sizeof(aacProfile)); + RETURN_IF_ERR(err); + + LOG("Config OMX_IndexParamAudioAac, channel %d, sample rate %d, profile %d", + aacProfile.nChannels, aacProfile.nSampleRate, aacProfile.eAACProfile); + + return ConfigAudioOutputPort(aOmx, aInfo); + } +}; + +class OmxMp3Config : public OmxAudioConfig +{ +public: + OMX_ERRORTYPE Apply(OmxPlatformLayer& aOmx, const AudioInfo& aInfo) override + { + OMX_ERRORTYPE err; + + OMX_AUDIO_PARAM_MP3TYPE mp3Param; + InitOmxParameter(&mp3Param); + mp3Param.nPortIndex = aOmx.InputPortIndex(); + err = aOmx.GetParameter(OMX_IndexParamAudioMp3, &mp3Param, sizeof(mp3Param)); + RETURN_IF_ERR(err); + + mp3Param.nChannels = aInfo.mChannels; + mp3Param.nSampleRate = aInfo.mRate; + err = aOmx.SetParameter(OMX_IndexParamAudioMp3, &mp3Param, sizeof(mp3Param)); + RETURN_IF_ERR(err); + + LOG("Config OMX_IndexParamAudioMp3, channel %d, sample rate %d", + mp3Param.nChannels, mp3Param.nSampleRate); + + return ConfigAudioOutputPort(aOmx, aInfo); + } +}; + +enum OmxAmrSampleRate { + kNarrowBand = 8000, + kWideBand = 16000, +}; + +template <OmxAmrSampleRate R> +class OmxAmrConfig : public OmxAudioConfig +{ +public: + OMX_ERRORTYPE Apply(OmxPlatformLayer& aOmx, const AudioInfo& aInfo) override + { + OMX_ERRORTYPE err; + + OMX_AUDIO_PARAM_AMRTYPE def; + InitOmxParameter(&def); + def.nPortIndex = aOmx.InputPortIndex(); + err = aOmx.GetParameter(OMX_IndexParamAudioAmr, &def, sizeof(def)); + RETURN_IF_ERR(err); + + def.eAMRFrameFormat = OMX_AUDIO_AMRFrameFormatFSF; + err = aOmx.SetParameter(OMX_IndexParamAudioAmr, &def, sizeof(def)); + RETURN_IF_ERR(err); + + MOZ_ASSERT(aInfo.mChannels == 1); + MOZ_ASSERT(aInfo.mRate == R); + + return ConfigAudioOutputPort(aOmx, aInfo); + } +}; + +template<> +UniquePtr<OmxAudioConfig> +ConfigForMime(const nsACString& aMimeType) +{ + UniquePtr<OmxAudioConfig> conf; + + if (OmxPlatformLayer::SupportsMimeType(aMimeType)) { + if (aMimeType.EqualsLiteral("audio/mp4a-latm")) { + conf.reset(new OmxAacConfig()); + } else if (aMimeType.EqualsLiteral("audio/mp3") || + aMimeType.EqualsLiteral("audio/mpeg")) { + conf.reset(new OmxMp3Config()); + } else if (aMimeType.EqualsLiteral("audio/3gpp")) { + conf.reset(new OmxAmrConfig<OmxAmrSampleRate::kNarrowBand>()); + } else if (aMimeType.EqualsLiteral("audio/amr-wb")) { + conf.reset(new OmxAmrConfig<OmxAmrSampleRate::kWideBand>()); + } + } + return Move(conf); +} + +// There should be a better way to calculate it. +#define MIN_VIDEO_INPUT_BUFFER_SIZE 64 * 1024 + +class OmxCommonVideoConfig : public OmxVideoConfig +{ +public: + explicit OmxCommonVideoConfig() + : OmxVideoConfig() + {} + + OMX_ERRORTYPE Apply(OmxPlatformLayer& aOmx, const VideoInfo& aInfo) override + { + OMX_ERRORTYPE err; + OMX_PARAM_PORTDEFINITIONTYPE def; + + // Set up in/out port definition. + nsTArray<uint32_t> ports; + aOmx.GetPortIndices(ports); + for (auto idx : ports) { + InitOmxParameter(&def); + def.nPortIndex = idx; + err = aOmx.GetParameter(OMX_IndexParamPortDefinition, &def, sizeof(def)); + RETURN_IF_ERR(err); + + def.format.video.nFrameWidth = aInfo.mDisplay.width; + def.format.video.nFrameHeight = aInfo.mDisplay.height; + def.format.video.nStride = aInfo.mImage.width; + def.format.video.nSliceHeight = aInfo.mImage.height; + + if (def.eDir == OMX_DirInput) { + def.format.video.eCompressionFormat = aOmx.CompressionFormat(); + def.format.video.eColorFormat = OMX_COLOR_FormatUnused; + if (def.nBufferSize < MIN_VIDEO_INPUT_BUFFER_SIZE) { + def.nBufferSize = aInfo.mImage.width * aInfo.mImage.height; + LOG("Change input buffer size to %d", def.nBufferSize); + } + } else { + def.format.video.eCompressionFormat = OMX_VIDEO_CodingUnused; + } + + err = aOmx.SetParameter(OMX_IndexParamPortDefinition, &def, sizeof(def)); + } + return err; + } +}; + +template<> +UniquePtr<OmxVideoConfig> +ConfigForMime(const nsACString& aMimeType) +{ + UniquePtr<OmxVideoConfig> conf; + + if (OmxPlatformLayer::SupportsMimeType(aMimeType)) { + conf.reset(new OmxCommonVideoConfig()); + } + return Move(conf); +} + +OMX_ERRORTYPE +OmxPlatformLayer::Config() +{ + MOZ_ASSERT(mInfo); + + OMX_PORT_PARAM_TYPE portParam; + InitOmxParameter(&portParam); + if (mInfo->IsAudio()) { + GetParameter(OMX_IndexParamAudioInit, &portParam, sizeof(portParam)); + mStartPortNumber = portParam.nStartPortNumber; + UniquePtr<OmxAudioConfig> conf(ConfigForMime<OmxAudioConfig>(mInfo->mMimeType)); + MOZ_ASSERT(conf.get()); + return conf->Apply(*this, *(mInfo->GetAsAudioInfo())); + } else if (mInfo->IsVideo()) { + GetParameter(OMX_IndexParamVideoInit, &portParam, sizeof(portParam)); + UniquePtr<OmxVideoConfig> conf(ConfigForMime<OmxVideoConfig>(mInfo->mMimeType)); + MOZ_ASSERT(conf.get()); + return conf->Apply(*this, *(mInfo->GetAsVideoInfo())); + } else { + MOZ_ASSERT_UNREACHABLE("non-AV data (text?) is not supported."); + return OMX_ErrorNotImplemented; + } +} + +OMX_VIDEO_CODINGTYPE +OmxPlatformLayer::CompressionFormat() +{ + MOZ_ASSERT(mInfo); + + if (mInfo->mMimeType.EqualsLiteral("video/avc")) { + return OMX_VIDEO_CodingAVC; + } else if (mInfo->mMimeType.EqualsLiteral("video/mp4v-es") || + mInfo->mMimeType.EqualsLiteral("video/mp4")) { + return OMX_VIDEO_CodingMPEG4; + } else if (mInfo->mMimeType.EqualsLiteral("video/3gpp")) { + return OMX_VIDEO_CodingH263; + } else if (VPXDecoder::IsVP8(mInfo->mMimeType)) { + return static_cast<OMX_VIDEO_CODINGTYPE>(OMX_VIDEO_CodingVP8); + } else { + MOZ_ASSERT_UNREACHABLE("Unsupported compression format"); + return OMX_VIDEO_CodingUnused; + } +} + +// Implementations for different platforms will be defined in their own files. +#ifdef OMX_PLATFORM_GONK + +bool +OmxPlatformLayer::SupportsMimeType(const nsACString& aMimeType) +{ + return GonkOmxPlatformLayer::FindComponents(aMimeType); +} + +OmxPlatformLayer* +OmxPlatformLayer::Create(OmxDataDecoder* aDataDecoder, + OmxPromiseLayer* aPromiseLayer, + TaskQueue* aTaskQueue, + layers::ImageContainer* aImageContainer) +{ + return new GonkOmxPlatformLayer(aDataDecoder, aPromiseLayer, aTaskQueue, aImageContainer); +} + +#else // For platforms without OMX IL support. + +bool +OmxPlatformLayer::SupportsMimeType(const nsACString& aMimeType) +{ + return false; +} + +OmxPlatformLayer* +OmxPlatformLayer::Create(OmxDataDecoder* aDataDecoder, + OmxPromiseLayer* aPromiseLayer, + TaskQueue* aTaskQueue, + layers::ImageContainer* aImageContainer) +{ + return nullptr; +} + +#endif + +} diff --git a/dom/media/platforms/omx/OmxPlatformLayer.h b/dom/media/platforms/omx/OmxPlatformLayer.h new file mode 100644 index 000000000..67d9e448f --- /dev/null +++ b/dom/media/platforms/omx/OmxPlatformLayer.h @@ -0,0 +1,100 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#if !defined(OmxPlatformLayer_h_) +#define OmxPlatformLayer_h_ + +#include "OMX_Core.h" +#include "OMX_Types.h" + +#include "OmxPromiseLayer.h" + +class nsACString; + +namespace mozilla { + +class TaskQueue; +class TrackInfo; + +/* + * This class the the abstract layer of the platform OpenMax IL implementation. + * + * For some platform like andoird, it exposures its OpenMax IL via IOMX which + * is definitions are different comparing to standard. + * For other platforms like Raspberry Pi, it will be easy to implement this layer + * with the standard OpenMax IL api. + */ +class OmxPlatformLayer { +public: + typedef OmxPromiseLayer::BUFFERLIST BUFFERLIST; + typedef OmxPromiseLayer::BufferData BufferData; + + virtual OMX_ERRORTYPE InitOmxToStateLoaded(const TrackInfo* aInfo) = 0; + + OMX_ERRORTYPE Config(); + + virtual OMX_ERRORTYPE EmptyThisBuffer(BufferData* aData) = 0; + + virtual OMX_ERRORTYPE FillThisBuffer(BufferData* aData) = 0; + + virtual OMX_ERRORTYPE SendCommand(OMX_COMMANDTYPE aCmd, + OMX_U32 aParam1, + OMX_PTR aCmdData) = 0; + + // Buffer could be platform dependent; for example, video decoding needs gralloc + // on Gonk. Therefore, derived class needs to implement its owned buffer + // allocate/release API according to its platform type. + virtual nsresult AllocateOmxBuffer(OMX_DIRTYPE aType, BUFFERLIST* aBufferList) = 0; + + virtual nsresult ReleaseOmxBuffer(OMX_DIRTYPE aType, BUFFERLIST* aBufferList) = 0; + + virtual OMX_ERRORTYPE GetState(OMX_STATETYPE* aType) = 0; + + virtual OMX_ERRORTYPE GetParameter(OMX_INDEXTYPE aParamIndex, + OMX_PTR aComponentParameterStructure, + OMX_U32 aComponentParameterSize) = 0; + + virtual OMX_ERRORTYPE SetParameter(OMX_INDEXTYPE nIndex, + OMX_PTR aComponentParameterStructure, + OMX_U32 aComponentParameterSize) = 0; + + virtual nsresult Shutdown() = 0; + + virtual ~OmxPlatformLayer() {} + + // For decoders, input port index is start port number and output port is next. + // See OpenMAX IL spec v1.1.2 section 8.6.1 & 8.8.1. + OMX_U32 InputPortIndex() { return mStartPortNumber; } + + OMX_U32 OutputPortIndex() { return mStartPortNumber + 1; } + + void GetPortIndices(nsTArray<uint32_t>& aPortIndex) { + aPortIndex.AppendElement(InputPortIndex()); + aPortIndex.AppendElement(OutputPortIndex()); + } + + virtual OMX_VIDEO_CODINGTYPE CompressionFormat(); + + // Check if the platform implementation supports given MIME type. + static bool SupportsMimeType(const nsACString& aMimeType); + + // Hide the details of creating implementation objects for different platforms. + static OmxPlatformLayer* Create(OmxDataDecoder* aDataDecoder, + OmxPromiseLayer* aPromiseLayer, + TaskQueue* aTaskQueue, + layers::ImageContainer* aImageContainer); + +protected: + OmxPlatformLayer() : mInfo(nullptr), mStartPortNumber(0) {} + + // The pointee is held by |OmxDataDecoder::mTrackInfo| and will outlive this pointer. + const TrackInfo* mInfo; + OMX_U32 mStartPortNumber; +}; + +} + +#endif // OmxPlatformLayer_h_ diff --git a/dom/media/platforms/omx/OmxPromiseLayer.cpp b/dom/media/platforms/omx/OmxPromiseLayer.cpp new file mode 100644 index 000000000..0c794289d --- /dev/null +++ b/dom/media/platforms/omx/OmxPromiseLayer.cpp @@ -0,0 +1,382 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "OmxPromiseLayer.h" + +#include "ImageContainer.h" + +#include "OmxDataDecoder.h" +#include "OmxPlatformLayer.h" + + +#ifdef LOG +#undef LOG +#endif + +#define LOG(arg, ...) MOZ_LOG(sPDMLog, mozilla::LogLevel::Debug, ("OmxPromiseLayer(%p)::%s: " arg, this, __func__, ##__VA_ARGS__)) + +namespace mozilla { + +OmxPromiseLayer::OmxPromiseLayer(TaskQueue* aTaskQueue, + OmxDataDecoder* aDataDecoder, + layers::ImageContainer* aImageContainer) + : mTaskQueue(aTaskQueue) +{ + mPlatformLayer = OmxPlatformLayer::Create(aDataDecoder, + this, + aTaskQueue, + aImageContainer); + MOZ_ASSERT(!!mPlatformLayer); +} + +RefPtr<OmxPromiseLayer::OmxCommandPromise> +OmxPromiseLayer::Init(const TrackInfo* aInfo) +{ + MOZ_ASSERT(mTaskQueue->IsCurrentThreadIn()); + + OMX_ERRORTYPE err = mPlatformLayer->InitOmxToStateLoaded(aInfo); + if (err != OMX_ErrorNone) { + OmxCommandFailureHolder failure(OMX_ErrorUndefined, OMX_CommandStateSet); + return OmxCommandPromise::CreateAndReject(failure, __func__); + } + + OMX_STATETYPE state = GetState(); + if (state == OMX_StateLoaded) { + return OmxCommandPromise::CreateAndResolve(OMX_CommandStateSet, __func__); + } if (state == OMX_StateIdle) { + return SendCommand(OMX_CommandStateSet, OMX_StateIdle, nullptr); + } + + OmxCommandFailureHolder failure(OMX_ErrorUndefined, OMX_CommandStateSet); + return OmxCommandPromise::CreateAndReject(failure, __func__); +} + +OMX_ERRORTYPE +OmxPromiseLayer::Config() +{ + MOZ_ASSERT(GetState() == OMX_StateLoaded); + + return mPlatformLayer->Config(); +} + +RefPtr<OmxPromiseLayer::OmxBufferPromise> +OmxPromiseLayer::FillBuffer(BufferData* aData) +{ + MOZ_ASSERT(mTaskQueue->IsCurrentThreadIn()); + LOG("buffer %p", aData->mBuffer); + + RefPtr<OmxBufferPromise> p = aData->mPromise.Ensure(__func__); + + OMX_ERRORTYPE err = mPlatformLayer->FillThisBuffer(aData); + + if (err != OMX_ErrorNone) { + OmxBufferFailureHolder failure(err, aData); + aData->mPromise.Reject(Move(failure), __func__); + } else { + aData->mStatus = BufferData::BufferStatus::OMX_COMPONENT; + GetBufferHolders(OMX_DirOutput)->AppendElement(aData); + } + + return p; +} + +RefPtr<OmxPromiseLayer::OmxBufferPromise> +OmxPromiseLayer::EmptyBuffer(BufferData* aData) +{ + MOZ_ASSERT(mTaskQueue->IsCurrentThreadIn()); + LOG("buffer %p, size %d", aData->mBuffer, aData->mBuffer->nFilledLen); + + RefPtr<OmxBufferPromise> p = aData->mPromise.Ensure(__func__); + + OMX_ERRORTYPE err = mPlatformLayer->EmptyThisBuffer(aData); + + if (err != OMX_ErrorNone) { + OmxBufferFailureHolder failure(err, aData); + aData->mPromise.Reject(Move(failure), __func__); + } else { + if (aData->mRawData) { + mRawDatas.AppendElement(Move(aData->mRawData)); + } + aData->mStatus = BufferData::BufferStatus::OMX_COMPONENT; + GetBufferHolders(OMX_DirInput)->AppendElement(aData); + } + + return p; +} + +OmxPromiseLayer::BUFFERLIST* +OmxPromiseLayer::GetBufferHolders(OMX_DIRTYPE aType) +{ + MOZ_ASSERT(aType == OMX_DirInput || aType == OMX_DirOutput); + + if (aType == OMX_DirInput) { + return &mInbufferHolders; + } + + return &mOutbufferHolders; +} + +already_AddRefed<MediaRawData> +OmxPromiseLayer::FindAndRemoveRawData(OMX_TICKS aTimecode) +{ + for (auto raw : mRawDatas) { + if (raw->mTime == aTimecode) { + mRawDatas.RemoveElement(raw); + return raw.forget(); + } + } + return nullptr; +} + +already_AddRefed<BufferData> +OmxPromiseLayer::FindAndRemoveBufferHolder(OMX_DIRTYPE aType, + BufferData::BufferID aId) +{ + MOZ_ASSERT(mTaskQueue->IsCurrentThreadIn()); + + RefPtr<BufferData> holder; + BUFFERLIST* holders = GetBufferHolders(aType); + + for (uint32_t i = 0; i < holders->Length(); i++) { + if (holders->ElementAt(i)->ID() == aId) { + holder = holders->ElementAt(i); + holders->RemoveElementAt(i); + return holder.forget(); + } + } + + return nullptr; +} + +already_AddRefed<BufferData> +OmxPromiseLayer::FindBufferById(OMX_DIRTYPE aType, BufferData::BufferID aId) +{ + MOZ_ASSERT(mTaskQueue->IsCurrentThreadIn()); + + RefPtr<BufferData> holder; + BUFFERLIST* holders = GetBufferHolders(aType); + + for (uint32_t i = 0; i < holders->Length(); i++) { + if (holders->ElementAt(i)->ID() == aId) { + holder = holders->ElementAt(i); + return holder.forget(); + } + } + + return nullptr; +} + +void +OmxPromiseLayer::EmptyFillBufferDone(OMX_DIRTYPE aType, BufferData* aData) +{ + if (aData) { + LOG("type %d, buffer %p", aType, aData->mBuffer); + if (aType == OMX_DirOutput) { + aData->mRawData = nullptr; + aData->mRawData = FindAndRemoveRawData(aData->mBuffer->nTimeStamp); + } + aData->mStatus = BufferData::BufferStatus::OMX_CLIENT; + aData->mPromise.Resolve(aData, __func__); + } else { + LOG("type %d, no buffer", aType); + } +} + +void +OmxPromiseLayer::EmptyFillBufferDone(OMX_DIRTYPE aType, BufferData::BufferID aID) +{ + RefPtr<BufferData> holder = FindAndRemoveBufferHolder(aType, aID); + EmptyFillBufferDone(aType, holder); +} + +RefPtr<OmxPromiseLayer::OmxCommandPromise> +OmxPromiseLayer::SendCommand(OMX_COMMANDTYPE aCmd, OMX_U32 aParam1, OMX_PTR aCmdData) +{ + if (aCmd == OMX_CommandFlush) { + // It doesn't support another flush commands before previous one is completed. + MOZ_RELEASE_ASSERT(!mFlushCommands.Length()); + + // Some coomponents don't send event with OMX_ALL, they send flush complete + // event with input port and another event for output port. + // In prupose of better compatibility, we interpret the OMX_ALL to OMX_DirInput + // and OMX_DirOutput flush separately. + OMX_DIRTYPE types[] = {OMX_DIRTYPE::OMX_DirInput, OMX_DIRTYPE::OMX_DirOutput}; + for(const auto type : types) { + if ((aParam1 == type) || (aParam1 == OMX_ALL)) { + mFlushCommands.AppendElement(FlushCommand({type, aCmdData})); + } + + if (type == OMX_DirInput) { + // Clear all buffered raw data. + mRawDatas.Clear(); + } + } + + // Don't overlay more than one flush command, some components can't overlay flush commands. + // So here we send another flush after receiving the previous flush completed event. + if (mFlushCommands.Length()) { + OMX_ERRORTYPE err = + mPlatformLayer->SendCommand(OMX_CommandFlush, + mFlushCommands.ElementAt(0).type, + mFlushCommands.ElementAt(0).cmd); + if (err != OMX_ErrorNone) { + OmxCommandFailureHolder failure(OMX_ErrorNotReady, OMX_CommandFlush); + return OmxCommandPromise::CreateAndReject(failure, __func__); + } + } else { + LOG("OMX_CommandFlush parameter error"); + OmxCommandFailureHolder failure(OMX_ErrorNotReady, OMX_CommandFlush); + return OmxCommandPromise::CreateAndReject(failure, __func__); + } + } else { + OMX_ERRORTYPE err = mPlatformLayer->SendCommand(aCmd, aParam1, aCmdData); + if (err != OMX_ErrorNone) { + OmxCommandFailureHolder failure(OMX_ErrorNotReady, aCmd); + return OmxCommandPromise::CreateAndReject(failure, __func__); + } + } + + RefPtr<OmxCommandPromise> p; + if (aCmd == OMX_CommandStateSet) { + p = mCommandStatePromise.Ensure(__func__); + } else if (aCmd == OMX_CommandFlush) { + p = mFlushPromise.Ensure(__func__); + } else if (aCmd == OMX_CommandPortEnable) { + p = mPortEnablePromise.Ensure(__func__); + } else if (aCmd == OMX_CommandPortDisable) { + p = mPortDisablePromise.Ensure(__func__); + } else { + LOG("error unsupport command"); + MOZ_ASSERT(0); + } + + return p; +} + +bool +OmxPromiseLayer::Event(OMX_EVENTTYPE aEvent, OMX_U32 aData1, OMX_U32 aData2) +{ + OMX_COMMANDTYPE cmd = (OMX_COMMANDTYPE) aData1; + switch (aEvent) { + case OMX_EventCmdComplete: + { + if (cmd == OMX_CommandStateSet) { + mCommandStatePromise.Resolve(OMX_CommandStateSet, __func__); + } else if (cmd == OMX_CommandFlush) { + MOZ_RELEASE_ASSERT(mFlushCommands.ElementAt(0).type == aData2); + LOG("OMX_CommandFlush completed port type %d", aData2); + mFlushCommands.RemoveElementAt(0); + + // Sending next flush command. + if (mFlushCommands.Length()) { + OMX_ERRORTYPE err = + mPlatformLayer->SendCommand(OMX_CommandFlush, + mFlushCommands.ElementAt(0).type, + mFlushCommands.ElementAt(0).cmd); + if (err != OMX_ErrorNone) { + OmxCommandFailureHolder failure(OMX_ErrorNotReady, OMX_CommandFlush); + mFlushPromise.Reject(failure, __func__); + } + } else { + mFlushPromise.Resolve(OMX_CommandFlush, __func__); + } + } else if (cmd == OMX_CommandPortDisable) { + mPortDisablePromise.Resolve(OMX_CommandPortDisable, __func__); + } else if (cmd == OMX_CommandPortEnable) { + mPortEnablePromise.Resolve(OMX_CommandPortEnable, __func__); + } + break; + } + case OMX_EventError: + { + if (cmd == OMX_CommandStateSet) { + OmxCommandFailureHolder failure(OMX_ErrorUndefined, OMX_CommandStateSet); + mCommandStatePromise.Reject(failure, __func__); + } else if (cmd == OMX_CommandFlush) { + OmxCommandFailureHolder failure(OMX_ErrorUndefined, OMX_CommandFlush); + mFlushPromise.Reject(failure, __func__); + } else if (cmd == OMX_CommandPortDisable) { + OmxCommandFailureHolder failure(OMX_ErrorUndefined, OMX_CommandPortDisable); + mPortDisablePromise.Reject(failure, __func__); + } else if (cmd == OMX_CommandPortEnable) { + OmxCommandFailureHolder failure(OMX_ErrorUndefined, OMX_CommandPortEnable); + mPortEnablePromise.Reject(failure, __func__); + } else { + return false; + } + break; + } + default: + { + return false; + } + } + return true; +} + +nsresult +OmxPromiseLayer::AllocateOmxBuffer(OMX_DIRTYPE aType, BUFFERLIST* aBuffers) +{ + return mPlatformLayer->AllocateOmxBuffer(aType, aBuffers); +} + +nsresult +OmxPromiseLayer::ReleaseOmxBuffer(OMX_DIRTYPE aType, BUFFERLIST* aBuffers) +{ + return mPlatformLayer->ReleaseOmxBuffer(aType, aBuffers); +} + +OMX_STATETYPE +OmxPromiseLayer::GetState() +{ + OMX_STATETYPE state; + OMX_ERRORTYPE err = mPlatformLayer->GetState(&state); + return err == OMX_ErrorNone ? state : OMX_StateInvalid; +} + +OMX_ERRORTYPE +OmxPromiseLayer::GetParameter(OMX_INDEXTYPE aParamIndex, + OMX_PTR aComponentParameterStructure, + OMX_U32 aComponentParameterSize) +{ + return mPlatformLayer->GetParameter(aParamIndex, + aComponentParameterStructure, + aComponentParameterSize); +} + +OMX_ERRORTYPE +OmxPromiseLayer::SetParameter(OMX_INDEXTYPE aParamIndex, + OMX_PTR aComponentParameterStructure, + OMX_U32 aComponentParameterSize) +{ + return mPlatformLayer->SetParameter(aParamIndex, + aComponentParameterStructure, + aComponentParameterSize); +} + +OMX_U32 +OmxPromiseLayer::InputPortIndex() +{ + return mPlatformLayer->InputPortIndex(); +} + +OMX_U32 +OmxPromiseLayer::OutputPortIndex() +{ + return mPlatformLayer->OutputPortIndex(); +} + +nsresult +OmxPromiseLayer::Shutdown() +{ + LOG(""); + MOZ_ASSERT(mTaskQueue->IsCurrentThreadIn()); + MOZ_ASSERT(!GetBufferHolders(OMX_DirInput)->Length()); + MOZ_ASSERT(!GetBufferHolders(OMX_DirOutput)->Length()); + return mPlatformLayer->Shutdown(); +} + +} diff --git a/dom/media/platforms/omx/OmxPromiseLayer.h b/dom/media/platforms/omx/OmxPromiseLayer.h new file mode 100644 index 000000000..ecc6b2a9d --- /dev/null +++ b/dom/media/platforms/omx/OmxPromiseLayer.h @@ -0,0 +1,252 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#if !defined(OmxPromiseLayer_h_) +#define OmxPromiseLayer_h_ + +#include "mozilla/MozPromise.h" +#include "mozilla/TaskQueue.h" +#include "nsAutoPtr.h" + +#include "OMX_Core.h" +#include "OMX_Types.h" + +namespace mozilla { + +namespace layers +{ +class ImageContainer; +} + +class MediaData; +class MediaRawData; +class OmxDataDecoder; +class OmxPlatformLayer; +class TrackInfo; + +/* This class acts as a middle layer between OmxDataDecoder and the underlying + * OmxPlatformLayer. + * + * This class has two purposes: + * 1. Using promise instead of OpenMax async callback function. + * For example, OmxCommandPromise is used for OpenMax IL SendCommand. + * 2. Manage the buffer exchanged between client and component. + * Because omx buffer works crossing threads, so each omx buffer has its own + * promise, it is defined in BufferData. + * + * All of functions and members in this class should be run in the same + * TaskQueue. + */ +class OmxPromiseLayer { +protected: + virtual ~OmxPromiseLayer() {} + +public: + NS_INLINE_DECL_THREADSAFE_REFCOUNTING(OmxPromiseLayer) + + OmxPromiseLayer(TaskQueue* aTaskQueue, + OmxDataDecoder* aDataDecoder, + layers::ImageContainer* aImageContainer); + + class BufferData; + + typedef nsTArray<RefPtr<BufferData>> BUFFERLIST; + + class OmxBufferFailureHolder { + public: + OmxBufferFailureHolder(OMX_ERRORTYPE aError, BufferData* aBuffer) + : mError(aError) + , mBuffer(aBuffer) + {} + + OMX_ERRORTYPE mError; + BufferData* mBuffer; + }; + + typedef MozPromise<BufferData*, OmxBufferFailureHolder, /* IsExclusive = */ false> OmxBufferPromise; + + class OmxCommandFailureHolder { + public: + OmxCommandFailureHolder(OMX_ERRORTYPE aErrorType, + OMX_COMMANDTYPE aCommandType) + : mErrorType(aErrorType) + , mCommandType(aCommandType) + {} + + OMX_ERRORTYPE mErrorType; + OMX_COMMANDTYPE mCommandType; + }; + + typedef MozPromise<OMX_COMMANDTYPE, OmxCommandFailureHolder, /* IsExclusive = */ true> OmxCommandPromise; + + typedef MozPromise<uint32_t, bool, /* IsExclusive = */ true> OmxPortConfigPromise; + + // TODO: maybe a generic promise is good enough for this case? + RefPtr<OmxCommandPromise> Init(const TrackInfo* aInfo); + + OMX_ERRORTYPE Config(); + + RefPtr<OmxBufferPromise> FillBuffer(BufferData* aData); + + RefPtr<OmxBufferPromise> EmptyBuffer(BufferData* aData); + + RefPtr<OmxCommandPromise> SendCommand(OMX_COMMANDTYPE aCmd, + OMX_U32 aParam1, + OMX_PTR aCmdData); + + nsresult AllocateOmxBuffer(OMX_DIRTYPE aType, BUFFERLIST* aBuffers); + + nsresult ReleaseOmxBuffer(OMX_DIRTYPE aType, BUFFERLIST* aBuffers); + + OMX_STATETYPE GetState(); + + OMX_ERRORTYPE GetParameter(OMX_INDEXTYPE aParamIndex, + OMX_PTR aComponentParameterStructure, + OMX_U32 aComponentParameterSize); + + OMX_ERRORTYPE SetParameter(OMX_INDEXTYPE nIndex, + OMX_PTR aComponentParameterStructure, + OMX_U32 aComponentParameterSize); + + OMX_U32 InputPortIndex(); + + OMX_U32 OutputPortIndex(); + + nsresult Shutdown(); + + // BufferData maintains the status of OMX buffer (OMX_BUFFERHEADERTYPE). + // mStatus tracks the buffer owner. + // And a promise because OMX buffer working among different threads. + class BufferData { + protected: + virtual ~BufferData() {} + + public: + explicit BufferData(OMX_BUFFERHEADERTYPE* aBuffer) + : mEos(false) + , mStatus(BufferStatus::FREE) + , mBuffer(aBuffer) + {} + + typedef void* BufferID; + + NS_INLINE_DECL_THREADSAFE_REFCOUNTING(BufferData) + + // In most cases, the ID of this buffer is the pointer address of mBuffer. + // However, in platform like gonk, it is another value. + virtual BufferID ID() + { + return mBuffer; + } + + // Return the platform dependent MediaData(). + // For example, it returns the MediaData with Gralloc texture. + // If it returns nullptr, then caller uses the normal way to + // create MediaData(). + virtual already_AddRefed<MediaData> GetPlatformMediaData() + { + return nullptr; + } + + // The buffer could be used by several objects. And only one object owns the + // buffer the same time. + // FREE: + // nobody uses it. + // + // OMX_COMPONENT: + // buffer is used by OMX component (OmxPlatformLayer). + // + // OMX_CLIENT: + // buffer is used by client which is wait for audio/video playing + // (OmxDataDecoder) + // + // OMX_CLIENT_OUTPUT: + // used by client to output decoded data (for example, Gecko layer in + // this case) + // + // For output port buffer, the status transition is: + // FREE -> OMX_COMPONENT -> OMX_CLIENT -> OMX_CLIENT_OUTPUT -> FREE + // + // For input port buffer, the status transition is: + // FREE -> OMX_COMPONENT -> OMX_CLIENT -> FREE + // + enum BufferStatus { + FREE, + OMX_COMPONENT, + OMX_CLIENT, + OMX_CLIENT_OUTPUT, + INVALID + }; + + bool mEos; + + // The raw keeps in OmxPromiseLayer after EmptyBuffer and then passing to + // output decoded buffer in EmptyFillBufferDone. It is used to keep the + // records of the original data from demuxer, like duration, stream offset...etc. + RefPtr<MediaRawData> mRawData; + + // Because OMX buffer works across threads, so it uses a promise + // for each buffer when the buffer is used by Omx component. + MozPromiseHolder<OmxBufferPromise> mPromise; + BufferStatus mStatus; + OMX_BUFFERHEADERTYPE* mBuffer; + }; + + void EmptyFillBufferDone(OMX_DIRTYPE aType, BufferData::BufferID aID); + + void EmptyFillBufferDone(OMX_DIRTYPE aType, BufferData* aData); + + already_AddRefed<BufferData> + FindBufferById(OMX_DIRTYPE aType, BufferData::BufferID aId); + + already_AddRefed<BufferData> + FindAndRemoveBufferHolder(OMX_DIRTYPE aType, BufferData::BufferID aId); + + // Return true if event is handled. + bool Event(OMX_EVENTTYPE aEvent, OMX_U32 aData1, OMX_U32 aData2); + +protected: + struct FlushCommand { + OMX_DIRTYPE type; + OMX_PTR cmd; + }; + + BUFFERLIST* GetBufferHolders(OMX_DIRTYPE aType); + + already_AddRefed<MediaRawData> FindAndRemoveRawData(OMX_TICKS aTimecode); + + RefPtr<TaskQueue> mTaskQueue; + + MozPromiseHolder<OmxCommandPromise> mCommandStatePromise; + + MozPromiseHolder<OmxCommandPromise> mPortDisablePromise; + + MozPromiseHolder<OmxCommandPromise> mPortEnablePromise; + + MozPromiseHolder<OmxCommandPromise> mFlushPromise; + + nsTArray<FlushCommand> mFlushCommands; + + nsAutoPtr<OmxPlatformLayer> mPlatformLayer; + +private: + // Elements are added to holders when FillBuffer() or FillBuffer(). And + // removing element when the promise is resolved. Buffers in these lists + // should NOT be used by other component; for example, output it to audio + // output. These lists should be empty when engine is about to shutdown. + // + // Note: + // There bufferlist should not be used by other class directly. + BUFFERLIST mInbufferHolders; + + BUFFERLIST mOutbufferHolders; + + nsTArray<RefPtr<MediaRawData>> mRawDatas; +}; + +} + +#endif /* OmxPromiseLayer_h_ */ diff --git a/dom/media/platforms/omx/moz.build b/dom/media/platforms/omx/moz.build new file mode 100644 index 000000000..9f641d937 --- /dev/null +++ b/dom/media/platforms/omx/moz.build @@ -0,0 +1,57 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +EXPORTS += [ + 'OmxDecoderModule.h', +] + +UNIFIED_SOURCES += [ + 'OmxDataDecoder.cpp', + 'OmxDecoderModule.cpp', + 'OmxPlatformLayer.cpp', + 'OmxPromiseLayer.cpp', +] + +LOCAL_INCLUDES += [ + '/media/openmax_il/il112', +] + +include('/ipc/chromium/chromium-config.mozbuild') + +if CONFIG['MOZ_WIDGET_TOOLKIT'] == 'gonk' and (CONFIG['ANDROID_VERSION'] == '19' or CONFIG['ANDROID_VERSION'] == '20'): + # Suppress some GCC/clang warnings being treated as errors: + # - about attributes on forward declarations for types that are already + # defined, which complains about an important MOZ_EXPORT for android::AString + # - about multi-character constants which are used in codec-related code + if CONFIG['GNU_CC'] or CONFIG['CLANG_CL']: + CXXFLAGS += [ + '-Wno-error=attributes', + '-Wno-error=multichar' + ] + CXXFLAGS += [ + '-I%s/%s' % (CONFIG['ANDROID_SOURCE'], d) for d in [ + 'frameworks/base/include/binder', + 'frameworks/base/include/utils', + ] + ] + UNIFIED_SOURCES += [ + 'GonkOmxPlatformLayer.cpp', + ] + EXTRA_DSO_LDOPTS += [ + '-libbinder', + ] + +FINAL_LIBRARY = 'xul' + +if CONFIG['GNU_CXX']: + CXXFLAGS += ['-Wno-error=shadow'] + +if CONFIG['_MSC_VER']: + # Avoid warnings from third-party code that we can not modify. + if CONFIG['CLANG_CL']: + CXXFLAGS += ['-Wno-invalid-source-encoding'] + else: + CXXFLAGS += ['-validate-charset-'] |