From 5f8de423f190bbb79a62f804151bc24824fa32d8 Mon Sep 17 00:00:00 2001 From: "Matt A. Tobin" Date: Fri, 2 Feb 2018 04:16:08 -0500 Subject: Add m-esr52 at 52.6.0 --- dom/media/fmp4/MP4Demuxer.cpp | 502 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 502 insertions(+) create mode 100644 dom/media/fmp4/MP4Demuxer.cpp (limited to 'dom/media/fmp4/MP4Demuxer.cpp') diff --git a/dom/media/fmp4/MP4Demuxer.cpp b/dom/media/fmp4/MP4Demuxer.cpp new file mode 100644 index 000000000..70b176699 --- /dev/null +++ b/dom/media/fmp4/MP4Demuxer.cpp @@ -0,0 +1,502 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include +#include +#include + +#include "MP4Demuxer.h" +#include "mp4_demuxer/MoofParser.h" +#include "mp4_demuxer/MP4Metadata.h" +#include "mp4_demuxer/ResourceStream.h" +#include "mp4_demuxer/BufferStream.h" +#include "mp4_demuxer/Index.h" +#include "nsPrintfCString.h" + +// Used for telemetry +#include "mozilla/Telemetry.h" +#include "mp4_demuxer/AnnexB.h" +#include "mp4_demuxer/H264.h" + +#include "nsAutoPtr.h" + +extern mozilla::LazyLogModule gMediaDemuxerLog; +mozilla::LogModule* GetDemuxerLog() { + return gMediaDemuxerLog; +} + +#define LOG(arg, ...) MOZ_LOG(gMediaDemuxerLog, mozilla::LogLevel::Debug, ("MP4Demuxer(%p)::%s: " arg, this, __func__, ##__VA_ARGS__)) + +namespace mozilla { + +class MP4TrackDemuxer : public MediaTrackDemuxer +{ +public: + MP4TrackDemuxer(MP4Demuxer* aParent, + UniquePtr&& aInfo, + const nsTArray& indices); + + UniquePtr GetInfo() const override; + + RefPtr Seek(media::TimeUnit aTime) override; + + RefPtr GetSamples(int32_t aNumSamples = 1) override; + + void Reset() override; + + nsresult GetNextRandomAccessPoint(media::TimeUnit* aTime) override; + + RefPtr SkipToNextRandomAccessPoint(media::TimeUnit aTimeThreshold) override; + + media::TimeIntervals GetBuffered() override; + + void BreakCycles() override; + + void NotifyDataRemoved(); + +private: + friend class MP4Demuxer; + void NotifyDataArrived(); + already_AddRefed GetNextSample(); + void EnsureUpToDateIndex(); + void SetNextKeyFrameTime(); + RefPtr mParent; + RefPtr mStream; + UniquePtr mInfo; + RefPtr mIndex; + UniquePtr mIterator; + Maybe mNextKeyframeTime; + // Queued samples extracted by the demuxer, but not yet returned. + RefPtr mQueuedSample; + bool mNeedReIndex; + bool mNeedSPSForTelemetry; + bool mIsH264 = false; +}; + + +// Returns true if no SPS was found and search for it should continue. +bool +AccumulateSPSTelemetry(const MediaByteBuffer* aExtradata) +{ + mp4_demuxer::SPSData spsdata; + if (mp4_demuxer::H264::DecodeSPSFromExtraData(aExtradata, spsdata)) { + uint8_t constraints = (spsdata.constraint_set0_flag ? (1 << 0) : 0) | + (spsdata.constraint_set1_flag ? (1 << 1) : 0) | + (spsdata.constraint_set2_flag ? (1 << 2) : 0) | + (spsdata.constraint_set3_flag ? (1 << 3) : 0) | + (spsdata.constraint_set4_flag ? (1 << 4) : 0) | + (spsdata.constraint_set5_flag ? (1 << 5) : 0); + Telemetry::Accumulate(Telemetry::VIDEO_DECODED_H264_SPS_CONSTRAINT_SET_FLAG, + constraints); + + // Collect profile_idc values up to 244, otherwise 0 for unknown. + Telemetry::Accumulate(Telemetry::VIDEO_DECODED_H264_SPS_PROFILE, + spsdata.profile_idc <= 244 ? spsdata.profile_idc : 0); + + // Make sure level_idc represents a value between levels 1 and 5.2, + // otherwise collect 0 for unknown level. + Telemetry::Accumulate(Telemetry::VIDEO_DECODED_H264_SPS_LEVEL, + (spsdata.level_idc >= 10 && spsdata.level_idc <= 52) ? + spsdata.level_idc : 0); + + // max_num_ref_frames should be between 0 and 16, anything larger will + // be treated as invalid. + Telemetry::Accumulate(Telemetry::VIDEO_H264_SPS_MAX_NUM_REF_FRAMES, + std::min(spsdata.max_num_ref_frames, 17u)); + + return false; + } + + return true; +} + +MP4Demuxer::MP4Demuxer(MediaResource* aResource) + : mResource(aResource) + , mStream(new mp4_demuxer::ResourceStream(aResource)) + , mInitData(new MediaByteBuffer) +{ +} + +RefPtr +MP4Demuxer::Init() +{ + AutoPinned stream(mStream); + + // Check that we have enough data to read the metadata. + if (!mp4_demuxer::MP4Metadata::HasCompleteMetadata(stream)) { + return InitPromise::CreateAndReject(NS_ERROR_DOM_MEDIA_DEMUXER_ERR, __func__); + } + + mInitData = mp4_demuxer::MP4Metadata::Metadata(stream); + if (!mInitData) { + // OOM + return InitPromise::CreateAndReject(NS_ERROR_DOM_MEDIA_DEMUXER_ERR, __func__); + } + + RefPtr bufferstream = + new mp4_demuxer::BufferStream(mInitData); + + mMetadata = MakeUnique(bufferstream); + + if (!mMetadata->GetNumberTracks(mozilla::TrackInfo::kAudioTrack) && + !mMetadata->GetNumberTracks(mozilla::TrackInfo::kVideoTrack)) { + return InitPromise::CreateAndReject(NS_ERROR_DOM_MEDIA_DEMUXER_ERR, __func__); + } + + return InitPromise::CreateAndResolve(NS_OK, __func__); +} + +bool +MP4Demuxer::HasTrackType(TrackInfo::TrackType aType) const +{ + return !!GetNumberTracks(aType); +} + +uint32_t +MP4Demuxer::GetNumberTracks(TrackInfo::TrackType aType) const +{ + return mMetadata->GetNumberTracks(aType); +} + +already_AddRefed +MP4Demuxer::GetTrackDemuxer(TrackInfo::TrackType aType, uint32_t aTrackNumber) +{ + if (mMetadata->GetNumberTracks(aType) <= aTrackNumber) { + return nullptr; + } + UniquePtr info = mMetadata->GetTrackInfo(aType, aTrackNumber); + if (!info) { + return nullptr; + } + FallibleTArray indices; + if (!mMetadata->ReadTrackIndex(indices, info->mTrackId)) { + return nullptr; + } + RefPtr e = new MP4TrackDemuxer(this, Move(info), indices); + mDemuxers.AppendElement(e); + + return e.forget(); +} + +bool +MP4Demuxer::IsSeekable() const +{ + return mMetadata->CanSeek(); +} + +void +MP4Demuxer::NotifyDataArrived() +{ + for (uint32_t i = 0; i < mDemuxers.Length(); i++) { + mDemuxers[i]->NotifyDataArrived(); + } +} + +void +MP4Demuxer::NotifyDataRemoved() +{ + for (uint32_t i = 0; i < mDemuxers.Length(); i++) { + mDemuxers[i]->NotifyDataRemoved(); + } +} + +UniquePtr +MP4Demuxer::GetCrypto() +{ + const mp4_demuxer::CryptoFile& cryptoFile = mMetadata->Crypto(); + if (!cryptoFile.valid) { + return nullptr; + } + + const nsTArray& psshs = cryptoFile.pssh; + nsTArray initData; + for (uint32_t i = 0; i < psshs.Length(); i++) { + initData.AppendElements(psshs[i].data); + } + + if (initData.IsEmpty()) { + return nullptr; + } + + auto crypto = MakeUnique(); + crypto->AddInitData(NS_LITERAL_STRING("cenc"), Move(initData)); + + return crypto; +} + +MP4TrackDemuxer::MP4TrackDemuxer(MP4Demuxer* aParent, + UniquePtr&& aInfo, + const nsTArray& indices) + : mParent(aParent) + , mStream(new mp4_demuxer::ResourceStream(mParent->mResource)) + , mInfo(Move(aInfo)) + , mIndex(new mp4_demuxer::Index(indices, + mStream, + mInfo->mTrackId, + mInfo->IsAudio())) + , mIterator(MakeUnique(mIndex)) + , mNeedReIndex(true) +{ + EnsureUpToDateIndex(); // Force update of index + + VideoInfo* videoInfo = mInfo->GetAsVideoInfo(); + // Collect telemetry from h264 AVCC SPS. + if (videoInfo && + (mInfo->mMimeType.EqualsLiteral("video/mp4") || + mInfo->mMimeType.EqualsLiteral("video/avc"))) { + mIsH264 = true; + RefPtr extraData = videoInfo->mExtraData; + mNeedSPSForTelemetry = AccumulateSPSTelemetry(extraData); + mp4_demuxer::SPSData spsdata; + if (mp4_demuxer::H264::DecodeSPSFromExtraData(extraData, spsdata) && + spsdata.pic_width > 0 && spsdata.pic_height > 0 && + mp4_demuxer::H264::EnsureSPSIsSane(spsdata)) { + videoInfo->mImage.width = spsdata.pic_width; + videoInfo->mImage.height = spsdata.pic_height; + videoInfo->mDisplay.width = spsdata.display_width; + videoInfo->mDisplay.height = spsdata.display_height; + } + } else { + // No SPS to be found. + mNeedSPSForTelemetry = false; + } +} + +UniquePtr +MP4TrackDemuxer::GetInfo() const +{ + return mInfo->Clone(); +} + +void +MP4TrackDemuxer::EnsureUpToDateIndex() +{ + if (!mNeedReIndex) { + return; + } + AutoPinned resource(mParent->mResource); + MediaByteRangeSet byteRanges; + nsresult rv = resource->GetCachedRanges(byteRanges); + if (NS_FAILED(rv)) { + return; + } + mIndex->UpdateMoofIndex(byteRanges); + mNeedReIndex = false; +} + +RefPtr +MP4TrackDemuxer::Seek(media::TimeUnit aTime) +{ + int64_t seekTime = aTime.ToMicroseconds(); + mQueuedSample = nullptr; + + mIterator->Seek(seekTime); + + // Check what time we actually seeked to. + mQueuedSample = GetNextSample(); + if (mQueuedSample) { + seekTime = mQueuedSample->mTime; + } + + SetNextKeyFrameTime(); + + return SeekPromise::CreateAndResolve(media::TimeUnit::FromMicroseconds(seekTime), __func__); +} + +already_AddRefed +MP4TrackDemuxer::GetNextSample() +{ + RefPtr sample = mIterator->GetNext(); + if (!sample) { + return nullptr; + } + if (mInfo->GetAsVideoInfo()) { + sample->mExtraData = mInfo->GetAsVideoInfo()->mExtraData; + if (mIsH264) { + mp4_demuxer::H264::FrameType type = + mp4_demuxer::H264::GetFrameType(sample); + switch (type) { + case mp4_demuxer::H264::FrameType::I_FRAME: MOZ_FALLTHROUGH; + case mp4_demuxer::H264::FrameType::OTHER: + { + bool keyframe = type == mp4_demuxer::H264::FrameType::I_FRAME; + if (sample->mKeyframe != keyframe) { + NS_WARNING(nsPrintfCString("Frame incorrectly marked as %skeyframe @ pts:%lld dur:%u dts:%lld", + keyframe ? "" : "non-", + sample->mTime, + sample->mDuration, + sample->mTimecode).get()); + sample->mKeyframe = keyframe; + } + break; + } + case mp4_demuxer::H264::FrameType::INVALID: + NS_WARNING(nsPrintfCString("Invalid H264 frame @ pts:%lld dur:%u dts:%lld", + sample->mTime, + sample->mDuration, + sample->mTimecode).get()); + // We could reject the sample now, however demuxer errors are fatal. + // So we keep the invalid frame, relying on the H264 decoder to + // handle the error later. + // TODO: make demuxer errors non-fatal. + break; + } + } + } + if (sample->mCrypto.mValid) { + nsAutoPtr writer(sample->CreateWriter()); + writer->mCrypto.mMode = mInfo->mCrypto.mMode; + writer->mCrypto.mIVSize = mInfo->mCrypto.mIVSize; + writer->mCrypto.mKeyId.AppendElements(mInfo->mCrypto.mKeyId); + } + return sample.forget(); +} + +RefPtr +MP4TrackDemuxer::GetSamples(int32_t aNumSamples) +{ + EnsureUpToDateIndex(); + RefPtr samples = new SamplesHolder; + if (!aNumSamples) { + return SamplesPromise::CreateAndReject(NS_ERROR_DOM_MEDIA_DEMUXER_ERR, __func__); + } + + if (mQueuedSample) { + MOZ_ASSERT(mQueuedSample->mKeyframe, + "mQueuedSample must be a keyframe"); + samples->mSamples.AppendElement(mQueuedSample); + mQueuedSample = nullptr; + aNumSamples--; + } + RefPtr sample; + while (aNumSamples && (sample = GetNextSample())) { + if (!sample->Size()) { + continue; + } + samples->mSamples.AppendElement(sample); + aNumSamples--; + } + + if (samples->mSamples.IsEmpty()) { + return SamplesPromise::CreateAndReject(NS_ERROR_DOM_MEDIA_END_OF_STREAM, __func__); + } else { + for (const auto& sample : samples->mSamples) { + // Collect telemetry from h264 Annex B SPS. + if (mNeedSPSForTelemetry && mp4_demuxer::AnnexB::HasSPS(sample)) { + RefPtr extradata = + mp4_demuxer::AnnexB::ExtractExtraData(sample); + mNeedSPSForTelemetry = AccumulateSPSTelemetry(extradata); + } + } + + if (mNextKeyframeTime.isNothing() || + samples->mSamples.LastElement()->mTime >= mNextKeyframeTime.value().ToMicroseconds()) { + SetNextKeyFrameTime(); + } + return SamplesPromise::CreateAndResolve(samples, __func__); + } +} + +void +MP4TrackDemuxer::SetNextKeyFrameTime() +{ + mNextKeyframeTime.reset(); + mp4_demuxer::Microseconds frameTime = mIterator->GetNextKeyframeTime(); + if (frameTime != -1) { + mNextKeyframeTime.emplace( + media::TimeUnit::FromMicroseconds(frameTime)); + } +} + +void +MP4TrackDemuxer::Reset() +{ + mQueuedSample = nullptr; + // TODO, Seek to first frame available, which isn't always 0. + mIterator->Seek(0); + SetNextKeyFrameTime(); +} + +nsresult +MP4TrackDemuxer::GetNextRandomAccessPoint(media::TimeUnit* aTime) +{ + if (mNextKeyframeTime.isNothing()) { + // There's no next key frame. + *aTime = + media::TimeUnit::FromMicroseconds(std::numeric_limits::max()); + } else { + *aTime = mNextKeyframeTime.value(); + } + return NS_OK; +} + +RefPtr +MP4TrackDemuxer::SkipToNextRandomAccessPoint(media::TimeUnit aTimeThreshold) +{ + mQueuedSample = nullptr; + // Loop until we reach the next keyframe after the threshold. + uint32_t parsed = 0; + bool found = false; + RefPtr sample; + while (!found && (sample = GetNextSample())) { + parsed++; + if (sample->mKeyframe && sample->mTime >= aTimeThreshold.ToMicroseconds()) { + found = true; + mQueuedSample = sample; + } + } + SetNextKeyFrameTime(); + if (found) { + return SkipAccessPointPromise::CreateAndResolve(parsed, __func__); + } else { + SkipFailureHolder failure(NS_ERROR_DOM_MEDIA_END_OF_STREAM, parsed); + return SkipAccessPointPromise::CreateAndReject(Move(failure), __func__); + } +} + +media::TimeIntervals +MP4TrackDemuxer::GetBuffered() +{ + EnsureUpToDateIndex(); + AutoPinned resource(mParent->mResource); + MediaByteRangeSet byteRanges; + nsresult rv = resource->GetCachedRanges(byteRanges); + + if (NS_FAILED(rv)) { + return media::TimeIntervals(); + } + + return mIndex->ConvertByteRangesToTimeRanges(byteRanges); +} + +void +MP4TrackDemuxer::NotifyDataArrived() +{ + mNeedReIndex = true; +} + +void +MP4TrackDemuxer::NotifyDataRemoved() +{ + AutoPinned resource(mParent->mResource); + MediaByteRangeSet byteRanges; + nsresult rv = resource->GetCachedRanges(byteRanges); + if (NS_FAILED(rv)) { + return; + } + mIndex->UpdateMoofIndex(byteRanges, true /* can evict */); + mNeedReIndex = false; +} + +void +MP4TrackDemuxer::BreakCycles() +{ + mParent = nullptr; +} + +} // namespace mozilla + +#undef LOG -- cgit v1.2.3