diff options
Diffstat (limited to 'third_party/aom/test/error_resilience_test.cc')
| -rw-r--r-- | third_party/aom/test/error_resilience_test.cc | 438 |
1 files changed, 438 insertions, 0 deletions
diff --git a/third_party/aom/test/error_resilience_test.cc b/third_party/aom/test/error_resilience_test.cc new file mode 100644 index 000000000..13ac0bf93 --- /dev/null +++ b/third_party/aom/test/error_resilience_test.cc @@ -0,0 +1,438 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include "third_party/googletest/src/googletest/include/gtest/gtest.h" +#include "test/codec_factory.h" +#include "test/encode_test_driver.h" +#include "test/i420_video_source.h" +#include "test/util.h" + +namespace { + +const int kMaxErrorFrames = 12; +const int kMaxInvisibleErrorFrames = 12; +const int kMaxDroppableFrames = 12; +const int kMaxErrorResilientFrames = 12; +const int kMaxNoMFMVFrames = 12; +const int kMaxPrimRefNoneFrames = 12; +const int kMaxSFrames = 12; +const int kCpuUsed = 1; + +class ErrorResilienceTestLarge + : public ::libaom_test::CodecTestWithParam<libaom_test::TestMode>, + public ::libaom_test::EncoderTest { + protected: + ErrorResilienceTestLarge() + : EncoderTest(GET_PARAM(0)), psnr_(0.0), nframes_(0), mismatch_psnr_(0.0), + mismatch_nframes_(0), encoding_mode_(GET_PARAM(1)), allow_mismatch_(0) { + Reset(); + } + + virtual ~ErrorResilienceTestLarge() {} + + void Reset() { + error_nframes_ = 0; + invisible_error_nframes_ = 0; + droppable_nframes_ = 0; + error_resilient_nframes_ = 0; + nomfmv_nframes_ = 0; + prim_ref_none_nframes_ = 0; + s_nframes_ = 0; + } + + void SetupEncoder(int bitrate, int lag) { + const aom_rational timebase = { 33333333, 1000000000 }; + cfg_.g_timebase = timebase; + cfg_.rc_target_bitrate = bitrate; + cfg_.kf_mode = AOM_KF_DISABLED; + cfg_.g_lag_in_frames = lag; + init_flags_ = AOM_CODEC_USE_PSNR; + } + + virtual void SetUp() { + InitializeConfig(); + SetMode(encoding_mode_); + } + + virtual void BeginPassHook(unsigned int /*pass*/) { + psnr_ = 0.0; + nframes_ = 0; + decoded_nframes_ = 0; + mismatch_psnr_ = 0.0; + mismatch_nframes_ = 0; + } + + virtual void PSNRPktHook(const aom_codec_cx_pkt_t *pkt) { + psnr_ += pkt->data.psnr.psnr[0]; + nframes_++; + } + + virtual void PreEncodeFrameHook(libaom_test::VideoSource *video, + libaom_test::Encoder *encoder) { + if (video->frame() == 0) encoder->Control(AOME_SET_CPUUSED, kCpuUsed); + frame_flags_ &= + ~(AOM_EFLAG_NO_UPD_LAST | AOM_EFLAG_NO_UPD_GF | AOM_EFLAG_NO_UPD_ARF | + AOM_EFLAG_NO_REF_FRAME_MVS | AOM_EFLAG_ERROR_RESILIENT | + AOM_EFLAG_SET_S_FRAME | AOM_EFLAG_SET_PRIMARY_REF_NONE); + if (droppable_nframes_ > 0 && + (cfg_.g_pass == AOM_RC_LAST_PASS || cfg_.g_pass == AOM_RC_ONE_PASS)) { + for (unsigned int i = 0; i < droppable_nframes_; ++i) { + if (droppable_frames_[i] == video->frame()) { + std::cout << " Encoding droppable frame: " + << droppable_frames_[i] << "\n"; + frame_flags_ |= (AOM_EFLAG_NO_UPD_LAST | AOM_EFLAG_NO_UPD_GF | + AOM_EFLAG_NO_UPD_ARF); + break; + } + } + } + + if (error_resilient_nframes_ > 0 && + (cfg_.g_pass == AOM_RC_LAST_PASS || cfg_.g_pass == AOM_RC_ONE_PASS)) { + for (unsigned int i = 0; i < error_resilient_nframes_; ++i) { + if (error_resilient_frames_[i] == video->frame()) { + std::cout << " Encoding error_resilient frame: " + << error_resilient_frames_[i] << "\n"; + frame_flags_ |= AOM_EFLAG_ERROR_RESILIENT; + break; + } + } + } + + if (nomfmv_nframes_ > 0 && + (cfg_.g_pass == AOM_RC_LAST_PASS || cfg_.g_pass == AOM_RC_ONE_PASS)) { + for (unsigned int i = 0; i < nomfmv_nframes_; ++i) { + if (nomfmv_frames_[i] == video->frame()) { + std::cout << " Encoding no mfmv frame: " + << nomfmv_frames_[i] << "\n"; + frame_flags_ |= AOM_EFLAG_NO_REF_FRAME_MVS; + break; + } + } + } + + if (prim_ref_none_nframes_ > 0 && + (cfg_.g_pass == AOM_RC_LAST_PASS || cfg_.g_pass == AOM_RC_ONE_PASS)) { + for (unsigned int i = 0; i < prim_ref_none_nframes_; ++i) { + if (prim_ref_none_frames_[i] == video->frame()) { + std::cout << " Encoding no PRIMARY_REF_NONE frame: " + << prim_ref_none_frames_[i] << "\n"; + frame_flags_ |= AOM_EFLAG_SET_PRIMARY_REF_NONE; + break; + } + } + } + + encoder->Control(AV1E_SET_S_FRAME_MODE, 0); + if (s_nframes_ > 0 && + (cfg_.g_pass == AOM_RC_LAST_PASS || cfg_.g_pass == AOM_RC_ONE_PASS)) { + for (unsigned int i = 0; i < s_nframes_; ++i) { + if (s_frames_[i] == video->frame()) { + std::cout << " Encoding S frame: " << s_frames_[i] + << "\n"; + frame_flags_ |= AOM_EFLAG_SET_S_FRAME; + break; + } + } + } + } + + double GetAveragePsnr() const { + if (nframes_) return psnr_ / nframes_; + return 0.0; + } + + double GetAverageMismatchPsnr() const { + if (mismatch_nframes_) return mismatch_psnr_ / mismatch_nframes_; + return 0.0; + } + + virtual bool DoDecode() const { + if (error_nframes_ > 0 && + (cfg_.g_pass == AOM_RC_LAST_PASS || cfg_.g_pass == AOM_RC_ONE_PASS)) { + for (unsigned int i = 0; i < error_nframes_; ++i) { + if (error_frames_[i] == nframes_ - 1) { + std::cout << " Skipping decoding frame: " + << error_frames_[i] << "\n"; + return 0; + } + } + } + return 1; + } + + virtual bool DoDecodeInvisible() const { + if (invisible_error_nframes_ > 0 && + (cfg_.g_pass == AOM_RC_LAST_PASS || cfg_.g_pass == AOM_RC_ONE_PASS)) { + for (unsigned int i = 0; i < invisible_error_nframes_; ++i) { + if (invisible_error_frames_[i] == nframes_ - 1) { + std::cout << " Skipping decoding all invisible frames in " + "frame pkt: " + << invisible_error_frames_[i] << "\n"; + return 0; + } + } + } + return 1; + } + + virtual void MismatchHook(const aom_image_t *img1, const aom_image_t *img2) { + if (allow_mismatch_) { + double mismatch_psnr = compute_psnr(img1, img2); + mismatch_psnr_ += mismatch_psnr; + ++mismatch_nframes_; + // std::cout << "Mismatch frame psnr: " << mismatch_psnr << "\n"; + } else { + ::libaom_test::EncoderTest::MismatchHook(img1, img2); + } + } + + virtual void DecompressedFrameHook(const aom_image_t &img, + aom_codec_pts_t pts) { + (void)img; + (void)pts; + ++decoded_nframes_; + } + + void SetErrorFrames(int num, unsigned int *list) { + if (num > kMaxErrorFrames) + num = kMaxErrorFrames; + else if (num < 0) + num = 0; + error_nframes_ = num; + for (unsigned int i = 0; i < error_nframes_; ++i) + error_frames_[i] = list[i]; + } + + void SetInvisibleErrorFrames(int num, unsigned int *list) { + if (num > kMaxInvisibleErrorFrames) + num = kMaxInvisibleErrorFrames; + else if (num < 0) + num = 0; + invisible_error_nframes_ = num; + for (unsigned int i = 0; i < invisible_error_nframes_; ++i) + invisible_error_frames_[i] = list[i]; + } + + void SetDroppableFrames(int num, unsigned int *list) { + if (num > kMaxDroppableFrames) + num = kMaxDroppableFrames; + else if (num < 0) + num = 0; + droppable_nframes_ = num; + for (unsigned int i = 0; i < droppable_nframes_; ++i) + droppable_frames_[i] = list[i]; + } + + void SetErrorResilientFrames(int num, unsigned int *list) { + if (num > kMaxErrorResilientFrames) + num = kMaxErrorResilientFrames; + else if (num < 0) + num = 0; + error_resilient_nframes_ = num; + for (unsigned int i = 0; i < error_resilient_nframes_; ++i) + error_resilient_frames_[i] = list[i]; + } + + void SetNoMFMVFrames(int num, unsigned int *list) { + if (num > kMaxNoMFMVFrames) + num = kMaxNoMFMVFrames; + else if (num < 0) + num = 0; + nomfmv_nframes_ = num; + for (unsigned int i = 0; i < nomfmv_nframes_; ++i) + nomfmv_frames_[i] = list[i]; + } + + void SetPrimaryRefNoneFrames(int num, unsigned int *list) { + if (num > kMaxPrimRefNoneFrames) + num = kMaxPrimRefNoneFrames; + else if (num < 0) + num = 0; + prim_ref_none_nframes_ = num; + for (unsigned int i = 0; i < prim_ref_none_nframes_; ++i) + prim_ref_none_frames_[i] = list[i]; + } + + void SetSFrames(int num, unsigned int *list) { + if (num > kMaxSFrames) + num = kMaxSFrames; + else if (num < 0) + num = 0; + s_nframes_ = num; + for (unsigned int i = 0; i < s_nframes_; ++i) s_frames_[i] = list[i]; + } + + unsigned int GetMismatchFrames() { return mismatch_nframes_; } + unsigned int GetEncodedFrames() { return nframes_; } + unsigned int GetDecodedFrames() { return decoded_nframes_; } + + void SetAllowMismatch(int allow) { allow_mismatch_ = allow; } + + private: + double psnr_; + unsigned int nframes_; + unsigned int decoded_nframes_; + unsigned int error_nframes_; + unsigned int invisible_error_nframes_; + unsigned int droppable_nframes_; + unsigned int error_resilient_nframes_; + unsigned int nomfmv_nframes_; + unsigned int prim_ref_none_nframes_; + unsigned int s_nframes_; + double mismatch_psnr_; + unsigned int mismatch_nframes_; + unsigned int error_frames_[kMaxErrorFrames]; + unsigned int invisible_error_frames_[kMaxInvisibleErrorFrames]; + unsigned int droppable_frames_[kMaxDroppableFrames]; + unsigned int error_resilient_frames_[kMaxErrorResilientFrames]; + unsigned int nomfmv_frames_[kMaxNoMFMVFrames]; + unsigned int prim_ref_none_frames_[kMaxPrimRefNoneFrames]; + unsigned int s_frames_[kMaxSFrames]; + libaom_test::TestMode encoding_mode_; + int allow_mismatch_; +}; + +TEST_P(ErrorResilienceTestLarge, OnVersusOff) { + SetupEncoder(2000, 10); + libaom_test::I420VideoSource video("hantro_collage_w352h288.yuv", 352, 288, + cfg_.g_timebase.den, cfg_.g_timebase.num, + 0, 12); + + // Global error resilient mode OFF. + cfg_.g_error_resilient = 0; + ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); + const double psnr_resilience_off = GetAveragePsnr(); + EXPECT_GT(psnr_resilience_off, 25.0); + + Reset(); + // Error resilient mode ON for certain frames + unsigned int num_error_resilient_frames = 5; + unsigned int error_resilient_frame_list[] = { 3, 5, 6, 9, 11 }; + SetErrorResilientFrames(num_error_resilient_frames, + error_resilient_frame_list); + ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); + const double psnr_resilience_on = GetAveragePsnr(); + EXPECT_GT(psnr_resilience_on, 25.0); + + // Test that turning on error resilient mode hurts by 10% at most. + if (psnr_resilience_off > 0.0) { + const double psnr_ratio = psnr_resilience_on / psnr_resilience_off; + EXPECT_GE(psnr_ratio, 0.9); + EXPECT_LE(psnr_ratio, 1.1); + } +} + +// Check for successful decoding and no encoder/decoder mismatch +// if we lose (i.e., drop before decoding) a set of droppable +// frames (i.e., frames that don't update any reference buffers). +TEST_P(ErrorResilienceTestLarge, DropFramesWithoutRecovery) { + SetupEncoder(500, 10); + libaom_test::I420VideoSource video("hantro_collage_w352h288.yuv", 352, 288, + cfg_.g_timebase.den, cfg_.g_timebase.num, + 0, 20); + + // Set an arbitrary set of error frames same as droppable frames. + unsigned int num_droppable_frames = 3; + unsigned int droppable_frame_list[] = { 5, 10, 13 }; + SetDroppableFrames(num_droppable_frames, droppable_frame_list); + SetErrorFrames(num_droppable_frames, droppable_frame_list); + ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); + // Test that no mismatches have been found + std::cout << " Encoded frames: " << GetEncodedFrames() << "\n"; + std::cout << " Decoded frames: " << GetDecodedFrames() << "\n"; + std::cout << " Mismatch frames: " << GetMismatchFrames() << "\n"; + EXPECT_EQ(GetEncodedFrames() - GetDecodedFrames(), num_droppable_frames); +} + +// Check for ParseAbility property of an error-resilient frame. +// Encode a frame in error-resilient mode (E-frame), and disallow all +// subsequent frames from using MFMV. If frames are dropped before the +// E frame, all frames starting from the E frame should be parse-able. +TEST_P(ErrorResilienceTestLarge, ParseAbilityTest) { + SetupEncoder(500, 10); + + libaom_test::I420VideoSource video("hantro_collage_w352h288.yuv", 352, 288, + cfg_.g_timebase.den, cfg_.g_timebase.num, + 0, 15); + + SetAllowMismatch(1); + + // Note that an E-frame cannot be forced on a frame that is a + // show_existing_frame, or a frame that comes directly after an invisible + // frame. Currently, this will cause an assertion failure. + // Set an arbitrary error resilient (E) frame + unsigned int num_error_resilient_frames = 1; + unsigned int error_resilient_frame_list[] = { 8 }; + SetErrorResilientFrames(num_error_resilient_frames, + error_resilient_frame_list); + // Ensure that any invisible frames before the E frame are dropped + SetInvisibleErrorFrames(num_error_resilient_frames, + error_resilient_frame_list); + // Set all frames after the error resilient frame to not allow MFMV + unsigned int num_post_error_resilient_frames = 6; + unsigned int post_error_resilient_frame_list[] = { 9, 10, 11, 12, 13, 14 }; + SetNoMFMVFrames(num_post_error_resilient_frames, + post_error_resilient_frame_list); + + // Set a few frames before the E frame that are lost (not decoded) + unsigned int num_error_frames = 5; + unsigned int error_frame_list[] = { 3, 4, 5, 6, 7 }; + SetErrorFrames(num_error_frames, error_frame_list); + + ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); + std::cout << " Encoded frames: " << GetEncodedFrames() << "\n"; + std::cout << " Decoded frames: " << GetDecodedFrames() << "\n"; + std::cout << " Mismatch frames: " << GetMismatchFrames() << "\n"; + EXPECT_EQ(GetEncodedFrames() - GetDecodedFrames(), num_error_frames); + // All frames following the E-frame and the E-frame are expected to have + // mismatches, but still be parse-able. + EXPECT_LE(GetMismatchFrames(), num_post_error_resilient_frames + 1); +} + +// Check for ParseAbility property of an S frame. +// Encode an S-frame. If frames are dropped before the S-frame, all frames +// starting from the S frame should be parse-able. +TEST_P(ErrorResilienceTestLarge, SFrameTest) { + SetupEncoder(500, 10); + + libaom_test::I420VideoSource video("hantro_collage_w352h288.yuv", 352, 288, + cfg_.g_timebase.den, cfg_.g_timebase.num, + 0, 15); + + SetAllowMismatch(1); + + // Note that an S-frame cannot be forced on a frame that is a + // show_existing_frame. This issue still needs to be addressed. + // Set an arbitrary S-frame + unsigned int num_s_frames = 1; + unsigned int s_frame_list[] = { 6 }; + SetSFrames(num_s_frames, s_frame_list); + // Ensure that any invisible frames before the S frame are dropped + SetInvisibleErrorFrames(num_s_frames, s_frame_list); + + // Set a few frames before the S frame that are lost (not decoded) + unsigned int num_error_frames = 4; + unsigned int error_frame_list[] = { 2, 3, 4, 5 }; + SetErrorFrames(num_error_frames, error_frame_list); + + ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); + std::cout << " Encoded frames: " << GetEncodedFrames() << "\n"; + std::cout << " Decoded frames: " << GetDecodedFrames() << "\n"; + std::cout << " Mismatch frames: " << GetMismatchFrames() << "\n"; + EXPECT_EQ(GetEncodedFrames() - GetDecodedFrames(), num_error_frames); + // All frames following the S-frame and the S-frame are expected to have + // mismatches, but still be parse-able. + EXPECT_LE(GetMismatchFrames(), GetEncodedFrames() - s_frame_list[0]); +} + +AV1_INSTANTIATE_TEST_CASE(ErrorResilienceTestLarge, NONREALTIME_TEST_MODES); +} // namespace |