diff options
author | Matt A. Tobin <email@mattatobin.com> | 2020-04-07 23:30:51 -0400 |
---|---|---|
committer | wolfbeast <mcwerewolf@wolfbeast.com> | 2020-04-14 13:26:42 +0200 |
commit | 277f2116b6660e9bbe7f5d67524be57eceb49b8b (patch) | |
tree | 4595f7cc71418f71b9a97dfaeb03a30aa60f336a /third_party/aom/av1/common/arm | |
parent | d270404436f6e84ffa3b92af537ac721bf10d66e (diff) | |
download | UXP-277f2116b6660e9bbe7f5d67524be57eceb49b8b.tar UXP-277f2116b6660e9bbe7f5d67524be57eceb49b8b.tar.gz UXP-277f2116b6660e9bbe7f5d67524be57eceb49b8b.tar.lz UXP-277f2116b6660e9bbe7f5d67524be57eceb49b8b.tar.xz UXP-277f2116b6660e9bbe7f5d67524be57eceb49b8b.zip |
Move aom source to a sub-directory under media/libaom
There is no damned reason to treat this differently than any other media lib given its license and there never was.
Diffstat (limited to 'third_party/aom/av1/common/arm')
-rw-r--r-- | third_party/aom/av1/common/arm/av1_inv_txfm_neon.c | 3231 | ||||
-rw-r--r-- | third_party/aom/av1/common/arm/av1_inv_txfm_neon.h | 154 | ||||
-rw-r--r-- | third_party/aom/av1/common/arm/av1_txfm_neon.c | 28 | ||||
-rw-r--r-- | third_party/aom/av1/common/arm/blend_a64_hmask_neon.c | 134 | ||||
-rw-r--r-- | third_party/aom/av1/common/arm/blend_a64_vmask_neon.c | 141 | ||||
-rw-r--r-- | third_party/aom/av1/common/arm/cfl_neon.c | 584 | ||||
-rw-r--r-- | third_party/aom/av1/common/arm/convolve_neon.c | 1455 | ||||
-rw-r--r-- | third_party/aom/av1/common/arm/convolve_neon.h | 228 | ||||
-rw-r--r-- | third_party/aom/av1/common/arm/jnt_convolve_neon.c | 1740 | ||||
-rw-r--r-- | third_party/aom/av1/common/arm/mem_neon.h | 494 | ||||
-rw-r--r-- | third_party/aom/av1/common/arm/reconinter_neon.c | 86 | ||||
-rw-r--r-- | third_party/aom/av1/common/arm/selfguided_neon.c | 1508 | ||||
-rw-r--r-- | third_party/aom/av1/common/arm/transpose_neon.h | 537 | ||||
-rw-r--r-- | third_party/aom/av1/common/arm/warp_plane_neon.c | 714 | ||||
-rw-r--r-- | third_party/aom/av1/common/arm/wiener_convolve_neon.c | 530 |
15 files changed, 0 insertions, 11564 deletions
diff --git a/third_party/aom/av1/common/arm/av1_inv_txfm_neon.c b/third_party/aom/av1/common/arm/av1_inv_txfm_neon.c deleted file mode 100644 index bad411743..000000000 --- a/third_party/aom/av1/common/arm/av1_inv_txfm_neon.c +++ /dev/null @@ -1,3231 +0,0 @@ -/* - * Copyright (c) 2018, Alliance for Open Media. All rights reserved - * - * This source code is subject to the terms of the BSD 2 Clause License and - * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License - * was not distributed with this source code in the LICENSE file, you can - * obtain it at www.aomedia.org/license/software. If the Alliance for Open - * Media Patent License 1.0 was not distributed with this source code in the - * PATENTS file, you can obtain it at www.aomedia.org/license/patent. - */ - -#include <arm_neon.h> - -#include "config/aom_config.h" -#include "config/aom_dsp_rtcd.h" -#include "config/av1_rtcd.h" - -#include "av1/common/av1_inv_txfm1d.h" -#include "av1/common/av1_inv_txfm1d_cfg.h" -#include "av1/common/av1_txfm.h" -#include "av1/common/enums.h" -#include "av1/common/idct.h" -#include "av1/common/arm/av1_inv_txfm_neon.h" -#include "av1/common/arm/transpose_neon.h" - -// 1D itx types -typedef enum ATTRIBUTE_PACKED { - IDCT_1D, - IADST_1D, - IFLIPADST_1D = IADST_1D, - IIDENTITY_1D, - ITX_TYPES_1D, -} ITX_TYPE_1D; - -static const ITX_TYPE_1D vitx_1d_tab[TX_TYPES] = { - IDCT_1D, IADST_1D, IDCT_1D, IADST_1D, - IFLIPADST_1D, IDCT_1D, IFLIPADST_1D, IADST_1D, - IFLIPADST_1D, IIDENTITY_1D, IDCT_1D, IIDENTITY_1D, - IADST_1D, IIDENTITY_1D, IFLIPADST_1D, IIDENTITY_1D, -}; - -static const ITX_TYPE_1D hitx_1d_tab[TX_TYPES] = { - IDCT_1D, IDCT_1D, IADST_1D, IADST_1D, - IDCT_1D, IFLIPADST_1D, IFLIPADST_1D, IFLIPADST_1D, - IADST_1D, IIDENTITY_1D, IIDENTITY_1D, IDCT_1D, - IIDENTITY_1D, IADST_1D, IIDENTITY_1D, IFLIPADST_1D, -}; - -// 1D functions -static const transform_1d_neon lowbd_txfm_all_1d_arr[TX_SIZES][ITX_TYPES_1D] = { - { av1_idct4_new, av1_iadst4_new, av1_iidentity4_c }, - { av1_idct8_new, av1_iadst8_new, av1_iidentity8_c }, - { av1_idct16_new, av1_iadst16_new, av1_iidentity16_c }, - { av1_idct32_new, NULL, NULL }, - { av1_idct64_new, NULL, NULL }, -}; - -static INLINE void lowbd_add_flip_buffer_8xn_neon(int16x8_t *in, - uint8_t *output, int stride, - int flipud, - const int height) { - int j = flipud ? (height - 1) : 0; - const int step = flipud ? -1 : 1; - int16x8_t temp_output; - for (int i = 0; i < height; ++i, j += step) { - temp_output = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(output))); - temp_output = vaddq_s16(temp_output, in[j]); - vst1_u8(output, vqmovun_s16(temp_output)); - output += stride; - } -} - -static INLINE uint8x16_t lowbd_get_recon_16x16_neon(const uint8x16_t pred, - int16x8_t res0, - int16x8_t res1) { - int16x8_t temp_output[2]; - uint8x16_t temp_output_8q; - temp_output[0] = vreinterpretq_s16_u16(vmovl_u8(vget_low_u8(pred))); - temp_output[0] = vaddq_s16(temp_output[0], res0); - temp_output[1] = vreinterpretq_s16_u16(vmovl_u8(vget_high_u8(pred))); - temp_output[1] = vaddq_s16(temp_output[1], res1); - temp_output_8q = - vcombine_u8(vqmovun_s16(temp_output[0]), vqmovun_s16(temp_output[1])); - return temp_output_8q; -} - -static INLINE void lowbd_add_flip_buffer_16xn_neon(int16x8_t *in, - uint8_t *output, int stride, - int flipud, int height) { - uint8x16_t temp_output_8q; - int j = flipud ? (height - 1) : 0; - const int step = flipud ? -1 : 1; - for (int i = 0; i < height; ++i, j += step) { - temp_output_8q = vld1q_u8(output + i * stride); - temp_output_8q = - lowbd_get_recon_16x16_neon(temp_output_8q, in[j], in[j + height]); - vst1q_u8((output + i * stride), temp_output_8q); - } -} - -static INLINE void lowbd_inv_txfm2d_memset_neon(int16x8_t *a, int size, - int value) { - for (int i = 0; i < size; i++) { - a[i] = vdupq_n_s16((int16_t)value); - } -} - -static INLINE void btf_16_lane_0_1_neon(const int16x8_t in0, - const int16x8_t in1, const int16x4_t c, - int16x8_t *t0, int16x8_t *t1) { - int32x4_t s0[2], s1[2]; - int16x4_t v0[2], v1[2]; - - s0[0] = vmull_lane_s16(vget_low_s16(in0), c, 0); - s0[1] = vmull_lane_s16(vget_high_s16(in0), c, 0); - s1[0] = vmull_lane_s16(vget_low_s16(in0), c, 1); - s1[1] = vmull_lane_s16(vget_high_s16(in0), c, 1); - - s0[0] = vmlal_lane_s16(s0[0], vget_low_s16(in1), c, 1); - s0[1] = vmlal_lane_s16(s0[1], vget_high_s16(in1), c, 1); - s1[0] = vmlsl_lane_s16(s1[0], vget_low_s16(in1), c, 0); - s1[1] = vmlsl_lane_s16(s1[1], vget_high_s16(in1), c, 0); - - v0[0] = vrshrn_n_s32(s0[0], INV_COS_BIT); - v0[1] = vrshrn_n_s32(s0[1], INV_COS_BIT); - v1[0] = vrshrn_n_s32(s1[0], INV_COS_BIT); - v1[1] = vrshrn_n_s32(s1[1], INV_COS_BIT); - - *t0 = vcombine_s16(v0[0], v0[1]); - *t1 = vcombine_s16(v1[0], v1[1]); -} - -static INLINE void btf_16_lane_1_0_neon(const int16x8_t in0, - const int16x8_t in1, const int16x4_t c, - int16x8_t *t0, int16x8_t *t1) { - int32x4_t s0[2], s1[2]; - int16x4_t v0[2], v1[2]; - - s0[0] = vmull_lane_s16(vget_low_s16(in0), c, 1); - s0[1] = vmull_lane_s16(vget_high_s16(in0), c, 1); - s1[0] = vmull_lane_s16(vget_low_s16(in0), c, 0); - s1[1] = vmull_lane_s16(vget_high_s16(in0), c, 0); - - s0[0] = vmlal_lane_s16(s0[0], vget_low_s16(in1), c, 0); - s0[1] = vmlal_lane_s16(s0[1], vget_high_s16(in1), c, 0); - s1[0] = vmlsl_lane_s16(s1[0], vget_low_s16(in1), c, 1); - s1[1] = vmlsl_lane_s16(s1[1], vget_high_s16(in1), c, 1); - - v0[0] = vrshrn_n_s32(s0[0], INV_COS_BIT); - v0[1] = vrshrn_n_s32(s0[1], INV_COS_BIT); - v1[0] = vrshrn_n_s32(s1[0], INV_COS_BIT); - v1[1] = vrshrn_n_s32(s1[1], INV_COS_BIT); - - *t0 = vcombine_s16(v0[0], v0[1]); - *t1 = vcombine_s16(v1[0], v1[1]); -} - -static INLINE void btf_16_lane_2_3_neon(const int16x8_t in0, - const int16x8_t in1, const int16x4_t c, - int16x8_t *t0, int16x8_t *t1) { - int32x4_t s0[2], s1[2]; - int16x4_t v0[2], v1[2]; - - s0[0] = vmull_lane_s16(vget_low_s16(in0), c, 2); - s0[1] = vmull_lane_s16(vget_high_s16(in0), c, 2); - s1[0] = vmull_lane_s16(vget_low_s16(in0), c, 3); - s1[1] = vmull_lane_s16(vget_high_s16(in0), c, 3); - - s0[0] = vmlal_lane_s16(s0[0], vget_low_s16(in1), c, 3); - s0[1] = vmlal_lane_s16(s0[1], vget_high_s16(in1), c, 3); - s1[0] = vmlsl_lane_s16(s1[0], vget_low_s16(in1), c, 2); - s1[1] = vmlsl_lane_s16(s1[1], vget_high_s16(in1), c, 2); - - v0[0] = vrshrn_n_s32(s0[0], INV_COS_BIT); - v0[1] = vrshrn_n_s32(s0[1], INV_COS_BIT); - v1[0] = vrshrn_n_s32(s1[0], INV_COS_BIT); - v1[1] = vrshrn_n_s32(s1[1], INV_COS_BIT); - - *t0 = vcombine_s16(v0[0], v0[1]); - *t1 = vcombine_s16(v1[0], v1[1]); -} - -static INLINE void btf_16_neon(const int16x8_t in0, int16_t coef1, - int16_t coef2, int16x8_t *t0, int16x8_t *t1) { - int32x4_t s0_l, s0_h, s1_l, s1_h; - int16x4_t v0[2], v1[2]; - - s0_l = vmull_n_s16(vget_low_s16(in0), coef1); - s0_h = vmull_n_s16(vget_high_s16(in0), coef1); - s1_l = vmull_n_s16(vget_low_s16(in0), coef2); - s1_h = vmull_n_s16(vget_high_s16(in0), coef2); - - v0[0] = vrshrn_n_s32(s0_l, INV_COS_BIT); - v0[1] = vrshrn_n_s32(s0_h, INV_COS_BIT); - v1[0] = vrshrn_n_s32(s1_l, INV_COS_BIT); - v1[1] = vrshrn_n_s32(s1_h, INV_COS_BIT); - - *t0 = vcombine_s16(v0[0], v0[1]); - *t1 = vcombine_s16(v1[0], v1[1]); -} - -static INLINE void btf_16_lane_3_2_neon(const int16x8_t in0, - const int16x8_t in1, const int16x4_t c, - int16x8_t *t0, int16x8_t *t1) { - int32x4_t s0[2], s1[2]; - int16x4_t v0[2], v1[2]; - - s0[0] = vmull_lane_s16(vget_low_s16(in0), c, 3); - s0[1] = vmull_lane_s16(vget_high_s16(in0), c, 3); - s1[0] = vmull_lane_s16(vget_low_s16(in0), c, 2); - s1[1] = vmull_lane_s16(vget_high_s16(in0), c, 2); - - s0[0] = vmlal_lane_s16(s0[0], vget_low_s16(in1), c, 2); - s0[1] = vmlal_lane_s16(s0[1], vget_high_s16(in1), c, 2); - s1[0] = vmlsl_lane_s16(s1[0], vget_low_s16(in1), c, 3); - s1[1] = vmlsl_lane_s16(s1[1], vget_high_s16(in1), c, 3); - - v0[0] = vrshrn_n_s32(s0[0], INV_COS_BIT); - v0[1] = vrshrn_n_s32(s0[1], INV_COS_BIT); - v1[0] = vrshrn_n_s32(s1[0], INV_COS_BIT); - v1[1] = vrshrn_n_s32(s1[1], INV_COS_BIT); - - *t0 = vcombine_s16(v0[0], v0[1]); - *t1 = vcombine_s16(v1[0], v1[1]); -} - -static INLINE void btf_16_half_neon(int16x8_t *const x, const int16x4_t c) { - int32x4_t t0[2], t1[2]; - int16x4_t v0[2], v1[2]; - - // Don't add/sub before multiply, which will overflow in iadst8. - const int32x4_t x0_lo = vmull_lane_s16(vget_low_s16(x[0]), c, 0); - const int32x4_t x0_hi = vmull_lane_s16(vget_high_s16(x[0]), c, 0); - const int32x4_t x1_lo = vmull_lane_s16(vget_low_s16(x[1]), c, 0); - const int32x4_t x1_hi = vmull_lane_s16(vget_high_s16(x[1]), c, 0); - - t0[0] = vaddq_s32(x0_lo, x1_lo); - t0[1] = vaddq_s32(x0_hi, x1_hi); - t1[0] = vsubq_s32(x0_lo, x1_lo); - t1[1] = vsubq_s32(x0_hi, x1_hi); - - v0[0] = vrshrn_n_s32(t0[0], INV_COS_BIT); - v0[1] = vrshrn_n_s32(t0[1], INV_COS_BIT); - v1[0] = vrshrn_n_s32(t1[0], INV_COS_BIT); - v1[1] = vrshrn_n_s32(t1[1], INV_COS_BIT); - - x[0] = vcombine_s16(v0[0], v0[1]); - x[1] = vcombine_s16(v1[0], v1[1]); -} - -static INLINE int16x4_t create_s16x4_neon(int16_t *const c0, int16_t *const c1, - int16_t *const c2, - int16_t *const c3) { - int16x4_t val = vdup_n_s16((int16_t)0); - val = vld1_lane_s16(c0, val, 0); - val = vld1_lane_s16(c1, val, 1); - val = vld1_lane_s16(c2, val, 2); - val = vld1_lane_s16(c3, val, 3); - return val; -} - -static INLINE void iadst8_new_neon(int16x8_t *const in, int16x8_t *out, - int8_t cos_bit, int bit) { - (void)bit; - const int32_t *cospi = cospi_arr(cos_bit); - - const int16x4_t c0 = - create_s16x4_neon((int16_t *)(cospi + 4), (int16_t *)(cospi + 60), - (int16_t *)(cospi + 20), (int16_t *)(cospi + 44)); - const int16x4_t c1 = - create_s16x4_neon((int16_t *)(cospi + 36), (int16_t *)(cospi + 28), - (int16_t *)(cospi + 52), (int16_t *)(cospi + 12)); - const int16x4_t c2 = - create_s16x4_neon((int16_t *)(cospi + 32), (int16_t *)(cospi + 32), - (int16_t *)(cospi + 16), (int16_t *)(cospi + 48)); - - int16x8_t x[8]; - int16x8_t s0, s1, s2, s3, s4, s5, s6, s7; - - // Stage 1 - x[0] = in[7]; - x[1] = in[0]; - x[2] = in[5]; - x[3] = in[2]; - x[4] = in[3]; - x[5] = in[4]; - x[6] = in[1]; - x[7] = in[6]; - - // Stage 2 - btf_16_lane_0_1_neon(x[0], x[1], c0, &s0, &s1); - btf_16_lane_2_3_neon(x[2], x[3], c0, &s2, &s3); - btf_16_lane_0_1_neon(x[4], x[5], c1, &s4, &s5); - btf_16_lane_2_3_neon(x[6], x[7], c1, &s6, &s7); - - // Stage 3 - x[0] = vqaddq_s16(s0, s4); - x[1] = vqaddq_s16(s1, s5); - x[2] = vqaddq_s16(s2, s6); - x[3] = vqaddq_s16(s3, s7); - x[4] = vqsubq_s16(s0, s4); - x[5] = vqsubq_s16(s1, s5); - x[6] = vqsubq_s16(s2, s6); - x[7] = vqsubq_s16(s3, s7); - - // Stage 4 - s0 = x[0]; - s1 = x[1]; - s2 = x[2]; - s3 = x[3]; - btf_16_lane_2_3_neon(x[4], x[5], c2, &s4, &s5); - btf_16_lane_3_2_neon(x[7], x[6], c2, &s7, &s6); - - // Stage 5 - x[0] = vqaddq_s16(s0, s2); - x[1] = vqaddq_s16(s1, s3); - x[2] = vqsubq_s16(s0, s2); - x[3] = vqsubq_s16(s1, s3); - x[4] = vqaddq_s16(s4, s6); - x[5] = vqaddq_s16(s5, s7); - x[6] = vqsubq_s16(s4, s6); - x[7] = vqsubq_s16(s5, s7); - - // stage 6 - btf_16_half_neon(x + 2, c2); - btf_16_half_neon(x + 6, c2); - - // Stage 7 - out[0] = x[0]; - out[1] = vnegq_s16(x[4]); - out[2] = x[6]; - out[3] = vnegq_s16(x[2]); - out[4] = x[3]; - out[5] = vnegq_s16(x[7]); - out[6] = x[5]; - out[7] = vnegq_s16(x[1]); -} - -static INLINE void iadst8_low1_new_neon(int16x8_t *const in, int16x8_t *out, - int8_t cos_bit, int bit) { - (void)bit; - const int32_t *cospi = cospi_arr(cos_bit); - const int16x4_t c2 = - create_s16x4_neon((int16_t *)(cospi + 32), (int16_t *)(cospi + 32), - (int16_t *)(cospi + 16), (int16_t *)(cospi + 48)); - - int16x8_t x[8]; - int16x8_t s0, s1, s4, s5; - - // Stage 1 - x[1] = in[0]; - - // Stage 2 - - btf_16_neon(x[1], cospi[60], -cospi[4], &s0, &s1); - - // Stage 3 - x[0] = s0; - x[1] = s1; - x[4] = s0; - x[5] = s1; - - // Stage 4 - s0 = x[0]; - s1 = x[1]; - btf_16_lane_2_3_neon(x[4], x[5], c2, &s4, &s5); - - // Stage 5 - x[0] = s0; - x[1] = s1; - x[2] = s0; - x[3] = s1; - x[4] = s4; - x[5] = s5; - x[6] = s4; - x[7] = s5; - - // stage 6 - btf_16_half_neon(x + 2, c2); - btf_16_half_neon(x + 6, c2); - - // Stage 7 - out[0] = x[0]; - out[1] = vnegq_s16(x[4]); - out[2] = x[6]; - out[3] = vnegq_s16(x[2]); - out[4] = x[3]; - out[5] = vnegq_s16(x[7]); - out[6] = x[5]; - out[7] = vnegq_s16(x[1]); -} - -static INLINE void idct8_new_neon(int16x8_t *in, int16x8_t *out, int8_t cos_bit, - int bit) { - (void)bit; - const int32_t *cospi = cospi_arr(cos_bit); - int16x8_t step1[8], step2[8]; - const int16x4_t c0 = - create_s16x4_neon((int16_t *)(cospi + 8), (int16_t *)(cospi + 56), - (int16_t *)(cospi + 40), (int16_t *)(cospi + 24)); - const int16x4_t c2 = - create_s16x4_neon((int16_t *)(cospi + 32), (int16_t *)(cospi + 32), - (int16_t *)(cospi + 16), (int16_t *)(cospi + 48)); - - // stage 2 - btf_16_lane_0_1_neon(in[1], in[7], c0, &step1[7], &step1[4]); - btf_16_lane_2_3_neon(in[5], in[3], c0, &step1[6], &step1[5]); - - // stage 3 - btf_16_lane_0_1_neon(in[0], in[4], c2, &step2[0], &step2[1]); - btf_16_lane_2_3_neon(in[2], in[6], c2, &step2[3], &step2[2]); - step2[4] = vqaddq_s16(step1[4], step1[5]); - step2[5] = vqsubq_s16(step1[4], step1[5]); - step2[6] = vqsubq_s16(step1[7], step1[6]); - step2[7] = vqaddq_s16(step1[7], step1[6]); - - // stage 4 - step1[0] = vqaddq_s16(step2[0], step2[3]); - step1[1] = vqaddq_s16(step2[1], step2[2]); - step1[2] = vqsubq_s16(step2[1], step2[2]); - step1[3] = vqsubq_s16(step2[0], step2[3]); - btf_16_lane_0_1_neon(step2[6], step2[5], c2, &step1[6], &step1[5]); - - // stage 5 - out[0] = vqaddq_s16(step1[0], step2[7]); - out[1] = vqaddq_s16(step1[1], step1[6]); - out[2] = vqaddq_s16(step1[2], step1[5]); - out[3] = vqaddq_s16(step1[3], step2[4]); - out[4] = vqsubq_s16(step1[3], step2[4]); - out[5] = vqsubq_s16(step1[2], step1[5]); - out[6] = vqsubq_s16(step1[1], step1[6]); - out[7] = vqsubq_s16(step1[0], step2[7]); -} - -static INLINE void idct8_low1_new_neon(int16x8_t *in, int16x8_t *out, - int8_t cos_bit, int bit) { - (void)bit; - const int32_t *cospi = cospi_arr(cos_bit); - int16x8_t step1; - int32x4_t t32[2]; - - // stage 1 - // stage 2 - // stage 3 - t32[0] = vmull_n_s16(vget_low_s16(in[0]), (int16_t)cospi[32]); - t32[1] = vmull_n_s16(vget_high_s16(in[0]), (int16_t)cospi[32]); - - step1 = vcombine_s16(vrshrn_n_s32(t32[0], INV_COS_BIT), - vrshrn_n_s32(t32[1], INV_COS_BIT)); - - // stage 4 - // stage 5 - out[0] = step1; - out[1] = step1; - out[2] = step1; - out[3] = step1; - out[4] = step1; - out[5] = step1; - out[6] = step1; - out[7] = step1; -} - -void av1_round_shift_array_16_neon(int16x8_t *arr, int size, int bit) { - assert(!(size % 4)); - if (!bit) return; - const int16x8_t dup_bits_n_16x8 = vdupq_n_s16((int16_t)(-bit)); - for (int i = 0; i < size; i++) { - arr[i] = vrshlq_s16(arr[i], dup_bits_n_16x8); - } -} - -static INLINE void flip_buf_ud_neon(int16x8_t *input, int size) { - int16x8_t temp[8]; - for (int i = 0; i < size; ++i) { - temp[i] = input[size - 1 - i]; - } - for (int i = 0; i < size; ++i) { - input[i] = temp[i]; - } -} - -static INLINE void load_buffer_32bit_to_16bit_neon(const int32_t *input, - int16x8_t *const a, - int out_size) { - for (int i = 0; i < 8; ++i) { - a[i] = vcombine_s16(vmovn_s32(vld1q_s32(input)), - vmovn_s32(vld1q_s32(input + 4))); - input += out_size; - } -} - -static INLINE void identity8_new_neon(int16x8_t *input, int16x8_t *output, - int8_t cos_bit, int bit) { - (void)bit; - (void)cos_bit; - - output[0] = vmulq_n_s16(input[0], (int16_t)2); - output[1] = vmulq_n_s16(input[1], (int16_t)2); - output[2] = vmulq_n_s16(input[2], (int16_t)2); - output[3] = vmulq_n_s16(input[3], (int16_t)2); - output[4] = vmulq_n_s16(input[4], (int16_t)2); - output[5] = vmulq_n_s16(input[5], (int16_t)2); - output[6] = vmulq_n_s16(input[6], (int16_t)2); - output[7] = vmulq_n_s16(input[7], (int16_t)2); -} - -static INLINE void round_shift_for_rect(int16x8_t *input, int16x8_t *output, - int size) { - int32x4_t out_low, out_high; - int16x4_t low, high; - - for (int z = 0; z < size; ++z) { - out_low = vmull_n_s16(vget_low_s16(input[z]), (int16_t)NewInvSqrt2); - out_high = vmull_n_s16(vget_high_s16(input[z]), (int16_t)NewInvSqrt2); - - low = vqrshrn_n_s32(out_low, (int32_t)NewSqrt2Bits); - high = vqrshrn_n_s32(out_high, (int32_t)NewSqrt2Bits); - - output[z] = vcombine_s16(low, high); - } -} - -static INLINE void identity16_new_neon(int16x8_t *input, int16x8_t *output, - int8_t cos_bit, int bit) { - (void)bit; - (void)cos_bit; - - int32x4_t out_low, out_high; - int16x4_t low, high; - int16_t scale = (int16_t)(2 * NewSqrt2); - - for (int z = 0; z < 16; ++z) { - out_low = vmull_n_s16(vget_low_s16(input[z]), scale); - out_high = vmull_n_s16(vget_high_s16(input[z]), scale); - - low = vqrshrn_n_s32(out_low, (int32_t)NewSqrt2Bits); - high = vqrshrn_n_s32(out_high, (int32_t)NewSqrt2Bits); - - output[z] = vcombine_s16(low, high); - } -} - -static INLINE void identity32_new_neon(int16x8_t *input, int16x8_t *output, - int8_t cos_bit, int bit) { - (void)bit; - (void)cos_bit; - - for (int z = 0; z < 32; ++z) { - output[z] = vmulq_n_s16(input[z], (int16_t)4); - } -} - -static INLINE void idct16_low1_new_neon(int16x8_t *in, int16x8_t *out, - int8_t cos_bit, int bit) { - (void)bit; - const int32_t *cospi = cospi_arr(cos_bit); - int16x8_t step1; - int32x4_t t32[2]; - - // stage 4 - - t32[0] = vmull_n_s16(vget_low_s16(in[0]), cospi[32]); - t32[1] = vmull_n_s16(vget_high_s16(in[0]), cospi[32]); - step1 = vcombine_s16(vrshrn_n_s32(t32[0], INV_COS_BIT), - vrshrn_n_s32(t32[1], INV_COS_BIT)); - - // stage 6 - // stage 7 - out[0] = step1; - out[1] = step1; - out[2] = step1; - out[3] = step1; - out[4] = step1; - out[5] = step1; - out[6] = step1; - out[7] = step1; - out[8] = step1; - out[9] = step1; - out[10] = step1; - out[11] = step1; - out[12] = step1; - out[13] = step1; - out[14] = step1; - out[15] = step1; -} - -static INLINE void idct16_new_neon(int16x8_t *in, int16x8_t *out, - int8_t cos_bit, int bit) { - (void)bit; - const int32_t *cospi = cospi_arr(cos_bit); - int16x8_t step1[16], step2[16]; - - const int16x4_t c0 = - create_s16x4_neon((int16_t *)(cospi + 4), (int16_t *)(cospi + 60), - (int16_t *)(cospi + 36), (int16_t *)(cospi + 28)); - const int16x4_t c1 = - create_s16x4_neon((int16_t *)(cospi + 20), (int16_t *)(cospi + 44), - (int16_t *)(cospi + 52), (int16_t *)(cospi + 12)); - const int16x4_t c2 = - create_s16x4_neon((int16_t *)(cospi + 8), (int16_t *)(cospi + 56), - (int16_t *)(cospi + 40), (int16_t *)(cospi + 24)); - const int16x4_t c3 = - create_s16x4_neon((int16_t *)(cospi + 32), (int16_t *)(cospi + 32), - (int16_t *)(cospi + 16), (int16_t *)(cospi + 48)); - - // stage 2 - - btf_16_lane_0_1_neon(in[1], in[15], c0, &step2[15], &step2[8]); - btf_16_lane_2_3_neon(in[9], in[7], c0, &step2[14], &step2[9]); - btf_16_lane_0_1_neon(in[5], in[11], c1, &step2[13], &step2[10]); - btf_16_lane_2_3_neon(in[13], in[3], c1, &step2[12], &step2[11]); - - step2[0] = in[0]; - step2[1] = in[8]; - step2[2] = in[4]; - step2[3] = in[12]; - step2[4] = in[2]; - step2[5] = in[10]; - step2[6] = in[6]; - step2[7] = in[14]; - - // stage 3 - - btf_16_lane_0_1_neon(step2[4], step2[7], c2, &step1[7], &step1[4]); - btf_16_lane_2_3_neon(step2[5], step2[6], c2, &step1[6], &step1[5]); - - step1[0] = step2[0]; - step1[1] = step2[1]; - step1[2] = step2[2]; - step1[3] = step2[3]; - step1[8] = vqaddq_s16(step2[8], step2[9]); - step1[9] = vqsubq_s16(step2[8], step2[9]); - step1[10] = vqsubq_s16(step2[11], step2[10]); - step1[11] = vqaddq_s16(step2[11], step2[10]); - step1[12] = vqaddq_s16(step2[12], step2[13]); - step1[13] = vqsubq_s16(step2[12], step2[13]); - step1[14] = vqsubq_s16(step2[15], step2[14]); - step1[15] = vqaddq_s16(step2[15], step2[14]); - - // stage 4 - - btf_16_lane_0_1_neon(step1[0], step1[1], c3, &step2[0], &step2[1]); - btf_16_lane_2_3_neon(step1[2], step1[3], c3, &step2[3], &step2[2]); - btf_16_lane_2_3_neon(step1[14], step1[9], c3, &step2[14], &step2[9]); - btf_16_lane_3_2_neon(vnegq_s16(step1[10]), vnegq_s16(step1[13]), c3, - &step2[10], &step2[13]); - - step2[4] = vqaddq_s16(step1[4], step1[5]); - step2[5] = vqsubq_s16(step1[4], step1[5]); - step2[6] = vqsubq_s16(step1[7], step1[6]); - step2[7] = vqaddq_s16(step1[7], step1[6]); - step2[8] = step1[8]; - step2[11] = step1[11]; - step2[12] = step1[12]; - step2[15] = step1[15]; - - // stage 5 - - btf_16_lane_0_1_neon(step2[6], step2[5], c3, &step1[6], &step1[5]); - - step1[0] = vqaddq_s16(step2[0], step2[3]); - step1[1] = vqaddq_s16(step2[1], step2[2]); - step1[2] = vqsubq_s16(step2[1], step2[2]); - step1[3] = vqsubq_s16(step2[0], step2[3]); - step1[4] = step2[4]; - step1[7] = step2[7]; - step1[8] = vqaddq_s16(step2[8], step2[11]); - step1[9] = vqaddq_s16(step2[9], step2[10]); - step1[10] = vqsubq_s16(step2[9], step2[10]); - step1[11] = vqsubq_s16(step2[8], step2[11]); - step1[12] = vqsubq_s16(step2[15], step2[12]); - step1[13] = vqsubq_s16(step2[14], step2[13]); - step1[14] = vqaddq_s16(step2[14], step2[13]); - step1[15] = vqaddq_s16(step2[15], step2[12]); - - // stage 6 - - btf_16_lane_0_1_neon(step1[13], step1[10], c3, &step2[13], &step2[10]); - btf_16_lane_0_1_neon(step1[12], step1[11], c3, &step2[12], &step2[11]); - - step2[0] = vqaddq_s16(step1[0], step1[7]); - step2[1] = vqaddq_s16(step1[1], step1[6]); - step2[2] = vqaddq_s16(step1[2], step1[5]); - step2[3] = vqaddq_s16(step1[3], step1[4]); - step2[4] = vqsubq_s16(step1[3], step1[4]); - step2[5] = vqsubq_s16(step1[2], step1[5]); - step2[6] = vqsubq_s16(step1[1], step1[6]); - step2[7] = vqsubq_s16(step1[0], step1[7]); - step2[8] = step1[8]; - step2[9] = step1[9]; - step2[14] = step1[14]; - step2[15] = step1[15]; - - // stage 7 - out[0] = vqaddq_s16(step2[0], step2[15]); - out[1] = vqaddq_s16(step2[1], step2[14]); - out[2] = vqaddq_s16(step2[2], step2[13]); - out[3] = vqaddq_s16(step2[3], step2[12]); - out[4] = vqaddq_s16(step2[4], step2[11]); - out[5] = vqaddq_s16(step2[5], step2[10]); - out[6] = vqaddq_s16(step2[6], step2[9]); - out[7] = vqaddq_s16(step2[7], step2[8]); - out[8] = vqsubq_s16(step2[7], step2[8]); - out[9] = vqsubq_s16(step2[6], step2[9]); - out[10] = vqsubq_s16(step2[5], step2[10]); - out[11] = vqsubq_s16(step2[4], step2[11]); - out[12] = vqsubq_s16(step2[3], step2[12]); - out[13] = vqsubq_s16(step2[2], step2[13]); - out[14] = vqsubq_s16(step2[1], step2[14]); - out[15] = vqsubq_s16(step2[0], step2[15]); -} - -static INLINE void idct16_low8_new_neon(int16x8_t *in, int16x8_t *out, - int8_t cos_bit, int bit) { - (void)bit; - const int32_t *cospi = cospi_arr(cos_bit); - int16x8_t step1[16], step2[16]; - const int16x4_t c0 = - create_s16x4_neon((int16_t *)(cospi + 32), (int16_t *)(cospi + 32), - (int16_t *)(cospi + 16), (int16_t *)(cospi + 48)); - - // stage 1 - // stage 2 - - step2[0] = in[0]; - step2[2] = in[4]; - step2[4] = in[2]; - step2[6] = in[6]; - - btf_16_neon(in[1], cospi[60], cospi[4], &step2[8], &step2[15]); - btf_16_neon(in[7], -cospi[36], cospi[28], &step2[9], &step2[14]); - btf_16_neon(in[5], cospi[44], cospi[20], &step2[10], &step2[13]); - btf_16_neon(in[3], -cospi[52], cospi[12], &step2[11], &step2[12]); - - // stage 3 - - btf_16_neon(step2[4], cospi[56], cospi[8], &step1[4], &step1[7]); - btf_16_neon(step2[6], -cospi[40], cospi[24], &step1[5], &step1[6]); - - step1[0] = step2[0]; - step1[2] = step2[2]; - step1[8] = vqaddq_s16(step2[8], step2[9]); - step1[9] = vqsubq_s16(step2[8], step2[9]); - step1[10] = vqsubq_s16(step2[11], step2[10]); - step1[11] = vqaddq_s16(step2[11], step2[10]); - step1[12] = vqaddq_s16(step2[12], step2[13]); - step1[13] = vqsubq_s16(step2[12], step2[13]); - step1[14] = vqsubq_s16(step2[15], step2[14]); - step1[15] = vqaddq_s16(step2[15], step2[14]); - - // stage 4 - - btf_16_neon(step1[0], cospi[32], cospi[32], &step2[0], &step2[1]); - btf_16_neon(step1[2], cospi[48], cospi[16], &step2[2], &step2[3]); - btf_16_lane_2_3_neon(step1[14], step1[9], c0, &step2[14], &step2[9]); - btf_16_lane_3_2_neon(vnegq_s16(step1[10]), vnegq_s16(step1[13]), c0, - &step2[10], &step2[13]); - - step2[4] = vqaddq_s16(step1[4], step1[5]); - step2[5] = vqsubq_s16(step1[4], step1[5]); - step2[6] = vqsubq_s16(step1[7], step1[6]); - step2[7] = vqaddq_s16(step1[7], step1[6]); - step2[8] = step1[8]; - step2[11] = step1[11]; - step2[12] = step1[12]; - step2[15] = step1[15]; - - // stage 5 - - btf_16_lane_0_1_neon(step2[6], step2[5], c0, &step1[6], &step1[5]); - step1[0] = vqaddq_s16(step2[0], step2[3]); - step1[1] = vqaddq_s16(step2[1], step2[2]); - step1[2] = vqsubq_s16(step2[1], step2[2]); - step1[3] = vqsubq_s16(step2[0], step2[3]); - step1[4] = step2[4]; - step1[7] = step2[7]; - step1[8] = vqaddq_s16(step2[8], step2[11]); - step1[9] = vqaddq_s16(step2[9], step2[10]); - step1[10] = vqsubq_s16(step2[9], step2[10]); - step1[11] = vqsubq_s16(step2[8], step2[11]); - step1[12] = vqsubq_s16(step2[15], step2[12]); - step1[13] = vqsubq_s16(step2[14], step2[13]); - step1[14] = vqaddq_s16(step2[14], step2[13]); - step1[15] = vqaddq_s16(step2[15], step2[12]); - - // stage 6 - btf_16_lane_0_1_neon(step1[13], step1[10], c0, &step2[13], &step2[10]); - btf_16_lane_0_1_neon(step1[12], step1[11], c0, &step2[12], &step2[11]); - - step2[0] = vqaddq_s16(step1[0], step1[7]); - step2[1] = vqaddq_s16(step1[1], step1[6]); - step2[2] = vqaddq_s16(step1[2], step1[5]); - step2[3] = vqaddq_s16(step1[3], step1[4]); - step2[4] = vqsubq_s16(step1[3], step1[4]); - step2[5] = vqsubq_s16(step1[2], step1[5]); - step2[6] = vqsubq_s16(step1[1], step1[6]); - step2[7] = vqsubq_s16(step1[0], step1[7]); - step2[8] = step1[8]; - step2[9] = step1[9]; - step2[14] = step1[14]; - step2[15] = step1[15]; - - // stage 7 - - out[0] = vqaddq_s16(step2[0], step2[15]); - out[1] = vqaddq_s16(step2[1], step2[14]); - out[2] = vqaddq_s16(step2[2], step2[13]); - out[3] = vqaddq_s16(step2[3], step2[12]); - out[4] = vqaddq_s16(step2[4], step2[11]); - out[5] = vqaddq_s16(step2[5], step2[10]); - out[6] = vqaddq_s16(step2[6], step2[9]); - out[7] = vqaddq_s16(step2[7], step2[8]); - out[8] = vqsubq_s16(step2[7], step2[8]); - out[9] = vqsubq_s16(step2[6], step2[9]); - out[10] = vqsubq_s16(step2[5], step2[10]); - out[11] = vqsubq_s16(step2[4], step2[11]); - out[12] = vqsubq_s16(step2[3], step2[12]); - out[13] = vqsubq_s16(step2[2], step2[13]); - out[14] = vqsubq_s16(step2[1], step2[14]); - out[15] = vqsubq_s16(step2[0], step2[15]); -} - -static INLINE void iadst16_new_neon(int16x8_t *const in, int16x8_t *out, - int8_t cos_bit, int bit) { - (void)bit; - const int32_t *cospi = cospi_arr(cos_bit); - - const int16x4_t c0 = - create_s16x4_neon((int16_t *)(cospi + 2), (int16_t *)(cospi + 62), - (int16_t *)(cospi + 10), (int16_t *)(cospi + 54)); - const int16x4_t c1 = - create_s16x4_neon((int16_t *)(cospi + 18), (int16_t *)(cospi + 46), - (int16_t *)(cospi + 26), (int16_t *)(cospi + 38)); - const int16x4_t c2 = - create_s16x4_neon((int16_t *)(cospi + 34), (int16_t *)(cospi + 30), - (int16_t *)(cospi + 42), (int16_t *)(cospi + 22)); - const int16x4_t c3 = - create_s16x4_neon((int16_t *)(cospi + 50), (int16_t *)(cospi + 14), - (int16_t *)(cospi + 58), (int16_t *)(cospi + 6)); - const int16x4_t c4 = - create_s16x4_neon((int16_t *)(cospi + 8), (int16_t *)(cospi + 56), - (int16_t *)(cospi + 40), (int16_t *)(cospi + 24)); - - const int16x4_t c = - create_s16x4_neon((int16_t *)(cospi + 32), (int16_t *)(cospi + 32), - (int16_t *)(cospi + 16), (int16_t *)(cospi + 48)); - - int16x8_t x[16]; - int16x8_t t[14]; - int16x8_t s0, s1, s2, s3, s4, s5, s6, s7; - int16x8_t s8, s9, s10, s11, s12, s13, s14, s15; - - // Stage 1 - x[0] = in[15]; - x[1] = in[0]; - x[2] = in[13]; - x[3] = in[2]; - x[4] = in[11]; - x[5] = in[4]; - x[6] = in[9]; - x[7] = in[6]; - x[8] = in[7]; - x[9] = in[8]; - x[10] = in[5]; - x[11] = in[10]; - x[12] = in[3]; - x[13] = in[12]; - x[14] = in[1]; - x[15] = in[14]; - - // Stage 2 - btf_16_lane_0_1_neon(x[0], x[1], c0, &s0, &s1); - btf_16_lane_2_3_neon(x[2], x[3], c0, &s2, &s3); - btf_16_lane_0_1_neon(x[4], x[5], c1, &s4, &s5); - btf_16_lane_2_3_neon(x[6], x[7], c1, &s6, &s7); - btf_16_lane_0_1_neon(x[8], x[9], c2, &s8, &s9); - btf_16_lane_2_3_neon(x[10], x[11], c2, &s10, &s11); - btf_16_lane_0_1_neon(x[12], x[13], c3, &s12, &s13); - btf_16_lane_2_3_neon(x[14], x[15], c3, &s14, &s15); - - // Stage 3 - x[0] = vqaddq_s16(s0, s8); - x[1] = vqaddq_s16(s1, s9); - x[2] = vqaddq_s16(s2, s10); - x[3] = vqaddq_s16(s3, s11); - x[4] = vqaddq_s16(s4, s12); - x[5] = vqaddq_s16(s5, s13); - x[6] = vqaddq_s16(s6, s14); - x[7] = vqaddq_s16(s7, s15); - x[8] = vqsubq_s16(s0, s8); - x[9] = vqsubq_s16(s1, s9); - x[10] = vqsubq_s16(s2, s10); - x[11] = vqsubq_s16(s3, s11); - x[12] = vqsubq_s16(s4, s12); - x[13] = vqsubq_s16(s5, s13); - x[14] = vqsubq_s16(s6, s14); - x[15] = vqsubq_s16(s7, s15); - - // Stage 4 - t[0] = x[0]; - t[1] = x[1]; - t[2] = x[2]; - t[3] = x[3]; - t[4] = x[4]; - t[5] = x[5]; - t[6] = x[6]; - t[7] = x[7]; - btf_16_lane_0_1_neon(x[8], x[9], c4, &s8, &s9); - btf_16_lane_2_3_neon(x[10], x[11], c4, &s10, &s11); - btf_16_lane_1_0_neon(x[13], x[12], c4, &s13, &s12); - btf_16_lane_3_2_neon(x[15], x[14], c4, &s15, &s14); - - // Stage 5 - x[0] = vqaddq_s16(t[0], t[4]); - x[1] = vqaddq_s16(t[1], t[5]); - x[2] = vqaddq_s16(t[2], t[6]); - x[3] = vqaddq_s16(t[3], t[7]); - x[4] = vqsubq_s16(t[0], t[4]); - x[5] = vqsubq_s16(t[1], t[5]); - x[6] = vqsubq_s16(t[2], t[6]); - x[7] = vqsubq_s16(t[3], t[7]); - x[8] = vqaddq_s16(s8, s12); - x[9] = vqaddq_s16(s9, s13); - x[10] = vqaddq_s16(s10, s14); - x[11] = vqaddq_s16(s11, s15); - x[12] = vqsubq_s16(s8, s12); - x[13] = vqsubq_s16(s9, s13); - x[14] = vqsubq_s16(s10, s14); - x[15] = vqsubq_s16(s11, s15); - - // stage 6 - t[0] = x[0]; - t[1] = x[1]; - t[2] = x[2]; - t[3] = x[3]; - btf_16_lane_2_3_neon(x[4], x[5], c, &s4, &s5); - btf_16_lane_3_2_neon(x[7], x[6], c, &s7, &s6); - t[8] = x[8]; - t[9] = x[9]; - t[10] = x[10]; - t[11] = x[11]; - btf_16_lane_2_3_neon(x[12], x[13], c, &s12, &s13); - btf_16_lane_3_2_neon(x[15], x[14], c, &s15, &s14); - - // Stage 7 - x[0] = vqaddq_s16(t[0], t[2]); - x[1] = vqaddq_s16(t[1], t[3]); - x[2] = vqsubq_s16(t[0], t[2]); - x[3] = vqsubq_s16(t[1], t[3]); - x[4] = vqaddq_s16(s4, s6); - x[5] = vqaddq_s16(s5, s7); - x[6] = vqsubq_s16(s4, s6); - x[7] = vqsubq_s16(s5, s7); - x[8] = vqaddq_s16(t[8], t[10]); - x[9] = vqaddq_s16(t[9], t[11]); - x[10] = vqsubq_s16(t[8], t[10]); - x[11] = vqsubq_s16(t[9], t[11]); - x[12] = vqaddq_s16(s12, s14); - x[13] = vqaddq_s16(s13, s15); - x[14] = vqsubq_s16(s12, s14); - x[15] = vqsubq_s16(s13, s15); - - // Stage 8 - btf_16_half_neon(x + 2, c); - btf_16_half_neon(x + 6, c); - btf_16_half_neon(x + 10, c); - btf_16_half_neon(x + 14, c); - - // Stage 9 - out[0] = x[0]; - out[1] = vnegq_s16(x[8]); - out[2] = x[12]; - out[3] = vnegq_s16(x[4]); - out[4] = x[6]; - out[5] = vnegq_s16(x[14]); - out[6] = x[10]; - out[7] = vnegq_s16(x[2]); - out[8] = x[3]; - out[9] = vnegq_s16(x[11]); - out[10] = x[15]; - out[11] = vnegq_s16(x[7]); - out[12] = x[5]; - out[13] = vnegq_s16(x[13]); - out[14] = x[9]; - out[15] = vnegq_s16(x[1]); -} - -static INLINE void iadst16_low1_new_neon(int16x8_t *const in, int16x8_t *out, - int8_t cos_bit, int bit) { - (void)bit; - const int32_t *cospi = cospi_arr(cos_bit); - const int16x4_t c4 = - create_s16x4_neon((int16_t *)(cospi + 8), (int16_t *)(cospi + 56), - (int16_t *)(cospi + 40), (int16_t *)(cospi + 24)); - const int16x4_t c = - create_s16x4_neon((int16_t *)(cospi + 32), (int16_t *)(cospi + 32), - (int16_t *)(cospi + 16), (int16_t *)(cospi + 48)); - - int16x8_t x[16]; - int16x8_t t[10]; - int16x8_t s0, s1, s4, s5; - int16x8_t s8, s9, s12, s13; - - // Stage 1 - x[1] = in[0]; - - // Stage 2 - btf_16_neon(x[1], cospi[62], -cospi[2], &s0, &s1); - - // Stage 3 - x[0] = s0; - x[1] = s1; - x[8] = s0; - x[9] = s1; - - // Stage 4 - t[0] = x[0]; - t[1] = x[1]; - btf_16_lane_0_1_neon(x[8], x[9], c4, &s8, &s9); - - // Stage 5 - x[0] = t[0]; - x[1] = t[1]; - x[4] = t[0]; - x[5] = t[1]; - x[8] = s8; - x[9] = s9; - x[12] = s8; - x[13] = s9; - - // stage 6 - t[0] = x[0]; - t[1] = x[1]; - btf_16_lane_2_3_neon(x[4], x[5], c, &s4, &s5); - t[8] = x[8]; - t[9] = x[9]; - btf_16_lane_2_3_neon(x[12], x[13], c, &s12, &s13); - - // Stage 7 - x[0] = t[0]; - x[1] = t[1]; - x[2] = t[0]; - x[3] = t[1]; - x[4] = s4; - x[5] = s5; - x[6] = s4; - x[7] = s5; - x[8] = t[8]; - x[9] = t[9]; - x[10] = t[8]; - x[11] = t[9]; - x[12] = s12; - x[13] = s13; - x[14] = s12; - x[15] = s13; - - // Stage 8 - btf_16_half_neon(x + 2, c); - btf_16_half_neon(x + 6, c); - btf_16_half_neon(x + 10, c); - btf_16_half_neon(x + 14, c); - - // Stage 9 - out[0] = x[0]; - out[1] = vnegq_s16(x[8]); - out[2] = x[12]; - out[3] = vnegq_s16(x[4]); - out[4] = x[6]; - out[5] = vnegq_s16(x[14]); - out[6] = x[10]; - out[7] = vnegq_s16(x[2]); - out[8] = x[3]; - out[9] = vnegq_s16(x[11]); - out[10] = x[15]; - out[11] = vnegq_s16(x[7]); - out[12] = x[5]; - out[13] = vnegq_s16(x[13]); - out[14] = x[9]; - out[15] = vnegq_s16(x[1]); -} - -static INLINE void iadst16_low8_new_neon(int16x8_t *const in, int16x8_t *out, - int8_t cos_bit, int bit) { - (void)bit; - const int32_t *cospi = cospi_arr(cos_bit); - - const int16x4_t c4 = - create_s16x4_neon((int16_t *)(cospi + 8), (int16_t *)(cospi + 56), - (int16_t *)(cospi + 40), (int16_t *)(cospi + 24)); - const int16x4_t c = - create_s16x4_neon((int16_t *)(cospi + 32), (int16_t *)(cospi + 32), - (int16_t *)(cospi + 16), (int16_t *)(cospi + 48)); - - int16x8_t x[16]; - int16x8_t t[14]; - int16x8_t s0, s1, s2, s3, s4, s5, s6, s7; - int16x8_t s8, s9, s10, s11, s12, s13, s14, s15; - - // Stage 1 - x[1] = in[0]; - x[3] = in[2]; - x[5] = in[4]; - x[7] = in[6]; - x[8] = in[7]; - x[10] = in[5]; - x[12] = in[3]; - x[14] = in[1]; - - // Stage 2 - btf_16_neon(x[1], cospi[62], -cospi[2], &s0, &s1); - btf_16_neon(x[3], cospi[54], -cospi[10], &s2, &s3); - btf_16_neon(x[5], cospi[46], -cospi[18], &s4, &s5); - btf_16_neon(x[7], cospi[38], -cospi[26], &s6, &s7); - - btf_16_neon(x[8], cospi[34], cospi[30], &s8, &s9); - btf_16_neon(x[10], cospi[42], cospi[22], &s10, &s11); - btf_16_neon(x[12], cospi[50], cospi[14], &s12, &s13); - btf_16_neon(x[14], cospi[58], cospi[6], &s14, &s15); - - // Stage 3 - x[0] = vqaddq_s16(s0, s8); - x[1] = vqaddq_s16(s1, s9); - x[2] = vqaddq_s16(s2, s10); - x[3] = vqaddq_s16(s3, s11); - x[4] = vqaddq_s16(s4, s12); - x[5] = vqaddq_s16(s5, s13); - x[6] = vqaddq_s16(s6, s14); - x[7] = vqaddq_s16(s7, s15); - x[8] = vqsubq_s16(s0, s8); - x[9] = vqsubq_s16(s1, s9); - x[10] = vqsubq_s16(s2, s10); - x[11] = vqsubq_s16(s3, s11); - x[12] = vqsubq_s16(s4, s12); - x[13] = vqsubq_s16(s5, s13); - x[14] = vqsubq_s16(s6, s14); - x[15] = vqsubq_s16(s7, s15); - - // Stage 4 - t[0] = x[0]; - t[1] = x[1]; - t[2] = x[2]; - t[3] = x[3]; - t[4] = x[4]; - t[5] = x[5]; - t[6] = x[6]; - t[7] = x[7]; - btf_16_lane_0_1_neon(x[8], x[9], c4, &s8, &s9); - btf_16_lane_2_3_neon(x[10], x[11], c4, &s10, &s11); - btf_16_lane_1_0_neon(x[13], x[12], c4, &s13, &s12); - btf_16_lane_3_2_neon(x[15], x[14], c4, &s15, &s14); - - // Stage 5 - x[0] = vqaddq_s16(t[0], t[4]); - x[1] = vqaddq_s16(t[1], t[5]); - x[2] = vqaddq_s16(t[2], t[6]); - x[3] = vqaddq_s16(t[3], t[7]); - x[4] = vqsubq_s16(t[0], t[4]); - x[5] = vqsubq_s16(t[1], t[5]); - x[6] = vqsubq_s16(t[2], t[6]); - x[7] = vqsubq_s16(t[3], t[7]); - x[8] = vqaddq_s16(s8, s12); - x[9] = vqaddq_s16(s9, s13); - x[10] = vqaddq_s16(s10, s14); - x[11] = vqaddq_s16(s11, s15); - x[12] = vqsubq_s16(s8, s12); - x[13] = vqsubq_s16(s9, s13); - x[14] = vqsubq_s16(s10, s14); - x[15] = vqsubq_s16(s11, s15); - - // stage 6 - t[0] = x[0]; - t[1] = x[1]; - t[2] = x[2]; - t[3] = x[3]; - btf_16_lane_2_3_neon(x[4], x[5], c, &s4, &s5); - btf_16_lane_3_2_neon(x[7], x[6], c, &s7, &s6); - t[8] = x[8]; - t[9] = x[9]; - t[10] = x[10]; - t[11] = x[11]; - btf_16_lane_2_3_neon(x[12], x[13], c, &s12, &s13); - btf_16_lane_3_2_neon(x[15], x[14], c, &s15, &s14); - - // Stage 7 - x[0] = vqaddq_s16(t[0], t[2]); - x[1] = vqaddq_s16(t[1], t[3]); - x[2] = vqsubq_s16(t[0], t[2]); - x[3] = vqsubq_s16(t[1], t[3]); - x[4] = vqaddq_s16(s4, s6); - x[5] = vqaddq_s16(s5, s7); - x[6] = vqsubq_s16(s4, s6); - x[7] = vqsubq_s16(s5, s7); - x[8] = vqaddq_s16(t[8], t[10]); - x[9] = vqaddq_s16(t[9], t[11]); - x[10] = vqsubq_s16(t[8], t[10]); - x[11] = vqsubq_s16(t[9], t[11]); - x[12] = vqaddq_s16(s12, s14); - x[13] = vqaddq_s16(s13, s15); - x[14] = vqsubq_s16(s12, s14); - x[15] = vqsubq_s16(s13, s15); - - // Stage 8 - btf_16_half_neon(x + 2, c); - btf_16_half_neon(x + 6, c); - btf_16_half_neon(x + 10, c); - btf_16_half_neon(x + 14, c); - - // Stage 9 - out[0] = x[0]; - out[1] = vnegq_s16(x[8]); - out[2] = x[12]; - out[3] = vnegq_s16(x[4]); - out[4] = x[6]; - out[5] = vnegq_s16(x[14]); - out[6] = x[10]; - out[7] = vnegq_s16(x[2]); - out[8] = x[3]; - out[9] = vnegq_s16(x[11]); - out[10] = x[15]; - out[11] = vnegq_s16(x[7]); - out[12] = x[5]; - out[13] = vnegq_s16(x[13]); - out[14] = x[9]; - out[15] = vnegq_s16(x[1]); -} - -static INLINE void idct32_new_neon(int16x8_t *in, int16x8_t *out, - int8_t cos_bit, int bit) { - (void)bit; - const int32_t *cospi = cospi_arr(cos_bit); - int16x8_t step1[32], step2[32]; - - const int16x4_t c0 = - create_s16x4_neon((int16_t *)(cospi + 2), (int16_t *)(cospi + 62), - (int16_t *)(cospi + 34), (int16_t *)(cospi + 30)); - const int16x4_t c1 = - create_s16x4_neon((int16_t *)(cospi + 18), (int16_t *)(cospi + 46), - (int16_t *)(cospi + 50), (int16_t *)(cospi + 14)); - const int16x4_t c2 = - create_s16x4_neon((int16_t *)(cospi + 10), (int16_t *)(cospi + 54), - (int16_t *)(cospi + 42), (int16_t *)(cospi + 22)); - const int16x4_t c3 = - create_s16x4_neon((int16_t *)(cospi + 26), (int16_t *)(cospi + 38), - (int16_t *)(cospi + 58), (int16_t *)(cospi + 6)); - const int16x4_t c4 = - create_s16x4_neon((int16_t *)(cospi + 4), (int16_t *)(cospi + 60), - (int16_t *)(cospi + 36), (int16_t *)(cospi + 28)); - const int16x4_t c5 = - create_s16x4_neon((int16_t *)(cospi + 20), (int16_t *)(cospi + 44), - (int16_t *)(cospi + 52), (int16_t *)(cospi + 12)); - const int16x4_t c6 = - create_s16x4_neon((int16_t *)(cospi + 8), (int16_t *)(cospi + 56), - (int16_t *)(cospi + 40), (int16_t *)(cospi + 24)); - const int16x4_t c7 = - create_s16x4_neon((int16_t *)(cospi + 32), (int16_t *)(cospi + 32), - (int16_t *)(cospi + 16), (int16_t *)(cospi + 48)); - - // stage 2 - - btf_16_lane_0_1_neon(in[1], in[31], c0, &step2[31], &step2[16]); - btf_16_lane_2_3_neon(in[17], in[15], c0, &step2[30], &step2[17]); - btf_16_lane_0_1_neon(in[9], in[23], c1, &step2[29], &step2[18]); - btf_16_lane_2_3_neon(in[25], in[7], c1, &step2[28], &step2[19]); - btf_16_lane_0_1_neon(in[5], in[27], c2, &step2[27], &step2[20]); - btf_16_lane_2_3_neon(in[21], in[11], c2, &step2[26], &step2[21]); - btf_16_lane_0_1_neon(in[13], in[19], c3, &step2[25], &step2[22]); - btf_16_lane_2_3_neon(in[29], in[3], c3, &step2[24], &step2[23]); - - step2[0] = in[0]; - step2[1] = in[16]; - step2[2] = in[8]; - step2[3] = in[24]; - step2[4] = in[4]; - step2[5] = in[20]; - step2[6] = in[12]; - step2[7] = in[28]; - step2[8] = in[2]; - step2[9] = in[18]; - step2[10] = in[10]; - step2[11] = in[26]; - step2[12] = in[6]; - step2[13] = in[22]; - step2[14] = in[14]; - step2[15] = in[30]; - - // stage 3 - - btf_16_lane_0_1_neon(step2[8], step2[15], c4, &step1[15], &step1[8]); - btf_16_lane_2_3_neon(step2[9], step2[14], c4, &step1[14], &step1[9]); - btf_16_lane_0_1_neon(step2[10], step2[13], c5, &step1[13], &step1[10]); - btf_16_lane_2_3_neon(step2[11], step2[12], c5, &step1[12], &step1[11]); - - step1[0] = step2[0]; - step1[1] = step2[1]; - step1[2] = step2[2]; - step1[3] = step2[3]; - step1[4] = step2[4]; - step1[5] = step2[5]; - step1[6] = step2[6]; - step1[7] = step2[7]; - - step1[16] = vqaddq_s16(step2[16], step2[17]); - step1[17] = vqsubq_s16(step2[16], step2[17]); - step1[18] = vqsubq_s16(step2[19], step2[18]); - step1[19] = vqaddq_s16(step2[19], step2[18]); - step1[20] = vqaddq_s16(step2[20], step2[21]); - step1[21] = vqsubq_s16(step2[20], step2[21]); - step1[22] = vqsubq_s16(step2[23], step2[22]); - step1[23] = vqaddq_s16(step2[23], step2[22]); - step1[24] = vqaddq_s16(step2[24], step2[25]); - step1[25] = vqsubq_s16(step2[24], step2[25]); - step1[26] = vqsubq_s16(step2[27], step2[26]); - step1[27] = vqaddq_s16(step2[27], step2[26]); - step1[28] = vqaddq_s16(step2[28], step2[29]); - step1[29] = vqsubq_s16(step2[28], step2[29]); - step1[30] = vqsubq_s16(step2[31], step2[30]); - step1[31] = vqaddq_s16(step2[31], step2[30]); - - // stage 4 - - btf_16_lane_0_1_neon(step1[4], step1[7], c6, &step2[7], &step2[4]); - btf_16_lane_2_3_neon(step1[5], step1[6], c6, &step2[6], &step2[5]); - btf_16_lane_0_1_neon(step1[30], step1[17], c6, &step2[30], &step2[17]); - btf_16_lane_1_0_neon(vnegq_s16(step1[18]), vnegq_s16(step1[29]), c6, - &step2[18], &step2[29]); - btf_16_lane_2_3_neon(step1[26], step1[21], c6, &step2[26], &step2[21]); - btf_16_lane_3_2_neon(vnegq_s16(step1[22]), vnegq_s16(step1[25]), c6, - &step2[22], &step2[25]); - - step2[0] = step1[0]; - step2[1] = step1[1]; - step2[2] = step1[2]; - step2[3] = step1[3]; - step2[8] = vqaddq_s16(step1[8], step1[9]); - step2[9] = vqsubq_s16(step1[8], step1[9]); - step2[10] = vqsubq_s16(step1[11], step1[10]); - step2[11] = vqaddq_s16(step1[11], step1[10]); - step2[12] = vqaddq_s16(step1[12], step1[13]); - step2[13] = vqsubq_s16(step1[12], step1[13]); - step2[14] = vqsubq_s16(step1[15], step1[14]); - step2[15] = vqaddq_s16(step1[15], step1[14]); - step2[16] = step1[16]; - step2[19] = step1[19]; - step2[20] = step1[20]; - step2[23] = step1[23]; - step2[24] = step1[24]; - step2[27] = step1[27]; - step2[28] = step1[28]; - step2[31] = step1[31]; - - // stage 5 - - btf_16_lane_0_1_neon(step2[0], step2[1], c7, &step1[0], &step1[1]); - btf_16_lane_2_3_neon(step2[2], step2[3], c7, &step1[3], &step1[2]); - btf_16_lane_2_3_neon(step2[14], step2[9], c7, &step1[14], &step1[9]); - btf_16_lane_3_2_neon(vnegq_s16(step2[10]), vnegq_s16(step2[13]), c7, - &step1[10], &step1[13]); - - step1[4] = vqaddq_s16(step2[4], step2[5]); - step1[5] = vqsubq_s16(step2[4], step2[5]); - step1[6] = vqsubq_s16(step2[7], step2[6]); - step1[7] = vqaddq_s16(step2[7], step2[6]); - step1[8] = step2[8]; - step1[11] = step2[11]; - step1[12] = step2[12]; - step1[15] = step2[15]; - step1[16] = vqaddq_s16(step2[16], step2[19]); - step1[17] = vqaddq_s16(step2[17], step2[18]); - step1[18] = vqsubq_s16(step2[17], step2[18]); - step1[19] = vqsubq_s16(step2[16], step2[19]); - step1[20] = vqsubq_s16(step2[23], step2[20]); - step1[21] = vqsubq_s16(step2[22], step2[21]); - step1[22] = vqaddq_s16(step2[22], step2[21]); - step1[23] = vqaddq_s16(step2[23], step2[20]); - step1[24] = vqaddq_s16(step2[24], step2[27]); - step1[25] = vqaddq_s16(step2[25], step2[26]); - step1[26] = vqsubq_s16(step2[25], step2[26]); - step1[27] = vqsubq_s16(step2[24], step2[27]); - step1[28] = vqsubq_s16(step2[31], step2[28]); - step1[29] = vqsubq_s16(step2[30], step2[29]); - step1[30] = vqaddq_s16(step2[30], step2[29]); - step1[31] = vqaddq_s16(step2[31], step2[28]); - - // stage 6 - - btf_16_lane_0_1_neon(step1[6], step1[5], c7, &step2[6], &step2[5]); - btf_16_lane_2_3_neon(step1[29], step1[18], c7, &step2[29], &step2[18]); - btf_16_lane_2_3_neon(step1[28], step1[19], c7, &step2[28], &step2[19]); - btf_16_lane_3_2_neon(vnegq_s16(step1[20]), vnegq_s16(step1[27]), c7, - &step2[20], &step2[27]); - btf_16_lane_3_2_neon(vnegq_s16(step1[21]), vnegq_s16(step1[26]), c7, - &step2[21], &step2[26]); - - step2[0] = vqaddq_s16(step1[0], step1[3]); - step2[1] = vqaddq_s16(step1[1], step1[2]); - step2[2] = vqsubq_s16(step1[1], step1[2]); - step2[3] = vqsubq_s16(step1[0], step1[3]); - step2[4] = step1[4]; - step2[7] = step1[7]; - step2[8] = vqaddq_s16(step1[8], step1[11]); - step2[9] = vqaddq_s16(step1[9], step1[10]); - step2[10] = vqsubq_s16(step1[9], step1[10]); - step2[11] = vqsubq_s16(step1[8], step1[11]); - step2[12] = vqsubq_s16(step1[15], step1[12]); - step2[13] = vqsubq_s16(step1[14], step1[13]); - step2[14] = vqaddq_s16(step1[14], step1[13]); - step2[15] = vqaddq_s16(step1[15], step1[12]); - step2[16] = step1[16]; - step2[17] = step1[17]; - step2[22] = step1[22]; - step2[23] = step1[23]; - step2[24] = step1[24]; - step2[25] = step1[25]; - step2[30] = step1[30]; - step2[31] = step1[31]; - - // stage 7 - - btf_16_lane_0_1_neon(step2[13], step2[10], c7, &step1[13], &step1[10]); - btf_16_lane_0_1_neon(step2[12], step2[11], c7, &step1[12], &step1[11]); - - step1[0] = vqaddq_s16(step2[0], step2[7]); - step1[1] = vqaddq_s16(step2[1], step2[6]); - step1[2] = vqaddq_s16(step2[2], step2[5]); - step1[3] = vqaddq_s16(step2[3], step2[4]); - step1[4] = vqsubq_s16(step2[3], step2[4]); - step1[5] = vqsubq_s16(step2[2], step2[5]); - step1[6] = vqsubq_s16(step2[1], step2[6]); - step1[7] = vqsubq_s16(step2[0], step2[7]); - step1[8] = step2[8]; - step1[9] = step2[9]; - step1[14] = step2[14]; - step1[15] = step2[15]; - step1[16] = vqaddq_s16(step2[16], step2[23]); - step1[17] = vqaddq_s16(step2[17], step2[22]); - step1[18] = vqaddq_s16(step2[18], step2[21]); - step1[19] = vqaddq_s16(step2[19], step2[20]); - step1[20] = vqsubq_s16(step2[19], step2[20]); - step1[21] = vqsubq_s16(step2[18], step2[21]); - step1[22] = vqsubq_s16(step2[17], step2[22]); - step1[23] = vqsubq_s16(step2[16], step2[23]); - step1[24] = vqsubq_s16(step2[31], step2[24]); - step1[25] = vqsubq_s16(step2[30], step2[25]); - step1[26] = vqsubq_s16(step2[29], step2[26]); - step1[27] = vqsubq_s16(step2[28], step2[27]); - step1[28] = vqaddq_s16(step2[27], step2[28]); - step1[29] = vqaddq_s16(step2[26], step2[29]); - step1[30] = vqaddq_s16(step2[25], step2[30]); - step1[31] = vqaddq_s16(step2[24], step2[31]); - - // stage 8 - - btf_16_lane_0_1_neon(step1[27], step1[20], c7, &step2[27], &step2[20]); - btf_16_lane_0_1_neon(step1[26], step1[21], c7, &step2[26], &step2[21]); - btf_16_lane_0_1_neon(step1[25], step1[22], c7, &step2[25], &step2[22]); - btf_16_lane_0_1_neon(step1[24], step1[23], c7, &step2[24], &step2[23]); - - step2[0] = vqaddq_s16(step1[0], step1[15]); - step2[1] = vqaddq_s16(step1[1], step1[14]); - step2[2] = vqaddq_s16(step1[2], step1[13]); - step2[3] = vqaddq_s16(step1[3], step1[12]); - step2[4] = vqaddq_s16(step1[4], step1[11]); - step2[5] = vqaddq_s16(step1[5], step1[10]); - step2[6] = vqaddq_s16(step1[6], step1[9]); - step2[7] = vqaddq_s16(step1[7], step1[8]); - step2[8] = vqsubq_s16(step1[7], step1[8]); - step2[9] = vqsubq_s16(step1[6], step1[9]); - step2[10] = vqsubq_s16(step1[5], step1[10]); - step2[11] = vqsubq_s16(step1[4], step1[11]); - step2[12] = vqsubq_s16(step1[3], step1[12]); - step2[13] = vqsubq_s16(step1[2], step1[13]); - step2[14] = vqsubq_s16(step1[1], step1[14]); - step2[15] = vqsubq_s16(step1[0], step1[15]); - step2[16] = step1[16]; - step2[17] = step1[17]; - step2[18] = step1[18]; - step2[19] = step1[19]; - step2[28] = step1[28]; - step2[29] = step1[29]; - step2[30] = step1[30]; - step2[31] = step1[31]; - - // stage 9 - - out[0] = vqaddq_s16(step2[0], step2[31]); - out[1] = vqaddq_s16(step2[1], step2[30]); - out[2] = vqaddq_s16(step2[2], step2[29]); - out[3] = vqaddq_s16(step2[3], step2[28]); - out[4] = vqaddq_s16(step2[4], step2[27]); - out[5] = vqaddq_s16(step2[5], step2[26]); - out[6] = vqaddq_s16(step2[6], step2[25]); - out[7] = vqaddq_s16(step2[7], step2[24]); - out[8] = vqaddq_s16(step2[8], step2[23]); - out[9] = vqaddq_s16(step2[9], step2[22]); - out[10] = vqaddq_s16(step2[10], step2[21]); - out[11] = vqaddq_s16(step2[11], step2[20]); - out[12] = vqaddq_s16(step2[12], step2[19]); - out[13] = vqaddq_s16(step2[13], step2[18]); - out[14] = vqaddq_s16(step2[14], step2[17]); - out[15] = vqaddq_s16(step2[15], step2[16]); - out[16] = vqsubq_s16(step2[15], step2[16]); - out[17] = vqsubq_s16(step2[14], step2[17]); - out[18] = vqsubq_s16(step2[13], step2[18]); - out[19] = vqsubq_s16(step2[12], step2[19]); - out[20] = vqsubq_s16(step2[11], step2[20]); - out[21] = vqsubq_s16(step2[10], step2[21]); - out[22] = vqsubq_s16(step2[9], step2[22]); - out[23] = vqsubq_s16(step2[8], step2[23]); - out[24] = vqsubq_s16(step2[7], step2[24]); - out[25] = vqsubq_s16(step2[6], step2[25]); - out[26] = vqsubq_s16(step2[5], step2[26]); - out[27] = vqsubq_s16(step2[4], step2[27]); - out[28] = vqsubq_s16(step2[3], step2[28]); - out[29] = vqsubq_s16(step2[2], step2[29]); - out[30] = vqsubq_s16(step2[1], step2[30]); - out[31] = vqsubq_s16(step2[0], step2[31]); -} - -static INLINE void idct32_low1_new_neon(int16x8_t *in, int16x8_t *out, - int8_t cos_bit, int bit) { - (void)bit; - const int32_t *cospi = cospi_arr(cos_bit); - int16x8_t step1; - int32x4_t t32[2]; - - // stage 1 - // stage 2 - // stage 3 - // stage 4 - // stage 5 - - t32[0] = vmull_n_s16(vget_low_s16(in[0]), cospi[32]); - t32[1] = vmull_n_s16(vget_high_s16(in[0]), cospi[32]); - step1 = vcombine_s16(vrshrn_n_s32(t32[0], INV_COS_BIT), - vrshrn_n_s32(t32[1], INV_COS_BIT)); - - // stage 6 - // stage 7 - // stage 8 - // stage 9 - - out[0] = step1; - out[1] = step1; - out[2] = step1; - out[3] = step1; - out[4] = step1; - out[5] = step1; - out[6] = step1; - out[7] = step1; - out[8] = step1; - out[9] = step1; - out[10] = step1; - out[11] = step1; - out[12] = step1; - out[13] = step1; - out[14] = step1; - out[15] = step1; - out[16] = step1; - out[17] = step1; - out[18] = step1; - out[19] = step1; - out[20] = step1; - out[21] = step1; - out[22] = step1; - out[23] = step1; - out[24] = step1; - out[25] = step1; - out[26] = step1; - out[27] = step1; - out[28] = step1; - out[29] = step1; - out[30] = step1; - out[31] = step1; -} - -static INLINE void idct32_low8_new_neon(int16x8_t *in, int16x8_t *out, - int8_t cos_bit, int bit) { - (void)bit; - const int32_t *cospi = cospi_arr(cos_bit); - int16x8_t step1[32], step2[32]; - int32x4_t t32[16]; - const int16x4_t c0 = - create_s16x4_neon((int16_t *)(cospi + 8), (int16_t *)(cospi + 56), - (int16_t *)(cospi + 40), (int16_t *)(cospi + 24)); - const int16x4_t c1 = - create_s16x4_neon((int16_t *)(cospi + 32), (int16_t *)(cospi + 32), - (int16_t *)(cospi + 16), (int16_t *)(cospi + 48)); - - // stage 1 - // stage 2 - - step2[0] = in[0]; - step2[4] = in[4]; - step2[8] = in[2]; - step2[12] = in[6]; - - btf_16_neon(in[1], cospi[62], cospi[2], &step2[16], &step2[31]); - btf_16_neon(in[7], -cospi[50], cospi[14], &step2[19], &step2[28]); - btf_16_neon(in[5], cospi[54], cospi[10], &step2[20], &step2[27]); - btf_16_neon(in[3], -cospi[58], cospi[6], &step2[23], &step2[24]); - - // stage 3 - step1[0] = step2[0]; - step1[4] = step2[4]; - - btf_16_neon(step2[8], cospi[60], cospi[4], &step1[8], &step1[15]); - btf_16_neon(step2[12], -cospi[52], cospi[12], &step1[11], &step1[12]); - - step1[16] = step2[16]; - step1[17] = step2[16]; - step1[18] = step2[19]; - step1[19] = step2[19]; - step1[20] = step2[20]; - step1[21] = step2[20]; - step1[22] = step2[23]; - step1[23] = step2[23]; - step1[24] = step2[24]; - step1[25] = step2[24]; - step1[26] = step2[27]; - step1[27] = step2[27]; - step1[28] = step2[28]; - step1[29] = step2[28]; - step1[30] = step2[31]; - step1[31] = step2[31]; - - // stage 4 - - btf_16_neon(step1[4], cospi[56], cospi[8], &step2[4], &step2[7]); - btf_16_lane_0_1_neon(step1[30], step1[17], c0, &step2[30], &step2[17]); - btf_16_lane_1_0_neon(vnegq_s16(step1[18]), vnegq_s16(step1[29]), c0, - &step2[18], &step2[29]); - btf_16_lane_2_3_neon(step1[26], step1[21], c0, &step2[26], &step2[21]); - btf_16_lane_3_2_neon(vnegq_s16(step1[22]), vnegq_s16(step1[25]), c0, - &step2[22], &step2[25]); - - step2[0] = step1[0]; - step2[8] = step1[8]; - step2[9] = step1[8]; - step2[10] = step1[11]; - step2[11] = step1[11]; - step2[12] = step1[12]; - step2[13] = step1[12]; - step2[14] = step1[15]; - step2[15] = step1[15]; - step2[16] = step1[16]; - step2[19] = step1[19]; - step2[20] = step1[20]; - step2[23] = step1[23]; - step2[24] = step1[24]; - step2[27] = step1[27]; - step2[28] = step1[28]; - step2[31] = step1[31]; - - // stage 5 - - t32[0] = vmull_n_s16(vget_low_s16(step2[0]), cospi[32]); - t32[1] = vmull_n_s16(vget_high_s16(step2[0]), cospi[32]); - step1[0] = vcombine_s16(vrshrn_n_s32(t32[0], INV_COS_BIT), - vrshrn_n_s32(t32[1], INV_COS_BIT)); - - btf_16_lane_2_3_neon(step2[14], step2[9], c1, &step1[14], &step1[9]); - btf_16_lane_3_2_neon(vnegq_s16(step2[10]), vnegq_s16(step2[13]), c1, - &step1[10], &step1[13]); - - step1[4] = step2[4]; - step1[5] = step2[4]; - step1[6] = step2[7]; - step1[7] = step2[7]; - step1[8] = step2[8]; - step1[11] = step2[11]; - step1[12] = step2[12]; - step1[15] = step2[15]; - step1[16] = vqaddq_s16(step2[16], step2[19]); - step1[17] = vqaddq_s16(step2[17], step2[18]); - step1[18] = vqsubq_s16(step2[17], step2[18]); - step1[19] = vqsubq_s16(step2[16], step2[19]); - step1[20] = vqsubq_s16(step2[23], step2[20]); - step1[21] = vqsubq_s16(step2[22], step2[21]); - step1[22] = vqaddq_s16(step2[22], step2[21]); - step1[23] = vqaddq_s16(step2[23], step2[20]); - step1[24] = vqaddq_s16(step2[24], step2[27]); - step1[25] = vqaddq_s16(step2[25], step2[26]); - step1[26] = vqsubq_s16(step2[25], step2[26]); - step1[27] = vqsubq_s16(step2[24], step2[27]); - step1[28] = vqsubq_s16(step2[31], step2[28]); - step1[29] = vqsubq_s16(step2[30], step2[29]); - step1[30] = vqaddq_s16(step2[30], step2[29]); - step1[31] = vqaddq_s16(step2[31], step2[28]); - - // stage 6 - - btf_16_lane_0_1_neon(step1[6], step1[5], c1, &step2[6], &step2[5]); - btf_16_lane_2_3_neon(step1[29], step1[18], c1, &step2[29], &step2[18]); - btf_16_lane_2_3_neon(step1[28], step1[19], c1, &step2[28], &step2[19]); - btf_16_lane_3_2_neon(vnegq_s16(step1[20]), vnegq_s16(step1[27]), c1, - &step2[20], &step2[27]); - btf_16_lane_3_2_neon(vnegq_s16(step1[21]), vnegq_s16(step1[26]), c1, - &step2[21], &step2[26]); - - step2[0] = step1[0]; - step2[1] = step1[0]; - step2[2] = step1[0]; - step2[3] = step1[0]; - step2[4] = step1[4]; - step2[7] = step1[7]; - step2[8] = vqaddq_s16(step1[8], step1[11]); - step2[9] = vqaddq_s16(step1[9], step1[10]); - step2[10] = vqsubq_s16(step1[9], step1[10]); - step2[11] = vqsubq_s16(step1[8], step1[11]); - step2[12] = vqsubq_s16(step1[15], step1[12]); - step2[13] = vqsubq_s16(step1[14], step1[13]); - step2[14] = vqaddq_s16(step1[14], step1[13]); - step2[15] = vqaddq_s16(step1[15], step1[12]); - step2[16] = step1[16]; - step2[17] = step1[17]; - step2[22] = step1[22]; - step2[23] = step1[23]; - step2[24] = step1[24]; - step2[25] = step1[25]; - step2[30] = step1[30]; - step2[31] = step1[31]; - - // stage 7 - - btf_16_lane_0_1_neon(step2[13], step2[10], c1, &step1[13], &step1[10]); - btf_16_lane_0_1_neon(step2[12], step2[11], c1, &step1[12], &step1[11]); - - step1[0] = vqaddq_s16(step2[0], step2[7]); - step1[1] = vqaddq_s16(step2[1], step2[6]); - step1[2] = vqaddq_s16(step2[2], step2[5]); - step1[3] = vqaddq_s16(step2[3], step2[4]); - step1[4] = vqsubq_s16(step2[3], step2[4]); - step1[5] = vqsubq_s16(step2[2], step2[5]); - step1[6] = vqsubq_s16(step2[1], step2[6]); - step1[7] = vqsubq_s16(step2[0], step2[7]); - step1[8] = step2[8]; - step1[9] = step2[9]; - step1[14] = step2[14]; - step1[15] = step2[15]; - step1[16] = vqaddq_s16(step2[16], step2[23]); - step1[17] = vqaddq_s16(step2[17], step2[22]); - step1[18] = vqaddq_s16(step2[18], step2[21]); - step1[19] = vqaddq_s16(step2[19], step2[20]); - step1[20] = vqsubq_s16(step2[19], step2[20]); - step1[21] = vqsubq_s16(step2[18], step2[21]); - step1[22] = vqsubq_s16(step2[17], step2[22]); - step1[23] = vqsubq_s16(step2[16], step2[23]); - step1[24] = vqsubq_s16(step2[31], step2[24]); - step1[25] = vqsubq_s16(step2[30], step2[25]); - step1[26] = vqsubq_s16(step2[29], step2[26]); - step1[27] = vqsubq_s16(step2[28], step2[27]); - step1[28] = vqaddq_s16(step2[27], step2[28]); - step1[29] = vqaddq_s16(step2[26], step2[29]); - step1[30] = vqaddq_s16(step2[25], step2[30]); - step1[31] = vqaddq_s16(step2[24], step2[31]); - - // stage 8 - - btf_16_lane_0_1_neon(step1[27], step1[20], c1, &step2[27], &step2[20]); - btf_16_lane_0_1_neon(step1[26], step1[21], c1, &step2[26], &step2[21]); - btf_16_lane_0_1_neon(step1[25], step1[22], c1, &step2[25], &step2[22]); - btf_16_lane_0_1_neon(step1[24], step1[23], c1, &step2[24], &step2[23]); - - step2[0] = vqaddq_s16(step1[0], step1[15]); - step2[1] = vqaddq_s16(step1[1], step1[14]); - step2[2] = vqaddq_s16(step1[2], step1[13]); - step2[3] = vqaddq_s16(step1[3], step1[12]); - step2[4] = vqaddq_s16(step1[4], step1[11]); - step2[5] = vqaddq_s16(step1[5], step1[10]); - step2[6] = vqaddq_s16(step1[6], step1[9]); - step2[7] = vqaddq_s16(step1[7], step1[8]); - step2[8] = vqsubq_s16(step1[7], step1[8]); - step2[9] = vqsubq_s16(step1[6], step1[9]); - step2[10] = vqsubq_s16(step1[5], step1[10]); - step2[11] = vqsubq_s16(step1[4], step1[11]); - step2[12] = vqsubq_s16(step1[3], step1[12]); - step2[13] = vqsubq_s16(step1[2], step1[13]); - step2[14] = vqsubq_s16(step1[1], step1[14]); - step2[15] = vqsubq_s16(step1[0], step1[15]); - step2[16] = step1[16]; - step2[17] = step1[17]; - step2[18] = step1[18]; - step2[19] = step1[19]; - step2[28] = step1[28]; - step2[29] = step1[29]; - step2[30] = step1[30]; - step2[31] = step1[31]; - - // stage 9 - - out[0] = vqaddq_s16(step2[0], step2[31]); - out[1] = vqaddq_s16(step2[1], step2[30]); - out[2] = vqaddq_s16(step2[2], step2[29]); - out[3] = vqaddq_s16(step2[3], step2[28]); - out[4] = vqaddq_s16(step2[4], step2[27]); - out[5] = vqaddq_s16(step2[5], step2[26]); - out[6] = vqaddq_s16(step2[6], step2[25]); - out[7] = vqaddq_s16(step2[7], step2[24]); - out[8] = vqaddq_s16(step2[8], step2[23]); - out[9] = vqaddq_s16(step2[9], step2[22]); - out[10] = vqaddq_s16(step2[10], step2[21]); - out[11] = vqaddq_s16(step2[11], step2[20]); - out[12] = vqaddq_s16(step2[12], step2[19]); - out[13] = vqaddq_s16(step2[13], step2[18]); - out[14] = vqaddq_s16(step2[14], step2[17]); - out[15] = vqaddq_s16(step2[15], step2[16]); - out[16] = vqsubq_s16(step2[15], step2[16]); - out[17] = vqsubq_s16(step2[14], step2[17]); - out[18] = vqsubq_s16(step2[13], step2[18]); - out[19] = vqsubq_s16(step2[12], step2[19]); - out[20] = vqsubq_s16(step2[11], step2[20]); - out[21] = vqsubq_s16(step2[10], step2[21]); - out[22] = vqsubq_s16(step2[9], step2[22]); - out[23] = vqsubq_s16(step2[8], step2[23]); - out[24] = vqsubq_s16(step2[7], step2[24]); - out[25] = vqsubq_s16(step2[6], step2[25]); - out[26] = vqsubq_s16(step2[5], step2[26]); - out[27] = vqsubq_s16(step2[4], step2[27]); - out[28] = vqsubq_s16(step2[3], step2[28]); - out[29] = vqsubq_s16(step2[2], step2[29]); - out[30] = vqsubq_s16(step2[1], step2[30]); - out[31] = vqsubq_s16(step2[0], step2[31]); -} - -static INLINE void idct32_low16_new_neon(int16x8_t *in, int16x8_t *out, - int8_t cos_bit, int bit) { - (void)bit; - const int32_t *cospi = cospi_arr(cos_bit); - int16x8_t step1[32], step2[32]; - int32x4_t t32[16]; - const int16x4_t c0 = - create_s16x4_neon((int16_t *)(cospi + 8), (int16_t *)(cospi + 56), - (int16_t *)(cospi + 40), (int16_t *)(cospi + 24)); - const int16x4_t c1 = - create_s16x4_neon((int16_t *)(cospi + 32), (int16_t *)(cospi + 32), - (int16_t *)(cospi + 16), (int16_t *)(cospi + 48)); - - // stage 1 - // stage 2 - - btf_16_neon(in[1], cospi[62], cospi[2], &step2[16], &step2[31]); - btf_16_neon(in[15], -cospi[34], cospi[30], &step2[17], &step2[30]); - btf_16_neon(in[9], cospi[46], cospi[18], &step2[18], &step2[29]); - btf_16_neon(in[7], -cospi[50], cospi[14], &step2[19], &step2[28]); - btf_16_neon(in[5], cospi[54], cospi[10], &step2[20], &step2[27]); - btf_16_neon(in[11], -cospi[42], cospi[22], &step2[21], &step2[26]); - btf_16_neon(in[13], cospi[38], cospi[26], &step2[22], &step2[25]); - btf_16_neon(in[3], -cospi[58], cospi[6], &step2[23], &step2[24]); - - step2[0] = in[0]; - step2[2] = in[8]; - step2[4] = in[4]; - step2[6] = in[12]; - step2[8] = in[2]; - step2[10] = in[10]; - step2[12] = in[6]; - step2[14] = in[14]; - - // stage 3 - - btf_16_neon(step2[8], cospi[60], cospi[4], &step1[8], &step1[15]); - btf_16_neon(step2[14], -cospi[36], cospi[28], &step1[9], &step1[14]); - btf_16_neon(step2[10], cospi[44], cospi[20], &step1[10], &step1[13]); - btf_16_neon(step2[12], -cospi[52], cospi[12], &step1[11], &step1[12]); - - step1[0] = step2[0]; - step1[2] = step2[2]; - step1[4] = step2[4]; - step1[6] = step2[6]; - step1[16] = vqaddq_s16(step2[16], step2[17]); - step1[17] = vqsubq_s16(step2[16], step2[17]); - step1[18] = vqsubq_s16(step2[19], step2[18]); - step1[19] = vqaddq_s16(step2[19], step2[18]); - step1[20] = vqaddq_s16(step2[20], step2[21]); - step1[21] = vqsubq_s16(step2[20], step2[21]); - step1[22] = vqsubq_s16(step2[23], step2[22]); - step1[23] = vqaddq_s16(step2[23], step2[22]); - step1[24] = vqaddq_s16(step2[24], step2[25]); - step1[25] = vqsubq_s16(step2[24], step2[25]); - step1[26] = vqsubq_s16(step2[27], step2[26]); - step1[27] = vqaddq_s16(step2[27], step2[26]); - step1[28] = vqaddq_s16(step2[28], step2[29]); - step1[29] = vqsubq_s16(step2[28], step2[29]); - step1[30] = vqsubq_s16(step2[31], step2[30]); - step1[31] = vqaddq_s16(step2[31], step2[30]); - - // stage 4 - - btf_16_neon(step1[4], cospi[56], cospi[8], &step2[4], &step2[7]); - btf_16_neon(step1[6], -cospi[40], cospi[24], &step2[5], &step2[6]); - btf_16_lane_0_1_neon(step1[30], step1[17], c0, &step2[30], &step2[17]); - btf_16_lane_1_0_neon(vnegq_s16(step1[18]), vnegq_s16(step1[29]), c0, - &step2[18], &step2[29]); - btf_16_lane_2_3_neon(step1[26], step1[21], c0, &step2[26], &step2[21]); - btf_16_lane_3_2_neon(vnegq_s16(step1[22]), vnegq_s16(step1[25]), c0, - &step2[22], &step2[25]); - - step2[0] = step1[0]; - step2[2] = step1[2]; - step2[8] = vqaddq_s16(step1[8], step1[9]); - step2[9] = vqsubq_s16(step1[8], step1[9]); - step2[10] = vqsubq_s16(step1[11], step1[10]); - step2[11] = vqaddq_s16(step1[11], step1[10]); - step2[12] = vqaddq_s16(step1[12], step1[13]); - step2[13] = vqsubq_s16(step1[12], step1[13]); - step2[14] = vqsubq_s16(step1[15], step1[14]); - step2[15] = vqaddq_s16(step1[15], step1[14]); - step2[16] = step1[16]; - step2[19] = step1[19]; - step2[20] = step1[20]; - step2[23] = step1[23]; - step2[24] = step1[24]; - step2[27] = step1[27]; - step2[28] = step1[28]; - step2[31] = step1[31]; - - // stage 5 - - t32[0] = vmull_n_s16(vget_low_s16(step2[0]), cospi[32]); - t32[1] = vmull_n_s16(vget_high_s16(step2[0]), cospi[32]); - - step1[0] = vcombine_s16(vrshrn_n_s32(t32[0], INV_COS_BIT), - vrshrn_n_s32(t32[1], INV_COS_BIT)); - - btf_16_neon(step2[2], cospi[48], cospi[16], &step1[2], &step1[3]); - btf_16_lane_2_3_neon(step2[14], step2[9], c1, &step1[14], &step1[9]); - btf_16_lane_3_2_neon(vnegq_s16(step2[10]), vnegq_s16(step2[13]), c1, - &step1[10], &step1[13]); - - step1[4] = vqaddq_s16(step2[4], step2[5]); - step1[5] = vqsubq_s16(step2[4], step2[5]); - step1[6] = vqsubq_s16(step2[7], step2[6]); - step1[7] = vqaddq_s16(step2[7], step2[6]); - step1[8] = step2[8]; - step1[11] = step2[11]; - step1[12] = step2[12]; - step1[15] = step2[15]; - step1[16] = vqaddq_s16(step2[16], step2[19]); - step1[17] = vqaddq_s16(step2[17], step2[18]); - step1[18] = vqsubq_s16(step2[17], step2[18]); - step1[19] = vqsubq_s16(step2[16], step2[19]); - step1[20] = vqsubq_s16(step2[23], step2[20]); - step1[21] = vqsubq_s16(step2[22], step2[21]); - step1[22] = vqaddq_s16(step2[22], step2[21]); - step1[23] = vqaddq_s16(step2[23], step2[20]); - step1[24] = vqaddq_s16(step2[24], step2[27]); - step1[25] = vqaddq_s16(step2[25], step2[26]); - step1[26] = vqsubq_s16(step2[25], step2[26]); - step1[27] = vqsubq_s16(step2[24], step2[27]); - step1[28] = vqsubq_s16(step2[31], step2[28]); - step1[29] = vqsubq_s16(step2[30], step2[29]); - step1[30] = vqaddq_s16(step2[30], step2[29]); - step1[31] = vqaddq_s16(step2[31], step2[28]); - - // stage 6 - - btf_16_lane_0_1_neon(step1[6], step1[5], c1, &step2[6], &step2[5]); - btf_16_lane_2_3_neon(step1[29], step1[18], c1, &step2[29], &step2[18]); - btf_16_lane_2_3_neon(step1[28], step1[19], c1, &step2[28], &step2[19]); - btf_16_lane_3_2_neon(vnegq_s16(step1[20]), vnegq_s16(step1[27]), c1, - &step2[20], &step2[27]); - btf_16_lane_3_2_neon(vnegq_s16(step1[21]), vnegq_s16(step1[26]), c1, - &step2[21], &step2[26]); - - step2[0] = vqaddq_s16(step1[0], step1[3]); - step2[1] = vqaddq_s16(step1[0], step1[2]); - step2[2] = vqsubq_s16(step1[0], step1[2]); - step2[3] = vqsubq_s16(step1[0], step1[3]); - step2[4] = step1[4]; - step2[7] = step1[7]; - step2[8] = vqaddq_s16(step1[8], step1[11]); - step2[9] = vqaddq_s16(step1[9], step1[10]); - step2[10] = vqsubq_s16(step1[9], step1[10]); - step2[11] = vqsubq_s16(step1[8], step1[11]); - step2[12] = vqsubq_s16(step1[15], step1[12]); - step2[13] = vqsubq_s16(step1[14], step1[13]); - step2[14] = vqaddq_s16(step1[14], step1[13]); - step2[15] = vqaddq_s16(step1[15], step1[12]); - step2[16] = step1[16]; - step2[17] = step1[17]; - step2[22] = step1[22]; - step2[23] = step1[23]; - step2[24] = step1[24]; - step2[25] = step1[25]; - step2[30] = step1[30]; - step2[31] = step1[31]; - - // stage 7 - - btf_16_lane_0_1_neon(step2[13], step2[10], c1, &step1[13], &step1[10]); - btf_16_lane_0_1_neon(step2[12], step2[11], c1, &step1[12], &step1[11]); - - step1[0] = vqaddq_s16(step2[0], step2[7]); - step1[1] = vqaddq_s16(step2[1], step2[6]); - step1[2] = vqaddq_s16(step2[2], step2[5]); - step1[3] = vqaddq_s16(step2[3], step2[4]); - step1[4] = vqsubq_s16(step2[3], step2[4]); - step1[5] = vqsubq_s16(step2[2], step2[5]); - step1[6] = vqsubq_s16(step2[1], step2[6]); - step1[7] = vqsubq_s16(step2[0], step2[7]); - step1[8] = step2[8]; - step1[9] = step2[9]; - step1[14] = step2[14]; - step1[15] = step2[15]; - step1[16] = vqaddq_s16(step2[16], step2[23]); - step1[17] = vqaddq_s16(step2[17], step2[22]); - step1[18] = vqaddq_s16(step2[18], step2[21]); - step1[19] = vqaddq_s16(step2[19], step2[20]); - step1[20] = vqsubq_s16(step2[19], step2[20]); - step1[21] = vqsubq_s16(step2[18], step2[21]); - step1[22] = vqsubq_s16(step2[17], step2[22]); - step1[23] = vqsubq_s16(step2[16], step2[23]); - step1[24] = vqsubq_s16(step2[31], step2[24]); - step1[25] = vqsubq_s16(step2[30], step2[25]); - step1[26] = vqsubq_s16(step2[29], step2[26]); - step1[27] = vqsubq_s16(step2[28], step2[27]); - step1[28] = vqaddq_s16(step2[27], step2[28]); - step1[29] = vqaddq_s16(step2[26], step2[29]); - step1[30] = vqaddq_s16(step2[25], step2[30]); - step1[31] = vqaddq_s16(step2[24], step2[31]); - - // stage 8 - - btf_16_lane_0_1_neon(step1[27], step1[20], c1, &step2[27], &step2[20]); - btf_16_lane_0_1_neon(step1[26], step1[21], c1, &step2[26], &step2[21]); - btf_16_lane_0_1_neon(step1[25], step1[22], c1, &step2[25], &step2[22]); - btf_16_lane_0_1_neon(step1[24], step1[23], c1, &step2[24], &step2[23]); - - step2[0] = vqaddq_s16(step1[0], step1[15]); - step2[1] = vqaddq_s16(step1[1], step1[14]); - step2[2] = vqaddq_s16(step1[2], step1[13]); - step2[3] = vqaddq_s16(step1[3], step1[12]); - step2[4] = vqaddq_s16(step1[4], step1[11]); - step2[5] = vqaddq_s16(step1[5], step1[10]); - step2[6] = vqaddq_s16(step1[6], step1[9]); - step2[7] = vqaddq_s16(step1[7], step1[8]); - step2[8] = vqsubq_s16(step1[7], step1[8]); - step2[9] = vqsubq_s16(step1[6], step1[9]); - step2[10] = vqsubq_s16(step1[5], step1[10]); - step2[11] = vqsubq_s16(step1[4], step1[11]); - step2[12] = vqsubq_s16(step1[3], step1[12]); - step2[13] = vqsubq_s16(step1[2], step1[13]); - step2[14] = vqsubq_s16(step1[1], step1[14]); - step2[15] = vqsubq_s16(step1[0], step1[15]); - step2[16] = step1[16]; - step2[17] = step1[17]; - step2[18] = step1[18]; - step2[19] = step1[19]; - step2[28] = step1[28]; - step2[29] = step1[29]; - step2[30] = step1[30]; - step2[31] = step1[31]; - - // stage 9 - - out[0] = vqaddq_s16(step2[0], step2[31]); - out[1] = vqaddq_s16(step2[1], step2[30]); - out[2] = vqaddq_s16(step2[2], step2[29]); - out[3] = vqaddq_s16(step2[3], step2[28]); - out[4] = vqaddq_s16(step2[4], step2[27]); - out[5] = vqaddq_s16(step2[5], step2[26]); - out[6] = vqaddq_s16(step2[6], step2[25]); - out[7] = vqaddq_s16(step2[7], step2[24]); - out[8] = vqaddq_s16(step2[8], step2[23]); - out[9] = vqaddq_s16(step2[9], step2[22]); - out[10] = vqaddq_s16(step2[10], step2[21]); - out[11] = vqaddq_s16(step2[11], step2[20]); - out[12] = vqaddq_s16(step2[12], step2[19]); - out[13] = vqaddq_s16(step2[13], step2[18]); - out[14] = vqaddq_s16(step2[14], step2[17]); - out[15] = vqaddq_s16(step2[15], step2[16]); - out[16] = vqsubq_s16(step2[15], step2[16]); - out[17] = vqsubq_s16(step2[14], step2[17]); - out[18] = vqsubq_s16(step2[13], step2[18]); - out[19] = vqsubq_s16(step2[12], step2[19]); - out[20] = vqsubq_s16(step2[11], step2[20]); - out[21] = vqsubq_s16(step2[10], step2[21]); - out[22] = vqsubq_s16(step2[9], step2[22]); - out[23] = vqsubq_s16(step2[8], step2[23]); - out[24] = vqsubq_s16(step2[7], step2[24]); - out[25] = vqsubq_s16(step2[6], step2[25]); - out[26] = vqsubq_s16(step2[5], step2[26]); - out[27] = vqsubq_s16(step2[4], step2[27]); - out[28] = vqsubq_s16(step2[3], step2[28]); - out[29] = vqsubq_s16(step2[2], step2[29]); - out[30] = vqsubq_s16(step2[1], step2[30]); - out[31] = vqsubq_s16(step2[0], step2[31]); -} - -// Functions for blocks with eob at DC and within -// topleft 8x8, 16x16, 32x32 corner -static const transform_1d_neon - lowbd_txfm_all_1d_zeros_w8_arr[TX_SIZES][ITX_TYPES_1D][4] = { - { - { av1_idct4_new, av1_idct4_new, NULL, NULL }, - { av1_iadst4_new, av1_iadst4_new, NULL, NULL }, - { av1_iidentity4_c, av1_iidentity4_c, NULL, NULL }, - }, - { { av1_idct8_new, av1_idct8_new, NULL, NULL }, - { av1_iadst8_new, av1_iadst8_new, NULL, NULL }, - { av1_iidentity8_c, av1_iidentity8_c, NULL, NULL } }, - { - { av1_idct16_new, av1_idct16_new, av1_idct16_new, NULL }, - { av1_iadst16_new, av1_iadst16_new, av1_iadst16_new, NULL }, - { av1_iidentity16_c, av1_iidentity16_c, av1_iidentity16_c, NULL }, - }, - { { av1_idct32_new, av1_idct32_new, av1_idct32_new, av1_idct32_new }, - { NULL, NULL, NULL, NULL }, - { av1_iidentity32_c, av1_iidentity32_c, av1_iidentity32_c, - av1_iidentity32_c } }, - { { av1_idct64_new, av1_idct64_new, av1_idct64_new, av1_idct64_new }, - { NULL, NULL, NULL, NULL }, - { NULL, NULL, NULL, NULL } } - }; - -static const transform_neon - lowbd_txfm_all_1d_zeros_w_arr[TX_SIZES][ITX_TYPES_1D][4] = { - { - { NULL, NULL, NULL, NULL }, - { NULL, NULL, NULL, NULL }, - { NULL, NULL, NULL, NULL }, - }, - { { idct8_low1_new_neon, idct8_new_neon, NULL, NULL }, - { iadst8_low1_new_neon, iadst8_new_neon, NULL, NULL }, - { identity8_new_neon, identity8_new_neon, NULL, NULL } }, - { - { idct16_low1_new_neon, idct16_low8_new_neon, idct16_new_neon, NULL }, - { iadst16_low1_new_neon, iadst16_low8_new_neon, iadst16_new_neon, - NULL }, - { identity16_new_neon, identity16_new_neon, identity16_new_neon, - NULL }, - }, - { { idct32_low1_new_neon, idct32_low8_new_neon, idct32_low16_new_neon, - idct32_new_neon }, - { NULL, NULL, NULL, NULL }, - { identity32_new_neon, identity32_new_neon, identity32_new_neon, - identity32_new_neon } }, - { { NULL, NULL, NULL, NULL }, - { NULL, NULL, NULL, NULL }, - { NULL, NULL, NULL, NULL } } - }; - -static INLINE void lowbd_inv_txfm2d_add_wxh_idtx_neon( - const int32_t *input, uint8_t *output, int stride, TX_TYPE tx_type, - TX_SIZE tx_size, int eob) { - DECLARE_ALIGNED(32, int, txfm_buf[32 * 32 + 32 + 32]); - int32_t *temp_in = txfm_buf; - - int eobx, eoby; - get_eobx_eoby_scan_default(&eobx, &eoby, tx_size, eob); - const int8_t *shift = inv_txfm_shift_ls[tx_size]; - const int txw_idx = get_txw_idx(tx_size); - const int txh_idx = get_txh_idx(tx_size); - const int cos_bit_col = inv_cos_bit_col[txw_idx][txh_idx]; - const int cos_bit_row = inv_cos_bit_row[txw_idx][txh_idx]; - const int txfm_size_col = tx_size_wide[tx_size]; - const int txfm_size_row = tx_size_high[tx_size]; - const int buf_size_nonzero_h_div8 = (eoby + 8) >> 3; - - const int rect_type = get_rect_tx_log_ratio(txfm_size_col, txfm_size_row); - const int buf_offset = AOMMAX(txfm_size_row, txfm_size_col); - - int32_t *temp_out = temp_in + buf_offset; - int32_t *buf = temp_out + buf_offset; - int32_t *buf_ptr = buf; - const int8_t stage_range[MAX_TXFM_STAGE_NUM] = { 16 }; - int r, bd = 8; - - const int fun_idx_x = lowbd_txfm_all_1d_zeros_idx[eobx]; - const int fun_idx_y = lowbd_txfm_all_1d_zeros_idx[eoby]; - const transform_1d_neon row_txfm = - lowbd_txfm_all_1d_zeros_w8_arr[txw_idx][hitx_1d_tab[tx_type]][fun_idx_x]; - const transform_1d_neon col_txfm = - lowbd_txfm_all_1d_zeros_w8_arr[txh_idx][vitx_1d_tab[tx_type]][fun_idx_y]; - - assert(col_txfm != NULL); - assert(row_txfm != NULL); - - // row tx - int row_start = (buf_size_nonzero_h_div8 * 8); - for (int i = 0; i < row_start; i++) { - if (abs(rect_type) == 1) { - for (int j = 0; j < txfm_size_col; j++) - temp_in[j] = round_shift((int64_t)input[j] * NewInvSqrt2, NewSqrt2Bits); - row_txfm(temp_in, buf_ptr, cos_bit_row, stage_range); - } else { - row_txfm(input, buf_ptr, cos_bit_row, stage_range); - } - av1_round_shift_array(buf_ptr, txfm_size_col, -shift[0]); - input += txfm_size_col; - buf_ptr += txfm_size_col; - } - - // Doing memset for the rows which are not processed in row transform. - memset(buf_ptr, 0, - sizeof(int32_t) * txfm_size_col * (txfm_size_row - row_start)); - - // col tx - for (int c = 0; c < txfm_size_col; c++) { - for (r = 0; r < txfm_size_row; ++r) temp_in[r] = buf[r * txfm_size_col + c]; - - col_txfm(temp_in, temp_out, cos_bit_col, stage_range); - av1_round_shift_array(temp_out, txfm_size_row, -shift[1]); - - for (r = 0; r < txfm_size_row; ++r) { - output[r * stride + c] = - highbd_clip_pixel_add(output[r * stride + c], temp_out[r], bd); - } - } -} - -static INLINE void lowbd_inv_txfm2d_add_idtx_neon(const int32_t *input, - uint8_t *output, int stride, - TX_TYPE tx_type, - TX_SIZE tx_size, int eob) { - int16x8_t a[32 * 4]; - int16x8_t b[32 * 4]; - int eobx, eoby; - get_eobx_eoby_scan_default(&eobx, &eoby, tx_size, eob); - const int8_t *shift = inv_txfm_shift_ls[tx_size]; - const int txw_idx = get_txw_idx(tx_size); - const int txh_idx = get_txh_idx(tx_size); - const int cos_bit_col = inv_cos_bit_col[txw_idx][txh_idx]; - const int cos_bit_row = inv_cos_bit_row[txw_idx][txh_idx]; - const int txfm_size_col = tx_size_wide[tx_size]; - const int txfm_size_row = tx_size_high[tx_size]; - lowbd_inv_txfm2d_memset_neon(&a[0], (txfm_size_col * (txfm_size_row) >> 3), - 0); - lowbd_inv_txfm2d_memset_neon(&b[0], (txfm_size_col * (txfm_size_row) >> 3), - 0); - const int buf_size_w_div8 = txfm_size_col >> 3; - const int rect_type = get_rect_tx_log_ratio(txfm_size_col, txfm_size_row); - const int buf_size_nonzero_h_div8 = (eoby + 8) >> 3; - const int buf_size_nonzero_w_div8 = (eobx + 8) >> 3; - const int fun_idx_x = lowbd_txfm_all_1d_zeros_idx[eobx]; - const int fun_idx_y = lowbd_txfm_all_1d_zeros_idx[eoby]; - const int32_t *input_1; - int temp_b = 0; - const transform_neon row_txfm = - lowbd_txfm_all_1d_zeros_w_arr[txw_idx][hitx_1d_tab[tx_type]][fun_idx_x]; - const transform_neon col_txfm = - lowbd_txfm_all_1d_zeros_w_arr[txh_idx][vitx_1d_tab[tx_type]][fun_idx_y]; - - assert(col_txfm != NULL); - assert(row_txfm != NULL); - - for (int i = 0; i < buf_size_nonzero_h_div8; i++) { - input_1 = input; - for (int j = 0; j < buf_size_nonzero_w_div8; ++j) { - int k = j * 8 + i * txfm_size_col; - load_buffer_32bit_to_16bit_neon(input_1, &a[k], txfm_size_col); - transpose_s16_8x8q(&a[k], &a[k]); - input_1 += 8; - } - input += (txfm_size_col * 8); - if (abs(rect_type) == 1) { - int y = i * txfm_size_col; - round_shift_for_rect(&a[y], &a[y], txfm_size_col); - } - row_txfm(&a[i * txfm_size_col], &a[i * txfm_size_col], cos_bit_row, 0); - av1_round_shift_array_16_neon(&a[i * txfm_size_col], txfm_size_col, - -shift[0]); - for (int j = 0; j < buf_size_w_div8; ++j) { - int k = j * 8 + i * txfm_size_col; - transpose_s16_8x8q(&a[k], &b[temp_b + txfm_size_row * j]); - } - temp_b += 8; - } - for (int j = 0; j < buf_size_w_div8; ++j) { - col_txfm(&b[j * txfm_size_row], &b[j * txfm_size_row], cos_bit_col, 0); - av1_round_shift_array_16_neon(&b[j * txfm_size_row], txfm_size_row, - -shift[1]); - } - if (txfm_size_col >= 16) { - for (int i = 0; i < (txfm_size_col >> 4); i++) { - lowbd_add_flip_buffer_16xn_neon( - &b[i * txfm_size_row * 2], output + 16 * i, stride, 0, txfm_size_row); - } - } else if (txfm_size_col == 8) { - lowbd_add_flip_buffer_8xn_neon(b, output, stride, 0, txfm_size_row); - } -} - -static INLINE void lowbd_inv_txfm2d_add_v_wxh_identity_neon( - const int32_t *input, uint8_t *output, int stride, TX_TYPE tx_type, - TX_SIZE tx_size, int eob) { - DECLARE_ALIGNED(32, int, txfm_buf[32 * 32 + 32 + 32]); - int32_t *temp_in = txfm_buf; - - int eobx, eoby; - get_eobx_eoby_scan_v_identity(&eobx, &eoby, tx_size, eob); - const int8_t *shift = inv_txfm_shift_ls[tx_size]; - const int txw_idx = get_txw_idx(tx_size); - const int txh_idx = get_txh_idx(tx_size); - const int cos_bit_col = inv_cos_bit_col[txw_idx][txh_idx]; - const int cos_bit_row = inv_cos_bit_row[txw_idx][txh_idx]; - const int txfm_size_col = tx_size_wide[tx_size]; - const int txfm_size_row = tx_size_high[tx_size]; - const int buf_size_nonzero_h_div8 = (eoby + 8) >> 3; - - const int rect_type = get_rect_tx_log_ratio(txfm_size_col, txfm_size_row); - const int buf_offset = AOMMAX(txfm_size_row, txfm_size_col); - - int32_t *temp_out = temp_in + buf_offset; - int32_t *buf = temp_out + buf_offset; - int32_t *buf_ptr = buf; - const int8_t stage_range[MAX_TXFM_STAGE_NUM] = { 16 }; - int r, bd = 8; - - const int fun_idx_x = lowbd_txfm_all_1d_zeros_idx[eobx]; - const int fun_idx_y = lowbd_txfm_all_1d_zeros_idx[eoby]; - const transform_1d_neon row_txfm = - lowbd_txfm_all_1d_zeros_w8_arr[txw_idx][hitx_1d_tab[tx_type]][fun_idx_x]; - const transform_1d_neon col_txfm = - lowbd_txfm_all_1d_zeros_w8_arr[txh_idx][vitx_1d_tab[tx_type]][fun_idx_y]; - - assert(col_txfm != NULL); - assert(row_txfm != NULL); - int ud_flip, lr_flip; - get_flip_cfg(tx_type, &ud_flip, &lr_flip); - - // row tx - int row_start = (buf_size_nonzero_h_div8 * 8); - for (int i = 0; i < row_start; i++) { - if (abs(rect_type) == 1) { - for (int j = 0; j < txfm_size_col; j++) - temp_in[j] = round_shift((int64_t)input[j] * NewInvSqrt2, NewSqrt2Bits); - row_txfm(temp_in, buf_ptr, cos_bit_row, stage_range); - } else { - row_txfm(input, buf_ptr, cos_bit_row, stage_range); - } - av1_round_shift_array(buf_ptr, txfm_size_col, -shift[0]); - input += txfm_size_col; - buf_ptr += txfm_size_col; - } - // Doing memset for the rows which are not processed in row transform. - memset(buf_ptr, 0, - sizeof(int32_t) * txfm_size_col * (txfm_size_row - row_start)); - - // col tx - for (int c = 0; c < txfm_size_col; c++) { - if (lr_flip == 0) { - for (r = 0; r < txfm_size_row; ++r) - temp_in[r] = buf[r * txfm_size_col + c]; - } else { - // flip left right - for (r = 0; r < txfm_size_row; ++r) - temp_in[r] = buf[r * txfm_size_col + (txfm_size_col - c - 1)]; - } - col_txfm(temp_in, temp_out, cos_bit_col, stage_range); - av1_round_shift_array(temp_out, txfm_size_row, -shift[1]); - - if (ud_flip == 0) { - for (r = 0; r < txfm_size_row; ++r) { - output[r * stride + c] = - highbd_clip_pixel_add(output[r * stride + c], temp_out[r], bd); - } - } else { - // flip upside down - for (r = 0; r < txfm_size_row; ++r) { - output[r * stride + c] = highbd_clip_pixel_add( - output[r * stride + c], temp_out[txfm_size_row - r - 1], bd); - } - } - } -} - -static INLINE void lowbd_inv_txfm2d_add_v_identity_neon( - const int32_t *input, uint8_t *output, int stride, TX_TYPE tx_type, - TX_SIZE tx_size, int eob) { - int16x8_t a[16 * 2]; - int16x8_t b[16 * 2]; - int eobx, eoby, ud_flip, lr_flip; - get_eobx_eoby_scan_v_identity(&eobx, &eoby, tx_size, eob); - const int8_t *shift = inv_txfm_shift_ls[tx_size]; - const int txw_idx = get_txw_idx(tx_size); - const int txh_idx = get_txh_idx(tx_size); - const int cos_bit_col = inv_cos_bit_col[txw_idx][txh_idx]; - const int cos_bit_row = inv_cos_bit_row[txw_idx][txh_idx]; - const int txfm_size_col = tx_size_wide[tx_size]; - const int txfm_size_row = tx_size_high[tx_size]; - lowbd_inv_txfm2d_memset_neon(&b[0], (txfm_size_col * (txfm_size_row) >> 3), - 0); - const int rect_type = get_rect_tx_log_ratio(txfm_size_col, txfm_size_row); - const int buf_size_w_div8 = txfm_size_col >> 3; - const int buf_size_nonzero_h_div8 = (eoby + 8) >> 3; - const int buf_size_nonzero_w_div8 = (eobx + 8) >> 3; - const int fun_idx_x = lowbd_txfm_all_1d_zeros_idx[eobx]; - const int fun_idx_y = lowbd_txfm_all_1d_zeros_idx[eoby]; - const int32_t *input_1; - int temp_b = 0; - const transform_neon row_txfm = - lowbd_txfm_all_1d_zeros_w_arr[txw_idx][hitx_1d_tab[tx_type]][fun_idx_x]; - const transform_neon col_txfm = - lowbd_txfm_all_1d_zeros_w_arr[txh_idx][vitx_1d_tab[tx_type]][fun_idx_y]; - - assert(col_txfm != NULL); - assert(row_txfm != NULL); - - get_flip_cfg(tx_type, &ud_flip, &lr_flip); - - for (int i = 0; i < buf_size_nonzero_h_div8; i++) { - input_1 = input; - for (int j = 0; j < buf_size_nonzero_w_div8; ++j) { - int k = j * 8 + i * txfm_size_col; - load_buffer_32bit_to_16bit_neon(input_1, &a[k], txfm_size_col); - transpose_s16_8x8q(&a[k], &a[k]); - input_1 += 8; - } - input += (txfm_size_col * 8); - if (abs(rect_type) == 1) { - int y = i * txfm_size_col; - round_shift_for_rect(&a[y], &a[y], txfm_size_col); - } - row_txfm(&a[i * txfm_size_col], &a[i * txfm_size_col], cos_bit_row, 0); - av1_round_shift_array_16_neon(&a[i * txfm_size_col], txfm_size_col, - -shift[0]); - if (lr_flip == 1) { - for (int j = 0; j < buf_size_w_div8; ++j) { - int k = j * 8 + i * txfm_size_col; - flip_buf_ud_neon(&a[k], 8); - transpose_s16_8x8q( - &a[k], &b[temp_b + txfm_size_row * (buf_size_w_div8 - 1 - j)]); - } - temp_b += 8; - } else { - for (int j = 0; j < buf_size_w_div8; ++j) { - int k = j * 8 + i * txfm_size_col; - transpose_s16_8x8q(&a[k], &b[temp_b + txfm_size_row * j]); - } - temp_b += 8; - } - } - for (int j = 0; j < buf_size_w_div8; ++j) { - col_txfm(&b[j * txfm_size_row], &b[j * txfm_size_row], cos_bit_col, 0); - av1_round_shift_array_16_neon(&b[j * txfm_size_row], txfm_size_row, - -shift[1]); - } - if (txfm_size_col >= 16) { - for (int i = 0; i < (txfm_size_col >> 4); i++) { - lowbd_add_flip_buffer_16xn_neon( - &b[i * txfm_size_row * 2], output + 16 * i, stride, 0, txfm_size_row); - } - } else if (txfm_size_col == 8) { - lowbd_add_flip_buffer_8xn_neon(b, output, stride, 0, txfm_size_row); - } -} - -static INLINE void lowbd_inv_txfm2d_add_h_wxh_identity_neon( - const int32_t *input, uint8_t *output, int stride, TX_TYPE tx_type, - TX_SIZE tx_size, int eob) { - DECLARE_ALIGNED(32, int, txfm_buf[32 * 32 + 32 + 32]); - int32_t *temp_in = txfm_buf; - - int eobx, eoby; - get_eobx_eoby_scan_h_identity(&eobx, &eoby, tx_size, eob); - const int8_t *shift = inv_txfm_shift_ls[tx_size]; - const int txw_idx = get_txw_idx(tx_size); - const int txh_idx = get_txh_idx(tx_size); - const int cos_bit_col = inv_cos_bit_col[txw_idx][txh_idx]; - const int cos_bit_row = inv_cos_bit_row[txw_idx][txh_idx]; - const int txfm_size_col = tx_size_wide[tx_size]; - const int txfm_size_row = tx_size_high[tx_size]; - const int buf_size_nonzero_h_div8 = (eoby + 8) >> 3; - - const int rect_type = get_rect_tx_log_ratio(txfm_size_col, txfm_size_row); - const int buf_offset = AOMMAX(txfm_size_row, txfm_size_col); - - int32_t *temp_out = temp_in + buf_offset; - int32_t *buf = temp_out + buf_offset; - int32_t *buf_ptr = buf; - const int8_t stage_range[MAX_TXFM_STAGE_NUM] = { 16 }; - int r, bd = 8; - - const int fun_idx_x = lowbd_txfm_all_1d_zeros_idx[eobx]; - const int fun_idx_y = lowbd_txfm_all_1d_zeros_idx[eoby]; - const transform_1d_neon row_txfm = - lowbd_txfm_all_1d_zeros_w8_arr[txw_idx][hitx_1d_tab[tx_type]][fun_idx_x]; - const transform_1d_neon col_txfm = - lowbd_txfm_all_1d_zeros_w8_arr[txh_idx][vitx_1d_tab[tx_type]][fun_idx_y]; - - assert(col_txfm != NULL); - assert(row_txfm != NULL); - int ud_flip, lr_flip; - get_flip_cfg(tx_type, &ud_flip, &lr_flip); - - // row tx - int row_start = (buf_size_nonzero_h_div8 * 8); - for (int i = 0; i < row_start; i++) { - if (abs(rect_type) == 1) { - for (int j = 0; j < txfm_size_col; j++) - temp_in[j] = round_shift((int64_t)input[j] * NewInvSqrt2, NewSqrt2Bits); - row_txfm(temp_in, buf_ptr, cos_bit_row, stage_range); - } else { - row_txfm(input, buf_ptr, cos_bit_row, stage_range); - } - av1_round_shift_array(buf_ptr, txfm_size_col, -shift[0]); - input += txfm_size_col; - buf_ptr += txfm_size_col; - } - // Doing memset for the rows which are not processed in row transform. - memset(buf_ptr, 0, - sizeof(int32_t) * txfm_size_col * (txfm_size_row - row_start)); - - // col tx - for (int c = 0; c < txfm_size_col; c++) { - if (lr_flip == 0) { - for (r = 0; r < txfm_size_row; ++r) - temp_in[r] = buf[r * txfm_size_col + c]; - } else { - // flip left right - for (r = 0; r < txfm_size_row; ++r) - temp_in[r] = buf[r * txfm_size_col + (txfm_size_col - c - 1)]; - } - col_txfm(temp_in, temp_out, cos_bit_col, stage_range); - av1_round_shift_array(temp_out, txfm_size_row, -shift[1]); - - if (ud_flip == 0) { - for (r = 0; r < txfm_size_row; ++r) { - output[r * stride + c] = - highbd_clip_pixel_add(output[r * stride + c], temp_out[r], bd); - } - } else { - // flip upside down - for (r = 0; r < txfm_size_row; ++r) { - output[r * stride + c] = highbd_clip_pixel_add( - output[r * stride + c], temp_out[txfm_size_row - r - 1], bd); - } - } - } -} - -static INLINE void lowbd_inv_txfm2d_add_h_identity_neon( - const int32_t *input, uint8_t *output, int stride, TX_TYPE tx_type, - TX_SIZE tx_size, int eob) { - int16x8_t a[16 * 2]; - int16x8_t b[16 * 2]; - int eobx, eoby, ud_flip, lr_flip; - get_eobx_eoby_scan_h_identity(&eobx, &eoby, tx_size, eob); - const int8_t *shift = inv_txfm_shift_ls[tx_size]; - const int txw_idx = get_txw_idx(tx_size); - const int txh_idx = get_txh_idx(tx_size); - const int cos_bit_col = inv_cos_bit_col[txw_idx][txh_idx]; - const int cos_bit_row = inv_cos_bit_row[txw_idx][txh_idx]; - const int txfm_size_col = tx_size_wide[tx_size]; - const int txfm_size_row = tx_size_high[tx_size]; - lowbd_inv_txfm2d_memset_neon(&a[0], (txfm_size_col * (txfm_size_row) >> 3), - 0); - const int buf_size_w_div8 = txfm_size_col >> 3; - const int rect_type = get_rect_tx_log_ratio(txfm_size_col, txfm_size_row); - const int buf_size_nonzero_h_div8 = (eoby + 8) >> 3; - const int buf_size_nonzero_w_div8 = (eobx + 8) >> 3; - const int fun_idx_x = lowbd_txfm_all_1d_zeros_idx[eobx]; - const int fun_idx_y = lowbd_txfm_all_1d_zeros_idx[eoby]; - const int32_t *input_1; - int temp_b = 0; - const transform_neon row_txfm = - lowbd_txfm_all_1d_zeros_w_arr[txw_idx][hitx_1d_tab[tx_type]][fun_idx_x]; - const transform_neon col_txfm = - lowbd_txfm_all_1d_zeros_w_arr[txh_idx][vitx_1d_tab[tx_type]][fun_idx_y]; - - assert(col_txfm != NULL); - assert(row_txfm != NULL); - - get_flip_cfg(tx_type, &ud_flip, &lr_flip); - - for (int i = 0; i < buf_size_nonzero_h_div8; i++) { - input_1 = input; - for (int j = 0; j < buf_size_nonzero_w_div8; ++j) { - int k = j * 8 + i * txfm_size_col; - load_buffer_32bit_to_16bit_neon(input_1, &a[k], txfm_size_col); - transpose_s16_8x8q(&a[k], &a[k]); - input_1 += 8; - } - input += (txfm_size_col * 8); - if (abs(rect_type) == 1) { - int y = i * txfm_size_col; - round_shift_for_rect(&a[y], &a[y], txfm_size_col); - } - row_txfm(&a[i * txfm_size_col], &a[i * txfm_size_col], cos_bit_row, 0); - av1_round_shift_array_16_neon(&a[i * txfm_size_col], txfm_size_col, - -shift[0]); - for (int j = 0; j < buf_size_w_div8; ++j) { - int k = j * 8 + i * txfm_size_col; - transpose_s16_8x8q(&a[k], &b[temp_b + txfm_size_row * j]); - } - temp_b += 8; - } - for (int j = 0; j < buf_size_w_div8; ++j) { - col_txfm(&b[j * txfm_size_row], &b[j * txfm_size_row], cos_bit_col, 0); - av1_round_shift_array_16_neon(&b[j * txfm_size_row], txfm_size_row, - -shift[1]); - } - if (txfm_size_col >= 16) { - for (int i = 0; i < (txfm_size_col >> 4); i++) { - lowbd_add_flip_buffer_16xn_neon(&b[i * txfm_size_row * 2], - output + 16 * i, stride, ud_flip, - txfm_size_row); - } - } else if (txfm_size_col == 8) { - lowbd_add_flip_buffer_8xn_neon(b, output, stride, ud_flip, txfm_size_row); - } -} - -static INLINE void lowbd_inv_txfm2d_add_4x4_neon(const int32_t *input, - uint8_t *output, int stride, - TX_TYPE tx_type, - TX_SIZE tx_size, int eob) { - (void)eob; - DECLARE_ALIGNED(32, int, txfm_buf[4 * 4 + 8 + 8]); - int32_t *temp_in = txfm_buf; - - const int8_t *shift = inv_txfm_shift_ls[tx_size]; - const int txw_idx = get_txw_idx(tx_size); - const int txh_idx = get_txh_idx(tx_size); - const int cos_bit_row = inv_cos_bit_row[txw_idx][txh_idx]; - const int cos_bit_col = inv_cos_bit_col[txw_idx][txh_idx]; - const int txfm_size_col = tx_size_wide[tx_size]; - const int txfm_size_row = tx_size_high[tx_size]; - const int buf_offset = AOMMAX(txfm_size_row, txfm_size_col); - int32_t *temp_out = temp_in + buf_offset; - int32_t *buf = temp_out + buf_offset; - int32_t *buf_ptr = buf; - const int8_t stage_range[MAX_TXFM_STAGE_NUM] = { 16 }; - int r, bd = 8; - const transform_1d_neon row_txfm = - lowbd_txfm_all_1d_arr[txw_idx][hitx_1d_tab[tx_type]]; - const transform_1d_neon col_txfm = - lowbd_txfm_all_1d_arr[txh_idx][vitx_1d_tab[tx_type]]; - - int ud_flip, lr_flip; - get_flip_cfg(tx_type, &ud_flip, &lr_flip); - - for (int i = 0; i < txfm_size_row; i++) { - row_txfm(input, buf_ptr, cos_bit_row, stage_range); - - input += txfm_size_col; - buf_ptr += txfm_size_col; - } - - for (int c = 0; c < txfm_size_col; ++c) { - if (lr_flip == 0) { - for (r = 0; r < txfm_size_row; ++r) - temp_in[r] = buf[r * txfm_size_col + c]; - } else { - // flip left right - for (r = 0; r < txfm_size_row; ++r) - temp_in[r] = buf[r * txfm_size_col + (txfm_size_col - c - 1)]; - } - col_txfm(temp_in, temp_out, cos_bit_col, stage_range); - av1_round_shift_array(temp_out, txfm_size_row, -shift[1]); - - if (ud_flip == 0) { - for (r = 0; r < txfm_size_row; ++r) { - output[r * stride + c] = - highbd_clip_pixel_add(output[r * stride + c], temp_out[r], bd); - } - } else { - // flip upside down - for (r = 0; r < txfm_size_row; ++r) { - output[r * stride + c] = highbd_clip_pixel_add( - output[r * stride + c], temp_out[txfm_size_row - r - 1], bd); - } - } - } -} - -void lowbd_inv_txfm2d_add_4x8_neon(const int32_t *input, uint8_t *output, - int stride, TX_TYPE tx_type, TX_SIZE tx_size, - int eob) { - (void)eob; - DECLARE_ALIGNED(32, int, txfm_buf[4 * 8 + 8 + 8]); - int32_t *temp_in = txfm_buf; - - const int8_t *shift = inv_txfm_shift_ls[tx_size]; - const int txw_idx = get_txw_idx(tx_size); - const int txh_idx = get_txh_idx(tx_size); - const int cos_bit_row = inv_cos_bit_row[txw_idx][txh_idx]; - const int cos_bit_col = inv_cos_bit_col[txw_idx][txh_idx]; - const int txfm_size_col = tx_size_wide[tx_size]; - const int txfm_size_row = tx_size_high[tx_size]; - const int buf_offset = AOMMAX(txfm_size_row, txfm_size_col); - int32_t *temp_out = temp_in + buf_offset; - int32_t *buf = temp_out + buf_offset; - int32_t *buf_ptr = buf; - const int8_t stage_range[MAX_TXFM_STAGE_NUM] = { 16 }; - int r, bd = 8; - const transform_1d_neon row_txfm = - lowbd_txfm_all_1d_arr[txw_idx][hitx_1d_tab[tx_type]]; - const transform_1d_neon col_txfm = - lowbd_txfm_all_1d_arr[txh_idx][vitx_1d_tab[tx_type]]; - - int ud_flip, lr_flip; - get_flip_cfg(tx_type, &ud_flip, &lr_flip); - - for (int i = 0; i < txfm_size_row; i++) { - for (int j = 0; j < txfm_size_col; j++) - temp_in[j] = round_shift((int64_t)input[j] * NewInvSqrt2, NewSqrt2Bits); - - row_txfm(temp_in, buf_ptr, cos_bit_row, stage_range); - input += txfm_size_col; - buf_ptr += txfm_size_col; - } - - for (int c = 0; c < txfm_size_col; ++c) { - if (lr_flip == 0) { - for (r = 0; r < txfm_size_row; ++r) - temp_in[r] = buf[r * txfm_size_col + c]; - } else { - // flip left right - for (r = 0; r < txfm_size_row; ++r) - temp_in[r] = buf[r * txfm_size_col + (txfm_size_col - c - 1)]; - } - col_txfm(temp_in, temp_out, cos_bit_col, stage_range); - av1_round_shift_array(temp_out, txfm_size_row, -shift[1]); - - if (ud_flip == 0) { - for (r = 0; r < txfm_size_row; ++r) { - output[r * stride + c] = - highbd_clip_pixel_add(output[r * stride + c], temp_out[r], bd); - } - } else { - // flip upside down - for (r = 0; r < txfm_size_row; ++r) { - output[r * stride + c] = highbd_clip_pixel_add( - output[r * stride + c], temp_out[txfm_size_row - r - 1], bd); - } - } - } -} - -void lowbd_inv_txfm2d_add_8x4_neon(const int32_t *input, uint8_t *output, - int stride, TX_TYPE tx_type, TX_SIZE tx_size, - int eob) { - (void)eob; - DECLARE_ALIGNED(32, int, txfm_buf[8 * 4 + 8 + 8]); - int32_t *temp_in = txfm_buf; - - const int8_t *shift = inv_txfm_shift_ls[tx_size]; - const int txw_idx = get_txw_idx(tx_size); - const int txh_idx = get_txh_idx(tx_size); - const int cos_bit_row = inv_cos_bit_row[txw_idx][txh_idx]; - const int cos_bit_col = inv_cos_bit_col[txw_idx][txh_idx]; - const int txfm_size_col = tx_size_wide[tx_size]; - const int txfm_size_row = tx_size_high[tx_size]; - const int buf_offset = AOMMAX(txfm_size_row, txfm_size_col); - int32_t *temp_out = temp_in + buf_offset; - int32_t *buf = temp_out + buf_offset; - int32_t *buf_ptr = buf; - const int8_t stage_range[MAX_TXFM_STAGE_NUM] = { 16 }; - int r, bd = 8; - const transform_1d_neon row_txfm = - lowbd_txfm_all_1d_arr[txw_idx][hitx_1d_tab[tx_type]]; - const transform_1d_neon col_txfm = - lowbd_txfm_all_1d_arr[txh_idx][vitx_1d_tab[tx_type]]; - - int ud_flip, lr_flip; - get_flip_cfg(tx_type, &ud_flip, &lr_flip); - - for (int i = 0; i < txfm_size_row; i++) { - for (int j = 0; j < txfm_size_col; j++) - temp_in[j] = round_shift((int64_t)input[j] * NewInvSqrt2, NewSqrt2Bits); - - row_txfm(temp_in, buf_ptr, cos_bit_row, stage_range); - input += txfm_size_col; - buf_ptr += txfm_size_col; - } - - for (int c = 0; c < txfm_size_col; ++c) { - if (lr_flip == 0) { - for (r = 0; r < txfm_size_row; ++r) - temp_in[r] = buf[r * txfm_size_col + c]; - } else { - // flip left right - for (r = 0; r < txfm_size_row; ++r) - temp_in[r] = buf[r * txfm_size_col + (txfm_size_col - c - 1)]; - } - col_txfm(temp_in, temp_out, cos_bit_col, stage_range); - av1_round_shift_array(temp_out, txfm_size_row, -shift[1]); - - if (ud_flip == 0) { - for (r = 0; r < txfm_size_row; ++r) { - output[r * stride + c] = - highbd_clip_pixel_add(output[r * stride + c], temp_out[r], bd); - } - } else { - // flip upside down - for (r = 0; r < txfm_size_row; ++r) { - output[r * stride + c] = highbd_clip_pixel_add( - output[r * stride + c], temp_out[txfm_size_row - r - 1], bd); - } - } - } -} - -void lowbd_inv_txfm2d_add_4x16_neon(const int32_t *input, uint8_t *output, - int stride, TX_TYPE tx_type, - TX_SIZE tx_size, int eob) { - (void)eob; - DECLARE_ALIGNED(32, int, txfm_buf[4 * 16 + 16 + 16]); - int32_t *temp_in = txfm_buf; - - const int8_t *shift = inv_txfm_shift_ls[tx_size]; - const int txw_idx = get_txw_idx(tx_size); - const int txh_idx = get_txh_idx(tx_size); - const int cos_bit_row = inv_cos_bit_row[txw_idx][txh_idx]; - const int cos_bit_col = inv_cos_bit_col[txw_idx][txh_idx]; - const int txfm_size_col = tx_size_wide[tx_size]; - const int txfm_size_row = tx_size_high[tx_size]; - const int buf_offset = AOMMAX(txfm_size_row, txfm_size_col); - int32_t *temp_out = temp_in + buf_offset; - int32_t *buf = temp_out + buf_offset; - int32_t *buf_ptr = buf; - const int8_t stage_range[MAX_TXFM_STAGE_NUM] = { 16 }; - int r, bd = 8; - const transform_1d_neon row_txfm = - lowbd_txfm_all_1d_arr[txw_idx][hitx_1d_tab[tx_type]]; - const transform_1d_neon col_txfm = - lowbd_txfm_all_1d_arr[txh_idx][vitx_1d_tab[tx_type]]; - - int ud_flip, lr_flip; - get_flip_cfg(tx_type, &ud_flip, &lr_flip); - - for (int i = 0; i < txfm_size_row; i++) { - row_txfm(input, buf_ptr, cos_bit_row, stage_range); - av1_round_shift_array(buf_ptr, txfm_size_col, -shift[0]); - input += txfm_size_col; - buf_ptr += txfm_size_col; - } - - for (int c = 0; c < txfm_size_col; ++c) { - if (lr_flip == 0) { - for (r = 0; r < txfm_size_row; ++r) - temp_in[r] = buf[r * txfm_size_col + c]; - } else { - // flip left right - for (r = 0; r < txfm_size_row; ++r) - temp_in[r] = buf[r * txfm_size_col + (txfm_size_col - c - 1)]; - } - col_txfm(temp_in, temp_out, cos_bit_col, stage_range); - av1_round_shift_array(temp_out, txfm_size_row, -shift[1]); - - if (ud_flip == 0) { - for (r = 0; r < txfm_size_row; ++r) { - output[r * stride + c] = - highbd_clip_pixel_add(output[r * stride + c], temp_out[r], bd); - } - } else { - // flip upside down - for (r = 0; r < txfm_size_row; ++r) { - output[r * stride + c] = highbd_clip_pixel_add( - output[r * stride + c], temp_out[txfm_size_row - r - 1], bd); - } - } - } -} - -void lowbd_inv_txfm2d_add_16x4_neon(const int32_t *input, uint8_t *output, - int stride, TX_TYPE tx_type, - TX_SIZE tx_size, int eob) { - (void)eob; - - DECLARE_ALIGNED(32, int, txfm_buf[16 * 4 + 16 + 16]); - int32_t *temp_in = txfm_buf; - - const int8_t *shift = inv_txfm_shift_ls[tx_size]; - const int txw_idx = get_txw_idx(tx_size); - const int txh_idx = get_txh_idx(tx_size); - const int cos_bit_row = inv_cos_bit_row[txw_idx][txh_idx]; - const int cos_bit_col = inv_cos_bit_col[txw_idx][txh_idx]; - const int txfm_size_col = tx_size_wide[tx_size]; - const int txfm_size_row = tx_size_high[tx_size]; - const int buf_offset = AOMMAX(txfm_size_row, txfm_size_col); - int32_t *temp_out = temp_in + buf_offset; - int32_t *buf = temp_out + buf_offset; - int32_t *buf_ptr = buf; - const int8_t stage_range[MAX_TXFM_STAGE_NUM] = { 16 }; - int r, bd = 8; - const transform_1d_neon row_txfm = - lowbd_txfm_all_1d_arr[txw_idx][hitx_1d_tab[tx_type]]; - const transform_1d_neon col_txfm = - lowbd_txfm_all_1d_arr[txh_idx][vitx_1d_tab[tx_type]]; - - int ud_flip, lr_flip; - get_flip_cfg(tx_type, &ud_flip, &lr_flip); - - for (int i = 0; i < txfm_size_row; i++) { - row_txfm(input, buf_ptr, cos_bit_row, stage_range); - av1_round_shift_array(buf_ptr, txfm_size_col, -shift[0]); - input += txfm_size_col; - buf_ptr += txfm_size_col; - } - - for (int c = 0; c < txfm_size_col; ++c) { - if (lr_flip == 0) { - for (r = 0; r < txfm_size_row; ++r) - temp_in[r] = buf[r * txfm_size_col + c]; - } else { - // flip left right - for (r = 0; r < txfm_size_row; ++r) - temp_in[r] = buf[r * txfm_size_col + (txfm_size_col - c - 1)]; - } - col_txfm(temp_in, temp_out, cos_bit_col, stage_range); - av1_round_shift_array(temp_out, txfm_size_row, -shift[1]); - - if (ud_flip == 0) { - for (r = 0; r < txfm_size_row; ++r) { - output[r * stride + c] = - highbd_clip_pixel_add(output[r * stride + c], temp_out[r], bd); - } - } else { - // flip upside down - for (r = 0; r < txfm_size_row; ++r) { - output[r * stride + c] = highbd_clip_pixel_add( - output[r * stride + c], temp_out[txfm_size_row - r - 1], bd); - } - } - } -} - -static INLINE void lowbd_inv_txfm2d_add_wxh_no_identity_neon( - const int32_t *input, uint8_t *output, int stride, TX_TYPE tx_type, - TX_SIZE tx_size, int eob) { - DECLARE_ALIGNED(32, int, txfm_buf[64 * 64 + 64 + 64]); - int32_t *temp_in = txfm_buf; - - int eobx, eoby, ud_flip, lr_flip, row_start; - get_eobx_eoby_scan_default(&eobx, &eoby, tx_size, eob); - const int8_t *shift = inv_txfm_shift_ls[tx_size]; - const int txw_idx = get_txw_idx(tx_size); - const int txh_idx = get_txh_idx(tx_size); - const int cos_bit_col = inv_cos_bit_col[txw_idx][txh_idx]; - const int cos_bit_row = inv_cos_bit_row[txw_idx][txh_idx]; - const int txfm_size_col = tx_size_wide[tx_size]; - const int txfm_size_row = tx_size_high[tx_size]; - const int buf_size_nonzero_h_div8 = (eoby + 8) >> 3; - const int rect_type = get_rect_tx_log_ratio(txfm_size_col, txfm_size_row); - const int buf_offset = AOMMAX(txfm_size_row, txfm_size_col); - - int32_t *temp_out = temp_in + buf_offset; - int32_t *buf = temp_out + buf_offset; - int32_t *buf_ptr = buf; - const int8_t stage_range[MAX_TXFM_STAGE_NUM] = { 16 }; - const int bd = 8; - int r; - - const int fun_idx_x = lowbd_txfm_all_1d_zeros_idx[eobx]; - const int fun_idx_y = lowbd_txfm_all_1d_zeros_idx[eoby]; - const transform_1d_neon row_txfm = - lowbd_txfm_all_1d_zeros_w8_arr[txw_idx][hitx_1d_tab[tx_type]][fun_idx_x]; - const transform_1d_neon col_txfm = - lowbd_txfm_all_1d_zeros_w8_arr[txh_idx][vitx_1d_tab[tx_type]][fun_idx_y]; - - assert(col_txfm != NULL); - assert(row_txfm != NULL); - - get_flip_cfg(tx_type, &ud_flip, &lr_flip); - row_start = (buf_size_nonzero_h_div8 << 3); - - for (int i = 0; i < row_start; i++) { - if (abs(rect_type) == 1) { - for (int j = 0; j < txfm_size_col; j++) - temp_in[j] = round_shift((int64_t)input[j] * NewInvSqrt2, NewSqrt2Bits); - row_txfm(temp_in, buf_ptr, cos_bit_row, stage_range); - } else { - row_txfm(input, buf_ptr, cos_bit_row, stage_range); - } - av1_round_shift_array(buf_ptr, txfm_size_col, -shift[0]); - input += txfm_size_col; - buf_ptr += txfm_size_col; - } - - // Doing memset for the rows which are not processed in row transform. - memset(buf_ptr, 0, - sizeof(int32_t) * txfm_size_col * (txfm_size_row - row_start)); - - for (int c = 0; c < txfm_size_col; c++) { - if (lr_flip == 0) { - for (r = 0; r < txfm_size_row; ++r) - temp_in[r] = buf[r * txfm_size_col + c]; - } else { - // flip left right - for (r = 0; r < txfm_size_row; ++r) - temp_in[r] = buf[r * txfm_size_col + (txfm_size_col - c - 1)]; - } - col_txfm(temp_in, temp_out, cos_bit_col, stage_range); - av1_round_shift_array(temp_out, txfm_size_row, -shift[1]); - - if (ud_flip == 0) { - for (r = 0; r < txfm_size_row; ++r) { - output[r * stride + c] = - highbd_clip_pixel_add(output[r * stride + c], temp_out[r], bd); - } - } else { - // flip upside down - for (r = 0; r < txfm_size_row; ++r) { - output[r * stride + c] = highbd_clip_pixel_add( - output[r * stride + c], temp_out[txfm_size_row - r - 1], bd); - } - } - } -} - -static INLINE void lowbd_inv_txfm2d_add_no_identity_neon( - const int32_t *input, uint8_t *output, int stride, TX_TYPE tx_type, - TX_SIZE tx_size, int eob) { - int16x8_t a[64 * 8]; - int16x8_t b[64 * 8]; - int eobx, eoby, ud_flip, lr_flip; - get_eobx_eoby_scan_default(&eobx, &eoby, tx_size, eob); - const int8_t *shift = inv_txfm_shift_ls[tx_size]; - const int txw_idx = get_txw_idx(tx_size); - const int txh_idx = get_txh_idx(tx_size); - const int cos_bit_col = inv_cos_bit_col[txw_idx][txh_idx]; - const int cos_bit_row = inv_cos_bit_row[txw_idx][txh_idx]; - const int txfm_size_col = tx_size_wide[tx_size]; - const int txfm_size_row = tx_size_high[tx_size]; - const int rect_type = get_rect_tx_log_ratio(txfm_size_col, txfm_size_row); - const int buf_size_w_div8 = txfm_size_col >> 3; - const int buf_size_nonzero_h_div8 = (eoby + 8) >> 3; - const int buf_size_nonzero_w_div8 = (eobx + 8) >> 3; - const int fun_idx_x = lowbd_txfm_all_1d_zeros_idx[eobx]; - const int fun_idx_y = lowbd_txfm_all_1d_zeros_idx[eoby]; - const int32_t *input_1; - int temp_b = 0; - - const transform_neon row_txfm = - lowbd_txfm_all_1d_zeros_w_arr[txw_idx][hitx_1d_tab[tx_type]][fun_idx_x]; - const transform_neon col_txfm = - lowbd_txfm_all_1d_zeros_w_arr[txh_idx][vitx_1d_tab[tx_type]][fun_idx_y]; - - assert(col_txfm != NULL); - assert(row_txfm != NULL); - - get_flip_cfg(tx_type, &ud_flip, &lr_flip); - - for (int i = 0; i < buf_size_nonzero_h_div8; i++) { - input_1 = input; - for (int j = 0; j < buf_size_nonzero_w_div8; ++j) { - int k = j * 8 + i * txfm_size_col; - load_buffer_32bit_to_16bit_neon(input_1, &a[k], txfm_size_col); - transpose_s16_8x8q(&a[k], &a[k]); - input_1 += 8; - } - input += (txfm_size_col * 8); - if (abs(rect_type) == 1) { - int y = i * txfm_size_col; - round_shift_for_rect(&a[y], &a[y], txfm_size_col); - } - row_txfm(&a[i * txfm_size_col], &a[i * txfm_size_col], cos_bit_row, 0); - av1_round_shift_array_16_neon(&a[i * txfm_size_col], txfm_size_col, - -shift[0]); - if (lr_flip == 1) { - for (int j = 0; j < buf_size_w_div8; ++j) { - int k = j * 8 + i * txfm_size_col; - flip_buf_ud_neon(&a[k], 8); - transpose_s16_8x8q( - &a[k], &b[temp_b + txfm_size_row * (buf_size_w_div8 - 1 - j)]); - } - temp_b += 8; - } else { - for (int j = 0; j < buf_size_w_div8; ++j) { - int k = j * 8 + i * txfm_size_col; - transpose_s16_8x8q(&a[k], &b[temp_b + txfm_size_row * j]); - } - temp_b += 8; - } - } - for (int j = 0; j < buf_size_w_div8; ++j) { - col_txfm(&b[j * txfm_size_row], &b[j * txfm_size_row], cos_bit_col, 0); - av1_round_shift_array_16_neon(&b[j * txfm_size_row], txfm_size_row, - -shift[1]); - } - - if (txfm_size_col >= 16) { - for (int i = 0; i < (txfm_size_col >> 4); i++) { - lowbd_add_flip_buffer_16xn_neon(&b[i * txfm_size_row * 2], - output + 16 * i, stride, ud_flip, - txfm_size_row); - } - } else if (txfm_size_col == 8) { - lowbd_add_flip_buffer_8xn_neon(b, output, stride, ud_flip, txfm_size_row); - } -} - -static INLINE void lowbd_inv_txfm2d_add_wxh_universe_neon( - const int32_t *input, uint8_t *output, int stride, TX_TYPE tx_type, - TX_SIZE tx_size, int eob) { - switch (tx_type) { - case IDTX: - lowbd_inv_txfm2d_add_wxh_idtx_neon(input, output, stride, tx_type, - tx_size, eob); - break; - - case H_DCT: - case H_ADST: - case H_FLIPADST: - lowbd_inv_txfm2d_add_v_wxh_identity_neon(input, output, stride, tx_type, - tx_size, eob); - break; - - case V_DCT: - case V_ADST: - case V_FLIPADST: - lowbd_inv_txfm2d_add_h_wxh_identity_neon(input, output, stride, tx_type, - tx_size, eob); - break; - - default: - lowbd_inv_txfm2d_add_wxh_no_identity_neon(input, output, stride, tx_type, - tx_size, eob); - break; - } -} - -static INLINE void lowbd_inv_txfm2d_add_universe_neon( - const int32_t *input, uint8_t *output, int stride, TX_TYPE tx_type, - TX_SIZE tx_size, int eob) { - switch (tx_type) { - case IDTX: - lowbd_inv_txfm2d_add_idtx_neon(input, output, stride, tx_type, tx_size, - eob); - break; - - case H_DCT: - case H_ADST: - case H_FLIPADST: - lowbd_inv_txfm2d_add_v_identity_neon(input, output, stride, tx_type, - tx_size, eob); - break; - - case V_DCT: - case V_ADST: - case V_FLIPADST: - lowbd_inv_txfm2d_add_h_identity_neon(input, output, stride, tx_type, - tx_size, eob); - break; - - default: - lowbd_inv_txfm2d_add_no_identity_neon(input, output, stride, tx_type, - tx_size, eob); - break; - } -} - -void av1_lowbd_inv_txfm2d_add_neon(const int32_t *input, uint8_t *output, - int stride, TX_TYPE tx_type, TX_SIZE tx_size, - int eob) { - int row; - switch (tx_size) { - case TX_4X4: - lowbd_inv_txfm2d_add_4x4_neon(input, output, stride, tx_type, tx_size, - eob); - break; - - case TX_4X8: - lowbd_inv_txfm2d_add_4x8_neon(input, output, stride, tx_type, tx_size, - eob); - break; - - case TX_8X4: - lowbd_inv_txfm2d_add_8x4_neon(input, output, stride, tx_type, tx_size, - eob); - break; - - case TX_4X16: - lowbd_inv_txfm2d_add_4x16_neon(input, output, stride, tx_type, tx_size, - eob); - break; - - case TX_16X4: - lowbd_inv_txfm2d_add_16x4_neon(input, output, stride, tx_type, tx_size, - eob); - break; - - case TX_16X64: { - lowbd_inv_txfm2d_add_wxh_universe_neon(input, output, stride, tx_type, - tx_size, eob); - } break; - - case TX_64X16: { - int32_t mod_input[64 * 16]; - for (row = 0; row < 16; ++row) { - memcpy(mod_input + row * 64, input + row * 32, 32 * sizeof(*mod_input)); - memset(mod_input + row * 64 + 32, 0, 32 * sizeof(*mod_input)); - } - lowbd_inv_txfm2d_add_wxh_universe_neon(mod_input, output, stride, tx_type, - tx_size, eob); - } break; - - case TX_32X64: { - lowbd_inv_txfm2d_add_wxh_universe_neon(input, output, stride, tx_type, - tx_size, eob); - } break; - - case TX_64X32: { - int32_t mod_input[64 * 32]; - for (row = 0; row < 32; ++row) { - memcpy(mod_input + row * 64, input + row * 32, 32 * sizeof(*mod_input)); - memset(mod_input + row * 64 + 32, 0, 32 * sizeof(*mod_input)); - } - lowbd_inv_txfm2d_add_wxh_universe_neon(mod_input, output, stride, tx_type, - tx_size, eob); - } break; - - case TX_64X64: { - int32_t mod_input[64 * 64]; - for (row = 0; row < 32; ++row) { - memcpy(mod_input + row * 64, input + row * 32, 32 * sizeof(*mod_input)); - memset(mod_input + row * 64 + 32, 0, 32 * sizeof(*mod_input)); - } - lowbd_inv_txfm2d_add_wxh_universe_neon(mod_input, output, stride, tx_type, - tx_size, eob); - } break; - - default: - lowbd_inv_txfm2d_add_universe_neon(input, output, stride, tx_type, - tx_size, eob); - break; - } -} -void av1_inv_txfm_add_neon(const tran_low_t *dqcoeff, uint8_t *dst, int stride, - const TxfmParam *txfm_param) { - const TX_TYPE tx_type = txfm_param->tx_type; - if (!txfm_param->lossless) { - av1_lowbd_inv_txfm2d_add_neon(dqcoeff, dst, stride, tx_type, - txfm_param->tx_size, txfm_param->eob); - } else { - av1_inv_txfm_add_c(dqcoeff, dst, stride, txfm_param); - } -} diff --git a/third_party/aom/av1/common/arm/av1_inv_txfm_neon.h b/third_party/aom/av1/common/arm/av1_inv_txfm_neon.h deleted file mode 100644 index 9ec658291..000000000 --- a/third_party/aom/av1/common/arm/av1_inv_txfm_neon.h +++ /dev/null @@ -1,154 +0,0 @@ -/* - * Copyright (c) 2018, Alliance for Open Media. All rights reserved - * - * This source code is subject to the terms of the BSD 2 Clause License and - * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License - * was not distributed with this source code in the LICENSE file, you can - * obtain it at www.aomedia.org/license/software. If the Alliance for Open - * Media Patent License 1.0 was not distributed with this source code in the - * PATENTS file, you can obtain it at www.aomedia.org/license/patent. - */ -#ifndef AOM_AV1_COMMON_ARM_AV1_INV_TXFM_NEON_H_ -#define AOM_AV1_COMMON_ARM_AV1_INV_TXFM_NEON_H_ - -#include "config/aom_config.h" -#include "config/av1_rtcd.h" - -#include "aom/aom_integer.h" -#include "av1/common/enums.h" -#include "av1/common/av1_inv_txfm1d.h" -#include "av1/common/av1_inv_txfm1d_cfg.h" -#include "av1/common/av1_txfm.h" - -typedef void (*transform_1d_neon)(const int32_t *input, int32_t *output, - const int8_t cos_bit, - const int8_t *stage_ptr); -typedef void (*transform_neon)(int16x8_t *input, int16x8_t *output, - int8_t cos_bit, int bit); - -DECLARE_ALIGNED(16, static const int16_t, av1_eob_to_eobxy_8x8_default[8]) = { - 0x0707, 0x0707, 0x0707, 0x0707, 0x0707, 0x0707, 0x0707, 0x0707, -}; - -DECLARE_ALIGNED(16, static const int16_t, - av1_eob_to_eobxy_16x16_default[16]) = { - 0x0707, 0x0707, 0x0f0f, 0x0f0f, 0x0f0f, 0x0f0f, 0x0f0f, 0x0f0f, - 0x0f0f, 0x0f0f, 0x0f0f, 0x0f0f, 0x0f0f, 0x0f0f, 0x0f0f, 0x0f0f, -}; - -DECLARE_ALIGNED(16, static const int16_t, - av1_eob_to_eobxy_32x32_default[32]) = { - 0x0707, 0x0f0f, 0x0f0f, 0x0f0f, 0x1f1f, 0x1f1f, 0x1f1f, 0x1f1f, - 0x1f1f, 0x1f1f, 0x1f1f, 0x1f1f, 0x1f1f, 0x1f1f, 0x1f1f, 0x1f1f, - 0x1f1f, 0x1f1f, 0x1f1f, 0x1f1f, 0x1f1f, 0x1f1f, 0x1f1f, 0x1f1f, - 0x1f1f, 0x1f1f, 0x1f1f, 0x1f1f, 0x1f1f, 0x1f1f, 0x1f1f, 0x1f1f, -}; - -DECLARE_ALIGNED(16, static const int16_t, av1_eob_to_eobxy_8x16_default[16]) = { - 0x0707, 0x0707, 0x0707, 0x0707, 0x0707, 0x0f07, 0x0f07, 0x0f07, - 0x0f07, 0x0f07, 0x0f07, 0x0f07, 0x0f07, 0x0f07, 0x0f07, 0x0f07, -}; - -DECLARE_ALIGNED(16, static const int16_t, av1_eob_to_eobxy_16x8_default[8]) = { - 0x0707, 0x0707, 0x070f, 0x070f, 0x070f, 0x070f, 0x070f, 0x070f, -}; - -DECLARE_ALIGNED(16, static const int16_t, - av1_eob_to_eobxy_16x32_default[32]) = { - 0x0707, 0x0707, 0x0f0f, 0x0f0f, 0x0f0f, 0x0f0f, 0x0f0f, 0x0f0f, - 0x0f0f, 0x1f0f, 0x1f0f, 0x1f0f, 0x1f0f, 0x1f0f, 0x1f0f, 0x1f0f, - 0x1f0f, 0x1f0f, 0x1f0f, 0x1f0f, 0x1f0f, 0x1f0f, 0x1f0f, 0x1f0f, - 0x1f0f, 0x1f0f, 0x1f0f, 0x1f0f, 0x1f0f, 0x1f0f, 0x1f0f, 0x1f0f, -}; - -DECLARE_ALIGNED(16, static const int16_t, - av1_eob_to_eobxy_32x16_default[16]) = { - 0x0707, 0x0f0f, 0x0f0f, 0x0f0f, 0x0f1f, 0x0f1f, 0x0f1f, 0x0f1f, - 0x0f1f, 0x0f1f, 0x0f1f, 0x0f1f, 0x0f1f, 0x0f1f, 0x0f1f, 0x0f1f, -}; - -DECLARE_ALIGNED(16, static const int16_t, av1_eob_to_eobxy_8x32_default[32]) = { - 0x0707, 0x0707, 0x0707, 0x0707, 0x0707, 0x0f07, 0x0f07, 0x0f07, - 0x0f07, 0x0f07, 0x0f07, 0x0f07, 0x0f07, 0x1f07, 0x1f07, 0x1f07, - 0x1f07, 0x1f07, 0x1f07, 0x1f07, 0x1f07, 0x1f07, 0x1f07, 0x1f07, - 0x1f07, 0x1f07, 0x1f07, 0x1f07, 0x1f07, 0x1f07, 0x1f07, 0x1f07, -}; - -DECLARE_ALIGNED(16, static const int16_t, av1_eob_to_eobxy_32x8_default[8]) = { - 0x0707, 0x070f, 0x070f, 0x071f, 0x071f, 0x071f, 0x071f, 0x071f, -}; - -DECLARE_ALIGNED(16, static const int16_t *, - av1_eob_to_eobxy_default[TX_SIZES_ALL]) = { - NULL, - av1_eob_to_eobxy_8x8_default, - av1_eob_to_eobxy_16x16_default, - av1_eob_to_eobxy_32x32_default, - av1_eob_to_eobxy_32x32_default, - NULL, - NULL, - av1_eob_to_eobxy_8x16_default, - av1_eob_to_eobxy_16x8_default, - av1_eob_to_eobxy_16x32_default, - av1_eob_to_eobxy_32x16_default, - av1_eob_to_eobxy_32x32_default, - av1_eob_to_eobxy_32x32_default, - NULL, - NULL, - av1_eob_to_eobxy_8x32_default, - av1_eob_to_eobxy_32x8_default, - av1_eob_to_eobxy_16x32_default, - av1_eob_to_eobxy_32x16_default, -}; - -static const int lowbd_txfm_all_1d_zeros_idx[32] = { - 0, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, - 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, -}; - -// Transform block width in log2 for eob (size of 64 map to 32) -static const int tx_size_wide_log2_eob[TX_SIZES_ALL] = { - 2, 3, 4, 5, 5, 2, 3, 3, 4, 4, 5, 5, 5, 2, 4, 3, 5, 4, 5, -}; - -static int eob_fill[32] = { - 0, 7, 7, 7, 7, 7, 7, 7, 15, 15, 15, 15, 15, 15, 15, 15, - 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, -}; - -static INLINE void get_eobx_eoby_scan_default(int *eobx, int *eoby, - TX_SIZE tx_size, int eob) { - if (eob == 1) { - *eobx = 0; - *eoby = 0; - return; - } - - const int tx_w_log2 = tx_size_wide_log2_eob[tx_size]; - const int eob_row = (eob - 1) >> tx_w_log2; - const int eobxy = av1_eob_to_eobxy_default[tx_size][eob_row]; - *eobx = eobxy & 0xFF; - *eoby = eobxy >> 8; -} - -static INLINE void get_eobx_eoby_scan_v_identity(int *eobx, int *eoby, - TX_SIZE tx_size, int eob) { - eob -= 1; - const int txfm_size_row = tx_size_high[tx_size]; - const int eoby_max = AOMMIN(32, txfm_size_row) - 1; - *eobx = eob / (eoby_max + 1); - *eoby = (eob >= eoby_max) ? eoby_max : eob_fill[eob]; -} - -static INLINE void get_eobx_eoby_scan_h_identity(int *eobx, int *eoby, - TX_SIZE tx_size, int eob) { - eob -= 1; - const int txfm_size_col = tx_size_wide[tx_size]; - const int eobx_max = AOMMIN(32, txfm_size_col) - 1; - *eobx = (eob >= eobx_max) ? eobx_max : eob_fill[eob]; - const int temp_eoby = eob / (eobx_max + 1); - assert(temp_eoby < 32); - *eoby = eob_fill[temp_eoby]; -} - -#endif // AOM_AV1_COMMON_ARM_AV1_INV_TXFM_NEON_H_ diff --git a/third_party/aom/av1/common/arm/av1_txfm_neon.c b/third_party/aom/av1/common/arm/av1_txfm_neon.c deleted file mode 100644 index de3c54724..000000000 --- a/third_party/aom/av1/common/arm/av1_txfm_neon.c +++ /dev/null @@ -1,28 +0,0 @@ -/* - * - * Copyright (c) 2018, Alliance for Open Media. All rights reserved - * - * This source code is subject to the terms of the BSD 2 Clause License and - * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License - * was not distributed with this source code in the LICENSE file, you can - * obtain it at www.aomedia.org/license/software. If the Alliance for Open - * Media Patent License 1.0 was not distributed with this source code in the - * PATENTS file, you can obtain it at www.aomedia.org/license/patent. - */ -#include <arm_neon.h> -#include <assert.h> - -#include "aom_ports/mem.h" -#include "av1/common/arm/mem_neon.h" - -void av1_round_shift_array_neon(int32_t *arr, int size, int bit) { - assert(!(size % 4)); - if (!bit) return; - const int32x4_t dup_bits_n_32x4 = vdupq_n_s32((int32_t)(-bit)); - for (int i = 0; i < size; i += 4) { - int32x4_t tmp_q_s32 = vld1q_s32(arr); - tmp_q_s32 = vrshlq_s32(tmp_q_s32, dup_bits_n_32x4); - vst1q_s32(arr, tmp_q_s32); - arr += 4; - } -} diff --git a/third_party/aom/av1/common/arm/blend_a64_hmask_neon.c b/third_party/aom/av1/common/arm/blend_a64_hmask_neon.c deleted file mode 100644 index 7134f183e..000000000 --- a/third_party/aom/av1/common/arm/blend_a64_hmask_neon.c +++ /dev/null @@ -1,134 +0,0 @@ -/* - * - * Copyright (c) 2018, Alliance for Open Media. All rights reserved - * - * This source code is subject to the terms of the BSD 2 Clause License and - * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License - * was not distributed with this source code in the LICENSE file, you can - * obtain it at www.aomedia.org/license/software. If the Alliance for Open - * Media Patent License 1.0 was not distributed with this source code in the - * PATENTS file, you can obtain it at www.aomedia.org/license/patent. - */ - -#include <arm_neon.h> -#include <assert.h> - -#include "aom/aom_integer.h" -#include "aom_dsp/blend.h" -#include "aom_ports/mem.h" -#include "av1/common/arm/mem_neon.h" -#include "aom_dsp/aom_dsp_common.h" -#include "config/aom_dsp_rtcd.h" - -void aom_blend_a64_hmask_neon(uint8_t *dst, uint32_t dst_stride, - const uint8_t *src0, uint32_t src0_stride, - const uint8_t *src1, uint32_t src1_stride, - const uint8_t *mask, int w, int h) { - assert(IMPLIES(src0 == dst, src0_stride == dst_stride)); - assert(IMPLIES(src1 == dst, src1_stride == dst_stride)); - - assert(h >= 2); - assert(w >= 2); - assert(IS_POWER_OF_TWO(h)); - assert(IS_POWER_OF_TWO(w)); - uint8x8_t tmp0, tmp1; - uint8x16_t res_q; - uint16x8_t res, res_low, res_high; - uint32x2_t tmp0_32 = vdup_n_u32(0), tmp1_32 = vdup_n_u32(0); - uint16x4_t tmp0_16 = vdup_n_u16(0), tmp1_16 = vdup_n_u16(0); - const uint8x8_t vdup_64 = vdup_n_u8((uint8_t)64); - - if (w >= 16) { - const uint8x16_t vdup_64_q = vdupq_n_u8((uint8_t)64); - for (int i = 0; i < h; ++i) { - for (int j = 0; j < w; j += 16) { - __builtin_prefetch(src0); - __builtin_prefetch(src1); - const uint8x16_t tmp0_q = vld1q_u8(src0); - const uint8x16_t tmp1_q = vld1q_u8(src1); - const uint8x16_t m_q = vld1q_u8(mask); - const uint8x16_t max_minus_m_q = vsubq_u8(vdup_64_q, m_q); - res_low = vmull_u8(vget_low_u8(m_q), vget_low_u8(tmp0_q)); - res_low = - vmlal_u8(res_low, vget_low_u8(max_minus_m_q), vget_low_u8(tmp1_q)); - res_high = vmull_u8(vget_high_u8(m_q), vget_high_u8(tmp0_q)); - res_high = vmlal_u8(res_high, vget_high_u8(max_minus_m_q), - vget_high_u8(tmp1_q)); - res_q = vcombine_u8(vrshrn_n_u16(res_low, AOM_BLEND_A64_ROUND_BITS), - vrshrn_n_u16(res_high, AOM_BLEND_A64_ROUND_BITS)); - vst1q_u8(dst, res_q); - src0 += 16; - src1 += 16; - dst += 16; - mask += 16; - } - src0 += src0_stride - w; - src1 += src1_stride - w; - dst += dst_stride - w; - mask -= w; - } - } else if (w == 8) { - const uint8x8_t m = vld1_u8(mask); - const uint8x8_t max_minus_m = vsub_u8(vdup_64, m); - for (int i = 0; i < h; ++i) { - __builtin_prefetch(src0); - __builtin_prefetch(src1); - tmp0 = vld1_u8(src0); - tmp1 = vld1_u8(src1); - res = vmull_u8(m, tmp0); - res = vmlal_u8(res, max_minus_m, tmp1); - vst1_u8(dst, vrshrn_n_u16(res, AOM_BLEND_A64_ROUND_BITS)); - src0 += src0_stride; - src1 += src1_stride; - dst += dst_stride; - } - } else if (w == 4) { - const uint8x8_t m = vreinterpret_u8_u32(vld1_dup_u32((uint32_t *)mask)); - const uint8x8_t max_minus_m = vsub_u8(vdup_64, m); - for (int i = 0; i < h; i += 2) { - __builtin_prefetch(src0 + 0 * src0_stride); - __builtin_prefetch(src0 + 1 * src0_stride); - __builtin_prefetch(src1 + 0 * src1_stride); - __builtin_prefetch(src1 + 1 * src1_stride); - load_unaligned_u8_4x2(src0, src0_stride, &tmp0_32); - tmp0 = vreinterpret_u8_u32(tmp0_32); - load_unaligned_u8_4x2(src1, src1_stride, &tmp1_32); - tmp1 = vreinterpret_u8_u32(tmp1_32); - res = vmull_u8(m, tmp0); - res = vmlal_u8(res, max_minus_m, tmp1); - vst1_lane_u32( - (uint32_t *)(dst + (0 * dst_stride)), - vreinterpret_u32_u8(vrshrn_n_u16(res, AOM_BLEND_A64_ROUND_BITS)), 0); - vst1_lane_u32( - (uint32_t *)(dst + (1 * dst_stride)), - vreinterpret_u32_u8(vrshrn_n_u16(res, AOM_BLEND_A64_ROUND_BITS)), 1); - src0 += (2 * src0_stride); - src1 += (2 * src1_stride); - dst += (2 * dst_stride); - } - } else if (w == 2) { - const uint8x8_t m = vreinterpret_u8_u16(vld1_dup_u16((uint16_t *)mask)); - const uint8x8_t max_minus_m = vsub_u8(vdup_64, m); - for (int i = 0; i < h; i += 2) { - __builtin_prefetch(src0 + 0 * src0_stride); - __builtin_prefetch(src0 + 1 * src0_stride); - __builtin_prefetch(src1 + 0 * src1_stride); - __builtin_prefetch(src1 + 1 * src1_stride); - load_unaligned_u8_2x2(src0, src0_stride, &tmp0_16); - tmp0 = vreinterpret_u8_u16(tmp0_16); - load_unaligned_u8_2x2(src1, src1_stride, &tmp1_16); - tmp1 = vreinterpret_u8_u16(tmp1_16); - res = vmull_u8(m, tmp0); - res = vmlal_u8(res, max_minus_m, tmp1); - vst1_lane_u16( - (uint16_t *)(dst + (0 * dst_stride)), - vreinterpret_u16_u8(vrshrn_n_u16(res, AOM_BLEND_A64_ROUND_BITS)), 0); - vst1_lane_u16( - (uint16_t *)(dst + (1 * dst_stride)), - vreinterpret_u16_u8(vrshrn_n_u16(res, AOM_BLEND_A64_ROUND_BITS)), 1); - src0 += (2 * src0_stride); - src1 += (2 * src1_stride); - dst += (2 * dst_stride); - } - } -} diff --git a/third_party/aom/av1/common/arm/blend_a64_vmask_neon.c b/third_party/aom/av1/common/arm/blend_a64_vmask_neon.c deleted file mode 100644 index 194e94c8c..000000000 --- a/third_party/aom/av1/common/arm/blend_a64_vmask_neon.c +++ /dev/null @@ -1,141 +0,0 @@ -/* - * - * Copyright (c) 2018, Alliance for Open Media. All rights reserved - * - * This source code is subject to the terms of the BSD 2 Clause License and - * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License - * was not distributed with this source code in the LICENSE file, you can - * obtain it at www.aomedia.org/license/software. If the Alliance for Open - * Media Patent License 1.0 was not distributed with this source code in the - * PATENTS file, you can obtain it at www.aomedia.org/license/patent. - */ - -#include <arm_neon.h> -#include <assert.h> - -#include "aom/aom_integer.h" -#include "aom_dsp/blend.h" -#include "aom_ports/mem.h" -#include "av1/common/arm/mem_neon.h" -#include "aom_dsp/aom_dsp_common.h" -#include "config/aom_dsp_rtcd.h" - -void aom_blend_a64_vmask_neon(uint8_t *dst, uint32_t dst_stride, - const uint8_t *src0, uint32_t src0_stride, - const uint8_t *src1, uint32_t src1_stride, - const uint8_t *mask, int w, int h) { - uint8x8_t tmp0, tmp1; - uint8x16_t tmp0_q, tmp1_q, res_q; - uint16x8_t res, res_low, res_high; - uint32x2_t tmp0_32 = vdup_n_u32(0), tmp1_32 = vdup_n_u32(0); - uint16x4_t tmp0_16 = vdup_n_u16(0), tmp1_16 = vdup_n_u16(0); - assert(IMPLIES(src0 == dst, src0_stride == dst_stride)); - assert(IMPLIES(src1 == dst, src1_stride == dst_stride)); - - assert(h >= 2); - assert(w >= 2); - assert(IS_POWER_OF_TWO(h)); - assert(IS_POWER_OF_TWO(w)); - - if (w >= 16) { - for (int i = 0; i < h; ++i) { - const uint8x8_t m = vdup_n_u8((uint8_t)mask[i]); - const uint8x8_t max_minus_m = vdup_n_u8(64 - (uint8_t)mask[i]); - for (int j = 0; j < w; j += 16) { - __builtin_prefetch(src0); - __builtin_prefetch(src1); - tmp0_q = vld1q_u8(src0); - tmp1_q = vld1q_u8(src1); - res_low = vmull_u8(m, vget_low_u8(tmp0_q)); - res_low = vmlal_u8(res_low, max_minus_m, vget_low_u8(tmp1_q)); - res_high = vmull_u8(m, vget_high_u8(tmp0_q)); - res_high = vmlal_u8(res_high, max_minus_m, vget_high_u8(tmp1_q)); - res_q = vcombine_u8(vrshrn_n_u16(res_low, AOM_BLEND_A64_ROUND_BITS), - vrshrn_n_u16(res_high, AOM_BLEND_A64_ROUND_BITS)); - vst1q_u8(dst, res_q); - src0 += 16; - src1 += 16; - dst += 16; - } - src0 += src0_stride - w; - src1 += src1_stride - w; - dst += dst_stride - w; - } - } else if (w == 8) { - for (int i = 0; i < h; ++i) { - __builtin_prefetch(src0); - __builtin_prefetch(src1); - const uint8x8_t m = vdup_n_u8((uint8_t)mask[i]); - const uint8x8_t max_minus_m = vdup_n_u8(64 - (uint8_t)mask[i]); - tmp0 = vld1_u8(src0); - tmp1 = vld1_u8(src1); - res = vmull_u8(m, tmp0); - res = vmlal_u8(res, max_minus_m, tmp1); - vst1_u8(dst, vrshrn_n_u16(res, AOM_BLEND_A64_ROUND_BITS)); - src0 += src0_stride; - src1 += src1_stride; - dst += dst_stride; - } - } else if (w == 4) { - for (int i = 0; i < h; i += 2) { - __builtin_prefetch(src0 + 0 * src0_stride); - __builtin_prefetch(src0 + 1 * src0_stride); - __builtin_prefetch(src1 + 0 * src1_stride); - __builtin_prefetch(src1 + 1 * src1_stride); - const uint16x4_t m1 = vdup_n_u16((uint16_t)mask[i]); - const uint16x4_t m2 = vdup_n_u16((uint16_t)mask[i + 1]); - const uint8x8_t m = vmovn_u16(vcombine_u16(m1, m2)); - const uint16x4_t max_minus_m1 = vdup_n_u16(64 - (uint16_t)mask[i]); - const uint16x4_t max_minus_m2 = vdup_n_u16(64 - (uint16_t)mask[i + 1]); - const uint8x8_t max_minus_m = - vmovn_u16(vcombine_u16(max_minus_m1, max_minus_m2)); - load_unaligned_u8_4x2(src0, src0_stride, &tmp0_32); - tmp0 = vreinterpret_u8_u32(tmp0_32); - load_unaligned_u8_4x2(src1, src1_stride, &tmp1_32); - tmp1 = vreinterpret_u8_u32(tmp1_32); - res = vmull_u8(m, tmp0); - res = vmlal_u8(res, max_minus_m, tmp1); - vst1_lane_u32( - (uint32_t *)(dst + (0 * dst_stride)), - vreinterpret_u32_u8(vrshrn_n_u16(res, AOM_BLEND_A64_ROUND_BITS)), 0); - vst1_lane_u32( - (uint32_t *)(dst + (1 * dst_stride)), - vreinterpret_u32_u8(vrshrn_n_u16(res, AOM_BLEND_A64_ROUND_BITS)), 1); - src0 += (2 * src0_stride); - src1 += (2 * src1_stride); - dst += (2 * dst_stride); - } - } else if (w == 2) { - for (int i = 0; i < h; i += 2) { - __builtin_prefetch(src0 + 0 * src0_stride); - __builtin_prefetch(src0 + 1 * src0_stride); - __builtin_prefetch(src1 + 0 * src1_stride); - __builtin_prefetch(src1 + 1 * src1_stride); - const uint8x8_t m1 = vdup_n_u8(mask[i]); - const uint8x8_t m2 = vdup_n_u8(mask[i + 1]); - const uint16x4x2_t m_trn = - vtrn_u16(vreinterpret_u16_u8(m1), vreinterpret_u16_u8(m2)); - const uint8x8_t m = vreinterpret_u8_u16(m_trn.val[0]); - const uint8x8_t max_minus_m1 = vdup_n_u8(64 - mask[i]); - const uint8x8_t max_minus_m2 = vdup_n_u8(64 - mask[i + 1]); - const uint16x4x2_t max_minus_m_trn = vtrn_u16( - vreinterpret_u16_u8(max_minus_m1), vreinterpret_u16_u8(max_minus_m2)); - const uint8x8_t max_minus_m = vreinterpret_u8_u16(max_minus_m_trn.val[0]); - load_unaligned_u8_2x2(src0, src0_stride, &tmp0_16); - tmp0 = vreinterpret_u8_u16(tmp0_16); - load_unaligned_u8_2x2(src1, src1_stride, &tmp1_16); - tmp1 = vreinterpret_u8_u16(tmp1_16); - res = vmull_u8(m, tmp0); - res = vmlal_u8(res, max_minus_m, tmp1); - vst1_lane_u16( - (uint16_t *)(dst + (0 * dst_stride)), - vreinterpret_u16_u8(vrshrn_n_u16(res, AOM_BLEND_A64_ROUND_BITS)), 0); - vst1_lane_u16( - (uint16_t *)(dst + (1 * dst_stride)), - vreinterpret_u16_u8(vrshrn_n_u16(res, AOM_BLEND_A64_ROUND_BITS)), 1); - src0 += (2 * src0_stride); - src1 += (2 * src1_stride); - dst += (2 * dst_stride); - } - } -} diff --git a/third_party/aom/av1/common/arm/cfl_neon.c b/third_party/aom/av1/common/arm/cfl_neon.c deleted file mode 100644 index 39025b5e5..000000000 --- a/third_party/aom/av1/common/arm/cfl_neon.c +++ /dev/null @@ -1,584 +0,0 @@ -/* - * Copyright (c) 2017, Alliance for Open Media. All rights reserved - * - * This source code is subject to the terms of the BSD 2 Clause License and - * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License - * was not distributed with this source code in the LICENSE file, you can - * obtain it at www.aomedia.org/license/software. If the Alliance for Open - * Media Patent License 1.0 was not distributed with this source code in the - * PATENTS file, you can obtain it at www.aomedia.org/license/patent. - */ -#include <arm_neon.h> - -#include "config/av1_rtcd.h" - -#include "av1/common/cfl.h" - -static INLINE void vldsubstq_s16(int16_t *dst, const uint16_t *src, int offset, - int16x8_t sub) { - vst1q_s16(dst + offset, - vsubq_s16(vreinterpretq_s16_u16(vld1q_u16(src + offset)), sub)); -} - -static INLINE uint16x8_t vldaddq_u16(const uint16_t *buf, size_t offset) { - return vaddq_u16(vld1q_u16(buf), vld1q_u16(buf + offset)); -} - -// Load half of a vector and duplicated in other half -static INLINE uint8x8_t vldh_dup_u8(const uint8_t *ptr) { - return vreinterpret_u8_u32(vld1_dup_u32((const uint32_t *)ptr)); -} - -// Store half of a vector. -static INLINE void vsth_u16(uint16_t *ptr, uint16x4_t val) { - *((uint32_t *)ptr) = vreinterpret_u32_u16(val)[0]; -} - -// Store half of a vector. -static INLINE void vsth_u8(uint8_t *ptr, uint8x8_t val) { - *((uint32_t *)ptr) = vreinterpret_u32_u8(val)[0]; -} - -static void cfl_luma_subsampling_420_lbd_neon(const uint8_t *input, - int input_stride, - uint16_t *pred_buf_q3, int width, - int height) { - const uint16_t *end = pred_buf_q3 + (height >> 1) * CFL_BUF_LINE; - const int luma_stride = input_stride << 1; - do { - if (width == 4) { - const uint16x4_t top = vpaddl_u8(vldh_dup_u8(input)); - const uint16x4_t sum = vpadal_u8(top, vldh_dup_u8(input + input_stride)); - vsth_u16(pred_buf_q3, vshl_n_u16(sum, 1)); - } else if (width == 8) { - const uint16x4_t top = vpaddl_u8(vld1_u8(input)); - const uint16x4_t sum = vpadal_u8(top, vld1_u8(input + input_stride)); - vst1_u16(pred_buf_q3, vshl_n_u16(sum, 1)); - } else if (width == 16) { - const uint16x8_t top = vpaddlq_u8(vld1q_u8(input)); - const uint16x8_t sum = vpadalq_u8(top, vld1q_u8(input + input_stride)); - vst1q_u16(pred_buf_q3, vshlq_n_u16(sum, 1)); - } else { - const uint8x8x4_t top = vld4_u8(input); - const uint8x8x4_t bot = vld4_u8(input + input_stride); - // equivalent to a vpaddlq_u8 (because vld4q interleaves) - const uint16x8_t top_0 = vaddl_u8(top.val[0], top.val[1]); - // equivalent to a vpaddlq_u8 (because vld4q interleaves) - const uint16x8_t bot_0 = vaddl_u8(bot.val[0], bot.val[1]); - // equivalent to a vpaddlq_u8 (because vld4q interleaves) - const uint16x8_t top_1 = vaddl_u8(top.val[2], top.val[3]); - // equivalent to a vpaddlq_u8 (because vld4q interleaves) - const uint16x8_t bot_1 = vaddl_u8(bot.val[2], bot.val[3]); - uint16x8x2_t sum; - sum.val[0] = vshlq_n_u16(vaddq_u16(top_0, bot_0), 1); - sum.val[1] = vshlq_n_u16(vaddq_u16(top_1, bot_1), 1); - vst2q_u16(pred_buf_q3, sum); - } - input += luma_stride; - } while ((pred_buf_q3 += CFL_BUF_LINE) < end); -} - -static void cfl_luma_subsampling_422_lbd_neon(const uint8_t *input, - int input_stride, - uint16_t *pred_buf_q3, int width, - int height) { - const uint16_t *end = pred_buf_q3 + height * CFL_BUF_LINE; - do { - if (width == 4) { - const uint16x4_t top = vpaddl_u8(vldh_dup_u8(input)); - vsth_u16(pred_buf_q3, vshl_n_u16(top, 2)); - } else if (width == 8) { - const uint16x4_t top = vpaddl_u8(vld1_u8(input)); - vst1_u16(pred_buf_q3, vshl_n_u16(top, 2)); - } else if (width == 16) { - const uint16x8_t top = vpaddlq_u8(vld1q_u8(input)); - vst1q_u16(pred_buf_q3, vshlq_n_u16(top, 2)); - } else { - const uint8x8x4_t top = vld4_u8(input); - uint16x8x2_t sum; - // vaddl_u8 is equivalent to a vpaddlq_u8 (because vld4q interleaves) - sum.val[0] = vshlq_n_u16(vaddl_u8(top.val[0], top.val[1]), 2); - sum.val[1] = vshlq_n_u16(vaddl_u8(top.val[2], top.val[3]), 2); - vst2q_u16(pred_buf_q3, sum); - } - input += input_stride; - } while ((pred_buf_q3 += CFL_BUF_LINE) < end); -} - -static void cfl_luma_subsampling_444_lbd_neon(const uint8_t *input, - int input_stride, - uint16_t *pred_buf_q3, int width, - int height) { - const uint16_t *end = pred_buf_q3 + height * CFL_BUF_LINE; - do { - if (width == 4) { - const uint16x8_t top = vshll_n_u8(vldh_dup_u8(input), 3); - vst1_u16(pred_buf_q3, vget_low_u16(top)); - } else if (width == 8) { - const uint16x8_t top = vshll_n_u8(vld1_u8(input), 3); - vst1q_u16(pred_buf_q3, top); - } else { - const uint8x16_t top = vld1q_u8(input); - vst1q_u16(pred_buf_q3, vshll_n_u8(vget_low_u8(top), 3)); - vst1q_u16(pred_buf_q3 + 8, vshll_n_u8(vget_high_u8(top), 3)); - if (width == 32) { - const uint8x16_t next_top = vld1q_u8(input + 16); - vst1q_u16(pred_buf_q3 + 16, vshll_n_u8(vget_low_u8(next_top), 3)); - vst1q_u16(pred_buf_q3 + 24, vshll_n_u8(vget_high_u8(next_top), 3)); - } - } - input += input_stride; - } while ((pred_buf_q3 += CFL_BUF_LINE) < end); -} - -#ifndef __aarch64__ -uint16x8_t vpaddq_u16(uint16x8_t a, uint16x8_t b) { - return vcombine_u16(vpadd_u16(vget_low_u16(a), vget_high_u16(a)), - vpadd_u16(vget_low_u16(b), vget_high_u16(b))); -} -#endif - -static void cfl_luma_subsampling_420_hbd_neon(const uint16_t *input, - int input_stride, - uint16_t *pred_buf_q3, int width, - int height) { - const uint16_t *end = pred_buf_q3 + (height >> 1) * CFL_BUF_LINE; - const int luma_stride = input_stride << 1; - do { - if (width == 4) { - const uint16x4_t top = vld1_u16(input); - const uint16x4_t bot = vld1_u16(input + input_stride); - const uint16x4_t sum = vadd_u16(top, bot); - const uint16x4_t hsum = vpadd_u16(sum, sum); - vsth_u16(pred_buf_q3, vshl_n_u16(hsum, 1)); - } else if (width < 32) { - const uint16x8_t top = vld1q_u16(input); - const uint16x8_t bot = vld1q_u16(input + input_stride); - const uint16x8_t sum = vaddq_u16(top, bot); - if (width == 8) { - const uint16x4_t hsum = vget_low_u16(vpaddq_u16(sum, sum)); - vst1_u16(pred_buf_q3, vshl_n_u16(hsum, 1)); - } else { - const uint16x8_t top_1 = vld1q_u16(input + 8); - const uint16x8_t bot_1 = vld1q_u16(input + 8 + input_stride); - const uint16x8_t sum_1 = vaddq_u16(top_1, bot_1); - const uint16x8_t hsum = vpaddq_u16(sum, sum_1); - vst1q_u16(pred_buf_q3, vshlq_n_u16(hsum, 1)); - } - } else { - const uint16x8x4_t top = vld4q_u16(input); - const uint16x8x4_t bot = vld4q_u16(input + input_stride); - // equivalent to a vpaddq_u16 (because vld4q interleaves) - const uint16x8_t top_0 = vaddq_u16(top.val[0], top.val[1]); - // equivalent to a vpaddq_u16 (because vld4q interleaves) - const uint16x8_t bot_0 = vaddq_u16(bot.val[0], bot.val[1]); - // equivalent to a vpaddq_u16 (because vld4q interleaves) - const uint16x8_t top_1 = vaddq_u16(top.val[2], top.val[3]); - // equivalent to a vpaddq_u16 (because vld4q interleaves) - const uint16x8_t bot_1 = vaddq_u16(bot.val[2], bot.val[3]); - uint16x8x2_t sum; - sum.val[0] = vshlq_n_u16(vaddq_u16(top_0, bot_0), 1); - sum.val[1] = vshlq_n_u16(vaddq_u16(top_1, bot_1), 1); - vst2q_u16(pred_buf_q3, sum); - } - input += luma_stride; - } while ((pred_buf_q3 += CFL_BUF_LINE) < end); -} - -static void cfl_luma_subsampling_422_hbd_neon(const uint16_t *input, - int input_stride, - uint16_t *pred_buf_q3, int width, - int height) { - const uint16_t *end = pred_buf_q3 + height * CFL_BUF_LINE; - do { - if (width == 4) { - const uint16x4_t top = vld1_u16(input); - const uint16x4_t hsum = vpadd_u16(top, top); - vsth_u16(pred_buf_q3, vshl_n_u16(hsum, 2)); - } else if (width == 8) { - const uint16x4x2_t top = vld2_u16(input); - // equivalent to a vpadd_u16 (because vld2 interleaves) - const uint16x4_t hsum = vadd_u16(top.val[0], top.val[1]); - vst1_u16(pred_buf_q3, vshl_n_u16(hsum, 2)); - } else if (width == 16) { - const uint16x8x2_t top = vld2q_u16(input); - // equivalent to a vpaddq_u16 (because vld2q interleaves) - const uint16x8_t hsum = vaddq_u16(top.val[0], top.val[1]); - vst1q_u16(pred_buf_q3, vshlq_n_u16(hsum, 2)); - } else { - const uint16x8x4_t top = vld4q_u16(input); - // equivalent to a vpaddq_u16 (because vld4q interleaves) - const uint16x8_t hsum_0 = vaddq_u16(top.val[0], top.val[1]); - // equivalent to a vpaddq_u16 (because vld4q interleaves) - const uint16x8_t hsum_1 = vaddq_u16(top.val[2], top.val[3]); - uint16x8x2_t result = { { vshlq_n_u16(hsum_0, 2), - vshlq_n_u16(hsum_1, 2) } }; - vst2q_u16(pred_buf_q3, result); - } - input += input_stride; - } while ((pred_buf_q3 += CFL_BUF_LINE) < end); -} - -static void cfl_luma_subsampling_444_hbd_neon(const uint16_t *input, - int input_stride, - uint16_t *pred_buf_q3, int width, - int height) { - const uint16_t *end = pred_buf_q3 + height * CFL_BUF_LINE; - do { - if (width == 4) { - const uint16x4_t top = vld1_u16(input); - vst1_u16(pred_buf_q3, vshl_n_u16(top, 3)); - } else if (width == 8) { - const uint16x8_t top = vld1q_u16(input); - vst1q_u16(pred_buf_q3, vshlq_n_u16(top, 3)); - } else if (width == 16) { - uint16x8x2_t top = vld2q_u16(input); - top.val[0] = vshlq_n_u16(top.val[0], 3); - top.val[1] = vshlq_n_u16(top.val[1], 3); - vst2q_u16(pred_buf_q3, top); - } else { - uint16x8x4_t top = vld4q_u16(input); - top.val[0] = vshlq_n_u16(top.val[0], 3); - top.val[1] = vshlq_n_u16(top.val[1], 3); - top.val[2] = vshlq_n_u16(top.val[2], 3); - top.val[3] = vshlq_n_u16(top.val[3], 3); - vst4q_u16(pred_buf_q3, top); - } - input += input_stride; - } while ((pred_buf_q3 += CFL_BUF_LINE) < end); -} - -CFL_GET_SUBSAMPLE_FUNCTION(neon) - -static INLINE void subtract_average_neon(const uint16_t *src, int16_t *dst, - int width, int height, - int round_offset, - const int num_pel_log2) { - const uint16_t *const end = src + height * CFL_BUF_LINE; - - // Round offset is not needed, because NEON will handle the rounding. - (void)round_offset; - - // To optimize the use of the CPU pipeline, we process 4 rows per iteration - const int step = 4 * CFL_BUF_LINE; - - // At this stage, the prediction buffer contains scaled reconstructed luma - // pixels, which are positive integer and only require 15 bits. By using - // unsigned integer for the sum, we can do one addition operation inside 16 - // bits (8 lanes) before having to convert to 32 bits (4 lanes). - const uint16_t *sum_buf = src; - uint32x4_t sum_32x4 = { 0, 0, 0, 0 }; - do { - // For all widths, we load, add and combine the data so it fits in 4 lanes. - if (width == 4) { - const uint16x4_t a0 = - vadd_u16(vld1_u16(sum_buf), vld1_u16(sum_buf + CFL_BUF_LINE)); - const uint16x4_t a1 = vadd_u16(vld1_u16(sum_buf + 2 * CFL_BUF_LINE), - vld1_u16(sum_buf + 3 * CFL_BUF_LINE)); - sum_32x4 = vaddq_u32(sum_32x4, vaddl_u16(a0, a1)); - } else if (width == 8) { - const uint16x8_t a0 = vldaddq_u16(sum_buf, CFL_BUF_LINE); - const uint16x8_t a1 = - vldaddq_u16(sum_buf + 2 * CFL_BUF_LINE, CFL_BUF_LINE); - sum_32x4 = vpadalq_u16(sum_32x4, a0); - sum_32x4 = vpadalq_u16(sum_32x4, a1); - } else { - const uint16x8_t row0 = vldaddq_u16(sum_buf, 8); - const uint16x8_t row1 = vldaddq_u16(sum_buf + CFL_BUF_LINE, 8); - const uint16x8_t row2 = vldaddq_u16(sum_buf + 2 * CFL_BUF_LINE, 8); - const uint16x8_t row3 = vldaddq_u16(sum_buf + 3 * CFL_BUF_LINE, 8); - sum_32x4 = vpadalq_u16(sum_32x4, row0); - sum_32x4 = vpadalq_u16(sum_32x4, row1); - sum_32x4 = vpadalq_u16(sum_32x4, row2); - sum_32x4 = vpadalq_u16(sum_32x4, row3); - - if (width == 32) { - const uint16x8_t row0_1 = vldaddq_u16(sum_buf + 16, 8); - const uint16x8_t row1_1 = vldaddq_u16(sum_buf + CFL_BUF_LINE + 16, 8); - const uint16x8_t row2_1 = - vldaddq_u16(sum_buf + 2 * CFL_BUF_LINE + 16, 8); - const uint16x8_t row3_1 = - vldaddq_u16(sum_buf + 3 * CFL_BUF_LINE + 16, 8); - - sum_32x4 = vpadalq_u16(sum_32x4, row0_1); - sum_32x4 = vpadalq_u16(sum_32x4, row1_1); - sum_32x4 = vpadalq_u16(sum_32x4, row2_1); - sum_32x4 = vpadalq_u16(sum_32x4, row3_1); - } - } - sum_buf += step; - } while (sum_buf < end); - - // Permute and add in such a way that each lane contains the block sum. - // [A+C+B+D, B+D+A+C, C+A+D+B, D+B+C+A] -#ifdef __aarch64__ - sum_32x4 = vpaddq_u32(sum_32x4, sum_32x4); - sum_32x4 = vpaddq_u32(sum_32x4, sum_32x4); -#else - uint32x4_t flip = - vcombine_u32(vget_high_u32(sum_32x4), vget_low_u32(sum_32x4)); - sum_32x4 = vaddq_u32(sum_32x4, flip); - sum_32x4 = vaddq_u32(sum_32x4, vrev64q_u32(sum_32x4)); -#endif - - // Computing the average could be done using scalars, but getting off the NEON - // engine introduces latency, so we use vqrshrn. - int16x4_t avg_16x4; - // Constant propagation makes for some ugly code. - switch (num_pel_log2) { - case 4: avg_16x4 = vreinterpret_s16_u16(vqrshrn_n_u32(sum_32x4, 4)); break; - case 5: avg_16x4 = vreinterpret_s16_u16(vqrshrn_n_u32(sum_32x4, 5)); break; - case 6: avg_16x4 = vreinterpret_s16_u16(vqrshrn_n_u32(sum_32x4, 6)); break; - case 7: avg_16x4 = vreinterpret_s16_u16(vqrshrn_n_u32(sum_32x4, 7)); break; - case 8: avg_16x4 = vreinterpret_s16_u16(vqrshrn_n_u32(sum_32x4, 8)); break; - case 9: avg_16x4 = vreinterpret_s16_u16(vqrshrn_n_u32(sum_32x4, 9)); break; - case 10: - avg_16x4 = vreinterpret_s16_u16(vqrshrn_n_u32(sum_32x4, 10)); - break; - default: assert(0); - } - - if (width == 4) { - do { - vst1_s16(dst, vsub_s16(vreinterpret_s16_u16(vld1_u16(src)), avg_16x4)); - src += CFL_BUF_LINE; - dst += CFL_BUF_LINE; - } while (src < end); - } else { - const int16x8_t avg_16x8 = vcombine_s16(avg_16x4, avg_16x4); - do { - vldsubstq_s16(dst, src, 0, avg_16x8); - vldsubstq_s16(dst, src, CFL_BUF_LINE, avg_16x8); - vldsubstq_s16(dst, src, 2 * CFL_BUF_LINE, avg_16x8); - vldsubstq_s16(dst, src, 3 * CFL_BUF_LINE, avg_16x8); - - if (width > 8) { - vldsubstq_s16(dst, src, 8, avg_16x8); - vldsubstq_s16(dst, src, 8 + CFL_BUF_LINE, avg_16x8); - vldsubstq_s16(dst, src, 8 + 2 * CFL_BUF_LINE, avg_16x8); - vldsubstq_s16(dst, src, 8 + 3 * CFL_BUF_LINE, avg_16x8); - } - if (width == 32) { - vldsubstq_s16(dst, src, 16, avg_16x8); - vldsubstq_s16(dst, src, 16 + CFL_BUF_LINE, avg_16x8); - vldsubstq_s16(dst, src, 16 + 2 * CFL_BUF_LINE, avg_16x8); - vldsubstq_s16(dst, src, 16 + 3 * CFL_BUF_LINE, avg_16x8); - vldsubstq_s16(dst, src, 24, avg_16x8); - vldsubstq_s16(dst, src, 24 + CFL_BUF_LINE, avg_16x8); - vldsubstq_s16(dst, src, 24 + 2 * CFL_BUF_LINE, avg_16x8); - vldsubstq_s16(dst, src, 24 + 3 * CFL_BUF_LINE, avg_16x8); - } - src += step; - dst += step; - } while (src < end); - } -} - -CFL_SUB_AVG_FN(neon) - -// Saturating negate 16-bit integers in a when the corresponding signed 16-bit -// integer in b is negative. -// Notes: -// * Negating INT16_MIN results in INT16_MIN. However, this cannot occur in -// practice, as scaled_luma is the multiplication of two absolute values. -// * In the Intel equivalent, elements in a are zeroed out when the -// corresponding elements in b are zero. Because vsign is used twice in a -// row, with b in the first call becoming a in the second call, there's no -// impact from not zeroing out. -static int16x4_t vsign_s16(int16x4_t a, int16x4_t b) { - const int16x4_t mask = vshr_n_s16(b, 15); - return veor_s16(vadd_s16(a, mask), mask); -} - -// Saturating negate 16-bit integers in a when the corresponding signed 16-bit -// integer in b is negative. -// Notes: -// * Negating INT16_MIN results in INT16_MIN. However, this cannot occur in -// practice, as scaled_luma is the multiplication of two absolute values. -// * In the Intel equivalent, elements in a are zeroed out when the -// corresponding elements in b are zero. Because vsignq is used twice in a -// row, with b in the first call becoming a in the second call, there's no -// impact from not zeroing out. -static int16x8_t vsignq_s16(int16x8_t a, int16x8_t b) { - const int16x8_t mask = vshrq_n_s16(b, 15); - return veorq_s16(vaddq_s16(a, mask), mask); -} - -static INLINE int16x4_t predict_w4(const int16_t *pred_buf_q3, - int16x4_t alpha_sign, int abs_alpha_q12, - int16x4_t dc) { - const int16x4_t ac_q3 = vld1_s16(pred_buf_q3); - const int16x4_t ac_sign = veor_s16(alpha_sign, ac_q3); - int16x4_t scaled_luma = vqrdmulh_n_s16(vabs_s16(ac_q3), abs_alpha_q12); - return vadd_s16(vsign_s16(scaled_luma, ac_sign), dc); -} - -static INLINE int16x8_t predict_w8(const int16_t *pred_buf_q3, - int16x8_t alpha_sign, int abs_alpha_q12, - int16x8_t dc) { - const int16x8_t ac_q3 = vld1q_s16(pred_buf_q3); - const int16x8_t ac_sign = veorq_s16(alpha_sign, ac_q3); - int16x8_t scaled_luma = vqrdmulhq_n_s16(vabsq_s16(ac_q3), abs_alpha_q12); - return vaddq_s16(vsignq_s16(scaled_luma, ac_sign), dc); -} - -static INLINE int16x8x2_t predict_w16(const int16_t *pred_buf_q3, - int16x8_t alpha_sign, int abs_alpha_q12, - int16x8_t dc) { - // vld2q_s16 interleaves, which is not useful for prediction. vst1q_s16_x2 - // does not interleave, but is not currently available in the compilier used - // by the AOM build system. - const int16x8x2_t ac_q3 = vld2q_s16(pred_buf_q3); - const int16x8_t ac_sign_0 = veorq_s16(alpha_sign, ac_q3.val[0]); - const int16x8_t ac_sign_1 = veorq_s16(alpha_sign, ac_q3.val[1]); - const int16x8_t scaled_luma_0 = - vqrdmulhq_n_s16(vabsq_s16(ac_q3.val[0]), abs_alpha_q12); - const int16x8_t scaled_luma_1 = - vqrdmulhq_n_s16(vabsq_s16(ac_q3.val[1]), abs_alpha_q12); - int16x8x2_t result; - result.val[0] = vaddq_s16(vsignq_s16(scaled_luma_0, ac_sign_0), dc); - result.val[1] = vaddq_s16(vsignq_s16(scaled_luma_1, ac_sign_1), dc); - return result; -} - -static INLINE int16x8x4_t predict_w32(const int16_t *pred_buf_q3, - int16x8_t alpha_sign, int abs_alpha_q12, - int16x8_t dc) { - // vld4q_s16 interleaves, which is not useful for prediction. vst1q_s16_x4 - // does not interleave, but is not currently available in the compilier used - // by the AOM build system. - const int16x8x4_t ac_q3 = vld4q_s16(pred_buf_q3); - const int16x8_t ac_sign_0 = veorq_s16(alpha_sign, ac_q3.val[0]); - const int16x8_t ac_sign_1 = veorq_s16(alpha_sign, ac_q3.val[1]); - const int16x8_t ac_sign_2 = veorq_s16(alpha_sign, ac_q3.val[2]); - const int16x8_t ac_sign_3 = veorq_s16(alpha_sign, ac_q3.val[3]); - const int16x8_t scaled_luma_0 = - vqrdmulhq_n_s16(vabsq_s16(ac_q3.val[0]), abs_alpha_q12); - const int16x8_t scaled_luma_1 = - vqrdmulhq_n_s16(vabsq_s16(ac_q3.val[1]), abs_alpha_q12); - const int16x8_t scaled_luma_2 = - vqrdmulhq_n_s16(vabsq_s16(ac_q3.val[2]), abs_alpha_q12); - const int16x8_t scaled_luma_3 = - vqrdmulhq_n_s16(vabsq_s16(ac_q3.val[3]), abs_alpha_q12); - int16x8x4_t result; - result.val[0] = vaddq_s16(vsignq_s16(scaled_luma_0, ac_sign_0), dc); - result.val[1] = vaddq_s16(vsignq_s16(scaled_luma_1, ac_sign_1), dc); - result.val[2] = vaddq_s16(vsignq_s16(scaled_luma_2, ac_sign_2), dc); - result.val[3] = vaddq_s16(vsignq_s16(scaled_luma_3, ac_sign_3), dc); - return result; -} - -static INLINE void cfl_predict_lbd_neon(const int16_t *pred_buf_q3, - uint8_t *dst, int dst_stride, - int alpha_q3, int width, int height) { - const int16_t abs_alpha_q12 = abs(alpha_q3) << 9; - const int16_t *const end = pred_buf_q3 + height * CFL_BUF_LINE; - if (width == 4) { - const int16x4_t alpha_sign = vdup_n_s16(alpha_q3); - const int16x4_t dc = vdup_n_s16(*dst); - do { - const int16x4_t pred = - predict_w4(pred_buf_q3, alpha_sign, abs_alpha_q12, dc); - vsth_u8(dst, vqmovun_s16(vcombine_s16(pred, pred))); - dst += dst_stride; - } while ((pred_buf_q3 += CFL_BUF_LINE) < end); - } else { - const int16x8_t alpha_sign = vdupq_n_s16(alpha_q3); - const int16x8_t dc = vdupq_n_s16(*dst); - do { - if (width == 8) { - vst1_u8(dst, vqmovun_s16(predict_w8(pred_buf_q3, alpha_sign, - abs_alpha_q12, dc))); - } else if (width == 16) { - const int16x8x2_t pred = - predict_w16(pred_buf_q3, alpha_sign, abs_alpha_q12, dc); - const uint8x8x2_t predun = { { vqmovun_s16(pred.val[0]), - vqmovun_s16(pred.val[1]) } }; - vst2_u8(dst, predun); - } else { - const int16x8x4_t pred = - predict_w32(pred_buf_q3, alpha_sign, abs_alpha_q12, dc); - const uint8x8x4_t predun = { - { vqmovun_s16(pred.val[0]), vqmovun_s16(pred.val[1]), - vqmovun_s16(pred.val[2]), vqmovun_s16(pred.val[3]) } - }; - vst4_u8(dst, predun); - } - dst += dst_stride; - } while ((pred_buf_q3 += CFL_BUF_LINE) < end); - } -} - -CFL_PREDICT_FN(neon, lbd) - -static INLINE uint16x4_t clamp_s16(int16x4_t a, int16x4_t max) { - return vreinterpret_u16_s16(vmax_s16(vmin_s16(a, max), vdup_n_s16(0))); -} - -static INLINE uint16x8_t clampq_s16(int16x8_t a, int16x8_t max) { - return vreinterpretq_u16_s16(vmaxq_s16(vminq_s16(a, max), vdupq_n_s16(0))); -} - -static INLINE uint16x8x2_t clamp2q_s16(int16x8x2_t a, int16x8_t max) { - uint16x8x2_t result; - result.val[0] = vreinterpretq_u16_s16( - vmaxq_s16(vminq_s16(a.val[0], max), vdupq_n_s16(0))); - result.val[1] = vreinterpretq_u16_s16( - vmaxq_s16(vminq_s16(a.val[1], max), vdupq_n_s16(0))); - return result; -} - -static INLINE uint16x8x4_t clamp4q_s16(int16x8x4_t a, int16x8_t max) { - uint16x8x4_t result; - result.val[0] = vreinterpretq_u16_s16( - vmaxq_s16(vminq_s16(a.val[0], max), vdupq_n_s16(0))); - result.val[1] = vreinterpretq_u16_s16( - vmaxq_s16(vminq_s16(a.val[1], max), vdupq_n_s16(0))); - result.val[2] = vreinterpretq_u16_s16( - vmaxq_s16(vminq_s16(a.val[2], max), vdupq_n_s16(0))); - result.val[3] = vreinterpretq_u16_s16( - vmaxq_s16(vminq_s16(a.val[3], max), vdupq_n_s16(0))); - return result; -} - -static INLINE void cfl_predict_hbd_neon(const int16_t *pred_buf_q3, - uint16_t *dst, int dst_stride, - int alpha_q3, int bd, int width, - int height) { - const int max = (1 << bd) - 1; - const int16_t abs_alpha_q12 = abs(alpha_q3) << 9; - const int16_t *const end = pred_buf_q3 + height * CFL_BUF_LINE; - if (width == 4) { - const int16x4_t alpha_sign = vdup_n_s16(alpha_q3); - const int16x4_t dc = vdup_n_s16(*dst); - const int16x4_t max_16x4 = vdup_n_s16(max); - do { - const int16x4_t scaled_luma = - predict_w4(pred_buf_q3, alpha_sign, abs_alpha_q12, dc); - vst1_u16(dst, clamp_s16(scaled_luma, max_16x4)); - dst += dst_stride; - } while ((pred_buf_q3 += CFL_BUF_LINE) < end); - } else { - const int16x8_t alpha_sign = vdupq_n_s16(alpha_q3); - const int16x8_t dc = vdupq_n_s16(*dst); - const int16x8_t max_16x8 = vdupq_n_s16(max); - do { - if (width == 8) { - const int16x8_t pred = - predict_w8(pred_buf_q3, alpha_sign, abs_alpha_q12, dc); - vst1q_u16(dst, clampq_s16(pred, max_16x8)); - } else if (width == 16) { - const int16x8x2_t pred = - predict_w16(pred_buf_q3, alpha_sign, abs_alpha_q12, dc); - vst2q_u16(dst, clamp2q_s16(pred, max_16x8)); - } else { - const int16x8x4_t pred = - predict_w32(pred_buf_q3, alpha_sign, abs_alpha_q12, dc); - vst4q_u16(dst, clamp4q_s16(pred, max_16x8)); - } - dst += dst_stride; - } while ((pred_buf_q3 += CFL_BUF_LINE) < end); - } -} - -CFL_PREDICT_FN(neon, hbd) diff --git a/third_party/aom/av1/common/arm/convolve_neon.c b/third_party/aom/av1/common/arm/convolve_neon.c deleted file mode 100644 index d0c4f8ff6..000000000 --- a/third_party/aom/av1/common/arm/convolve_neon.c +++ /dev/null @@ -1,1455 +0,0 @@ -/* - * - * Copyright (c) 2018, Alliance for Open Media. All rights reserved - * - * This source code is subject to the terms of the BSD 2 Clause License and - * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License - * was not distributed with this source code in the LICENSE file, you can - * obtain it at www.aomedia.org/license/software. If the Alliance for Open - * Media Patent License 1.0 was not distributed with this source code in the - * PATENTS file, you can obtain it at www.aomedia.org/license/patent. - */ - -#include <assert.h> -#include <arm_neon.h> - -#include "config/av1_rtcd.h" - -#include "aom_dsp/aom_dsp_common.h" -#include "aom_ports/mem.h" -#include "av1/common/convolve.h" -#include "av1/common/filter.h" -#include "av1/common/arm/convolve_neon.h" -#include "av1/common/arm/mem_neon.h" -#include "av1/common/arm/transpose_neon.h" - -static INLINE int16x4_t convolve8_4x4(const int16x4_t s0, const int16x4_t s1, - const int16x4_t s2, const int16x4_t s3, - const int16x4_t s4, const int16x4_t s5, - const int16x4_t s6, const int16x4_t s7, - const int16_t *filter) { - int16x4_t sum; - - sum = vmul_n_s16(s0, filter[0]); - sum = vmla_n_s16(sum, s1, filter[1]); - sum = vmla_n_s16(sum, s2, filter[2]); - sum = vmla_n_s16(sum, s5, filter[5]); - sum = vmla_n_s16(sum, s6, filter[6]); - sum = vmla_n_s16(sum, s7, filter[7]); - /* filter[3] can take a max value of 128. So the max value of the result : - * 128*255 + sum > 16 bits - */ - sum = vqadd_s16(sum, vmul_n_s16(s3, filter[3])); - sum = vqadd_s16(sum, vmul_n_s16(s4, filter[4])); - - return sum; -} - -static INLINE uint8x8_t convolve8_horiz_8x8( - const int16x8_t s0, const int16x8_t s1, const int16x8_t s2, - const int16x8_t s3, const int16x8_t s4, const int16x8_t s5, - const int16x8_t s6, const int16x8_t s7, const int16_t *filter, - const int16x8_t shift_round_0, const int16x8_t shift_by_bits) { - int16x8_t sum; - - sum = vmulq_n_s16(s0, filter[0]); - sum = vmlaq_n_s16(sum, s1, filter[1]); - sum = vmlaq_n_s16(sum, s2, filter[2]); - sum = vmlaq_n_s16(sum, s5, filter[5]); - sum = vmlaq_n_s16(sum, s6, filter[6]); - sum = vmlaq_n_s16(sum, s7, filter[7]); - /* filter[3] can take a max value of 128. So the max value of the result : - * 128*255 + sum > 16 bits - */ - sum = vqaddq_s16(sum, vmulq_n_s16(s3, filter[3])); - sum = vqaddq_s16(sum, vmulq_n_s16(s4, filter[4])); - - sum = vqrshlq_s16(sum, shift_round_0); - sum = vqrshlq_s16(sum, shift_by_bits); - - return vqmovun_s16(sum); -} - -#if !defined(__aarch64__) -static INLINE uint8x8_t convolve8_horiz_4x1( - const int16x4_t s0, const int16x4_t s1, const int16x4_t s2, - const int16x4_t s3, const int16x4_t s4, const int16x4_t s5, - const int16x4_t s6, const int16x4_t s7, const int16_t *filter, - const int16x4_t shift_round_0, const int16x4_t shift_by_bits) { - int16x4_t sum; - - sum = vmul_n_s16(s0, filter[0]); - sum = vmla_n_s16(sum, s1, filter[1]); - sum = vmla_n_s16(sum, s2, filter[2]); - sum = vmla_n_s16(sum, s5, filter[5]); - sum = vmla_n_s16(sum, s6, filter[6]); - sum = vmla_n_s16(sum, s7, filter[7]); - /* filter[3] can take a max value of 128. So the max value of the result : - * 128*255 + sum > 16 bits - */ - sum = vqadd_s16(sum, vmul_n_s16(s3, filter[3])); - sum = vqadd_s16(sum, vmul_n_s16(s4, filter[4])); - - sum = vqrshl_s16(sum, shift_round_0); - sum = vqrshl_s16(sum, shift_by_bits); - - return vqmovun_s16(vcombine_s16(sum, sum)); -} -#endif // !defined(__arch64__) - -static INLINE uint8x8_t convolve8_vert_8x4( - const int16x8_t s0, const int16x8_t s1, const int16x8_t s2, - const int16x8_t s3, const int16x8_t s4, const int16x8_t s5, - const int16x8_t s6, const int16x8_t s7, const int16_t *filter) { - int16x8_t sum; - - sum = vmulq_n_s16(s0, filter[0]); - sum = vmlaq_n_s16(sum, s1, filter[1]); - sum = vmlaq_n_s16(sum, s2, filter[2]); - sum = vmlaq_n_s16(sum, s5, filter[5]); - sum = vmlaq_n_s16(sum, s6, filter[6]); - sum = vmlaq_n_s16(sum, s7, filter[7]); - /* filter[3] can take a max value of 128. So the max value of the result : - * 128*255 + sum > 16 bits - */ - sum = vqaddq_s16(sum, vmulq_n_s16(s3, filter[3])); - sum = vqaddq_s16(sum, vmulq_n_s16(s4, filter[4])); - - return vqrshrun_n_s16(sum, FILTER_BITS); -} - -static INLINE uint16x4_t convolve8_vert_4x4_s32( - const int16x4_t s0, const int16x4_t s1, const int16x4_t s2, - const int16x4_t s3, const int16x4_t s4, const int16x4_t s5, - const int16x4_t s6, const int16x4_t s7, const int16_t *y_filter, - const int32x4_t round_shift_vec, const int32x4_t offset_const, - const int32x4_t sub_const_vec) { - int32x4_t sum0; - uint16x4_t res; - const int32x4_t zero = vdupq_n_s32(0); - - sum0 = vmull_n_s16(s0, y_filter[0]); - sum0 = vmlal_n_s16(sum0, s1, y_filter[1]); - sum0 = vmlal_n_s16(sum0, s2, y_filter[2]); - sum0 = vmlal_n_s16(sum0, s3, y_filter[3]); - sum0 = vmlal_n_s16(sum0, s4, y_filter[4]); - sum0 = vmlal_n_s16(sum0, s5, y_filter[5]); - sum0 = vmlal_n_s16(sum0, s6, y_filter[6]); - sum0 = vmlal_n_s16(sum0, s7, y_filter[7]); - - sum0 = vaddq_s32(sum0, offset_const); - sum0 = vqrshlq_s32(sum0, round_shift_vec); - sum0 = vsubq_s32(sum0, sub_const_vec); - sum0 = vmaxq_s32(sum0, zero); - - res = vmovn_u32(vreinterpretq_u32_s32(sum0)); - - return res; -} - -static INLINE uint8x8_t convolve8_vert_8x4_s32( - const int16x8_t s0, const int16x8_t s1, const int16x8_t s2, - const int16x8_t s3, const int16x8_t s4, const int16x8_t s5, - const int16x8_t s6, const int16x8_t s7, const int16_t *y_filter, - const int32x4_t round_shift_vec, const int32x4_t offset_const, - const int32x4_t sub_const_vec, const int16x8_t vec_round_bits) { - int32x4_t sum0, sum1; - uint16x8_t res; - const int32x4_t zero = vdupq_n_s32(0); - - sum0 = vmull_n_s16(vget_low_s16(s0), y_filter[0]); - sum0 = vmlal_n_s16(sum0, vget_low_s16(s1), y_filter[1]); - sum0 = vmlal_n_s16(sum0, vget_low_s16(s2), y_filter[2]); - sum0 = vmlal_n_s16(sum0, vget_low_s16(s3), y_filter[3]); - sum0 = vmlal_n_s16(sum0, vget_low_s16(s4), y_filter[4]); - sum0 = vmlal_n_s16(sum0, vget_low_s16(s5), y_filter[5]); - sum0 = vmlal_n_s16(sum0, vget_low_s16(s6), y_filter[6]); - sum0 = vmlal_n_s16(sum0, vget_low_s16(s7), y_filter[7]); - - sum1 = vmull_n_s16(vget_high_s16(s0), y_filter[0]); - sum1 = vmlal_n_s16(sum1, vget_high_s16(s1), y_filter[1]); - sum1 = vmlal_n_s16(sum1, vget_high_s16(s2), y_filter[2]); - sum1 = vmlal_n_s16(sum1, vget_high_s16(s3), y_filter[3]); - sum1 = vmlal_n_s16(sum1, vget_high_s16(s4), y_filter[4]); - sum1 = vmlal_n_s16(sum1, vget_high_s16(s5), y_filter[5]); - sum1 = vmlal_n_s16(sum1, vget_high_s16(s6), y_filter[6]); - sum1 = vmlal_n_s16(sum1, vget_high_s16(s7), y_filter[7]); - - sum0 = vaddq_s32(sum0, offset_const); - sum1 = vaddq_s32(sum1, offset_const); - sum0 = vqrshlq_s32(sum0, round_shift_vec); - sum1 = vqrshlq_s32(sum1, round_shift_vec); - sum0 = vsubq_s32(sum0, sub_const_vec); - sum1 = vsubq_s32(sum1, sub_const_vec); - sum0 = vmaxq_s32(sum0, zero); - sum1 = vmaxq_s32(sum1, zero); - res = vcombine_u16(vqmovn_u32(vreinterpretq_u32_s32(sum0)), - vqmovn_u32(vreinterpretq_u32_s32(sum1))); - - res = vqrshlq_u16(res, vec_round_bits); - - return vqmovn_u16(res); -} - -void av1_convolve_x_sr_neon(const uint8_t *src, int src_stride, uint8_t *dst, - int dst_stride, int w, int h, - const InterpFilterParams *filter_params_x, - const InterpFilterParams *filter_params_y, - const int subpel_x_q4, const int subpel_y_q4, - ConvolveParams *conv_params) { - const uint8_t horiz_offset = filter_params_x->taps / 2 - 1; - const int8_t bits = FILTER_BITS - conv_params->round_0; - - (void)subpel_y_q4; - (void)conv_params; - (void)filter_params_y; - - uint8x8_t t0; -#if defined(__aarch64__) - uint8x8_t t1, t2, t3; -#endif - - assert(bits >= 0); - assert((FILTER_BITS - conv_params->round_1) >= 0 || - ((conv_params->round_0 + conv_params->round_1) == 2 * FILTER_BITS)); - - const int16_t *x_filter = av1_get_interp_filter_subpel_kernel( - filter_params_x, subpel_x_q4 & SUBPEL_MASK); - - const int16x8_t shift_round_0 = vdupq_n_s16(-conv_params->round_0); - const int16x8_t shift_by_bits = vdupq_n_s16(-bits); - - src -= horiz_offset; -#if defined(__aarch64__) - if (h == 4) { - uint8x8_t d01, d23; - int16x4_t s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10, d0, d1, d2, d3; - int16x8_t d01_temp, d23_temp; - - __builtin_prefetch(src + 0 * src_stride); - __builtin_prefetch(src + 1 * src_stride); - __builtin_prefetch(src + 2 * src_stride); - __builtin_prefetch(src + 3 * src_stride); - - load_u8_8x4(src, src_stride, &t0, &t1, &t2, &t3); - transpose_u8_8x4(&t0, &t1, &t2, &t3); - - s0 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t0))); - s1 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t1))); - s2 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t2))); - s3 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t3))); - s4 = vget_high_s16(vreinterpretq_s16_u16(vmovl_u8(t0))); - s5 = vget_high_s16(vreinterpretq_s16_u16(vmovl_u8(t1))); - s6 = vget_high_s16(vreinterpretq_s16_u16(vmovl_u8(t2))); - __builtin_prefetch(dst + 0 * dst_stride); - __builtin_prefetch(dst + 1 * dst_stride); - __builtin_prefetch(dst + 2 * dst_stride); - __builtin_prefetch(dst + 3 * dst_stride); - src += 7; - - do { - load_u8_8x4(src, src_stride, &t0, &t1, &t2, &t3); - transpose_u8_8x4(&t0, &t1, &t2, &t3); - - s7 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t0))); - s8 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t1))); - s9 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t2))); - s10 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t3))); - - d0 = convolve8_4x4(s0, s1, s2, s3, s4, s5, s6, s7, x_filter); - - d1 = convolve8_4x4(s1, s2, s3, s4, s5, s6, s7, s8, x_filter); - - d2 = convolve8_4x4(s2, s3, s4, s5, s6, s7, s8, s9, x_filter); - - d3 = convolve8_4x4(s3, s4, s5, s6, s7, s8, s9, s10, x_filter); - - d01_temp = vqrshlq_s16(vcombine_s16(d0, d1), shift_round_0); - d23_temp = vqrshlq_s16(vcombine_s16(d2, d3), shift_round_0); - - d01_temp = vqrshlq_s16(d01_temp, shift_by_bits); - d23_temp = vqrshlq_s16(d23_temp, shift_by_bits); - - d01 = vqmovun_s16(d01_temp); - d23 = vqmovun_s16(d23_temp); - - transpose_u8_4x4(&d01, &d23); - - if (w != 2) { - vst1_lane_u32((uint32_t *)(dst + 0 * dst_stride), // 00 01 02 03 - vreinterpret_u32_u8(d01), 0); - vst1_lane_u32((uint32_t *)(dst + 1 * dst_stride), // 10 11 12 13 - vreinterpret_u32_u8(d23), 0); - vst1_lane_u32((uint32_t *)(dst + 2 * dst_stride), // 20 21 22 23 - vreinterpret_u32_u8(d01), 1); - vst1_lane_u32((uint32_t *)(dst + 3 * dst_stride), // 30 31 32 33 - vreinterpret_u32_u8(d23), 1); - } else { - vst1_lane_u16((uint16_t *)(dst + 0 * dst_stride), // 00 01 - vreinterpret_u16_u8(d01), 0); - vst1_lane_u16((uint16_t *)(dst + 1 * dst_stride), // 10 11 - vreinterpret_u16_u8(d23), 0); - vst1_lane_u16((uint16_t *)(dst + 2 * dst_stride), // 20 21 - vreinterpret_u16_u8(d01), 2); - vst1_lane_u16((uint16_t *)(dst + 3 * dst_stride), // 30 31 - vreinterpret_u16_u8(d23), 2); - } - - s0 = s4; - s1 = s5; - s2 = s6; - s3 = s7; - s4 = s8; - s5 = s9; - s6 = s10; - src += 4; - dst += 4; - w -= 4; - } while (w > 0); - } else { -#endif - int width; - const uint8_t *s; - int16x8_t s0, s1, s2, s3, s4, s5, s6, s7; - -#if defined(__aarch64__) - int16x8_t s8, s9, s10; - uint8x8_t t4, t5, t6, t7; -#endif - - if (w <= 4) { -#if defined(__aarch64__) - do { - load_u8_8x8(src, src_stride, &t0, &t1, &t2, &t3, &t4, &t5, &t6, &t7); - transpose_u8_8x8(&t0, &t1, &t2, &t3, &t4, &t5, &t6, &t7); - s0 = vreinterpretq_s16_u16(vmovl_u8(t0)); - s1 = vreinterpretq_s16_u16(vmovl_u8(t1)); - s2 = vreinterpretq_s16_u16(vmovl_u8(t2)); - s3 = vreinterpretq_s16_u16(vmovl_u8(t3)); - s4 = vreinterpretq_s16_u16(vmovl_u8(t4)); - s5 = vreinterpretq_s16_u16(vmovl_u8(t5)); - s6 = vreinterpretq_s16_u16(vmovl_u8(t6)); - - load_u8_8x8(src + 7, src_stride, &t0, &t1, &t2, &t3, &t4, &t5, &t6, - &t7); - src += 8 * src_stride; - __builtin_prefetch(dst + 0 * dst_stride); - __builtin_prefetch(dst + 1 * dst_stride); - __builtin_prefetch(dst + 2 * dst_stride); - __builtin_prefetch(dst + 3 * dst_stride); - __builtin_prefetch(dst + 4 * dst_stride); - __builtin_prefetch(dst + 5 * dst_stride); - __builtin_prefetch(dst + 6 * dst_stride); - __builtin_prefetch(dst + 7 * dst_stride); - - transpose_u8_4x8(&t0, &t1, &t2, &t3, t4, t5, t6, t7); - - s7 = vreinterpretq_s16_u16(vmovl_u8(t0)); - s8 = vreinterpretq_s16_u16(vmovl_u8(t1)); - s9 = vreinterpretq_s16_u16(vmovl_u8(t2)); - s10 = vreinterpretq_s16_u16(vmovl_u8(t3)); - - __builtin_prefetch(src + 0 * src_stride); - __builtin_prefetch(src + 1 * src_stride); - __builtin_prefetch(src + 2 * src_stride); - __builtin_prefetch(src + 3 * src_stride); - __builtin_prefetch(src + 4 * src_stride); - __builtin_prefetch(src + 5 * src_stride); - __builtin_prefetch(src + 6 * src_stride); - __builtin_prefetch(src + 7 * src_stride); - t0 = convolve8_horiz_8x8(s0, s1, s2, s3, s4, s5, s6, s7, x_filter, - shift_round_0, shift_by_bits); - t1 = convolve8_horiz_8x8(s1, s2, s3, s4, s5, s6, s7, s8, x_filter, - shift_round_0, shift_by_bits); - t2 = convolve8_horiz_8x8(s2, s3, s4, s5, s6, s7, s8, s9, x_filter, - shift_round_0, shift_by_bits); - t3 = convolve8_horiz_8x8(s3, s4, s5, s6, s7, s8, s9, s10, x_filter, - shift_round_0, shift_by_bits); - - transpose_u8_8x4(&t0, &t1, &t2, &t3); - - if ((w == 4) && (h > 4)) { - vst1_lane_u32((uint32_t *)dst, vreinterpret_u32_u8(t0), - 0); // 00 01 02 03 - dst += dst_stride; - vst1_lane_u32((uint32_t *)dst, vreinterpret_u32_u8(t1), - 0); // 10 11 12 13 - dst += dst_stride; - vst1_lane_u32((uint32_t *)dst, vreinterpret_u32_u8(t2), - 0); // 20 21 22 23 - dst += dst_stride; - vst1_lane_u32((uint32_t *)dst, vreinterpret_u32_u8(t3), - 0); // 30 31 32 33 - dst += dst_stride; - vst1_lane_u32((uint32_t *)dst, vreinterpret_u32_u8(t0), - 1); // 40 41 42 43 - dst += dst_stride; - vst1_lane_u32((uint32_t *)dst, vreinterpret_u32_u8(t1), - 1); // 50 51 52 53 - dst += dst_stride; - vst1_lane_u32((uint32_t *)dst, vreinterpret_u32_u8(t2), - 1); // 60 61 62 63 - dst += dst_stride; - vst1_lane_u32((uint32_t *)dst, vreinterpret_u32_u8(t3), - 1); // 70 71 72 73 - dst += dst_stride; - } else if ((w == 4) && (h == 2)) { - vst1_lane_u32((uint32_t *)dst, vreinterpret_u32_u8(t0), - 0); // 00 01 02 03 - dst += dst_stride; - vst1_lane_u32((uint32_t *)dst, vreinterpret_u32_u8(t1), - 0); // 10 11 12 13 - dst += dst_stride; - } else if ((w == 2) && (h > 4)) { - vst1_lane_u16((uint16_t *)dst, vreinterpret_u16_u8(t0), 0); // 00 01 - dst += dst_stride; - vst1_lane_u16((uint16_t *)dst, vreinterpret_u16_u8(t1), 0); // 10 11 - dst += dst_stride; - vst1_lane_u16((uint16_t *)dst, vreinterpret_u16_u8(t2), 0); // 20 21 - dst += dst_stride; - vst1_lane_u16((uint16_t *)dst, vreinterpret_u16_u8(t3), 0); // 30 31 - dst += dst_stride; - vst1_lane_u16((uint16_t *)dst, vreinterpret_u16_u8(t0), 2); // 40 41 - dst += dst_stride; - vst1_lane_u16((uint16_t *)dst, vreinterpret_u16_u8(t1), 2); // 50 51 - dst += dst_stride; - vst1_lane_u16((uint16_t *)dst, vreinterpret_u16_u8(t2), 2); // 60 61 - dst += dst_stride; - vst1_lane_u16((uint16_t *)dst, vreinterpret_u16_u8(t3), 2); // 70 71 - dst += dst_stride; - } else if ((w == 2) && (h == 2)) { - vst1_lane_u16((uint16_t *)dst, vreinterpret_u16_u8(t0), 0); // 00 01 - dst += dst_stride; - vst1_lane_u16((uint16_t *)dst, vreinterpret_u16_u8(t1), 0); // 10 11 - dst += dst_stride; - } - h -= 8; - } while (h > 0); -#else - int16x8_t tt0; - int16x4_t x0, x1, x2, x3, x4, x5, x6, x7; - const int16x4_t shift_round_0_low = vget_low_s16(shift_round_0); - const int16x4_t shift_by_bits_low = vget_low_s16(shift_by_bits); - do { - t0 = vld1_u8(src); // a0 a1 a2 a3 a4 a5 a6 a7 - tt0 = vreinterpretq_s16_u16(vmovl_u8(t0)); - x0 = vget_low_s16(tt0); // a0 a1 a2 a3 - x4 = vget_high_s16(tt0); // a4 a5 a6 a7 - - t0 = vld1_u8(src + 8); // a8 a9 a10 a11 a12 a13 a14 a15 - tt0 = vreinterpretq_s16_u16(vmovl_u8(t0)); - x7 = vget_low_s16(tt0); // a8 a9 a10 a11 - - x1 = vext_s16(x0, x4, 1); // a1 a2 a3 a4 - x2 = vext_s16(x0, x4, 2); // a2 a3 a4 a5 - x3 = vext_s16(x0, x4, 3); // a3 a4 a5 a6 - x5 = vext_s16(x4, x7, 1); // a5 a6 a7 a8 - x6 = vext_s16(x4, x7, 2); // a6 a7 a8 a9 - x7 = vext_s16(x4, x7, 3); // a7 a8 a9 a10 - - src += src_stride; - - t0 = convolve8_horiz_4x1(x0, x1, x2, x3, x4, x5, x6, x7, x_filter, - shift_round_0_low, shift_by_bits_low); - - if (w == 4) { - vst1_lane_u32((uint32_t *)dst, vreinterpret_u32_u8(t0), - 0); // 00 01 02 03 - dst += dst_stride; - } else if (w == 2) { - vst1_lane_u16((uint16_t *)dst, vreinterpret_u16_u8(t0), 0); // 00 01 - dst += dst_stride; - } - h -= 1; - } while (h > 0); -#endif - } else { - uint8_t *d; - int16x8_t s11; -#if defined(__aarch64__) - int16x8_t s12, s13, s14; - do { - __builtin_prefetch(src + 0 * src_stride); - __builtin_prefetch(src + 1 * src_stride); - __builtin_prefetch(src + 2 * src_stride); - __builtin_prefetch(src + 3 * src_stride); - __builtin_prefetch(src + 4 * src_stride); - __builtin_prefetch(src + 5 * src_stride); - __builtin_prefetch(src + 6 * src_stride); - __builtin_prefetch(src + 7 * src_stride); - load_u8_8x8(src, src_stride, &t0, &t1, &t2, &t3, &t4, &t5, &t6, &t7); - transpose_u8_8x8(&t0, &t1, &t2, &t3, &t4, &t5, &t6, &t7); - s0 = vreinterpretq_s16_u16(vmovl_u8(t0)); - s1 = vreinterpretq_s16_u16(vmovl_u8(t1)); - s2 = vreinterpretq_s16_u16(vmovl_u8(t2)); - s3 = vreinterpretq_s16_u16(vmovl_u8(t3)); - s4 = vreinterpretq_s16_u16(vmovl_u8(t4)); - s5 = vreinterpretq_s16_u16(vmovl_u8(t5)); - s6 = vreinterpretq_s16_u16(vmovl_u8(t6)); - - width = w; - s = src + 7; - d = dst; - __builtin_prefetch(dst + 0 * dst_stride); - __builtin_prefetch(dst + 1 * dst_stride); - __builtin_prefetch(dst + 2 * dst_stride); - __builtin_prefetch(dst + 3 * dst_stride); - __builtin_prefetch(dst + 4 * dst_stride); - __builtin_prefetch(dst + 5 * dst_stride); - __builtin_prefetch(dst + 6 * dst_stride); - __builtin_prefetch(dst + 7 * dst_stride); - - do { - load_u8_8x8(s, src_stride, &t0, &t1, &t2, &t3, &t4, &t5, &t6, &t7); - transpose_u8_8x8(&t0, &t1, &t2, &t3, &t4, &t5, &t6, &t7); - s7 = vreinterpretq_s16_u16(vmovl_u8(t0)); - s8 = vreinterpretq_s16_u16(vmovl_u8(t1)); - s9 = vreinterpretq_s16_u16(vmovl_u8(t2)); - s10 = vreinterpretq_s16_u16(vmovl_u8(t3)); - s11 = vreinterpretq_s16_u16(vmovl_u8(t4)); - s12 = vreinterpretq_s16_u16(vmovl_u8(t5)); - s13 = vreinterpretq_s16_u16(vmovl_u8(t6)); - s14 = vreinterpretq_s16_u16(vmovl_u8(t7)); - - t0 = convolve8_horiz_8x8(s0, s1, s2, s3, s4, s5, s6, s7, x_filter, - shift_round_0, shift_by_bits); - - t1 = convolve8_horiz_8x8(s1, s2, s3, s4, s5, s6, s7, s8, x_filter, - shift_round_0, shift_by_bits); - - t2 = convolve8_horiz_8x8(s2, s3, s4, s5, s6, s7, s8, s9, x_filter, - shift_round_0, shift_by_bits); - - t3 = convolve8_horiz_8x8(s3, s4, s5, s6, s7, s8, s9, s10, x_filter, - shift_round_0, shift_by_bits); - - t4 = convolve8_horiz_8x8(s4, s5, s6, s7, s8, s9, s10, s11, x_filter, - shift_round_0, shift_by_bits); - - t5 = convolve8_horiz_8x8(s5, s6, s7, s8, s9, s10, s11, s12, x_filter, - shift_round_0, shift_by_bits); - - t6 = convolve8_horiz_8x8(s6, s7, s8, s9, s10, s11, s12, s13, x_filter, - shift_round_0, shift_by_bits); - - t7 = convolve8_horiz_8x8(s7, s8, s9, s10, s11, s12, s13, s14, - x_filter, shift_round_0, shift_by_bits); - - transpose_u8_8x8(&t0, &t1, &t2, &t3, &t4, &t5, &t6, &t7); - if (h != 2) { - store_u8_8x8(d, dst_stride, t0, t1, t2, t3, t4, t5, t6, t7); - } else { - store_row2_u8_8x8(d, dst_stride, t0, t1); - } - s0 = s8; - s1 = s9; - s2 = s10; - s3 = s11; - s4 = s12; - s5 = s13; - s6 = s14; - s += 8; - d += 8; - width -= 8; - } while (width > 0); - src += 8 * src_stride; - dst += 8 * dst_stride; - h -= 8; - } while (h > 0); -#else - do { - t0 = vld1_u8(src); // a0 a1 a2 a3 a4 a5 a6 a7 - s0 = vreinterpretq_s16_u16(vmovl_u8(t0)); - - width = w; - s = src + 8; - d = dst; - __builtin_prefetch(dst); - - do { - t0 = vld1_u8(s); // a8 a9 a10 a11 a12 a13 a14 a15 - s7 = vreinterpretq_s16_u16(vmovl_u8(t0)); - s11 = s0; - s0 = s7; - - s1 = vextq_s16(s11, s7, 1); // a1 a2 a3 a4 a5 a6 a7 a8 - s2 = vextq_s16(s11, s7, 2); // a2 a3 a4 a5 a6 a7 a8 a9 - s3 = vextq_s16(s11, s7, 3); // a3 a4 a5 a6 a7 a8 a9 a10 - s4 = vextq_s16(s11, s7, 4); // a4 a5 a6 a7 a8 a9 a10 a11 - s5 = vextq_s16(s11, s7, 5); // a5 a6 a7 a8 a9 a10 a11 a12 - s6 = vextq_s16(s11, s7, 6); // a6 a7 a8 a9 a10 a11 a12 a13 - s7 = vextq_s16(s11, s7, 7); // a7 a8 a9 a10 a11 a12 a13 a14 - - t0 = convolve8_horiz_8x8(s11, s1, s2, s3, s4, s5, s6, s7, x_filter, - shift_round_0, shift_by_bits); - vst1_u8(d, t0); - - s += 8; - d += 8; - width -= 8; - } while (width > 0); - src += src_stride; - dst += dst_stride; - h -= 1; - } while (h > 0); -#endif - } -#if defined(__aarch64__) - } -#endif -} - -void av1_convolve_y_sr_neon(const uint8_t *src, int src_stride, uint8_t *dst, - int dst_stride, int w, int h, - const InterpFilterParams *filter_params_x, - const InterpFilterParams *filter_params_y, - const int subpel_x_q4, const int subpel_y_q4, - ConvolveParams *conv_params) { - const int vert_offset = filter_params_y->taps / 2 - 1; - - src -= vert_offset * src_stride; - - (void)filter_params_x; - (void)subpel_x_q4; - (void)conv_params; - - assert(conv_params->round_0 <= FILTER_BITS); - assert(((conv_params->round_0 + conv_params->round_1) <= (FILTER_BITS + 1)) || - ((conv_params->round_0 + conv_params->round_1) == (2 * FILTER_BITS))); - - const int16_t *y_filter = av1_get_interp_filter_subpel_kernel( - filter_params_y, subpel_y_q4 & SUBPEL_MASK); - - if (w <= 4) { - uint8x8_t d01; - int16x4_t s0, s1, s2, s3, s4, s5, s6, s7, d0; -#if defined(__aarch64__) - uint8x8_t d23; - int16x4_t s8, s9, s10, d1, d2, d3; -#endif - s0 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(vld1_u8(src)))); - src += src_stride; - s1 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(vld1_u8(src)))); - src += src_stride; - s2 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(vld1_u8(src)))); - src += src_stride; - s3 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(vld1_u8(src)))); - src += src_stride; - s4 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(vld1_u8(src)))); - src += src_stride; - s5 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(vld1_u8(src)))); - src += src_stride; - s6 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(vld1_u8(src)))); - src += src_stride; - - do { - s7 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(vld1_u8(src)))); - src += src_stride; -#if defined(__aarch64__) - s8 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(vld1_u8(src)))); - src += src_stride; - s9 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(vld1_u8(src)))); - src += src_stride; - s10 = vreinterpret_s16_u16(vget_low_u16(vmovl_u8(vld1_u8(src)))); - src += src_stride; - - __builtin_prefetch(dst + 0 * dst_stride); - __builtin_prefetch(dst + 1 * dst_stride); - __builtin_prefetch(dst + 2 * dst_stride); - __builtin_prefetch(dst + 3 * dst_stride); - __builtin_prefetch(src + 0 * src_stride); - __builtin_prefetch(src + 1 * src_stride); - __builtin_prefetch(src + 2 * src_stride); - __builtin_prefetch(src + 3 * src_stride); - d0 = convolve8_4x4(s0, s1, s2, s3, s4, s5, s6, s7, y_filter); - d1 = convolve8_4x4(s1, s2, s3, s4, s5, s6, s7, s8, y_filter); - d2 = convolve8_4x4(s2, s3, s4, s5, s6, s7, s8, s9, y_filter); - d3 = convolve8_4x4(s3, s4, s5, s6, s7, s8, s9, s10, y_filter); - - d01 = vqrshrun_n_s16(vcombine_s16(d0, d1), FILTER_BITS); - d23 = vqrshrun_n_s16(vcombine_s16(d2, d3), FILTER_BITS); - if ((w == 4) && (h != 2)) { - vst1_lane_u32((uint32_t *)dst, vreinterpret_u32_u8(d01), - 0); // 00 01 02 03 - dst += dst_stride; - vst1_lane_u32((uint32_t *)dst, vreinterpret_u32_u8(d01), - 1); // 10 11 12 13 - dst += dst_stride; - vst1_lane_u32((uint32_t *)dst, vreinterpret_u32_u8(d23), - 0); // 20 21 22 23 - dst += dst_stride; - vst1_lane_u32((uint32_t *)dst, vreinterpret_u32_u8(d23), - 1); // 30 31 32 33 - dst += dst_stride; - } else if ((w == 4) && (h == 2)) { - vst1_lane_u32((uint32_t *)dst, vreinterpret_u32_u8(d01), - 0); // 00 01 02 03 - dst += dst_stride; - vst1_lane_u32((uint32_t *)dst, vreinterpret_u32_u8(d01), - 1); // 10 11 12 13 - dst += dst_stride; - } else if ((w == 2) && (h != 2)) { - vst1_lane_u16((uint16_t *)dst, vreinterpret_u16_u8(d01), 0); // 00 01 - dst += dst_stride; - vst1_lane_u16((uint16_t *)dst, vreinterpret_u16_u8(d01), 2); // 10 11 - dst += dst_stride; - vst1_lane_u16((uint16_t *)dst, vreinterpret_u16_u8(d23), 0); // 20 21 - dst += dst_stride; - vst1_lane_u16((uint16_t *)dst, vreinterpret_u16_u8(d23), 2); // 30 31 - dst += dst_stride; - } else if ((w == 2) && (h == 2)) { - vst1_lane_u16((uint16_t *)dst, vreinterpret_u16_u8(d01), 0); // 00 01 - dst += dst_stride; - vst1_lane_u16((uint16_t *)dst, vreinterpret_u16_u8(d01), 2); // 10 11 - dst += dst_stride; - } - s0 = s4; - s1 = s5; - s2 = s6; - s3 = s7; - s4 = s8; - s5 = s9; - s6 = s10; - h -= 4; -#else - __builtin_prefetch(dst + 0 * dst_stride); - __builtin_prefetch(src + 0 * src_stride); - - d0 = convolve8_4x4(s0, s1, s2, s3, s4, s5, s6, s7, y_filter); - - d01 = vqrshrun_n_s16(vcombine_s16(d0, d0), FILTER_BITS); - - if (w == 4) { - vst1_lane_u32((uint32_t *)dst, vreinterpret_u32_u8(d01), 0); - dst += dst_stride; - } else if (w == 2) { - vst1_lane_u16((uint16_t *)dst, vreinterpret_u16_u8(d01), 0); - dst += dst_stride; - } - s0 = s1; - s1 = s2; - s2 = s3; - s3 = s4; - s4 = s5; - s5 = s6; - s6 = s7; - h -= 1; -#endif - } while (h > 0); - } else { - int height; - const uint8_t *s; - uint8_t *d; - uint8x8_t t0; - int16x8_t s0, s1, s2, s3, s4, s5, s6, s7; -#if defined(__aarch64__) - uint8x8_t t1, t2, t3; - int16x8_t s8, s9, s10; -#endif - do { - __builtin_prefetch(src + 0 * src_stride); - __builtin_prefetch(src + 1 * src_stride); - __builtin_prefetch(src + 2 * src_stride); - __builtin_prefetch(src + 3 * src_stride); - __builtin_prefetch(src + 4 * src_stride); - __builtin_prefetch(src + 5 * src_stride); - __builtin_prefetch(src + 6 * src_stride); - s = src; - s0 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(s))); - s += src_stride; - s1 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(s))); - s += src_stride; - s2 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(s))); - s += src_stride; - s3 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(s))); - s += src_stride; - s4 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(s))); - s += src_stride; - s5 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(s))); - s += src_stride; - s6 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(s))); - s += src_stride; - d = dst; - height = h; - - do { - s7 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(s))); - s += src_stride; -#if defined(__aarch64__) - s8 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(s))); - s += src_stride; - s9 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(s))); - s += src_stride; - s10 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(s))); - s += src_stride; - - __builtin_prefetch(d + 0 * dst_stride); - __builtin_prefetch(d + 1 * dst_stride); - __builtin_prefetch(d + 2 * dst_stride); - __builtin_prefetch(d + 3 * dst_stride); - __builtin_prefetch(s + 0 * src_stride); - __builtin_prefetch(s + 1 * src_stride); - __builtin_prefetch(s + 2 * src_stride); - __builtin_prefetch(s + 3 * src_stride); - t0 = convolve8_vert_8x4(s0, s1, s2, s3, s4, s5, s6, s7, y_filter); - t1 = convolve8_vert_8x4(s1, s2, s3, s4, s5, s6, s7, s8, y_filter); - t2 = convolve8_vert_8x4(s2, s3, s4, s5, s6, s7, s8, s9, y_filter); - t3 = convolve8_vert_8x4(s3, s4, s5, s6, s7, s8, s9, s10, y_filter); - if (h != 2) { - vst1_u8(d, t0); - d += dst_stride; - vst1_u8(d, t1); - d += dst_stride; - vst1_u8(d, t2); - d += dst_stride; - vst1_u8(d, t3); - d += dst_stride; - } else { - vst1_u8(d, t0); - d += dst_stride; - vst1_u8(d, t1); - d += dst_stride; - } - s0 = s4; - s1 = s5; - s2 = s6; - s3 = s7; - s4 = s8; - s5 = s9; - s6 = s10; - height -= 4; -#else - __builtin_prefetch(d); - __builtin_prefetch(s); - - t0 = convolve8_vert_8x4(s0, s1, s2, s3, s4, s5, s6, s7, y_filter); - - vst1_u8(d, t0); - d += dst_stride; - - s0 = s1; - s1 = s2; - s2 = s3; - s3 = s4; - s4 = s5; - s5 = s6; - s6 = s7; - height -= 1; -#endif - } while (height > 0); - src += 8; - dst += 8; - w -= 8; - } while (w > 0); - } -} - -void av1_convolve_2d_sr_neon(const uint8_t *src, int src_stride, uint8_t *dst, - int dst_stride, int w, int h, - const InterpFilterParams *filter_params_x, - const InterpFilterParams *filter_params_y, - const int subpel_x_q4, const int subpel_y_q4, - ConvolveParams *conv_params) { - int im_dst_stride; - int width, height; - uint8x8_t t0; -#if defined(__aarch64__) - uint8x8_t t1, t2, t3, t4, t5, t6, t7; -#endif - - DECLARE_ALIGNED(16, int16_t, - im_block[(MAX_SB_SIZE + HORIZ_EXTRA_ROWS) * MAX_SB_SIZE]); - - const int bd = 8; - const int im_h = h + filter_params_y->taps - 1; - const int im_stride = MAX_SB_SIZE; - const int vert_offset = filter_params_y->taps / 2 - 1; - const int horiz_offset = filter_params_x->taps / 2 - 1; - - const uint8_t *src_ptr = src - vert_offset * src_stride - horiz_offset; - const uint8_t *s; - int16_t *dst_ptr; - - dst_ptr = im_block; - im_dst_stride = im_stride; - height = im_h; - width = w; - - const int16_t round_bits = - FILTER_BITS * 2 - conv_params->round_0 - conv_params->round_1; - const int16x8_t vec_round_bits = vdupq_n_s16(-round_bits); - const int offset_bits = bd + 2 * FILTER_BITS - conv_params->round_0; - const int16_t *x_filter = av1_get_interp_filter_subpel_kernel( - filter_params_x, subpel_x_q4 & SUBPEL_MASK); - - int16_t x_filter_tmp[8]; - int16x8_t filter_x_coef = vld1q_s16(x_filter); - - // filter coeffs are even, so downshifting by 1 to reduce intermediate - // precision requirements. - filter_x_coef = vshrq_n_s16(filter_x_coef, 1); - vst1q_s16(&x_filter_tmp[0], filter_x_coef); - - assert(conv_params->round_0 > 0); - - if (w <= 4) { - int16x4_t s0, s1, s2, s3, s4, s5, s6, s7, d0; -#if defined(__aarch64__) - int16x4_t s8, s9, s10, d1, d2, d3; -#endif - - const int16x4_t horiz_const = vdup_n_s16((1 << (bd + FILTER_BITS - 2))); - const int16x4_t shift_round_0 = vdup_n_s16(-(conv_params->round_0 - 1)); - - do { - s = src_ptr; - -#if defined(__aarch64__) - __builtin_prefetch(s + 0 * src_stride); - __builtin_prefetch(s + 1 * src_stride); - __builtin_prefetch(s + 2 * src_stride); - __builtin_prefetch(s + 3 * src_stride); - - load_u8_8x4(s, src_stride, &t0, &t1, &t2, &t3); - transpose_u8_8x4(&t0, &t1, &t2, &t3); - - s0 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t0))); - s1 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t1))); - s2 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t2))); - s3 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t3))); - s4 = vget_high_s16(vreinterpretq_s16_u16(vmovl_u8(t0))); - s5 = vget_high_s16(vreinterpretq_s16_u16(vmovl_u8(t1))); - s6 = vget_high_s16(vreinterpretq_s16_u16(vmovl_u8(t2))); - - __builtin_prefetch(dst_ptr + 0 * im_dst_stride); - __builtin_prefetch(dst_ptr + 1 * im_dst_stride); - __builtin_prefetch(dst_ptr + 2 * im_dst_stride); - __builtin_prefetch(dst_ptr + 3 * im_dst_stride); - s += 7; - - load_u8_8x4(s, src_stride, &t0, &t1, &t2, &t3); - transpose_u8_8x4(&t0, &t1, &t2, &t3); - - s7 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t0))); - s8 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t1))); - s9 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t2))); - s10 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t3))); - - d0 = convolve8_4x4_s16(s0, s1, s2, s3, s4, s5, s6, s7, x_filter_tmp, - horiz_const, shift_round_0); - d1 = convolve8_4x4_s16(s1, s2, s3, s4, s5, s6, s7, s8, x_filter_tmp, - horiz_const, shift_round_0); - d2 = convolve8_4x4_s16(s2, s3, s4, s5, s6, s7, s8, s9, x_filter_tmp, - horiz_const, shift_round_0); - d3 = convolve8_4x4_s16(s3, s4, s5, s6, s7, s8, s9, s10, x_filter_tmp, - horiz_const, shift_round_0); - - transpose_s16_4x4d(&d0, &d1, &d2, &d3); - if (w == 4) { - vst1_s16((dst_ptr + 0 * im_dst_stride), d0); - vst1_s16((dst_ptr + 1 * im_dst_stride), d1); - vst1_s16((dst_ptr + 2 * im_dst_stride), d2); - vst1_s16((dst_ptr + 3 * im_dst_stride), d3); - } else if (w == 2) { - vst1_lane_u32((uint32_t *)(dst_ptr + 0 * im_dst_stride), - vreinterpret_u32_s16(d0), 0); - vst1_lane_u32((uint32_t *)(dst_ptr + 1 * im_dst_stride), - vreinterpret_u32_s16(d1), 0); - vst1_lane_u32((uint32_t *)(dst_ptr + 2 * im_dst_stride), - vreinterpret_u32_s16(d2), 0); - vst1_lane_u32((uint32_t *)(dst_ptr + 3 * im_dst_stride), - vreinterpret_u32_s16(d3), 0); - } - src_ptr += 4 * src_stride; - dst_ptr += 4 * im_dst_stride; - height -= 4; -#else - int16x8_t tt0; - - __builtin_prefetch(s); - - t0 = vld1_u8(s); // a0 a1 a2 a3 a4 a5 a6 a7 - tt0 = vreinterpretq_s16_u16(vmovl_u8(t0)); - s0 = vget_low_s16(tt0); - s4 = vget_high_s16(tt0); - - __builtin_prefetch(dst_ptr); - s += 8; - - t0 = vld1_u8(s); // a8 a9 a10 a11 a12 a13 a14 a15 - s7 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t0))); - - s1 = vext_s16(s0, s4, 1); // a1 a2 a3 a4 - s2 = vext_s16(s0, s4, 2); // a2 a3 a4 a5 - s3 = vext_s16(s0, s4, 3); // a3 a4 a5 a6 - s5 = vext_s16(s4, s7, 1); // a5 a6 a7 a8 - s6 = vext_s16(s4, s7, 2); // a6 a7 a8 a9 - s7 = vext_s16(s4, s7, 3); // a7 a8 a9 a10 - - d0 = convolve8_4x4_s16(s0, s1, s2, s3, s4, s5, s6, s7, x_filter_tmp, - horiz_const, shift_round_0); - - if (w == 4) { - vst1_s16(dst_ptr, d0); - dst_ptr += im_dst_stride; - } else if (w == 2) { - vst1_lane_u32((uint32_t *)dst_ptr, vreinterpret_u32_s16(d0), 0); - dst_ptr += im_dst_stride; - } - - src_ptr += src_stride; - height -= 1; -#endif - } while (height > 0); - } else { - int16_t *d_tmp; - int16x8_t s0, s1, s2, s3, s4, s5, s6, s7, res0; -#if defined(__aarch64__) - int16x8_t s8, s9, s10, res1, res2, res3, res4, res5, res6, res7; - int16x8_t s11, s12, s13, s14; -#endif - - const int16x8_t horiz_const = vdupq_n_s16((1 << (bd + FILTER_BITS - 2))); - const int16x8_t shift_round_0 = vdupq_n_s16(-(conv_params->round_0 - 1)); - -#if defined(__aarch64__) - do { - __builtin_prefetch(src_ptr + 0 * src_stride); - __builtin_prefetch(src_ptr + 1 * src_stride); - __builtin_prefetch(src_ptr + 2 * src_stride); - __builtin_prefetch(src_ptr + 3 * src_stride); - __builtin_prefetch(src_ptr + 4 * src_stride); - __builtin_prefetch(src_ptr + 5 * src_stride); - __builtin_prefetch(src_ptr + 6 * src_stride); - __builtin_prefetch(src_ptr + 7 * src_stride); - - load_u8_8x8(src_ptr, src_stride, &t0, &t1, &t2, &t3, &t4, &t5, &t6, &t7); - - transpose_u8_8x8(&t0, &t1, &t2, &t3, &t4, &t5, &t6, &t7); - - s0 = vreinterpretq_s16_u16(vmovl_u8(t0)); - s1 = vreinterpretq_s16_u16(vmovl_u8(t1)); - s2 = vreinterpretq_s16_u16(vmovl_u8(t2)); - s3 = vreinterpretq_s16_u16(vmovl_u8(t3)); - s4 = vreinterpretq_s16_u16(vmovl_u8(t4)); - s5 = vreinterpretq_s16_u16(vmovl_u8(t5)); - s6 = vreinterpretq_s16_u16(vmovl_u8(t6)); - - width = w; - s = src_ptr + 7; - d_tmp = dst_ptr; - - __builtin_prefetch(dst_ptr + 0 * im_dst_stride); - __builtin_prefetch(dst_ptr + 1 * im_dst_stride); - __builtin_prefetch(dst_ptr + 2 * im_dst_stride); - __builtin_prefetch(dst_ptr + 3 * im_dst_stride); - __builtin_prefetch(dst_ptr + 4 * im_dst_stride); - __builtin_prefetch(dst_ptr + 5 * im_dst_stride); - __builtin_prefetch(dst_ptr + 6 * im_dst_stride); - __builtin_prefetch(dst_ptr + 7 * im_dst_stride); - - do { - load_u8_8x8(s, src_stride, &t0, &t1, &t2, &t3, &t4, &t5, &t6, &t7); - - transpose_u8_8x8(&t0, &t1, &t2, &t3, &t4, &t5, &t6, &t7); - - s7 = vreinterpretq_s16_u16(vmovl_u8(t0)); - s8 = vreinterpretq_s16_u16(vmovl_u8(t1)); - s9 = vreinterpretq_s16_u16(vmovl_u8(t2)); - s10 = vreinterpretq_s16_u16(vmovl_u8(t3)); - s11 = vreinterpretq_s16_u16(vmovl_u8(t4)); - s12 = vreinterpretq_s16_u16(vmovl_u8(t5)); - s13 = vreinterpretq_s16_u16(vmovl_u8(t6)); - s14 = vreinterpretq_s16_u16(vmovl_u8(t7)); - - res0 = convolve8_8x8_s16(s0, s1, s2, s3, s4, s5, s6, s7, x_filter_tmp, - horiz_const, shift_round_0); - res1 = convolve8_8x8_s16(s1, s2, s3, s4, s5, s6, s7, s8, x_filter_tmp, - horiz_const, shift_round_0); - res2 = convolve8_8x8_s16(s2, s3, s4, s5, s6, s7, s8, s9, x_filter_tmp, - horiz_const, shift_round_0); - res3 = convolve8_8x8_s16(s3, s4, s5, s6, s7, s8, s9, s10, x_filter_tmp, - horiz_const, shift_round_0); - res4 = convolve8_8x8_s16(s4, s5, s6, s7, s8, s9, s10, s11, x_filter_tmp, - horiz_const, shift_round_0); - res5 = convolve8_8x8_s16(s5, s6, s7, s8, s9, s10, s11, s12, - x_filter_tmp, horiz_const, shift_round_0); - res6 = convolve8_8x8_s16(s6, s7, s8, s9, s10, s11, s12, s13, - x_filter_tmp, horiz_const, shift_round_0); - res7 = convolve8_8x8_s16(s7, s8, s9, s10, s11, s12, s13, s14, - x_filter_tmp, horiz_const, shift_round_0); - - transpose_s16_8x8(&res0, &res1, &res2, &res3, &res4, &res5, &res6, - &res7); - - store_s16_8x8(d_tmp, im_dst_stride, res0, res1, res2, res3, res4, res5, - res6, res7); - - s0 = s8; - s1 = s9; - s2 = s10; - s3 = s11; - s4 = s12; - s5 = s13; - s6 = s14; - s += 8; - d_tmp += 8; - width -= 8; - } while (width > 0); - src_ptr += 8 * src_stride; - dst_ptr += 8 * im_dst_stride; - height -= 8; - } while (height > 0); -#else - do { - t0 = vld1_u8(src_ptr); - s0 = vreinterpretq_s16_u16(vmovl_u8(t0)); // a0 a1 a2 a3 a4 a5 a6 a7 - - width = w; - s = src_ptr + 8; - d_tmp = dst_ptr; - - __builtin_prefetch(dst_ptr); - - do { - t0 = vld1_u8(s); // a8 a9 a10 a11 a12 a13 a14 a15 - s7 = vreinterpretq_s16_u16(vmovl_u8(t0)); - int16x8_t sum = s0; - s0 = s7; - - s1 = vextq_s16(sum, s7, 1); // a1 a2 a3 a4 a5 a6 a7 a8 - s2 = vextq_s16(sum, s7, 2); // a2 a3 a4 a5 a6 a7 a8 a9 - s3 = vextq_s16(sum, s7, 3); // a3 a4 a5 a6 a7 a8 a9 a10 - s4 = vextq_s16(sum, s7, 4); // a4 a5 a6 a7 a8 a9 a10 a11 - s5 = vextq_s16(sum, s7, 5); // a5 a6 a7 a8 a9 a10 a11 a12 - s6 = vextq_s16(sum, s7, 6); // a6 a7 a8 a9 a10 a11 a12 a13 - s7 = vextq_s16(sum, s7, 7); // a7 a8 a9 a10 a11 a12 a13 a14 - - res0 = convolve8_8x8_s16(sum, s1, s2, s3, s4, s5, s6, s7, x_filter_tmp, - horiz_const, shift_round_0); - - vst1q_s16(d_tmp, res0); - - s += 8; - d_tmp += 8; - width -= 8; - } while (width > 0); - src_ptr += src_stride; - dst_ptr += im_dst_stride; - height -= 1; - } while (height > 0); -#endif - } - - // vertical - { - uint8_t *dst_u8_ptr, *d_u8; - int16_t *v_src_ptr, *v_s; - - const int32_t sub_const = (1 << (offset_bits - conv_params->round_1)) + - (1 << (offset_bits - conv_params->round_1 - 1)); - const int16_t *y_filter = av1_get_interp_filter_subpel_kernel( - filter_params_y, subpel_y_q4 & SUBPEL_MASK); - - const int32x4_t round_shift_vec = vdupq_n_s32(-(conv_params->round_1)); - const int32x4_t offset_const = vdupq_n_s32(1 << offset_bits); - const int32x4_t sub_const_vec = vdupq_n_s32(sub_const); - - src_stride = im_stride; - v_src_ptr = im_block; - dst_u8_ptr = dst; - - height = h; - width = w; - - if (width <= 4) { - int16x4_t s0, s1, s2, s3, s4, s5, s6, s7; - uint16x4_t d0; - uint16x8_t dd0; - uint8x8_t d01; - -#if defined(__aarch64__) - int16x4_t s8, s9, s10; - uint16x4_t d1, d2, d3; - uint16x8_t dd1; - uint8x8_t d23; -#endif - - d_u8 = dst_u8_ptr; - v_s = v_src_ptr; - - __builtin_prefetch(v_s + 0 * im_stride); - __builtin_prefetch(v_s + 1 * im_stride); - __builtin_prefetch(v_s + 2 * im_stride); - __builtin_prefetch(v_s + 3 * im_stride); - __builtin_prefetch(v_s + 4 * im_stride); - __builtin_prefetch(v_s + 5 * im_stride); - __builtin_prefetch(v_s + 6 * im_stride); - __builtin_prefetch(v_s + 7 * im_stride); - - load_s16_4x8(v_s, im_stride, &s0, &s1, &s2, &s3, &s4, &s5, &s6, &s7); - v_s += (7 * im_stride); - - do { -#if defined(__aarch64__) - load_s16_4x4(v_s, im_stride, &s7, &s8, &s9, &s10); - v_s += (im_stride << 2); - - __builtin_prefetch(d_u8 + 0 * dst_stride); - __builtin_prefetch(d_u8 + 1 * dst_stride); - __builtin_prefetch(d_u8 + 2 * dst_stride); - __builtin_prefetch(d_u8 + 3 * dst_stride); - - d0 = convolve8_vert_4x4_s32(s0, s1, s2, s3, s4, s5, s6, s7, y_filter, - round_shift_vec, offset_const, - sub_const_vec); - d1 = convolve8_vert_4x4_s32(s1, s2, s3, s4, s5, s6, s7, s8, y_filter, - round_shift_vec, offset_const, - sub_const_vec); - d2 = convolve8_vert_4x4_s32(s2, s3, s4, s5, s6, s7, s8, s9, y_filter, - round_shift_vec, offset_const, - sub_const_vec); - d3 = convolve8_vert_4x4_s32(s3, s4, s5, s6, s7, s8, s9, s10, y_filter, - round_shift_vec, offset_const, - sub_const_vec); - - dd0 = vqrshlq_u16(vcombine_u16(d0, d1), vec_round_bits); - dd1 = vqrshlq_u16(vcombine_u16(d2, d3), vec_round_bits); - - d01 = vqmovn_u16(dd0); - d23 = vqmovn_u16(dd1); - - if ((w == 4) && (h != 2)) { - vst1_lane_u32((uint32_t *)d_u8, vreinterpret_u32_u8(d01), - 0); // 00 01 02 03 - d_u8 += dst_stride; - vst1_lane_u32((uint32_t *)d_u8, vreinterpret_u32_u8(d01), - 1); // 10 11 12 13 - d_u8 += dst_stride; - vst1_lane_u32((uint32_t *)d_u8, vreinterpret_u32_u8(d23), - 0); // 20 21 22 23 - d_u8 += dst_stride; - vst1_lane_u32((uint32_t *)d_u8, vreinterpret_u32_u8(d23), - 1); // 30 31 32 33 - d_u8 += dst_stride; - } else if ((w == 2) && (h != 2)) { - vst1_lane_u16((uint16_t *)d_u8, vreinterpret_u16_u8(d01), - 0); // 00 01 - d_u8 += dst_stride; - vst1_lane_u16((uint16_t *)d_u8, vreinterpret_u16_u8(d01), - 2); // 10 11 - d_u8 += dst_stride; - vst1_lane_u16((uint16_t *)d_u8, vreinterpret_u16_u8(d23), - 0); // 20 21 - d_u8 += dst_stride; - vst1_lane_u16((uint16_t *)d_u8, vreinterpret_u16_u8(d23), - 2); // 30 31 - d_u8 += dst_stride; - } else if ((w == 4) && (h == 2)) { - vst1_lane_u32((uint32_t *)d_u8, vreinterpret_u32_u8(d01), - 0); // 00 01 02 03 - d_u8 += dst_stride; - vst1_lane_u32((uint32_t *)d_u8, vreinterpret_u32_u8(d01), - 1); // 10 11 12 13 - d_u8 += dst_stride; - } else if ((w == 2) && (h == 2)) { - vst1_lane_u16((uint16_t *)d_u8, vreinterpret_u16_u8(d01), - 0); // 00 01 - d_u8 += dst_stride; - vst1_lane_u16((uint16_t *)d_u8, vreinterpret_u16_u8(d01), - 2); // 10 11 - d_u8 += dst_stride; - } - - s0 = s4; - s1 = s5; - s2 = s6; - s3 = s7; - s4 = s8; - s5 = s9; - s6 = s10; - height -= 4; -#else - s7 = vld1_s16(v_s); - v_s += im_stride; - - __builtin_prefetch(d_u8 + 0 * dst_stride); - - d0 = convolve8_vert_4x4_s32(s0, s1, s2, s3, s4, s5, s6, s7, y_filter, - round_shift_vec, offset_const, - sub_const_vec); - - dd0 = vqrshlq_u16(vcombine_u16(d0, d0), vec_round_bits); - d01 = vqmovn_u16(dd0); - - if (w == 4) { - vst1_lane_u32((uint32_t *)d_u8, vreinterpret_u32_u8(d01), - 0); // 00 01 02 03 - d_u8 += dst_stride; - - } else if (w == 2) { - vst1_lane_u16((uint16_t *)d_u8, vreinterpret_u16_u8(d01), - 0); // 00 01 - d_u8 += dst_stride; - } - - s0 = s1; - s1 = s2; - s2 = s3; - s3 = s4; - s4 = s5; - s5 = s6; - s6 = s7; - height -= 1; -#endif - } while (height > 0); - } else { - // if width is a multiple of 8 & height is a multiple of 4 - int16x8_t s0, s1, s2, s3, s4, s5, s6, s7; - uint8x8_t res0; -#if defined(__aarch64__) - int16x8_t s8, s9, s10; - uint8x8_t res1, res2, res3; -#endif - - do { - __builtin_prefetch(v_src_ptr + 0 * im_stride); - __builtin_prefetch(v_src_ptr + 1 * im_stride); - __builtin_prefetch(v_src_ptr + 2 * im_stride); - __builtin_prefetch(v_src_ptr + 3 * im_stride); - __builtin_prefetch(v_src_ptr + 4 * im_stride); - __builtin_prefetch(v_src_ptr + 5 * im_stride); - __builtin_prefetch(v_src_ptr + 6 * im_stride); - __builtin_prefetch(v_src_ptr + 7 * im_stride); - - v_s = v_src_ptr; - load_s16_8x8(v_s, im_stride, &s0, &s1, &s2, &s3, &s4, &s5, &s6, &s7); - v_s += (7 * im_stride); - - d_u8 = dst_u8_ptr; - height = h; - - do { -#if defined(__aarch64__) - load_s16_8x4(v_s, im_stride, &s7, &s8, &s9, &s10); - v_s += (im_stride << 2); - - __builtin_prefetch(d_u8 + 4 * dst_stride); - __builtin_prefetch(d_u8 + 5 * dst_stride); - __builtin_prefetch(d_u8 + 6 * dst_stride); - __builtin_prefetch(d_u8 + 7 * dst_stride); - - res0 = convolve8_vert_8x4_s32(s0, s1, s2, s3, s4, s5, s6, s7, - y_filter, round_shift_vec, offset_const, - sub_const_vec, vec_round_bits); - res1 = convolve8_vert_8x4_s32(s1, s2, s3, s4, s5, s6, s7, s8, - y_filter, round_shift_vec, offset_const, - sub_const_vec, vec_round_bits); - res2 = convolve8_vert_8x4_s32(s2, s3, s4, s5, s6, s7, s8, s9, - y_filter, round_shift_vec, offset_const, - sub_const_vec, vec_round_bits); - res3 = convolve8_vert_8x4_s32(s3, s4, s5, s6, s7, s8, s9, s10, - y_filter, round_shift_vec, offset_const, - sub_const_vec, vec_round_bits); - - if (h != 2) { - vst1_u8(d_u8, res0); - d_u8 += dst_stride; - vst1_u8(d_u8, res1); - d_u8 += dst_stride; - vst1_u8(d_u8, res2); - d_u8 += dst_stride; - vst1_u8(d_u8, res3); - d_u8 += dst_stride; - } else { - vst1_u8(d_u8, res0); - d_u8 += dst_stride; - vst1_u8(d_u8, res1); - d_u8 += dst_stride; - } - s0 = s4; - s1 = s5; - s2 = s6; - s3 = s7; - s4 = s8; - s5 = s9; - s6 = s10; - height -= 4; -#else - s7 = vld1q_s16(v_s); - v_s += im_stride; - - __builtin_prefetch(d_u8 + 0 * dst_stride); - - res0 = convolve8_vert_8x4_s32(s0, s1, s2, s3, s4, s5, s6, s7, - y_filter, round_shift_vec, offset_const, - sub_const_vec, vec_round_bits); - - vst1_u8(d_u8, res0); - d_u8 += dst_stride; - - s0 = s1; - s1 = s2; - s2 = s3; - s3 = s4; - s4 = s5; - s5 = s6; - s6 = s7; - height -= 1; -#endif - } while (height > 0); - v_src_ptr += 8; - dst_u8_ptr += 8; - w -= 8; - } while (w > 0); - } - } -} -void av1_convolve_2d_copy_sr_neon(const uint8_t *src, int src_stride, - uint8_t *dst, int dst_stride, int w, int h, - const InterpFilterParams *filter_params_x, - const InterpFilterParams *filter_params_y, - const int subpel_x_q4, const int subpel_y_q4, - ConvolveParams *conv_params) { - (void)filter_params_x; - (void)filter_params_y; - (void)subpel_x_q4; - (void)subpel_y_q4; - (void)conv_params; - - const uint8_t *src1; - uint8_t *dst1; - int y; - - if (!(w & 0x0F)) { - for (y = 0; y < h; ++y) { - src1 = src; - dst1 = dst; - for (int x = 0; x < (w >> 4); ++x) { - vst1q_u8(dst1, vld1q_u8(src1)); - src1 += 16; - dst1 += 16; - } - src += src_stride; - dst += dst_stride; - } - } else if (!(w & 0x07)) { - for (y = 0; y < h; ++y) { - vst1_u8(dst, vld1_u8(src)); - src += src_stride; - dst += dst_stride; - } - } else if (!(w & 0x03)) { - for (y = 0; y < h; ++y) { - vst1_lane_u32((uint32_t *)(dst), vreinterpret_u32_u8(vld1_u8(src)), 0); - src += src_stride; - dst += dst_stride; - } - } else if (!(w & 0x01)) { - for (y = 0; y < h; ++y) { - vst1_lane_u16((uint16_t *)(dst), vreinterpret_u16_u8(vld1_u8(src)), 0); - src += src_stride; - dst += dst_stride; - } - } -} diff --git a/third_party/aom/av1/common/arm/convolve_neon.h b/third_party/aom/av1/common/arm/convolve_neon.h deleted file mode 100644 index f382984f2..000000000 --- a/third_party/aom/av1/common/arm/convolve_neon.h +++ /dev/null @@ -1,228 +0,0 @@ -/* - * Copyright (c) 2018, Alliance for Open Media. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#ifndef AOM_AV1_COMMON_ARM_CONVOLVE_NEON_H_ -#define AOM_AV1_COMMON_ARM_CONVOLVE_NEON_H_ - -#include <arm_neon.h> - -#define HORIZ_EXTRA_ROWS ((SUBPEL_TAPS + 7) & ~0x07) - -static INLINE uint8x8_t wiener_convolve8_vert_4x8( - const int16x8_t s0, const int16x8_t s1, const int16x8_t s2, - const int16x8_t s3, const int16x8_t s4, const int16x8_t s5, - const int16x8_t s6, int16_t *filter_y, const int bd, - const int round1_bits) { - int16x8_t ss0, ss1, ss2; - int32x4_t sum0, sum1; - uint16x4_t tmp0, tmp1; - uint16x8_t tmp; - uint8x8_t res; - - const int32_t round_const = (1 << (bd + round1_bits - 1)); - const int32x4_t round_bits = vdupq_n_s32(-round1_bits); - const int32x4_t zero = vdupq_n_s32(0); - const int32x4_t round_vec = vdupq_n_s32(round_const); - - ss0 = vaddq_s16(s0, s6); - ss1 = vaddq_s16(s1, s5); - ss2 = vaddq_s16(s2, s4); - - sum0 = vmull_n_s16(vget_low_s16(ss0), filter_y[0]); - sum0 = vmlal_n_s16(sum0, vget_low_s16(ss1), filter_y[1]); - sum0 = vmlal_n_s16(sum0, vget_low_s16(ss2), filter_y[2]); - sum0 = vmlal_n_s16(sum0, vget_low_s16(s3), filter_y[3]); - - sum1 = vmull_n_s16(vget_high_s16(ss0), filter_y[0]); - sum1 = vmlal_n_s16(sum1, vget_high_s16(ss1), filter_y[1]); - sum1 = vmlal_n_s16(sum1, vget_high_s16(ss2), filter_y[2]); - sum1 = vmlal_n_s16(sum1, vget_high_s16(s3), filter_y[3]); - - sum0 = vsubq_s32(sum0, round_vec); - sum1 = vsubq_s32(sum1, round_vec); - - /* right shift & rounding */ - sum0 = vrshlq_s32(sum0, round_bits); - sum1 = vrshlq_s32(sum1, round_bits); - - sum0 = vmaxq_s32(sum0, zero); - sum1 = vmaxq_s32(sum1, zero); - - /* from int32x4_t to uint8x8_t */ - tmp0 = vqmovn_u32(vreinterpretq_u32_s32(sum0)); - tmp1 = vqmovn_u32(vreinterpretq_u32_s32(sum1)); - tmp = vcombine_u16(tmp0, tmp1); - res = vqmovn_u16(tmp); - - return res; -} - -static INLINE uint16x8_t wiener_convolve8_horiz_8x8( - const int16x8_t s0, const int16x8_t s1, const int16x8_t s2, - const int16x8_t s3, int16_t *filter_x, const int bd, - const int round0_bits) { - int16x8_t sum; - uint16x8_t res; - int32x4_t sum_0, sum_1; - int32x4_t s3_0, s3_1; - const int32_t round_const_0 = (1 << (bd + FILTER_BITS - 1)); - const int32_t round_const_1 = (1 << ((bd) + 1 + FILTER_BITS - round0_bits)); - - /* for the purpose of right shift by { conv_params->round_0 } */ - const int32x4_t round_bits = vdupq_n_s32(-round0_bits); - - const int32x4_t round_vec_0 = vdupq_n_s32(round_const_0); - const int32x4_t round_vec_1 = vdupq_n_s32(round_const_1); - - sum = vmulq_n_s16(s0, filter_x[0]); - sum = vmlaq_n_s16(sum, s1, filter_x[1]); - sum = vmlaq_n_s16(sum, s2, filter_x[2]); - - /* sum from 16x8 to 2 32x4 registers */ - sum_0 = vmovl_s16(vget_low_s16(sum)); - sum_1 = vmovl_s16(vget_high_s16(sum)); - - /* s[3]*128 -- and filter coef max can be 128 - * then max value possible = 128*128*255 exceeding 16 bit - */ - - s3_0 = vmull_n_s16(vget_low_s16(s3), filter_x[3]); - s3_1 = vmull_n_s16(vget_high_s16(s3), filter_x[3]); - sum_0 = vaddq_s32(sum_0, s3_0); - sum_1 = vaddq_s32(sum_1, s3_1); - - /* Add the constant value */ - sum_0 = vaddq_s32(sum_0, round_vec_0); - sum_1 = vaddq_s32(sum_1, round_vec_0); - - /* right shift & rounding & saturating */ - sum_0 = vqrshlq_s32(sum_0, round_bits); - sum_1 = vqrshlq_s32(sum_1, round_bits); - - /* Clipping to max value */ - sum_0 = vminq_s32(sum_0, round_vec_1); - sum_1 = vminq_s32(sum_1, round_vec_1); - - res = vcombine_u16(vqmovun_s32(sum_0), vqmovun_s32(sum_1)); - return res; -} - -static INLINE uint16x4_t wiener_convolve8_horiz_4x8( - const int16x4_t s0, const int16x4_t s1, const int16x4_t s2, - const int16x4_t s3, const int16x4_t s4, const int16x4_t s5, - const int16x4_t s6, int16_t *filter_x, const int bd, - const int round0_bits) { - uint16x4_t res; - int32x4_t sum_0, s3_0; - int16x4_t sum, temp0, temp1, temp2; - - const int32_t round_const_0 = (1 << (bd + FILTER_BITS - 1)); - const int32_t round_const_1 = (1 << ((bd) + 1 + FILTER_BITS - round0_bits)); - const int32x4_t round_bits = vdupq_n_s32(-round0_bits); - const int32x4_t zero = vdupq_n_s32(0); - const int32x4_t round_vec_0 = vdupq_n_s32(round_const_0); - const int32x4_t round_vec_1 = vdupq_n_s32(round_const_1); - - temp0 = vadd_s16(s0, s6); - temp1 = vadd_s16(s1, s5); - temp2 = vadd_s16(s2, s4); - - sum = vmul_n_s16(temp0, filter_x[0]); - sum = vmla_n_s16(sum, temp1, filter_x[1]); - sum = vmla_n_s16(sum, temp2, filter_x[2]); - sum_0 = vmovl_s16(sum); - - /* s[3]*128 -- and filter coff max can be 128. - * then max value possible = 128*128*255 Therefore, 32 bits are required to - * hold the result. - */ - s3_0 = vmull_n_s16(s3, filter_x[3]); - sum_0 = vaddq_s32(sum_0, s3_0); - - sum_0 = vaddq_s32(sum_0, round_vec_0); - sum_0 = vrshlq_s32(sum_0, round_bits); - - sum_0 = vmaxq_s32(sum_0, zero); - sum_0 = vminq_s32(sum_0, round_vec_1); - res = vqmovun_s32(sum_0); - return res; -} - -static INLINE int16x8_t -convolve8_8x8_s16(const int16x8_t s0, const int16x8_t s1, const int16x8_t s2, - const int16x8_t s3, const int16x8_t s4, const int16x8_t s5, - const int16x8_t s6, const int16x8_t s7, const int16_t *filter, - const int16x8_t horiz_const, const int16x8_t shift_round_0) { - int16x8_t sum; - int16x8_t res; - - sum = horiz_const; - sum = vmlaq_n_s16(sum, s0, filter[0]); - sum = vmlaq_n_s16(sum, s1, filter[1]); - sum = vmlaq_n_s16(sum, s2, filter[2]); - sum = vmlaq_n_s16(sum, s3, filter[3]); - sum = vmlaq_n_s16(sum, s4, filter[4]); - sum = vmlaq_n_s16(sum, s5, filter[5]); - sum = vmlaq_n_s16(sum, s6, filter[6]); - sum = vmlaq_n_s16(sum, s7, filter[7]); - - res = vqrshlq_s16(sum, shift_round_0); - - return res; -} - -static INLINE int16x4_t -convolve8_4x4_s16(const int16x4_t s0, const int16x4_t s1, const int16x4_t s2, - const int16x4_t s3, const int16x4_t s4, const int16x4_t s5, - const int16x4_t s6, const int16x4_t s7, const int16_t *filter, - const int16x4_t horiz_const, const int16x4_t shift_round_0) { - int16x4_t sum; - sum = horiz_const; - sum = vmla_n_s16(sum, s0, filter[0]); - sum = vmla_n_s16(sum, s1, filter[1]); - sum = vmla_n_s16(sum, s2, filter[2]); - sum = vmla_n_s16(sum, s3, filter[3]); - sum = vmla_n_s16(sum, s4, filter[4]); - sum = vmla_n_s16(sum, s5, filter[5]); - sum = vmla_n_s16(sum, s6, filter[6]); - sum = vmla_n_s16(sum, s7, filter[7]); - - sum = vqrshl_s16(sum, shift_round_0); - - return sum; -} - -static INLINE uint16x4_t convolve8_4x4_s32( - const int16x4_t s0, const int16x4_t s1, const int16x4_t s2, - const int16x4_t s3, const int16x4_t s4, const int16x4_t s5, - const int16x4_t s6, const int16x4_t s7, const int16_t *y_filter, - const int32x4_t round_shift_vec, const int32x4_t offset_const) { - int32x4_t sum0; - uint16x4_t res; - const int32x4_t zero = vdupq_n_s32(0); - - sum0 = vmull_n_s16(s0, y_filter[0]); - sum0 = vmlal_n_s16(sum0, s1, y_filter[1]); - sum0 = vmlal_n_s16(sum0, s2, y_filter[2]); - sum0 = vmlal_n_s16(sum0, s3, y_filter[3]); - sum0 = vmlal_n_s16(sum0, s4, y_filter[4]); - sum0 = vmlal_n_s16(sum0, s5, y_filter[5]); - sum0 = vmlal_n_s16(sum0, s6, y_filter[6]); - sum0 = vmlal_n_s16(sum0, s7, y_filter[7]); - - sum0 = vaddq_s32(sum0, offset_const); - sum0 = vqrshlq_s32(sum0, round_shift_vec); - sum0 = vmaxq_s32(sum0, zero); - res = vmovn_u32(vreinterpretq_u32_s32(sum0)); - - return res; -} - -#endif // AOM_AV1_COMMON_ARM_CONVOLVE_NEON_H_ diff --git a/third_party/aom/av1/common/arm/jnt_convolve_neon.c b/third_party/aom/av1/common/arm/jnt_convolve_neon.c deleted file mode 100644 index e5674ef7c..000000000 --- a/third_party/aom/av1/common/arm/jnt_convolve_neon.c +++ /dev/null @@ -1,1740 +0,0 @@ -/* - * Copyright (c) 2018, Alliance for Open Media. All rights reserved - * - * This source code is subject to the terms of the BSD 2 Clause License and - * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License - * was not distributed with this source code in the LICENSE file, you can - * obtain it at www.aomedia.org/license/software. If the Alliance for Open - * Media Patent License 1.0 was not distributed with this source code in the - * PATENTS file, you can obtain it at www.aomedia.org/license/patent. - */ - -#include <arm_neon.h> -#include <assert.h> - -#include "config/aom_config.h" -#include "config/av1_rtcd.h" - -#include "aom_dsp/txfm_common.h" -#include "aom_ports/mem.h" -#include "av1/common/common.h" -#include "av1/common/arm/convolve_neon.h" -#include "av1/common/arm/mem_neon.h" -#include "av1/common/arm/transpose_neon.h" - -#if !defined(__aarch64__) -static INLINE void compute_avg_4x1(uint16x4_t res0, uint16x4_t d0, - const uint16_t fwd_offset, - const uint16_t bck_offset, - const int16x4_t sub_const_vec, - const int16_t round_bits, - const int use_jnt_comp_avg, uint8x8_t *t0) { - int16x4_t tmp0; - uint16x4_t tmp_u0; - uint32x4_t sum0; - int32x4_t dst0; - int16x8_t tmp4; - - if (use_jnt_comp_avg) { - const int32x4_t round_bits_vec = vdupq_n_s32((int32_t)(-round_bits)); - - sum0 = vmull_n_u16(res0, fwd_offset); - sum0 = vmlal_n_u16(sum0, d0, bck_offset); - - sum0 = vshrq_n_u32(sum0, DIST_PRECISION_BITS); - - dst0 = vsubq_s32(vreinterpretq_s32_u32(sum0), vmovl_s16(sub_const_vec)); - - dst0 = vqrshlq_s32(dst0, round_bits_vec); - - tmp0 = vqmovn_s32(dst0); - tmp4 = vcombine_s16(tmp0, tmp0); - - *t0 = vqmovun_s16(tmp4); - } else { - const int16x4_t round_bits_vec = vdup_n_s16(-round_bits); - tmp_u0 = vhadd_u16(res0, d0); - - tmp0 = vsub_s16(vreinterpret_s16_u16(tmp_u0), sub_const_vec); - - tmp0 = vqrshl_s16(tmp0, round_bits_vec); - - tmp4 = vcombine_s16(tmp0, tmp0); - - *t0 = vqmovun_s16(tmp4); - } -} - -static INLINE void compute_avg_8x1(uint16x8_t res0, uint16x8_t d0, - const uint16_t fwd_offset, - const uint16_t bck_offset, - const int16x4_t sub_const, - const int16_t round_bits, - const int use_jnt_comp_avg, uint8x8_t *t0) { - int16x4_t tmp0, tmp2; - int16x8_t f0; - uint32x4_t sum0, sum2; - int32x4_t dst0, dst2; - - uint16x8_t tmp_u0; - - if (use_jnt_comp_avg) { - const int32x4_t sub_const_vec = vmovl_s16(sub_const); - const int32x4_t round_bits_vec = vdupq_n_s32(-(int32_t)round_bits); - - sum0 = vmull_n_u16(vget_low_u16(res0), fwd_offset); - sum0 = vmlal_n_u16(sum0, vget_low_u16(d0), bck_offset); - sum0 = vshrq_n_u32(sum0, DIST_PRECISION_BITS); - - sum2 = vmull_n_u16(vget_high_u16(res0), fwd_offset); - sum2 = vmlal_n_u16(sum2, vget_high_u16(d0), bck_offset); - sum2 = vshrq_n_u32(sum2, DIST_PRECISION_BITS); - - dst0 = vsubq_s32(vreinterpretq_s32_u32(sum0), sub_const_vec); - dst2 = vsubq_s32(vreinterpretq_s32_u32(sum2), sub_const_vec); - - dst0 = vqrshlq_s32(dst0, round_bits_vec); - dst2 = vqrshlq_s32(dst2, round_bits_vec); - - tmp0 = vqmovn_s32(dst0); - tmp2 = vqmovn_s32(dst2); - - f0 = vcombine_s16(tmp0, tmp2); - - *t0 = vqmovun_s16(f0); - - } else { - const int16x8_t sub_const_vec = vcombine_s16(sub_const, sub_const); - const int16x8_t round_bits_vec = vdupq_n_s16(-round_bits); - - tmp_u0 = vhaddq_u16(res0, d0); - - f0 = vsubq_s16(vreinterpretq_s16_u16(tmp_u0), sub_const_vec); - - f0 = vqrshlq_s16(f0, round_bits_vec); - - *t0 = vqmovun_s16(f0); - } -} -#endif // !defined(__arch64__) - -static INLINE void compute_avg_4x4( - uint16x4_t res0, uint16x4_t res1, uint16x4_t res2, uint16x4_t res3, - uint16x4_t d0, uint16x4_t d1, uint16x4_t d2, uint16x4_t d3, - const uint16_t fwd_offset, const uint16_t bck_offset, - const int16x4_t sub_const_vec, const int16_t round_bits, - const int use_jnt_comp_avg, uint8x8_t *t0, uint8x8_t *t1) { - int16x4_t tmp0, tmp1, tmp2, tmp3; - uint16x4_t tmp_u0, tmp_u1, tmp_u2, tmp_u3; - uint32x4_t sum0, sum1, sum2, sum3; - - int32x4_t dst0, dst1, dst2, dst3; - int16x8_t tmp4, tmp5; - const int16x8_t zero = vdupq_n_s16(0); - - if (use_jnt_comp_avg) { - const int32x4_t round_bits_vec = vdupq_n_s32((int32_t)(-round_bits)); - const int32x4_t const_vec = vmovl_s16(sub_const_vec); - - sum0 = vmull_n_u16(res0, fwd_offset); - sum0 = vmlal_n_u16(sum0, d0, bck_offset); - sum1 = vmull_n_u16(res1, fwd_offset); - sum1 = vmlal_n_u16(sum1, d1, bck_offset); - sum2 = vmull_n_u16(res2, fwd_offset); - sum2 = vmlal_n_u16(sum2, d2, bck_offset); - sum3 = vmull_n_u16(res3, fwd_offset); - sum3 = vmlal_n_u16(sum3, d3, bck_offset); - - sum0 = vshrq_n_u32(sum0, DIST_PRECISION_BITS); - sum1 = vshrq_n_u32(sum1, DIST_PRECISION_BITS); - sum2 = vshrq_n_u32(sum2, DIST_PRECISION_BITS); - sum3 = vshrq_n_u32(sum3, DIST_PRECISION_BITS); - - dst0 = vsubq_s32(vreinterpretq_s32_u32(sum0), const_vec); - dst1 = vsubq_s32(vreinterpretq_s32_u32(sum1), const_vec); - dst2 = vsubq_s32(vreinterpretq_s32_u32(sum2), const_vec); - dst3 = vsubq_s32(vreinterpretq_s32_u32(sum3), const_vec); - - dst0 = vqrshlq_s32(dst0, round_bits_vec); - dst1 = vqrshlq_s32(dst1, round_bits_vec); - dst2 = vqrshlq_s32(dst2, round_bits_vec); - dst3 = vqrshlq_s32(dst3, round_bits_vec); - - tmp0 = vqmovn_s32(dst0); - tmp1 = vqmovn_s32(dst1); - tmp2 = vqmovn_s32(dst2); - tmp3 = vqmovn_s32(dst3); - tmp4 = vcombine_s16(tmp0, tmp1); - tmp5 = vcombine_s16(tmp2, tmp3); - tmp4 = vmaxq_s16(tmp4, zero); - tmp5 = vmaxq_s16(tmp5, zero); - - *t0 = vqmovn_u16(vreinterpretq_u16_s16(tmp4)); - *t1 = vqmovn_u16(vreinterpretq_u16_s16(tmp5)); - } else { - const int16x4_t round_bits_vec = vdup_n_s16(-round_bits); - tmp_u0 = vhadd_u16(res0, d0); - tmp_u1 = vhadd_u16(res1, d1); - tmp_u2 = vhadd_u16(res2, d2); - tmp_u3 = vhadd_u16(res3, d3); - - tmp0 = vsub_s16(vreinterpret_s16_u16(tmp_u0), sub_const_vec); - tmp1 = vsub_s16(vreinterpret_s16_u16(tmp_u1), sub_const_vec); - tmp2 = vsub_s16(vreinterpret_s16_u16(tmp_u2), sub_const_vec); - tmp3 = vsub_s16(vreinterpret_s16_u16(tmp_u3), sub_const_vec); - - tmp0 = vqrshl_s16(tmp0, round_bits_vec); - tmp1 = vqrshl_s16(tmp1, round_bits_vec); - tmp2 = vqrshl_s16(tmp2, round_bits_vec); - tmp3 = vqrshl_s16(tmp3, round_bits_vec); - - tmp4 = vcombine_s16(tmp0, tmp1); - tmp5 = vcombine_s16(tmp2, tmp3); - tmp4 = vmaxq_s16(tmp4, zero); - tmp5 = vmaxq_s16(tmp5, zero); - - *t0 = vqmovn_u16(vreinterpretq_u16_s16(tmp4)); - *t1 = vqmovn_u16(vreinterpretq_u16_s16(tmp5)); - } -} - -static INLINE void compute_avg_8x4( - uint16x8_t res0, uint16x8_t res1, uint16x8_t res2, uint16x8_t res3, - uint16x8_t d0, uint16x8_t d1, uint16x8_t d2, uint16x8_t d3, - const uint16_t fwd_offset, const uint16_t bck_offset, - const int16x4_t sub_const, const int16_t round_bits, - const int use_jnt_comp_avg, uint8x8_t *t0, uint8x8_t *t1, uint8x8_t *t2, - uint8x8_t *t3) { - int16x4_t tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7; - int16x8_t f0, f1, f2, f3; - uint32x4_t sum0, sum1, sum2, sum3; - uint32x4_t sum4, sum5, sum6, sum7; - int32x4_t dst0, dst1, dst2, dst3; - int32x4_t dst4, dst5, dst6, dst7; - uint16x8_t tmp_u0, tmp_u1, tmp_u2, tmp_u3; - const int16x8_t zero = vdupq_n_s16(0); - - if (use_jnt_comp_avg) { - const int32x4_t sub_const_vec = vmovl_s16(sub_const); - const int32x4_t round_bits_vec = vdupq_n_s32(-(int32_t)round_bits); - - sum0 = vmull_n_u16(vget_low_u16(res0), fwd_offset); - sum0 = vmlal_n_u16(sum0, vget_low_u16(d0), bck_offset); - sum1 = vmull_n_u16(vget_low_u16(res1), fwd_offset); - sum1 = vmlal_n_u16(sum1, vget_low_u16(d1), bck_offset); - sum0 = vshrq_n_u32(sum0, DIST_PRECISION_BITS); - sum1 = vshrq_n_u32(sum1, DIST_PRECISION_BITS); - - sum2 = vmull_n_u16(vget_high_u16(res0), fwd_offset); - sum2 = vmlal_n_u16(sum2, vget_high_u16(d0), bck_offset); - sum3 = vmull_n_u16(vget_high_u16(res1), fwd_offset); - sum3 = vmlal_n_u16(sum3, vget_high_u16(d1), bck_offset); - sum2 = vshrq_n_u32(sum2, DIST_PRECISION_BITS); - sum3 = vshrq_n_u32(sum3, DIST_PRECISION_BITS); - - sum4 = vmull_n_u16(vget_low_u16(res2), fwd_offset); - sum4 = vmlal_n_u16(sum4, vget_low_u16(d2), bck_offset); - sum5 = vmull_n_u16(vget_low_u16(res3), fwd_offset); - sum5 = vmlal_n_u16(sum5, vget_low_u16(d3), bck_offset); - sum4 = vshrq_n_u32(sum4, DIST_PRECISION_BITS); - sum5 = vshrq_n_u32(sum5, DIST_PRECISION_BITS); - - sum6 = vmull_n_u16(vget_high_u16(res2), fwd_offset); - sum6 = vmlal_n_u16(sum6, vget_high_u16(d2), bck_offset); - sum7 = vmull_n_u16(vget_high_u16(res3), fwd_offset); - sum7 = vmlal_n_u16(sum7, vget_high_u16(d3), bck_offset); - sum6 = vshrq_n_u32(sum6, DIST_PRECISION_BITS); - sum7 = vshrq_n_u32(sum7, DIST_PRECISION_BITS); - - dst0 = vsubq_s32(vreinterpretq_s32_u32(sum0), sub_const_vec); - dst1 = vsubq_s32(vreinterpretq_s32_u32(sum1), sub_const_vec); - dst2 = vsubq_s32(vreinterpretq_s32_u32(sum2), sub_const_vec); - dst3 = vsubq_s32(vreinterpretq_s32_u32(sum3), sub_const_vec); - dst4 = vsubq_s32(vreinterpretq_s32_u32(sum4), sub_const_vec); - dst5 = vsubq_s32(vreinterpretq_s32_u32(sum5), sub_const_vec); - dst6 = vsubq_s32(vreinterpretq_s32_u32(sum6), sub_const_vec); - dst7 = vsubq_s32(vreinterpretq_s32_u32(sum7), sub_const_vec); - - dst0 = vqrshlq_s32(dst0, round_bits_vec); - dst1 = vqrshlq_s32(dst1, round_bits_vec); - dst2 = vqrshlq_s32(dst2, round_bits_vec); - dst3 = vqrshlq_s32(dst3, round_bits_vec); - dst4 = vqrshlq_s32(dst4, round_bits_vec); - dst5 = vqrshlq_s32(dst5, round_bits_vec); - dst6 = vqrshlq_s32(dst6, round_bits_vec); - dst7 = vqrshlq_s32(dst7, round_bits_vec); - - tmp0 = vqmovn_s32(dst0); - tmp1 = vqmovn_s32(dst1); - tmp2 = vqmovn_s32(dst2); - tmp3 = vqmovn_s32(dst3); - tmp4 = vqmovn_s32(dst4); - tmp5 = vqmovn_s32(dst5); - tmp6 = vqmovn_s32(dst6); - tmp7 = vqmovn_s32(dst7); - - f0 = vcombine_s16(tmp0, tmp2); - f1 = vcombine_s16(tmp1, tmp3); - f2 = vcombine_s16(tmp4, tmp6); - f3 = vcombine_s16(tmp5, tmp7); - - f0 = vmaxq_s16(f0, zero); - f1 = vmaxq_s16(f1, zero); - f2 = vmaxq_s16(f2, zero); - f3 = vmaxq_s16(f3, zero); - - *t0 = vqmovn_u16(vreinterpretq_u16_s16(f0)); - *t1 = vqmovn_u16(vreinterpretq_u16_s16(f1)); - *t2 = vqmovn_u16(vreinterpretq_u16_s16(f2)); - *t3 = vqmovn_u16(vreinterpretq_u16_s16(f3)); - - } else { - const int16x8_t sub_const_vec = vcombine_s16(sub_const, sub_const); - const int16x8_t round_bits_vec = vdupq_n_s16(-round_bits); - - tmp_u0 = vhaddq_u16(res0, d0); - tmp_u1 = vhaddq_u16(res1, d1); - tmp_u2 = vhaddq_u16(res2, d2); - tmp_u3 = vhaddq_u16(res3, d3); - - f0 = vsubq_s16(vreinterpretq_s16_u16(tmp_u0), sub_const_vec); - f1 = vsubq_s16(vreinterpretq_s16_u16(tmp_u1), sub_const_vec); - f2 = vsubq_s16(vreinterpretq_s16_u16(tmp_u2), sub_const_vec); - f3 = vsubq_s16(vreinterpretq_s16_u16(tmp_u3), sub_const_vec); - - f0 = vqrshlq_s16(f0, round_bits_vec); - f1 = vqrshlq_s16(f1, round_bits_vec); - f2 = vqrshlq_s16(f2, round_bits_vec); - f3 = vqrshlq_s16(f3, round_bits_vec); - - f0 = vmaxq_s16(f0, zero); - f1 = vmaxq_s16(f1, zero); - f2 = vmaxq_s16(f2, zero); - f3 = vmaxq_s16(f3, zero); - - *t0 = vqmovn_u16(vreinterpretq_u16_s16(f0)); - *t1 = vqmovn_u16(vreinterpretq_u16_s16(f1)); - *t2 = vqmovn_u16(vreinterpretq_u16_s16(f2)); - *t3 = vqmovn_u16(vreinterpretq_u16_s16(f3)); - } -} - -static INLINE void jnt_convolve_2d_horiz_neon( - const uint8_t *src, int src_stride, int16_t *im_block, const int im_stride, - int16_t *x_filter_tmp, const int im_h, int w, const int round_0) { - const int bd = 8; - const uint8_t *s; - int16_t *dst_ptr; - int dst_stride; - int width, height; - - dst_ptr = im_block; - dst_stride = im_stride; - height = im_h; - width = w; - - if (w == 4) { - int16x4_t s0, s1, s2, s3, s4, s5, s6, s7, d0; - int16x8_t tt0; - uint8x8_t t0; - - const int16x4_t horiz_const = vdup_n_s16((1 << (bd + FILTER_BITS - 2))); - const int16x4_t shift_round_0 = vdup_n_s16(-(round_0)); - -#if defined(__aarch64__) - int16x4_t s8, s9, s10, d1, d2, d3; - int16x8_t tt1, tt2, tt3; - uint8x8_t t1, t2, t3; -#endif - do { - s = src; - __builtin_prefetch(s + 0 * src_stride); -#if defined(__aarch64__) - __builtin_prefetch(s + 1 * src_stride); - __builtin_prefetch(s + 2 * src_stride); - __builtin_prefetch(s + 3 * src_stride); - - load_u8_8x4(s, src_stride, &t0, &t1, &t2, &t3); - transpose_u8_8x4(&t0, &t1, &t2, &t3); - tt0 = vreinterpretq_s16_u16(vmovl_u8(t0)); - tt1 = vreinterpretq_s16_u16(vmovl_u8(t1)); - tt2 = vreinterpretq_s16_u16(vmovl_u8(t2)); - tt3 = vreinterpretq_s16_u16(vmovl_u8(t3)); - s0 = vget_low_s16(tt0); - s1 = vget_low_s16(tt1); - s2 = vget_low_s16(tt2); - s3 = vget_low_s16(tt3); - s4 = vget_high_s16(tt0); - s5 = vget_high_s16(tt1); - s6 = vget_high_s16(tt2); - __builtin_prefetch(dst_ptr + 0 * dst_stride); - __builtin_prefetch(dst_ptr + 1 * dst_stride); - __builtin_prefetch(dst_ptr + 2 * dst_stride); - __builtin_prefetch(dst_ptr + 3 * dst_stride); - s += 7; - - load_u8_8x4(s, src_stride, &t0, &t1, &t2, &t3); - transpose_u8_8x4(&t0, &t1, &t2, &t3); - tt0 = vreinterpretq_s16_u16(vmovl_u8(t0)); - tt1 = vreinterpretq_s16_u16(vmovl_u8(t1)); - tt2 = vreinterpretq_s16_u16(vmovl_u8(t2)); - tt3 = vreinterpretq_s16_u16(vmovl_u8(t3)); - s7 = vget_low_s16(tt0); - s8 = vget_low_s16(tt1); - s9 = vget_low_s16(tt2); - s10 = vget_low_s16(tt3); - - d0 = convolve8_4x4_s16(s0, s1, s2, s3, s4, s5, s6, s7, x_filter_tmp, - horiz_const, shift_round_0); - d1 = convolve8_4x4_s16(s1, s2, s3, s4, s5, s6, s7, s8, x_filter_tmp, - horiz_const, shift_round_0); - d2 = convolve8_4x4_s16(s2, s3, s4, s5, s6, s7, s8, s9, x_filter_tmp, - horiz_const, shift_round_0); - d3 = convolve8_4x4_s16(s3, s4, s5, s6, s7, s8, s9, s10, x_filter_tmp, - horiz_const, shift_round_0); - - transpose_s16_4x4d(&d0, &d1, &d2, &d3); - - vst1_s16((dst_ptr + 0 * dst_stride), d0); - vst1_s16((dst_ptr + 1 * dst_stride), d1); - vst1_s16((dst_ptr + 2 * dst_stride), d2); - vst1_s16((dst_ptr + 3 * dst_stride), d3); - - src += 4 * src_stride; - dst_ptr += 4 * dst_stride; - height -= 4; -#else - t0 = vld1_u8(s); // a0 a1 a2 a3 a4 a5 a6 a7 - tt0 = vreinterpretq_s16_u16(vmovl_u8(t0)); // a0 a1 a2 a3 a4 a5 a6 a7 - s0 = vget_low_s16(tt0); // a0 a1 a2 a3 - s4 = vget_high_s16(tt0); // a4 a5 a6 a7 - __builtin_prefetch(dst_ptr); - s += 8; - t0 = vld1_u8(s); // a8 a9 a10 a11 - - // a8 a9 a10 a11 - s7 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t0))); - - s1 = vext_s16(s0, s4, 1); // a1 a2 a3 a4 - s2 = vext_s16(s0, s4, 2); // a2 a3 a4 a5 - s3 = vext_s16(s0, s4, 3); // a3 a4 a5 a6 - s5 = vext_s16(s4, s7, 1); // a5 a6 a7 a8 - s6 = vext_s16(s4, s7, 2); // a6 a7 a8 a9 - s7 = vext_s16(s4, s7, 3); // a7 a8 a9 a10 - - d0 = convolve8_4x4_s16(s0, s1, s2, s3, s4, s5, s6, s7, x_filter_tmp, - horiz_const, shift_round_0); - - vst1_s16(dst_ptr, d0); - - src += src_stride; - dst_ptr += dst_stride; - height -= 1; -#endif - } while (height > 0); - } else { - int16_t *d_tmp; - int16x8_t s0, s1, s2, s3, s4, s5, s6, s7; - int16x8_t res0; - uint8x8_t t0; - - const int16x8_t horiz_const = vdupq_n_s16((1 << (bd + FILTER_BITS - 2))); - const int16x8_t shift_round_0 = vdupq_n_s16(-(round_0)); - do { -#if defined(__aarch64__) - uint8x8_t t1, t2, t3, t4, t5, t6, t7; - int16x8_t s8, s9, s10, s11, s12, s13, s14; - int16x8_t res1, res2, res3, res4, res5, res6, res7; - __builtin_prefetch(src + 0 * src_stride); - __builtin_prefetch(src + 1 * src_stride); - __builtin_prefetch(src + 2 * src_stride); - __builtin_prefetch(src + 3 * src_stride); - __builtin_prefetch(src + 4 * src_stride); - __builtin_prefetch(src + 5 * src_stride); - __builtin_prefetch(src + 6 * src_stride); - __builtin_prefetch(src + 7 * src_stride); - load_u8_8x8(src, src_stride, &t0, &t1, &t2, &t3, &t4, &t5, &t6, &t7); - transpose_u8_8x8(&t0, &t1, &t2, &t3, &t4, &t5, &t6, &t7); - s0 = vreinterpretq_s16_u16(vmovl_u8(t0)); - s1 = vreinterpretq_s16_u16(vmovl_u8(t1)); - s2 = vreinterpretq_s16_u16(vmovl_u8(t2)); - s3 = vreinterpretq_s16_u16(vmovl_u8(t3)); - s4 = vreinterpretq_s16_u16(vmovl_u8(t4)); - s5 = vreinterpretq_s16_u16(vmovl_u8(t5)); - s6 = vreinterpretq_s16_u16(vmovl_u8(t6)); - - width = w; - s = src + 7; - d_tmp = dst_ptr; - __builtin_prefetch(dst_ptr + 0 * dst_stride); - __builtin_prefetch(dst_ptr + 1 * dst_stride); - __builtin_prefetch(dst_ptr + 2 * dst_stride); - __builtin_prefetch(dst_ptr + 3 * dst_stride); - __builtin_prefetch(dst_ptr + 4 * dst_stride); - __builtin_prefetch(dst_ptr + 5 * dst_stride); - __builtin_prefetch(dst_ptr + 6 * dst_stride); - __builtin_prefetch(dst_ptr + 7 * dst_stride); - - do { - load_u8_8x8(s, src_stride, &t0, &t1, &t2, &t3, &t4, &t5, &t6, &t7); - transpose_u8_8x8(&t0, &t1, &t2, &t3, &t4, &t5, &t6, &t7); - s7 = vreinterpretq_s16_u16(vmovl_u8(t0)); - s8 = vreinterpretq_s16_u16(vmovl_u8(t1)); - s9 = vreinterpretq_s16_u16(vmovl_u8(t2)); - s10 = vreinterpretq_s16_u16(vmovl_u8(t3)); - s11 = vreinterpretq_s16_u16(vmovl_u8(t4)); - s12 = vreinterpretq_s16_u16(vmovl_u8(t5)); - s13 = vreinterpretq_s16_u16(vmovl_u8(t6)); - s14 = vreinterpretq_s16_u16(vmovl_u8(t7)); - - res0 = convolve8_8x8_s16(s0, s1, s2, s3, s4, s5, s6, s7, x_filter_tmp, - horiz_const, shift_round_0); - res1 = convolve8_8x8_s16(s1, s2, s3, s4, s5, s6, s7, s8, x_filter_tmp, - horiz_const, shift_round_0); - res2 = convolve8_8x8_s16(s2, s3, s4, s5, s6, s7, s8, s9, x_filter_tmp, - horiz_const, shift_round_0); - res3 = convolve8_8x8_s16(s3, s4, s5, s6, s7, s8, s9, s10, x_filter_tmp, - horiz_const, shift_round_0); - res4 = convolve8_8x8_s16(s4, s5, s6, s7, s8, s9, s10, s11, x_filter_tmp, - horiz_const, shift_round_0); - res5 = convolve8_8x8_s16(s5, s6, s7, s8, s9, s10, s11, s12, - x_filter_tmp, horiz_const, shift_round_0); - res6 = convolve8_8x8_s16(s6, s7, s8, s9, s10, s11, s12, s13, - x_filter_tmp, horiz_const, shift_round_0); - res7 = convolve8_8x8_s16(s7, s8, s9, s10, s11, s12, s13, s14, - x_filter_tmp, horiz_const, shift_round_0); - - transpose_s16_8x8(&res0, &res1, &res2, &res3, &res4, &res5, &res6, - &res7); - - store_s16_8x8(d_tmp, dst_stride, res0, res1, res2, res3, res4, res5, - res6, res7); - s0 = s8; - s1 = s9; - s2 = s10; - s3 = s11; - s4 = s12; - s5 = s13; - s6 = s14; - s += 8; - d_tmp += 8; - width -= 8; - } while (width > 0); - src += 8 * src_stride; - dst_ptr += 8 * dst_stride; - height -= 8; -#else - int16x8_t temp_0; - t0 = vld1_u8(src); - s0 = vreinterpretq_s16_u16(vmovl_u8(t0)); // a0 a1 a2 a3 a4 a5 a6 a7 - - width = w; - s = src + 8; - d_tmp = dst_ptr; - __builtin_prefetch(dst_ptr); - - do { - t0 = vld1_u8(s); // a8 a9 a10 a11 a12 a13 a14 a15 - s7 = vreinterpretq_s16_u16(vmovl_u8(t0)); - temp_0 = s0; - s0 = s7; - - s1 = vextq_s16(temp_0, s7, 1); // a1 a2 a3 a4 a5 a6 a7 a8 - s2 = vextq_s16(temp_0, s7, 2); // a2 a3 a4 a5 a6 a7 a8 a9 - s3 = vextq_s16(temp_0, s7, 3); // a3 a4 a5 a6 a7 a8 a9 a10 - s4 = vextq_s16(temp_0, s7, 4); // a4 a5 a6 a7 a8 a9 a10 a11 - s5 = vextq_s16(temp_0, s7, 5); // a5 a6 a7 a8 a9 a10 a11 a12 - s6 = vextq_s16(temp_0, s7, 6); // a6 a7 a8 a9 a10 a11 a12 a13 - s7 = vextq_s16(temp_0, s7, 7); // a7 a8 a9 a10 a11 a12 a13 a14 - - res0 = convolve8_8x8_s16(temp_0, s1, s2, s3, s4, s5, s6, s7, - x_filter_tmp, horiz_const, shift_round_0); - vst1q_s16(d_tmp, res0); - - s += 8; - d_tmp += 8; - width -= 8; - } while (width > 0); - src += src_stride; - dst_ptr += dst_stride; - height -= 1; -#endif - } while (height > 0); - } -} - -static INLINE void jnt_convolve_2d_vert_neon( - int16_t *im_block, const int im_stride, uint8_t *dst8, int dst8_stride, - ConvolveParams *conv_params, const int16_t *y_filter, int h, int w) { - uint8_t *dst_u8_ptr, *d_u8; - CONV_BUF_TYPE *dst_ptr, *dst; - int16_t *src_ptr, *s; - uint16_t *d; - - const int bd = 8; - int height; - int dst_stride = conv_params->dst_stride; - const int offset_bits = bd + 2 * FILTER_BITS - conv_params->round_0; - const int16_t sub_const = (1 << (offset_bits - conv_params->round_1)) + - (1 << (offset_bits - conv_params->round_1 - 1)); - - const int16_t round_bits = - 2 * FILTER_BITS - conv_params->round_0 - conv_params->round_1; - const int offset = bd + 2 * FILTER_BITS - conv_params->round_0; - const int32x4_t round_shift_vec = vdupq_n_s32(-(conv_params->round_1)); - const int32x4_t offset_const = vdupq_n_s32(1 << offset); - const int16x4_t sub_const_vec = vdup_n_s16(sub_const); - const uint16_t fwd_offset = conv_params->fwd_offset; - const uint16_t bck_offset = conv_params->bck_offset; - const int do_average = conv_params->do_average; - const int use_jnt_comp_avg = conv_params->use_jnt_comp_avg; - - int16x4_t s0, s1, s2, s3, s4, s5, s6, s7; - uint16x4_t res4, d0; - uint8x8_t t0; - -#if defined(__aarch64__) - int16x4_t s8, s9, s10; - uint16x4_t res5, res6, res7, d1, d2, d3; - uint8x8_t t1; -#endif - - dst = conv_params->dst; - src_ptr = im_block; - dst_u8_ptr = dst8; - dst_ptr = dst; - height = h; - - do { - d = dst_ptr; - d_u8 = dst_u8_ptr; - s = src_ptr; - height = h; - - __builtin_prefetch(s + 0 * im_stride); - __builtin_prefetch(s + 1 * im_stride); - __builtin_prefetch(s + 2 * im_stride); - __builtin_prefetch(s + 3 * im_stride); - __builtin_prefetch(s + 4 * im_stride); - __builtin_prefetch(s + 5 * im_stride); - __builtin_prefetch(s + 6 * im_stride); - __builtin_prefetch(s + 7 * im_stride); - - load_s16_4x8(s, im_stride, &s0, &s1, &s2, &s3, &s4, &s5, &s6, &s7); - s += (7 * im_stride); - - do { -#if defined(__aarch64__) - load_s16_4x4(s, im_stride, &s7, &s8, &s9, &s10); - s += (im_stride << 2); - - __builtin_prefetch(d + 0 * dst_stride); - __builtin_prefetch(d + 1 * dst_stride); - __builtin_prefetch(d + 2 * dst_stride); - __builtin_prefetch(d + 3 * dst_stride); - - __builtin_prefetch(d_u8 + 4 * dst8_stride); - __builtin_prefetch(d_u8 + 5 * dst8_stride); - __builtin_prefetch(d_u8 + 6 * dst8_stride); - __builtin_prefetch(d_u8 + 7 * dst8_stride); - - d0 = convolve8_4x4_s32(s0, s1, s2, s3, s4, s5, s6, s7, y_filter, - round_shift_vec, offset_const); - d1 = convolve8_4x4_s32(s1, s2, s3, s4, s5, s6, s7, s8, y_filter, - round_shift_vec, offset_const); - d2 = convolve8_4x4_s32(s2, s3, s4, s5, s6, s7, s8, s9, y_filter, - round_shift_vec, offset_const); - d3 = convolve8_4x4_s32(s3, s4, s5, s6, s7, s8, s9, s10, y_filter, - round_shift_vec, offset_const); - - if (do_average) { - load_u16_4x4(d, dst_stride, &res4, &res5, &res6, &res7); - d += (dst_stride << 2); - - compute_avg_4x4(res4, res5, res6, res7, d0, d1, d2, d3, fwd_offset, - bck_offset, sub_const_vec, round_bits, use_jnt_comp_avg, - &t0, &t1); - - vst1_lane_u32((uint32_t *)d_u8, vreinterpret_u32_u8(t0), 0); - d_u8 += dst8_stride; - vst1_lane_u32((uint32_t *)d_u8, vreinterpret_u32_u8(t0), 1); - d_u8 += dst8_stride; - vst1_lane_u32((uint32_t *)d_u8, vreinterpret_u32_u8(t1), 0); - d_u8 += dst8_stride; - vst1_lane_u32((uint32_t *)d_u8, vreinterpret_u32_u8(t1), 1); - d_u8 += dst8_stride; - - } else { - store_u16_4x4(d, dst_stride, d0, d1, d2, d3); - d += (dst_stride << 2); - } - s0 = s4; - s1 = s5; - s2 = s6; - s3 = s7; - s4 = s8; - s5 = s9; - s6 = s10; - height -= 4; -#else - s7 = vld1_s16(s); - s += (im_stride); - - __builtin_prefetch(d + 0 * dst_stride); - __builtin_prefetch(d_u8 + 0 * dst8_stride); - - d0 = convolve8_4x4_s32(s0, s1, s2, s3, s4, s5, s6, s7, y_filter, - round_shift_vec, offset_const); - - if (do_average) { - res4 = vld1_u16(d); - d += (dst_stride); - - compute_avg_4x1(res4, d0, fwd_offset, bck_offset, sub_const_vec, - round_bits, use_jnt_comp_avg, &t0); - - vst1_lane_u32((uint32_t *)d_u8, vreinterpret_u32_u8(t0), 0); - d_u8 += dst8_stride; - - } else { - vst1_u16(d, d0); - d += (dst_stride); - } - s0 = s1; - s1 = s2; - s2 = s3; - s3 = s4; - s4 = s5; - s5 = s6; - s6 = s7; - height--; -#endif - } while (height > 0); - src_ptr += 4; - dst_ptr += 4; - dst_u8_ptr += 4; - w -= 4; - } while (w > 0); -} - -void av1_jnt_convolve_2d_neon(const uint8_t *src, int src_stride, uint8_t *dst8, - int dst8_stride, int w, int h, - const InterpFilterParams *filter_params_x, - const InterpFilterParams *filter_params_y, - const int subpel_x_q4, const int subpel_y_q4, - ConvolveParams *conv_params) { - assert(!(w % 4)); - assert(!(h % 4)); - - DECLARE_ALIGNED(16, int16_t, - im_block[(MAX_SB_SIZE + HORIZ_EXTRA_ROWS) * MAX_SB_SIZE]); - - const int im_h = h + filter_params_y->taps - 1; - const int im_stride = MAX_SB_SIZE; - const int vert_offset = filter_params_y->taps / 2 - 1; - const int horiz_offset = filter_params_x->taps / 2 - 1; - const int round_0 = conv_params->round_0 - 1; - const uint8_t *src_ptr = src - vert_offset * src_stride - horiz_offset; - const int16_t *x_filter = av1_get_interp_filter_subpel_kernel( - filter_params_x, subpel_x_q4 & SUBPEL_MASK); - const int16_t *y_filter = av1_get_interp_filter_subpel_kernel( - filter_params_y, subpel_y_q4 & SUBPEL_MASK); - - int16_t x_filter_tmp[8]; - int16x8_t filter_x_coef = vld1q_s16(x_filter); - - // filter coeffs are even, so downshifting by 1 to reduce intermediate - // precision requirements. - filter_x_coef = vshrq_n_s16(filter_x_coef, 1); - vst1q_s16(&x_filter_tmp[0], filter_x_coef); - - jnt_convolve_2d_horiz_neon(src_ptr, src_stride, im_block, im_stride, - x_filter_tmp, im_h, w, round_0); - - jnt_convolve_2d_vert_neon(im_block, im_stride, dst8, dst8_stride, conv_params, - y_filter, h, w); -} - -void av1_jnt_convolve_2d_copy_neon(const uint8_t *src, int src_stride, - uint8_t *dst8, int dst8_stride, int w, int h, - const InterpFilterParams *filter_params_x, - const InterpFilterParams *filter_params_y, - const int subpel_x_q4, const int subpel_y_q4, - ConvolveParams *conv_params) { - uint8x8_t res0_8, res1_8, res2_8, res3_8, tmp_shift0, tmp_shift1, tmp_shift2, - tmp_shift3; - uint16x8_t res_q0, res_q1, res_q2, res_q3, tmp_q0, tmp_q1, tmp_q2, tmp_q3; - uint16x4_t tmp4, tmp5, tmp6, tmp7, res4, res5, res6, res7; - const uint8_t *src1, *src2; - uint8_t *dst8_1; - CONV_BUF_TYPE *dst = conv_params->dst, *dst_1, *dst_2; - const int dst_stride = conv_params->dst_stride; - int x, y; - const int16_t bits = - FILTER_BITS * 2 - conv_params->round_1 - conv_params->round_0; - const int bd = 8; - const int offset_bits = bd + 2 * FILTER_BITS - conv_params->round_0; - const int round_offset = (1 << (offset_bits - conv_params->round_1)) + - (1 << (offset_bits - conv_params->round_1 - 1)); - const int16x4_t sub_const_vec = vdup_n_s16((int16_t)round_offset); - const uint16x8_t dup_round_offset16x8 = vdupq_n_u16((uint16_t)round_offset); - const int16x4_t dup_bits16x4 = vdup_n_s16(bits); - const int16x8_t dup_bits16x8 = vdupq_n_s16(bits); - - (void)filter_params_x; - (void)filter_params_y; - (void)subpel_x_q4; - (void)subpel_y_q4; - - if (!(w & 0x07)) { - for (y = 0; y < (h >> 2); ++y) { - src1 = src; - dst8_1 = dst8; - dst_1 = dst; - for (x = 0; x < (w >> 3); ++x) { - src2 = src1; - load_u8_8x4(src2, src_stride, &res0_8, &res1_8, &res2_8, &res3_8); - - res_q0 = vaddq_u16(vshlq_u16(vmovl_u8(res0_8), dup_bits16x8), - dup_round_offset16x8); - res_q1 = vaddq_u16(vshlq_u16(vmovl_u8(res1_8), dup_bits16x8), - dup_round_offset16x8); - res_q2 = vaddq_u16(vshlq_u16(vmovl_u8(res2_8), dup_bits16x8), - dup_round_offset16x8); - res_q3 = vaddq_u16(vshlq_u16(vmovl_u8(res3_8), dup_bits16x8), - dup_round_offset16x8); - - if (conv_params->do_average) { - dst_2 = dst_1; - load_u16_8x4(dst_2, dst_stride, &tmp_q0, &tmp_q1, &tmp_q2, &tmp_q3); - - compute_avg_8x4(tmp_q0, tmp_q1, tmp_q2, tmp_q3, res_q0, res_q1, - res_q2, res_q3, conv_params->fwd_offset, - conv_params->bck_offset, sub_const_vec, bits, - conv_params->use_jnt_comp_avg, &tmp_shift0, - &tmp_shift1, &tmp_shift2, &tmp_shift3); - - vst1_u8(dst8_1 + (0 * dst8_stride), tmp_shift0); - vst1_u8(dst8_1 + (1 * dst8_stride), tmp_shift1); - vst1_u8(dst8_1 + (2 * dst8_stride), tmp_shift2); - vst1_u8(dst8_1 + (3 * dst8_stride), tmp_shift3); - - } else { - vst1q_u16(dst_1 + (0 * dst_stride), res_q0); - vst1q_u16(dst_1 + (1 * dst_stride), res_q1); - vst1q_u16(dst_1 + (2 * dst_stride), res_q2); - vst1q_u16(dst_1 + (3 * dst_stride), res_q3); - } - src1 = src1 + 8; - dst_1 = dst_1 + 8; - dst8_1 = dst8_1 + 8; - } - src += src_stride * 4; - dst8 += dst8_stride * 4; - dst += dst_stride * 4; - } - } else if (!(w & 0x03)) { - for (y = 0; y < (h >> 2); ++y) { - src1 = src; - dst8_1 = dst8; - dst_1 = dst; - - load_u8_8x4(src1, src_stride, &res0_8, &res1_8, &res2_8, &res3_8); - - res4 = vadd_u16(vshl_u16(vget_low_u16(vmovl_u8(res0_8)), dup_bits16x4), - vreinterpret_u16_s16(sub_const_vec)); - res5 = vadd_u16(vshl_u16(vget_low_u16(vmovl_u8(res1_8)), dup_bits16x4), - vreinterpret_u16_s16(sub_const_vec)); - res6 = vadd_u16(vshl_u16(vget_low_u16(vmovl_u8(res2_8)), dup_bits16x4), - vreinterpret_u16_s16(sub_const_vec)); - res7 = vadd_u16(vshl_u16(vget_low_u16(vmovl_u8(res3_8)), dup_bits16x4), - vreinterpret_u16_s16(sub_const_vec)); - if (conv_params->do_average) { - load_u16_4x4(dst_1, dst_stride, &tmp4, &tmp5, &tmp6, &tmp7); - - compute_avg_4x4(tmp4, tmp5, tmp6, tmp7, res4, res5, res6, res7, - conv_params->fwd_offset, conv_params->bck_offset, - sub_const_vec, bits, conv_params->use_jnt_comp_avg, - &tmp_shift0, &tmp_shift1); - - vst1_lane_u32((uint32_t *)(dst8_1), vreinterpret_u32_u8(tmp_shift0), 0); - dst8_1 += dst8_stride; - vst1_lane_u32((uint32_t *)(dst8_1), vreinterpret_u32_u8(tmp_shift0), 1); - dst8_1 += dst8_stride; - vst1_lane_u32((uint32_t *)(dst8_1), vreinterpret_u32_u8(tmp_shift1), 0); - dst8_1 += dst8_stride; - vst1_lane_u32((uint32_t *)(dst8_1), vreinterpret_u32_u8(tmp_shift1), 1); - - } else { - vst1_u16(dst_1, res4); - dst_1 += dst_stride; - vst1_u16(dst_1, res5); - dst_1 += dst_stride; - vst1_u16(dst_1, res6); - dst_1 += dst_stride; - vst1_u16(dst_1, res7); - } - src += src_stride * 4; - dst += dst_stride * 4; - dst8 += dst8_stride * 4; - } - } -} - -void av1_jnt_convolve_x_neon(const uint8_t *src, int src_stride, uint8_t *dst8, - int dst8_stride, int w, int h, - const InterpFilterParams *filter_params_x, - const InterpFilterParams *filter_params_y, - const int subpel_x_q4, const int subpel_y_q4, - ConvolveParams *conv_params) { - assert(!(w % 4)); - assert(!(h % 4)); - - CONV_BUF_TYPE *dst = conv_params->dst; - int dst_stride = conv_params->dst_stride; - const int horiz_offset = filter_params_x->taps / 2 - 1; - const int bits = FILTER_BITS - conv_params->round_1; - const int bd = 8; - const int offset_bits = bd + 2 * FILTER_BITS - conv_params->round_0; - const int round_offset = (1 << (offset_bits - conv_params->round_1)) + - (1 << (offset_bits - conv_params->round_1 - 1)); - const int round_bits = - 2 * FILTER_BITS - conv_params->round_0 - conv_params->round_1; - const uint16_t fwd_offset = conv_params->fwd_offset; - const uint16_t bck_offset = conv_params->bck_offset; - const int use_jnt_comp_avg = conv_params->use_jnt_comp_avg; - - (void)filter_params_y; - (void)subpel_y_q4; - - // horizontal filter - const int16_t *x_filter = av1_get_interp_filter_subpel_kernel( - filter_params_x, subpel_x_q4 & SUBPEL_MASK); - - const uint8_t *src_ptr = src - horiz_offset; - - int16_t x_filter_tmp[8]; - int16x8_t filter_x_coef = vld1q_s16(x_filter); - - // filter coeffs are even, so downshifting by 1 to reduce intermediate - // precision requirements. - filter_x_coef = vshrq_n_s16(filter_x_coef, 1); - vst1q_s16(&x_filter_tmp[0], filter_x_coef); - - const uint8_t *s; - uint8_t *d_u8; - uint8_t *dst_u8_ptr; - CONV_BUF_TYPE *d, *dst_ptr; - int width, height; - uint8x8_t t0; -#if defined(__aarch64__) - uint8x8_t t1, t2, t3, t4, t5, t6, t7; -#endif - s = src_ptr; - dst_ptr = dst; - dst_u8_ptr = dst8; - width = w; - height = h; - - if ((w == 4) || (h == 4)) { - int16x4_t s0, s1, s2, s3, s4, s5, s6, s7, d0; - int16x8_t tt0; - uint16x4_t res4; -#if defined(__aarch64__) - int16x4_t s8, s9, s10, d1, d2, d3; - int16x8_t tt1, tt2, tt3; - uint16x4_t res5, res6, res7; - uint32x2_t tu0 = vdup_n_u32(0), tu1 = vdup_n_u32(0); - int16x8_t u0, u1; -#else - int16x4_t temp_0; -#endif - const int16x4_t zero = vdup_n_s16(0); - const int16x4_t round_offset_vec = vdup_n_s16(round_offset); - const int16x4_t shift_round_0 = vdup_n_s16(-conv_params->round_0 + 1); - const int16x4_t horiz_const = vdup_n_s16(bits); - do { - s = src_ptr; - d = dst_ptr; - d_u8 = dst_u8_ptr; - width = w; - __builtin_prefetch(s + 0 * src_stride); -#if defined(__aarch64__) - __builtin_prefetch(s + 1 * src_stride); - __builtin_prefetch(s + 2 * src_stride); - __builtin_prefetch(s + 3 * src_stride); - - load_u8_8x4(s, src_stride, &t0, &t1, &t2, &t3); - transpose_u8_8x4(&t0, &t1, &t2, &t3); - tt0 = vreinterpretq_s16_u16(vmovl_u8(t0)); - tt1 = vreinterpretq_s16_u16(vmovl_u8(t1)); - tt2 = vreinterpretq_s16_u16(vmovl_u8(t2)); - tt3 = vreinterpretq_s16_u16(vmovl_u8(t3)); - s0 = vget_low_s16(tt0); - s1 = vget_low_s16(tt1); - s2 = vget_low_s16(tt2); - s3 = vget_low_s16(tt3); - s4 = vget_high_s16(tt0); - s5 = vget_high_s16(tt1); - s6 = vget_high_s16(tt2); - __builtin_prefetch(d + 0 * dst_stride); - __builtin_prefetch(d + 1 * dst_stride); - __builtin_prefetch(d + 2 * dst_stride); - __builtin_prefetch(d + 3 * dst_stride); - s += 7; - do { - load_unaligned_u8_4x4(s, src_stride, &tu0, &tu1); - t0 = vreinterpret_u8_u32(tu0); - t1 = vreinterpret_u8_u32(tu1); - - transpose_u8_4x4(&t0, &t1); - u0 = vreinterpretq_s16_u16(vmovl_u8(t0)); - u1 = vreinterpretq_s16_u16(vmovl_u8(t1)); - - s7 = vget_low_s16(u0); - s8 = vget_low_s16(u1); - s9 = vget_high_s16(u0); - s10 = vget_high_s16(u1); - - d0 = convolve8_4x4_s16(s0, s1, s2, s3, s4, s5, s6, s7, x_filter_tmp, - zero, shift_round_0); - d0 = vrshl_s16(d0, horiz_const); - d0 = vadd_s16(d0, round_offset_vec); - d1 = convolve8_4x4_s16(s1, s2, s3, s4, s5, s6, s7, s8, x_filter_tmp, - zero, shift_round_0); - d1 = vrshl_s16(d1, horiz_const); - d1 = vadd_s16(d1, round_offset_vec); - d2 = convolve8_4x4_s16(s2, s3, s4, s5, s6, s7, s8, s9, x_filter_tmp, - zero, shift_round_0); - d2 = vrshl_s16(d2, horiz_const); - d2 = vadd_s16(d2, round_offset_vec); - d3 = convolve8_4x4_s16(s3, s4, s5, s6, s7, s8, s9, s10, x_filter_tmp, - zero, shift_round_0); - d3 = vrshl_s16(d3, horiz_const); - d3 = vadd_s16(d3, round_offset_vec); - - transpose_s16_4x4d(&d0, &d1, &d2, &d3); - - if (conv_params->do_average) { - __builtin_prefetch(d + 0 * dst_stride); - __builtin_prefetch(d + 1 * dst_stride); - __builtin_prefetch(d + 2 * dst_stride); - __builtin_prefetch(d + 3 * dst_stride); - - __builtin_prefetch(d_u8 + 0 * dst8_stride); - __builtin_prefetch(d_u8 + 1 * dst8_stride); - __builtin_prefetch(d_u8 + 2 * dst8_stride); - __builtin_prefetch(d_u8 + 3 * dst8_stride); - - load_u16_4x4(d, dst_stride, &res4, &res5, &res6, &res7); - - compute_avg_4x4(res4, res5, res6, res7, vreinterpret_u16_s16(d0), - vreinterpret_u16_s16(d1), vreinterpret_u16_s16(d2), - vreinterpret_u16_s16(d3), fwd_offset, bck_offset, - round_offset_vec, round_bits, use_jnt_comp_avg, &t0, - &t1); - - vst1_lane_u32((uint32_t *)d_u8, vreinterpret_u32_u8(t0), - 0); // 00 01 02 03 - vst1_lane_u32((uint32_t *)(d_u8 + dst8_stride), - vreinterpret_u32_u8(t0), - 1); // 10 11 12 13 - vst1_lane_u32((uint32_t *)(d_u8 + 2 * dst8_stride), - vreinterpret_u32_u8(t1), - 0); // 20 21 22 23 - vst1_lane_u32((uint32_t *)(d_u8 + 3 * dst8_stride), - vreinterpret_u32_u8(t1), - 1); // 30 31 32 33 - } else { - store_u16_4x4(d, dst_stride, vreinterpret_u16_s16(d0), - vreinterpret_u16_s16(d1), vreinterpret_u16_s16(d2), - vreinterpret_u16_s16(d3)); - } - - s0 = s4; - s1 = s5; - s2 = s6; - s3 = s7; - s4 = s8; - s5 = s9; - s6 = s10; - - s += 4; - width -= 4; - d += 4; - d_u8 += 4; - } while (width > 0); - src_ptr += (src_stride << 2); - dst_ptr += (dst_stride << 2); - dst_u8_ptr += (dst8_stride << 2); - height -= 4; -#else - t0 = vld1_u8(s); // a0 a1 a2 a3 a4 a5 a6 a7 - tt0 = vreinterpretq_s16_u16(vmovl_u8(t0)); // a0 a1 a2 a3 a4 a5 a6 a7 - s0 = vget_low_s16(tt0); // a0 a1 a2 a3 - s4 = vget_high_s16(tt0); // a4 a5 a6 a7 - __builtin_prefetch(d); - - s += 8; - do { - t0 = vld1_u8(s); // a8 a9 a10 a11 - - // a8 a9 a10 a11 - s7 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t0))); - temp_0 = s7; - s1 = vext_s16(s0, s4, 1); // a1 a2 a3 a4 - s2 = vext_s16(s0, s4, 2); // a2 a3 a4 a5 - s3 = vext_s16(s0, s4, 3); // a3 a4 a5 a6 - s5 = vext_s16(s4, s7, 1); // a5 a6 a7 a8 - s6 = vext_s16(s4, s7, 2); // a6 a7 a8 a9 - s7 = vext_s16(s4, s7, 3); // a7 a8 a9 a10 - - d0 = convolve8_4x4_s16(s0, s1, s2, s3, s4, s5, s6, s7, x_filter_tmp, - zero, shift_round_0); - d0 = vrshl_s16(d0, horiz_const); - d0 = vadd_s16(d0, round_offset_vec); - s0 = s4; - s4 = temp_0; - if (conv_params->do_average) { - __builtin_prefetch(d); - __builtin_prefetch(d_u8); - - res4 = vld1_u16(d); - - compute_avg_4x1(res4, vreinterpret_u16_s16(d0), fwd_offset, - bck_offset, round_offset_vec, round_bits, - use_jnt_comp_avg, &t0); - - vst1_lane_u32((uint32_t *)d_u8, vreinterpret_u32_u8(t0), - 0); // 00 01 02 03 - } else { - vst1_u16(d, vreinterpret_u16_s16(d0)); - } - - s += 4; - width -= 4; - d += 4; - d_u8 += 4; - } while (width > 0); - src_ptr += (src_stride); - dst_ptr += (dst_stride); - dst_u8_ptr += (dst8_stride); - height--; -#endif - } while (height > 0); - } else { - CONV_BUF_TYPE *d_tmp; - uint8_t *d_u8_tmp; - int16x8_t s0, s1, s2, s3, s4, s5, s6, s7; - int16x8_t res0; - uint16x8_t res8; - const int16x8_t round_offset128 = vdupq_n_s16(round_offset); - const int16x4_t round_offset64 = vdup_n_s16(round_offset); - const int16x8_t shift_round_0 = vdupq_n_s16(-conv_params->round_0 + 1); - const int16x8_t horiz_const = vdupq_n_s16(bits); - const int16x8_t zero = vdupq_n_s16(0); - - d = dst_ptr = dst; - d_u8 = dst_u8_ptr = dst8; - do { -#if defined(__aarch64__) - int16x8_t s11, s12, s13, s14; - int16x8_t s8, s9, s10; - int16x8_t res1, res2, res3, res4, res5, res6, res7; - uint16x8_t res9, res10, res11; - __builtin_prefetch(src_ptr + 0 * src_stride); - __builtin_prefetch(src_ptr + 1 * src_stride); - __builtin_prefetch(src_ptr + 2 * src_stride); - __builtin_prefetch(src_ptr + 3 * src_stride); - __builtin_prefetch(src_ptr + 4 * src_stride); - __builtin_prefetch(src_ptr + 5 * src_stride); - __builtin_prefetch(src_ptr + 6 * src_stride); - __builtin_prefetch(src_ptr + 7 * src_stride); - load_u8_8x8(src_ptr, src_stride, &t0, &t1, &t2, &t3, &t4, &t5, &t6, &t7); - transpose_u8_8x8(&t0, &t1, &t2, &t3, &t4, &t5, &t6, &t7); - s0 = vreinterpretq_s16_u16(vmovl_u8(t0)); - s1 = vreinterpretq_s16_u16(vmovl_u8(t1)); - s2 = vreinterpretq_s16_u16(vmovl_u8(t2)); - s3 = vreinterpretq_s16_u16(vmovl_u8(t3)); - s4 = vreinterpretq_s16_u16(vmovl_u8(t4)); - s5 = vreinterpretq_s16_u16(vmovl_u8(t5)); - s6 = vreinterpretq_s16_u16(vmovl_u8(t6)); - - width = w; - s = src_ptr + 7; - d = dst_ptr; - d_u8_tmp = dst_u8_ptr; - - __builtin_prefetch(dst_ptr + 0 * dst_stride); - __builtin_prefetch(dst_ptr + 1 * dst_stride); - __builtin_prefetch(dst_ptr + 2 * dst_stride); - __builtin_prefetch(dst_ptr + 3 * dst_stride); - __builtin_prefetch(dst_ptr + 4 * dst_stride); - __builtin_prefetch(dst_ptr + 5 * dst_stride); - __builtin_prefetch(dst_ptr + 6 * dst_stride); - __builtin_prefetch(dst_ptr + 7 * dst_stride); - - do { - d_u8 = d_u8_tmp; - d_tmp = d; - - load_u8_8x8(s, src_stride, &t0, &t1, &t2, &t3, &t4, &t5, &t6, &t7); - transpose_u8_8x8(&t0, &t1, &t2, &t3, &t4, &t5, &t6, &t7); - s7 = vreinterpretq_s16_u16(vmovl_u8(t0)); - s8 = vreinterpretq_s16_u16(vmovl_u8(t1)); - s9 = vreinterpretq_s16_u16(vmovl_u8(t2)); - s10 = vreinterpretq_s16_u16(vmovl_u8(t3)); - s11 = vreinterpretq_s16_u16(vmovl_u8(t4)); - s12 = vreinterpretq_s16_u16(vmovl_u8(t5)); - s13 = vreinterpretq_s16_u16(vmovl_u8(t6)); - s14 = vreinterpretq_s16_u16(vmovl_u8(t7)); - - res0 = convolve8_8x8_s16(s0, s1, s2, s3, s4, s5, s6, s7, x_filter_tmp, - zero, shift_round_0); - - res0 = vrshlq_s16(res0, horiz_const); - res0 = vaddq_s16(res0, round_offset128); - - res1 = convolve8_8x8_s16(s1, s2, s3, s4, s5, s6, s7, s8, x_filter_tmp, - zero, shift_round_0); - res1 = vrshlq_s16(res1, horiz_const); - res1 = vaddq_s16(res1, round_offset128); - res2 = convolve8_8x8_s16(s2, s3, s4, s5, s6, s7, s8, s9, x_filter_tmp, - zero, shift_round_0); - res2 = vrshlq_s16(res2, horiz_const); - res2 = vaddq_s16(res2, round_offset128); - res3 = convolve8_8x8_s16(s3, s4, s5, s6, s7, s8, s9, s10, x_filter_tmp, - zero, shift_round_0); - res3 = vrshlq_s16(res3, horiz_const); - res3 = vaddq_s16(res3, round_offset128); - res4 = convolve8_8x8_s16(s4, s5, s6, s7, s8, s9, s10, s11, x_filter_tmp, - zero, shift_round_0); - res4 = vrshlq_s16(res4, horiz_const); - res4 = vaddq_s16(res4, round_offset128); - res5 = convolve8_8x8_s16(s5, s6, s7, s8, s9, s10, s11, s12, - x_filter_tmp, zero, shift_round_0); - res5 = vrshlq_s16(res5, horiz_const); - res5 = vaddq_s16(res5, round_offset128); - res6 = convolve8_8x8_s16(s6, s7, s8, s9, s10, s11, s12, s13, - x_filter_tmp, zero, shift_round_0); - res6 = vrshlq_s16(res6, horiz_const); - res6 = vaddq_s16(res6, round_offset128); - res7 = convolve8_8x8_s16(s7, s8, s9, s10, s11, s12, s13, s14, - x_filter_tmp, zero, shift_round_0); - res7 = vrshlq_s16(res7, horiz_const); - res7 = vaddq_s16(res7, round_offset128); - - transpose_s16_8x8(&res0, &res1, &res2, &res3, &res4, &res5, &res6, - &res7); - - if (conv_params->do_average) { - load_u16_8x4(d_tmp, dst_stride, &res8, &res9, &res10, &res11); - d_tmp += (dst_stride << 2); - - compute_avg_8x4( - res8, res9, res10, res11, vreinterpretq_u16_s16(res0), - vreinterpretq_u16_s16(res1), vreinterpretq_u16_s16(res2), - vreinterpretq_u16_s16(res3), fwd_offset, bck_offset, - round_offset64, round_bits, use_jnt_comp_avg, &t0, &t1, &t2, &t3); - - store_u8_8x4(d_u8, dst8_stride, t0, t1, t2, t3); - d_u8 += (dst8_stride << 2); - - load_u16_8x4(d_tmp, dst_stride, &res8, &res9, &res10, &res11); - d_tmp += (dst_stride << 2); - - compute_avg_8x4( - res8, res9, res10, res11, vreinterpretq_u16_s16(res4), - vreinterpretq_u16_s16(res5), vreinterpretq_u16_s16(res6), - vreinterpretq_u16_s16(res7), fwd_offset, bck_offset, - round_offset64, round_bits, use_jnt_comp_avg, &t0, &t1, &t2, &t3); - - store_u8_8x4(d_u8, dst8_stride, t0, t1, t2, t3); - d_u8 += (dst8_stride << 2); - } else { - store_u16_8x8( - d_tmp, dst_stride, vreinterpretq_u16_s16(res0), - vreinterpretq_u16_s16(res1), vreinterpretq_u16_s16(res2), - vreinterpretq_u16_s16(res3), vreinterpretq_u16_s16(res4), - vreinterpretq_u16_s16(res5), vreinterpretq_u16_s16(res6), - vreinterpretq_u16_s16(res7)); - d_tmp += (dst_stride << 3); - } - - s0 = s8; - s1 = s9; - s2 = s10; - s3 = s11; - s4 = s12; - s5 = s13; - s6 = s14; - s += 8; - d += 8; - width -= 8; - d_u8_tmp += 8; - } while (width > 0); - src_ptr += 8 * src_stride; - dst_ptr += 8 * dst_stride; - dst_u8_ptr += 8 * dst8_stride; - height -= 8; -#else - int16x8_t temp_0; - __builtin_prefetch(src_ptr); - t0 = vld1_u8(src_ptr); - s0 = vreinterpretq_s16_u16(vmovl_u8(t0)); // a0 a1 a2 a3 a4 a5 a6 a7 - - width = w; - s = src_ptr + 8; - d = dst_ptr; - d_u8_tmp = dst_u8_ptr; - - __builtin_prefetch(dst_ptr); - - do { - d_u8 = d_u8_tmp; - d_tmp = d; - - t0 = vld1_u8(s); // a8 a9 a10 a11 a12 a13 a14 a15 - s7 = vreinterpretq_s16_u16(vmovl_u8(t0)); - temp_0 = s0; - s0 = s7; - - s1 = vextq_s16(temp_0, s7, 1); // a1 a2 a3 a4 a5 a6 a7 a8 - s2 = vextq_s16(temp_0, s7, 2); // a2 a3 a4 a5 a6 a7 a8 a9 - s3 = vextq_s16(temp_0, s7, 3); // a3 a4 a5 a6 a7 a8 a9 a10 - s4 = vextq_s16(temp_0, s7, 4); // a4 a5 a6 a7 a8 a9 a10 a11 - s5 = vextq_s16(temp_0, s7, 5); // a5 a6 a7 a8 a9 a10 a11 a12 - s6 = vextq_s16(temp_0, s7, 6); // a6 a7 a8 a9 a10 a11 a12 a13 - s7 = vextq_s16(temp_0, s7, 7); // a7 a8 a9 a10 a11 a12 a13 a14 - - res0 = convolve8_8x8_s16(temp_0, s1, s2, s3, s4, s5, s6, s7, - x_filter_tmp, zero, shift_round_0); - - res0 = vrshlq_s16(res0, horiz_const); - res0 = vaddq_s16(res0, round_offset128); - - if (conv_params->do_average) { - res8 = vld1q_u16(d_tmp); - d_tmp += (dst_stride); - - compute_avg_8x1(res8, vreinterpretq_u16_s16(res0), fwd_offset, - bck_offset, round_offset64, round_bits, - use_jnt_comp_avg, &t0); - - vst1_u8(d_u8, t0); - d_u8 += (dst8_stride); - } else { - vst1q_u16(d_tmp, vreinterpretq_u16_s16(res0)); - d_tmp += (dst_stride); - } - - s += 8; - d += 8; - width -= 8; - d_u8_tmp += 8; - } while (width > 0); - src_ptr += src_stride; - dst_ptr += dst_stride; - dst_u8_ptr += dst8_stride; - height--; -#endif - } while (height > 0); - } -} - -void av1_jnt_convolve_y_neon(const uint8_t *src, int src_stride, uint8_t *dst8, - int dst8_stride, int w, int h, - const InterpFilterParams *filter_params_x, - const InterpFilterParams *filter_params_y, - const int subpel_x_q4, const int subpel_y_q4, - ConvolveParams *conv_params) { - assert(!(w % 4)); - assert(!(h % 4)); - - CONV_BUF_TYPE *dst = conv_params->dst; - const int dst_stride = conv_params->dst_stride; - const int vert_offset = filter_params_y->taps / 2 - 1; - const int bits = FILTER_BITS - conv_params->round_0; - const int bd = 8; - const int offset_bits = bd + 2 * FILTER_BITS - conv_params->round_0; - const int round_offset = (1 << (offset_bits - conv_params->round_1)) + - (1 << (offset_bits - conv_params->round_1 - 1)); - const int round_bits = - 2 * FILTER_BITS - conv_params->round_0 - conv_params->round_1; - const uint16_t fwd_offset = conv_params->fwd_offset; - const uint16_t bck_offset = conv_params->bck_offset; - const int use_jnt_comp_avg = conv_params->use_jnt_comp_avg; - const int shift_value = (conv_params->round_1 - 1 - bits); - - (void)filter_params_x; - (void)subpel_x_q4; - - // vertical filter - const int16_t *y_filter = av1_get_interp_filter_subpel_kernel( - filter_params_y, subpel_y_q4 & SUBPEL_MASK); - - const uint8_t *src_ptr = src - (vert_offset * src_stride); - - int16_t y_filter_tmp[8]; - int16x8_t filter_y_coef = vld1q_s16(y_filter); - - // filter coeffs are even, so downshifting by 1 to reduce intermediate - // precision requirements. - filter_y_coef = vshrq_n_s16(filter_y_coef, 1); - vst1q_s16(&y_filter_tmp[0], filter_y_coef); - - const uint8_t *s; - uint8_t *d_u8; - uint8_t *dst_u8_ptr; - CONV_BUF_TYPE *d, *dst_ptr; - int width, height; - - s = src_ptr; - dst_ptr = dst; - dst_u8_ptr = dst8; - width = w; - height = h; - - // used to get rid of multiplication = (vertical filter output sum) * - // (1<<bits). - assert((conv_params->round_1 - 2) >= bits); - - if ((w == 4) || (h == 4)) { - int16x4_t s0, s1, s2, s3, s4, s5, s6, s7, d0; - uint16x4_t res4; - uint32x2_t tu0 = vdup_n_u32(0), tu1 = vdup_n_u32(0), tu2 = vdup_n_u32(0), - tu3 = vdup_n_u32(0); - int16x8_t u0, u1, u2, u3; - uint8x8_t t0; - -#if defined(__aarch64__) - int16x4_t s8, s9, s10, d1, d2, d3; - uint16x4_t res5, res6, res7; - uint8x8_t t1; -#endif - const int16x4_t round_offset64 = vdup_n_s16(round_offset); - const int16x4_t shift_vec = vdup_n_s16(-shift_value); - const int16x4_t zero = vdup_n_s16(0); - - do { - s = src_ptr; - d = dst_ptr; - d_u8 = dst_u8_ptr; - height = h; - __builtin_prefetch(s + 0 * src_stride); - __builtin_prefetch(s + 1 * src_stride); - __builtin_prefetch(s + 2 * src_stride); - __builtin_prefetch(s + 3 * src_stride); - - load_unaligned_u8_4x8(s, src_stride, &tu0, &tu1, &tu2, &tu3); - - u0 = vreinterpretq_s16_u16(vmovl_u8(vreinterpret_u8_u32(tu0))); - u1 = vreinterpretq_s16_u16(vmovl_u8(vreinterpret_u8_u32(tu1))); - u2 = vreinterpretq_s16_u16(vmovl_u8(vreinterpret_u8_u32(tu2))); - u3 = vreinterpretq_s16_u16(vmovl_u8(vreinterpret_u8_u32(tu3))); - - s0 = vget_low_s16(u0); - s1 = vget_high_s16(u0); - s2 = vget_low_s16(u1); - s3 = vget_high_s16(u1); - s4 = vget_low_s16(u2); - s5 = vget_high_s16(u2); - s6 = vget_low_s16(u3); - - __builtin_prefetch(d + 0 * dst_stride); - __builtin_prefetch(d + 1 * dst_stride); - __builtin_prefetch(d + 2 * dst_stride); - __builtin_prefetch(d + 3 * dst_stride); - - s += (7 * src_stride); - do { -#if defined(__aarch64__) - load_unaligned_u8_4x4(s, src_stride, &tu0, &tu1); - - u0 = vreinterpretq_s16_u16(vmovl_u8(vreinterpret_u8_u32(tu0))); - u1 = vreinterpretq_s16_u16(vmovl_u8(vreinterpret_u8_u32(tu1))); - - s7 = vget_low_s16(u0); - s8 = vget_high_s16(u0); - s9 = vget_low_s16(u1); - s10 = vget_high_s16(u1); - - d0 = convolve8_4x4_s16(s0, s1, s2, s3, s4, s5, s6, s7, y_filter_tmp, - zero, shift_vec); - d0 = vadd_s16(d0, round_offset64); - d1 = convolve8_4x4_s16(s1, s2, s3, s4, s5, s6, s7, s8, y_filter_tmp, - zero, shift_vec); - d1 = vadd_s16(d1, round_offset64); - d2 = convolve8_4x4_s16(s2, s3, s4, s5, s6, s7, s8, s9, y_filter_tmp, - zero, shift_vec); - d2 = vadd_s16(d2, round_offset64); - d3 = convolve8_4x4_s16(s3, s4, s5, s6, s7, s8, s9, s10, y_filter_tmp, - zero, shift_vec); - d3 = vadd_s16(d3, round_offset64); - - if (conv_params->do_average) { - __builtin_prefetch(d + 0 * dst_stride); - __builtin_prefetch(d + 1 * dst_stride); - __builtin_prefetch(d + 2 * dst_stride); - __builtin_prefetch(d + 3 * dst_stride); - - __builtin_prefetch(d_u8 + 0 * dst8_stride); - __builtin_prefetch(d_u8 + 1 * dst8_stride); - __builtin_prefetch(d_u8 + 2 * dst8_stride); - __builtin_prefetch(d_u8 + 3 * dst8_stride); - - load_u16_4x4(d, dst_stride, &res4, &res5, &res6, &res7); - d += (dst_stride << 2); - - compute_avg_4x4(res4, res5, res6, res7, vreinterpret_u16_s16(d0), - vreinterpret_u16_s16(d1), vreinterpret_u16_s16(d2), - vreinterpret_u16_s16(d3), fwd_offset, bck_offset, - round_offset64, round_bits, use_jnt_comp_avg, &t0, - &t1); - - vst1_lane_u32((uint32_t *)d_u8, vreinterpret_u32_u8(t0), 0); - d_u8 += dst8_stride; - vst1_lane_u32((uint32_t *)d_u8, vreinterpret_u32_u8(t0), 1); - d_u8 += dst8_stride; - vst1_lane_u32((uint32_t *)d_u8, vreinterpret_u32_u8(t1), 0); - d_u8 += dst8_stride; - vst1_lane_u32((uint32_t *)d_u8, vreinterpret_u32_u8(t1), 1); - d_u8 += dst8_stride; - } else { - store_u16_4x4(d, dst_stride, vreinterpret_u16_s16(d0), - vreinterpret_u16_s16(d1), vreinterpret_u16_s16(d2), - vreinterpret_u16_s16(d3)); - d += (dst_stride << 2); - } - - s0 = s4; - s1 = s5; - s2 = s6; - s3 = s7; - s4 = s8; - s5 = s9; - s6 = s10; - - s += (src_stride << 2); - height -= 4; -#else - load_unaligned_u8_4x1(s, src_stride, &tu0); - u0 = vreinterpretq_s16_u16(vmovl_u8(vreinterpret_u8_u32(tu0))); - s7 = vget_low_s16(u0); - - d0 = convolve8_4x4_s16(s0, s1, s2, s3, s4, s5, s6, s7, y_filter_tmp, - zero, shift_vec); - - d0 = vadd_s16(d0, round_offset64); - - if (conv_params->do_average) { - __builtin_prefetch(d); - - res4 = vld1_u16(d); - d += (dst_stride); - - compute_avg_4x1(res4, vreinterpret_u16_s16(d0), fwd_offset, - bck_offset, round_offset64, round_bits, - use_jnt_comp_avg, &t0); - - vst1_lane_u32((uint32_t *)d_u8, vreinterpret_u32_u8(t0), 0); - d_u8 += dst8_stride; - } else { - vst1_u16(d, vreinterpret_u16_s16(d0)); - d += (dst_stride); - } - - s0 = s1; - s1 = s2; - s2 = s3; - s3 = s4; - s4 = s5; - s5 = s6; - s6 = s7; - - s += (src_stride); - height--; -#endif - } while (height > 0); - src_ptr += 4; - dst_ptr += 4; - dst_u8_ptr += 4; - width -= 4; - } while (width > 0); - } else { - CONV_BUF_TYPE *d_tmp; - int16x8_t s0, s1, s2, s3, s4, s5, s6, s7; - int16x8_t res0; - uint16x8_t res8; - uint8x8_t t0, t1, t2, t3, t4, t5, t6, t7; - const int16x8_t round_offset128 = vdupq_n_s16(round_offset); - const int16x8_t shift_vec = vdupq_n_s16(-shift_value); - const int16x4_t round_offset64 = vdup_n_s16(round_offset); - const int16x8_t zero = vdupq_n_s16(0); -#if defined(__aarch64__) - int16x8_t s8, s9, s10, s11, s12, s13, s14; - int16x8_t res1, res2, res3, res4, res5, res6, res7; - uint16x8_t res10, res11, res9; -#endif - dst_ptr = dst; - dst_u8_ptr = dst8; - do { - __builtin_prefetch(src_ptr + 0 * src_stride); - __builtin_prefetch(src_ptr + 1 * src_stride); - __builtin_prefetch(src_ptr + 2 * src_stride); - __builtin_prefetch(src_ptr + 3 * src_stride); - __builtin_prefetch(src_ptr + 4 * src_stride); - __builtin_prefetch(src_ptr + 5 * src_stride); - __builtin_prefetch(src_ptr + 6 * src_stride); - __builtin_prefetch(src_ptr + 7 * src_stride); - load_u8_8x8(src_ptr, src_stride, &t0, &t1, &t2, &t3, &t4, &t5, &t6, &t7); - - s0 = vreinterpretq_s16_u16(vmovl_u8(t0)); - s1 = vreinterpretq_s16_u16(vmovl_u8(t1)); - s2 = vreinterpretq_s16_u16(vmovl_u8(t2)); - s3 = vreinterpretq_s16_u16(vmovl_u8(t3)); - s4 = vreinterpretq_s16_u16(vmovl_u8(t4)); - s5 = vreinterpretq_s16_u16(vmovl_u8(t5)); - s6 = vreinterpretq_s16_u16(vmovl_u8(t6)); - - height = h; - s = src_ptr + (7 * src_stride); - d_tmp = dst_ptr; - d_u8 = dst_u8_ptr; - - do { -#if defined(__aarch64__) - load_u8_8x8(s, src_stride, &t0, &t1, &t2, &t3, &t4, &t5, &t6, &t7); - - s7 = vreinterpretq_s16_u16(vmovl_u8(t0)); - s8 = vreinterpretq_s16_u16(vmovl_u8(t1)); - s9 = vreinterpretq_s16_u16(vmovl_u8(t2)); - s10 = vreinterpretq_s16_u16(vmovl_u8(t3)); - s11 = vreinterpretq_s16_u16(vmovl_u8(t4)); - s12 = vreinterpretq_s16_u16(vmovl_u8(t5)); - s13 = vreinterpretq_s16_u16(vmovl_u8(t6)); - s14 = vreinterpretq_s16_u16(vmovl_u8(t7)); - - __builtin_prefetch(dst_ptr + 0 * dst_stride); - __builtin_prefetch(dst_ptr + 1 * dst_stride); - __builtin_prefetch(dst_ptr + 2 * dst_stride); - __builtin_prefetch(dst_ptr + 3 * dst_stride); - - res0 = convolve8_8x8_s16(s0, s1, s2, s3, s4, s5, s6, s7, y_filter_tmp, - zero, shift_vec); - res0 = vaddq_s16(res0, round_offset128); - res1 = convolve8_8x8_s16(s1, s2, s3, s4, s5, s6, s7, s8, y_filter_tmp, - zero, shift_vec); - res1 = vaddq_s16(res1, round_offset128); - res2 = convolve8_8x8_s16(s2, s3, s4, s5, s6, s7, s8, s9, y_filter_tmp, - zero, shift_vec); - res2 = vaddq_s16(res2, round_offset128); - res3 = convolve8_8x8_s16(s3, s4, s5, s6, s7, s8, s9, s10, y_filter_tmp, - zero, shift_vec); - res3 = vaddq_s16(res3, round_offset128); - res4 = convolve8_8x8_s16(s4, s5, s6, s7, s8, s9, s10, s11, y_filter_tmp, - zero, shift_vec); - res4 = vaddq_s16(res4, round_offset128); - res5 = convolve8_8x8_s16(s5, s6, s7, s8, s9, s10, s11, s12, - y_filter_tmp, zero, shift_vec); - res5 = vaddq_s16(res5, round_offset128); - res6 = convolve8_8x8_s16(s6, s7, s8, s9, s10, s11, s12, s13, - y_filter_tmp, zero, shift_vec); - res6 = vaddq_s16(res6, round_offset128); - res7 = convolve8_8x8_s16(s7, s8, s9, s10, s11, s12, s13, s14, - y_filter_tmp, zero, shift_vec); - res7 = vaddq_s16(res7, round_offset128); - - if (conv_params->do_average) { - __builtin_prefetch(d_tmp + 0 * dst8_stride); - __builtin_prefetch(d_tmp + 1 * dst8_stride); - __builtin_prefetch(d_tmp + 2 * dst8_stride); - __builtin_prefetch(d_tmp + 3 * dst8_stride); - - load_u16_8x4(d_tmp, dst_stride, &res8, &res9, &res10, &res11); - d_tmp += (dst_stride << 2); - - compute_avg_8x4( - res8, res9, res10, res11, vreinterpretq_u16_s16(res0), - vreinterpretq_u16_s16(res1), vreinterpretq_u16_s16(res2), - vreinterpretq_u16_s16(res3), fwd_offset, bck_offset, - round_offset64, round_bits, use_jnt_comp_avg, &t0, &t1, &t2, &t3); - - store_u8_8x4(d_u8, dst8_stride, t0, t1, t2, t3); - d_u8 += (dst8_stride << 2); - - load_u16_8x4(d_tmp, dst_stride, &res8, &res9, &res10, &res11); - d_tmp += (dst_stride << 2); - - compute_avg_8x4( - res8, res9, res10, res11, vreinterpretq_u16_s16(res4), - vreinterpretq_u16_s16(res5), vreinterpretq_u16_s16(res6), - vreinterpretq_u16_s16(res7), fwd_offset, bck_offset, - round_offset64, round_bits, use_jnt_comp_avg, &t0, &t1, &t2, &t3); - - store_u8_8x4(d_u8, dst8_stride, t0, t1, t2, t3); - d_u8 += (dst8_stride << 2); - } else { - store_u16_8x8( - d_tmp, dst_stride, vreinterpretq_u16_s16(res0), - vreinterpretq_u16_s16(res1), vreinterpretq_u16_s16(res2), - vreinterpretq_u16_s16(res3), vreinterpretq_u16_s16(res4), - vreinterpretq_u16_s16(res5), vreinterpretq_u16_s16(res6), - vreinterpretq_u16_s16(res7)); - d_tmp += (dst_stride << 3); - } - - s0 = s8; - s1 = s9; - s2 = s10; - s3 = s11; - s4 = s12; - s5 = s13; - s6 = s14; - s += (8 * src_stride); - height -= 8; -#else - s7 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(s))); - - __builtin_prefetch(dst_ptr); - - res0 = convolve8_8x8_s16(s0, s1, s2, s3, s4, s5, s6, s7, y_filter_tmp, - zero, shift_vec); - res0 = vaddq_s16(res0, round_offset128); - - s0 = s1; - s1 = s2; - s2 = s3; - s3 = s4; - s4 = s5; - s5 = s6; - s6 = s7; - - if (conv_params->do_average) { - __builtin_prefetch(d_tmp); - - res8 = vld1q_u16(d_tmp); - d_tmp += (dst_stride); - - compute_avg_8x1(res8, vreinterpretq_u16_s16(res0), fwd_offset, - bck_offset, round_offset64, round_bits, - use_jnt_comp_avg, &t0); - - vst1_u8(d_u8, t0); - d_u8 += (dst8_stride); - } else { - vst1q_u16(d_tmp, vreinterpretq_u16_s16(res0)); - d_tmp += dst_stride; - } - - s += (src_stride); - height--; -#endif - } while (height > 0); - src_ptr += 8; - dst_ptr += 8; - dst_u8_ptr += 8; - width -= 8; - } while (width > 0); - } -} diff --git a/third_party/aom/av1/common/arm/mem_neon.h b/third_party/aom/av1/common/arm/mem_neon.h deleted file mode 100644 index c4ae2e784..000000000 --- a/third_party/aom/av1/common/arm/mem_neon.h +++ /dev/null @@ -1,494 +0,0 @@ -/* - * Copyright (c) 2018, Alliance for Open Media. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#ifndef AOM_AV1_COMMON_ARM_MEM_NEON_H_ -#define AOM_AV1_COMMON_ARM_MEM_NEON_H_ - -#include <arm_neon.h> -#include <string.h> - -static INLINE void store_row2_u8_8x8(uint8_t *s, int p, const uint8x8_t s0, - const uint8x8_t s1) { - vst1_u8(s, s0); - s += p; - vst1_u8(s, s1); - s += p; -} - -/* These intrinsics require immediate values, so we must use #defines - to enforce that. */ -#define load_u8_4x1(s, s0, lane) \ - do { \ - *(s0) = vreinterpret_u8_u32( \ - vld1_lane_u32((uint32_t *)(s), vreinterpret_u32_u8(*(s0)), lane)); \ - } while (0) - -static INLINE void load_u8_8x8(const uint8_t *s, ptrdiff_t p, - uint8x8_t *const s0, uint8x8_t *const s1, - uint8x8_t *const s2, uint8x8_t *const s3, - uint8x8_t *const s4, uint8x8_t *const s5, - uint8x8_t *const s6, uint8x8_t *const s7) { - *s0 = vld1_u8(s); - s += p; - *s1 = vld1_u8(s); - s += p; - *s2 = vld1_u8(s); - s += p; - *s3 = vld1_u8(s); - s += p; - *s4 = vld1_u8(s); - s += p; - *s5 = vld1_u8(s); - s += p; - *s6 = vld1_u8(s); - s += p; - *s7 = vld1_u8(s); -} - -static INLINE void load_u8_8x16(const uint8_t *s, ptrdiff_t p, - uint8x16_t *const s0, uint8x16_t *const s1, - uint8x16_t *const s2, uint8x16_t *const s3) { - *s0 = vld1q_u8(s); - s += p; - *s1 = vld1q_u8(s); - s += p; - *s2 = vld1q_u8(s); - s += p; - *s3 = vld1q_u8(s); -} - -static INLINE void load_u8_8x4(const uint8_t *s, const ptrdiff_t p, - uint8x8_t *const s0, uint8x8_t *const s1, - uint8x8_t *const s2, uint8x8_t *const s3) { - *s0 = vld1_u8(s); - s += p; - *s1 = vld1_u8(s); - s += p; - *s2 = vld1_u8(s); - s += p; - *s3 = vld1_u8(s); -} - -static INLINE void load_u16_4x4(const uint16_t *s, const ptrdiff_t p, - uint16x4_t *const s0, uint16x4_t *const s1, - uint16x4_t *const s2, uint16x4_t *const s3) { - *s0 = vld1_u16(s); - s += p; - *s1 = vld1_u16(s); - s += p; - *s2 = vld1_u16(s); - s += p; - *s3 = vld1_u16(s); - s += p; -} - -static INLINE void load_u16_8x4(const uint16_t *s, const ptrdiff_t p, - uint16x8_t *const s0, uint16x8_t *const s1, - uint16x8_t *const s2, uint16x8_t *const s3) { - *s0 = vld1q_u16(s); - s += p; - *s1 = vld1q_u16(s); - s += p; - *s2 = vld1q_u16(s); - s += p; - *s3 = vld1q_u16(s); - s += p; -} - -static INLINE void load_s16_4x8(const int16_t *s, ptrdiff_t p, - int16x4_t *const s0, int16x4_t *const s1, - int16x4_t *const s2, int16x4_t *const s3, - int16x4_t *const s4, int16x4_t *const s5, - int16x4_t *const s6, int16x4_t *const s7) { - *s0 = vld1_s16(s); - s += p; - *s1 = vld1_s16(s); - s += p; - *s2 = vld1_s16(s); - s += p; - *s3 = vld1_s16(s); - s += p; - *s4 = vld1_s16(s); - s += p; - *s5 = vld1_s16(s); - s += p; - *s6 = vld1_s16(s); - s += p; - *s7 = vld1_s16(s); -} - -static INLINE void load_s16_4x4(const int16_t *s, ptrdiff_t p, - int16x4_t *const s0, int16x4_t *const s1, - int16x4_t *const s2, int16x4_t *const s3) { - *s0 = vld1_s16(s); - s += p; - *s1 = vld1_s16(s); - s += p; - *s2 = vld1_s16(s); - s += p; - *s3 = vld1_s16(s); -} - -/* These intrinsics require immediate values, so we must use #defines - to enforce that. */ -#define store_u8_4x1(s, s0, lane) \ - do { \ - vst1_lane_u32((uint32_t *)(s), vreinterpret_u32_u8(s0), lane); \ - } while (0) - -static INLINE void store_u8_8x8(uint8_t *s, ptrdiff_t p, const uint8x8_t s0, - const uint8x8_t s1, const uint8x8_t s2, - const uint8x8_t s3, const uint8x8_t s4, - const uint8x8_t s5, const uint8x8_t s6, - const uint8x8_t s7) { - vst1_u8(s, s0); - s += p; - vst1_u8(s, s1); - s += p; - vst1_u8(s, s2); - s += p; - vst1_u8(s, s3); - s += p; - vst1_u8(s, s4); - s += p; - vst1_u8(s, s5); - s += p; - vst1_u8(s, s6); - s += p; - vst1_u8(s, s7); -} - -static INLINE void store_u8_8x4(uint8_t *s, ptrdiff_t p, const uint8x8_t s0, - const uint8x8_t s1, const uint8x8_t s2, - const uint8x8_t s3) { - vst1_u8(s, s0); - s += p; - vst1_u8(s, s1); - s += p; - vst1_u8(s, s2); - s += p; - vst1_u8(s, s3); -} - -static INLINE void store_u8_8x16(uint8_t *s, ptrdiff_t p, const uint8x16_t s0, - const uint8x16_t s1, const uint8x16_t s2, - const uint8x16_t s3) { - vst1q_u8(s, s0); - s += p; - vst1q_u8(s, s1); - s += p; - vst1q_u8(s, s2); - s += p; - vst1q_u8(s, s3); -} - -static INLINE void store_u16_8x8(uint16_t *s, ptrdiff_t dst_stride, - const uint16x8_t s0, const uint16x8_t s1, - const uint16x8_t s2, const uint16x8_t s3, - const uint16x8_t s4, const uint16x8_t s5, - const uint16x8_t s6, const uint16x8_t s7) { - vst1q_u16(s, s0); - s += dst_stride; - vst1q_u16(s, s1); - s += dst_stride; - vst1q_u16(s, s2); - s += dst_stride; - vst1q_u16(s, s3); - s += dst_stride; - vst1q_u16(s, s4); - s += dst_stride; - vst1q_u16(s, s5); - s += dst_stride; - vst1q_u16(s, s6); - s += dst_stride; - vst1q_u16(s, s7); -} - -static INLINE void store_u16_4x4(uint16_t *s, ptrdiff_t dst_stride, - const uint16x4_t s0, const uint16x4_t s1, - const uint16x4_t s2, const uint16x4_t s3) { - vst1_u16(s, s0); - s += dst_stride; - vst1_u16(s, s1); - s += dst_stride; - vst1_u16(s, s2); - s += dst_stride; - vst1_u16(s, s3); -} - -static INLINE void store_u16_8x4(uint16_t *s, ptrdiff_t dst_stride, - const uint16x8_t s0, const uint16x8_t s1, - const uint16x8_t s2, const uint16x8_t s3) { - vst1q_u16(s, s0); - s += dst_stride; - vst1q_u16(s, s1); - s += dst_stride; - vst1q_u16(s, s2); - s += dst_stride; - vst1q_u16(s, s3); -} - -static INLINE void store_s16_8x8(int16_t *s, ptrdiff_t dst_stride, - const int16x8_t s0, const int16x8_t s1, - const int16x8_t s2, const int16x8_t s3, - const int16x8_t s4, const int16x8_t s5, - const int16x8_t s6, const int16x8_t s7) { - vst1q_s16(s, s0); - s += dst_stride; - vst1q_s16(s, s1); - s += dst_stride; - vst1q_s16(s, s2); - s += dst_stride; - vst1q_s16(s, s3); - s += dst_stride; - vst1q_s16(s, s4); - s += dst_stride; - vst1q_s16(s, s5); - s += dst_stride; - vst1q_s16(s, s6); - s += dst_stride; - vst1q_s16(s, s7); -} - -static INLINE void store_s16_4x4(int16_t *s, ptrdiff_t dst_stride, - const int16x4_t s0, const int16x4_t s1, - const int16x4_t s2, const int16x4_t s3) { - vst1_s16(s, s0); - s += dst_stride; - vst1_s16(s, s1); - s += dst_stride; - vst1_s16(s, s2); - s += dst_stride; - vst1_s16(s, s3); -} - -static INLINE void store_s16_8x4(int16_t *s, ptrdiff_t dst_stride, - const int16x8_t s0, const int16x8_t s1, - const int16x8_t s2, const int16x8_t s3) { - vst1q_s16(s, s0); - s += dst_stride; - vst1q_s16(s, s1); - s += dst_stride; - vst1q_s16(s, s2); - s += dst_stride; - vst1q_s16(s, s3); -} - -static INLINE void load_s16_8x8(const int16_t *s, ptrdiff_t p, - int16x8_t *const s0, int16x8_t *const s1, - int16x8_t *const s2, int16x8_t *const s3, - int16x8_t *const s4, int16x8_t *const s5, - int16x8_t *const s6, int16x8_t *const s7) { - *s0 = vld1q_s16(s); - s += p; - *s1 = vld1q_s16(s); - s += p; - *s2 = vld1q_s16(s); - s += p; - *s3 = vld1q_s16(s); - s += p; - *s4 = vld1q_s16(s); - s += p; - *s5 = vld1q_s16(s); - s += p; - *s6 = vld1q_s16(s); - s += p; - *s7 = vld1q_s16(s); -} - -static INLINE void load_s16_8x4(const int16_t *s, ptrdiff_t p, - int16x8_t *const s0, int16x8_t *const s1, - int16x8_t *const s2, int16x8_t *const s3) { - *s0 = vld1q_s16(s); - s += p; - *s1 = vld1q_s16(s); - s += p; - *s2 = vld1q_s16(s); - s += p; - *s3 = vld1q_s16(s); -} - -static INLINE void load_unaligned_u8_4x8(const uint8_t *buf, int stride, - uint32x2_t *tu0, uint32x2_t *tu1, - uint32x2_t *tu2, uint32x2_t *tu3) { - uint32_t a; - - memcpy(&a, buf, 4); - buf += stride; - *tu0 = vset_lane_u32(a, *tu0, 0); - memcpy(&a, buf, 4); - buf += stride; - *tu0 = vset_lane_u32(a, *tu0, 1); - memcpy(&a, buf, 4); - buf += stride; - *tu1 = vset_lane_u32(a, *tu1, 0); - memcpy(&a, buf, 4); - buf += stride; - *tu1 = vset_lane_u32(a, *tu1, 1); - memcpy(&a, buf, 4); - buf += stride; - *tu2 = vset_lane_u32(a, *tu2, 0); - memcpy(&a, buf, 4); - buf += stride; - *tu2 = vset_lane_u32(a, *tu2, 1); - memcpy(&a, buf, 4); - buf += stride; - *tu3 = vset_lane_u32(a, *tu3, 0); - memcpy(&a, buf, 4); - *tu3 = vset_lane_u32(a, *tu3, 1); -} - -static INLINE void load_unaligned_u8_4x4(const uint8_t *buf, int stride, - uint32x2_t *tu0, uint32x2_t *tu1) { - uint32_t a; - - memcpy(&a, buf, 4); - buf += stride; - *tu0 = vset_lane_u32(a, *tu0, 0); - memcpy(&a, buf, 4); - buf += stride; - *tu0 = vset_lane_u32(a, *tu0, 1); - memcpy(&a, buf, 4); - buf += stride; - *tu1 = vset_lane_u32(a, *tu1, 0); - memcpy(&a, buf, 4); - *tu1 = vset_lane_u32(a, *tu1, 1); -} - -static INLINE void load_unaligned_u8_4x1(const uint8_t *buf, int stride, - uint32x2_t *tu0) { - uint32_t a; - - memcpy(&a, buf, 4); - buf += stride; - *tu0 = vset_lane_u32(a, *tu0, 0); -} - -static INLINE void load_unaligned_u8_4x2(const uint8_t *buf, int stride, - uint32x2_t *tu0) { - uint32_t a; - - memcpy(&a, buf, 4); - buf += stride; - *tu0 = vset_lane_u32(a, *tu0, 0); - memcpy(&a, buf, 4); - buf += stride; - *tu0 = vset_lane_u32(a, *tu0, 1); -} - -static INLINE void load_unaligned_u8_2x2(const uint8_t *buf, int stride, - uint16x4_t *tu0) { - uint16_t a; - - memcpy(&a, buf, 2); - buf += stride; - *tu0 = vset_lane_u16(a, *tu0, 0); - memcpy(&a, buf, 2); - buf += stride; - *tu0 = vset_lane_u16(a, *tu0, 1); -} - -static INLINE void load_u8_16x8(const uint8_t *s, ptrdiff_t p, - uint8x16_t *const s0, uint8x16_t *const s1, - uint8x16_t *const s2, uint8x16_t *const s3, - uint8x16_t *const s4, uint8x16_t *const s5, - uint8x16_t *const s6, uint8x16_t *const s7) { - *s0 = vld1q_u8(s); - s += p; - *s1 = vld1q_u8(s); - s += p; - *s2 = vld1q_u8(s); - s += p; - *s3 = vld1q_u8(s); - s += p; - *s4 = vld1q_u8(s); - s += p; - *s5 = vld1q_u8(s); - s += p; - *s6 = vld1q_u8(s); - s += p; - *s7 = vld1q_u8(s); -} - -static INLINE void load_u8_16x4(const uint8_t *s, ptrdiff_t p, - uint8x16_t *const s0, uint8x16_t *const s1, - uint8x16_t *const s2, uint8x16_t *const s3) { - *s0 = vld1q_u8(s); - s += p; - *s1 = vld1q_u8(s); - s += p; - *s2 = vld1q_u8(s); - s += p; - *s3 = vld1q_u8(s); -} - -static INLINE void load_unaligned_u16_4x4(const uint16_t *buf, uint32_t stride, - uint64x2_t *tu0, uint64x2_t *tu1) { - uint64_t a; - - memcpy(&a, buf, 8); - buf += stride; - *tu0 = vsetq_lane_u64(a, *tu0, 0); - memcpy(&a, buf, 8); - buf += stride; - *tu0 = vsetq_lane_u64(a, *tu0, 1); - memcpy(&a, buf, 8); - buf += stride; - *tu1 = vsetq_lane_u64(a, *tu1, 0); - memcpy(&a, buf, 8); - *tu1 = vsetq_lane_u64(a, *tu1, 1); -} - -static INLINE void load_s32_4x4(int32_t *s, int32_t p, int32x4_t *s1, - int32x4_t *s2, int32x4_t *s3, int32x4_t *s4) { - *s1 = vld1q_s32(s); - s += p; - *s2 = vld1q_s32(s); - s += p; - *s3 = vld1q_s32(s); - s += p; - *s4 = vld1q_s32(s); -} - -static INLINE void store_s32_4x4(int32_t *s, int32_t p, int32x4_t s1, - int32x4_t s2, int32x4_t s3, int32x4_t s4) { - vst1q_s32(s, s1); - s += p; - vst1q_s32(s, s2); - s += p; - vst1q_s32(s, s3); - s += p; - vst1q_s32(s, s4); -} - -static INLINE void load_u32_4x4(uint32_t *s, int32_t p, uint32x4_t *s1, - uint32x4_t *s2, uint32x4_t *s3, - uint32x4_t *s4) { - *s1 = vld1q_u32(s); - s += p; - *s2 = vld1q_u32(s); - s += p; - *s3 = vld1q_u32(s); - s += p; - *s4 = vld1q_u32(s); -} - -static INLINE void store_u32_4x4(uint32_t *s, int32_t p, uint32x4_t s1, - uint32x4_t s2, uint32x4_t s3, uint32x4_t s4) { - vst1q_u32(s, s1); - s += p; - vst1q_u32(s, s2); - s += p; - vst1q_u32(s, s3); - s += p; - vst1q_u32(s, s4); -} - -#endif // AOM_AV1_COMMON_ARM_MEM_NEON_H_ diff --git a/third_party/aom/av1/common/arm/reconinter_neon.c b/third_party/aom/av1/common/arm/reconinter_neon.c deleted file mode 100644 index 44e064195..000000000 --- a/third_party/aom/av1/common/arm/reconinter_neon.c +++ /dev/null @@ -1,86 +0,0 @@ -/* - * - * Copyright (c) 2018, Alliance for Open Media. All rights reserved - * - * This source code is subject to the terms of the BSD 2 Clause License and - * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License - * was not distributed with this source code in the LICENSE file, you can - * obtain it at www.aomedia.org/license/software. If the Alliance for Open - * Media Patent License 1.0 was not distributed with this source code in the - * PATENTS file, you can obtain it at www.aomedia.org/license/patent. - */ - -#include <arm_neon.h> -#include <assert.h> - -#include "aom/aom_integer.h" -#include "aom_dsp/blend.h" -#include "aom_ports/mem.h" -#include "av1/common/arm/mem_neon.h" -#include "av1/common/blockd.h" -#include "config/av1_rtcd.h" - -void av1_build_compound_diffwtd_mask_d16_neon( - uint8_t *mask, DIFFWTD_MASK_TYPE mask_type, const CONV_BUF_TYPE *src0, - int src0_stride, const CONV_BUF_TYPE *src1, int src1_stride, int h, int w, - ConvolveParams *conv_params, int bd) { - assert(h >= 4); - assert(w >= 4); - assert((mask_type == DIFFWTD_38_INV) || (mask_type == DIFFWTD_38)); - const int round = - 2 * FILTER_BITS - conv_params->round_0 - conv_params->round_1 + (bd - 8); - uint16x8_t diff_q, tmp0, tmp1; - uint8x8_t diff_d, diff_select; - const CONV_BUF_TYPE *src0_1, *src1_1; - const int16x8_t dup_round = vdupq_n_s16((int16_t)(-round)); - const uint8x8_t dup_38 = vdup_n_u8(38); - const uint8x8_t dup_64 = vdup_n_u8(AOM_BLEND_A64_MAX_ALPHA); - if (mask_type == DIFFWTD_38) { - diff_select = vdup_n_u8(255); - } else { - diff_select = vdup_n_u8(0); - } - if (w >= 8) { - for (int i = 0; i < h; ++i) { - src0_1 = src0; - src1_1 = src1; - for (int j = 0; j < w; j += 8) { - __builtin_prefetch(src0_1); - __builtin_prefetch(src1_1); - diff_q = vabdq_u16(vld1q_u16(src0_1), vld1q_u16(src1_1)); - diff_q = vrshlq_u16(diff_q, dup_round); - diff_d = vshrn_n_u16(diff_q, DIFF_FACTOR_LOG2); - diff_d = vmin_u8(vadd_u8(diff_d, dup_38), dup_64); - diff_d = vbsl_u8(diff_select, diff_d, vsub_u8(dup_64, diff_d)); - vst1_u8(mask, diff_d); - src0_1 += 8; - src1_1 += 8; - mask += 8; - } - src0 += src0_stride; - src1 += src1_stride; - } - } else if (w == 4) { - for (int i = 0; i < h; i += 2) { - src0_1 = src0; - src1_1 = src1; - __builtin_prefetch(src0_1 + 0 * src0_stride); - __builtin_prefetch(src0_1 + 1 * src0_stride); - __builtin_prefetch(src1_1 + 0 * src1_stride); - __builtin_prefetch(src1_1 + 1 * src1_stride); - tmp0 = vcombine_u16(vld1_u16(src0_1 + (0 * src0_stride)), - vld1_u16(src0_1 + (1 * src0_stride))); - tmp1 = vcombine_u16(vld1_u16(src1_1 + (0 * src1_stride)), - vld1_u16(src1_1 + (1 * src1_stride))); - diff_q = vabdq_u16(tmp0, tmp1); - diff_q = vrshlq_u16(diff_q, dup_round); - diff_d = vshrn_n_u16(diff_q, DIFF_FACTOR_LOG2); - diff_d = vmin_u8(vadd_u8(diff_d, dup_38), dup_64); - diff_d = vbsl_u8(diff_select, diff_d, vsub_u8(dup_64, diff_d)); - vst1_u8(mask, diff_d); - src0 += src0_stride * 2; - src1 += src1_stride * 2; - mask += w * 2; - } - } -} diff --git a/third_party/aom/av1/common/arm/selfguided_neon.c b/third_party/aom/av1/common/arm/selfguided_neon.c deleted file mode 100644 index b3a37c4cb..000000000 --- a/third_party/aom/av1/common/arm/selfguided_neon.c +++ /dev/null @@ -1,1508 +0,0 @@ -/* - * Copyright (c) 2018, Alliance for Open Media. All rights reserved - * - * This source code is subject to the terms of the BSD 2 Clause License and - * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License - * was not distributed with this source code in the LICENSE file, you can - * obtain it at www.aomedia.org/license/software. If the Alliance for Open - * Media Patent License 1.0 was not distributed with this source code in the - * PATENTS file, you can obtain it at www.aomedia.org/license/patent. - */ - -#include <arm_neon.h> -#include <assert.h> - -#include "config/aom_config.h" -#include "config/av1_rtcd.h" - -#include "aom_dsp/aom_dsp_common.h" -#include "aom_dsp/txfm_common.h" -#include "aom_mem/aom_mem.h" -#include "aom_ports/mem.h" -#include "av1/common/common.h" -#include "av1/common/onyxc_int.h" -#include "av1/common/resize.h" -#include "av1/common/restoration.h" -#include "av1/common/arm/mem_neon.h" -#include "av1/common/arm/transpose_neon.h" - -// Constants used for right shift in final_filter calculation. -#define NB_EVEN 5 -#define NB_ODD 4 - -static INLINE void calc_ab_fast_internal_common( - uint32x4_t s0, uint32x4_t s1, uint32x4_t s2, uint32x4_t s3, uint32x4_t s4, - uint32x4_t s5, uint32x4_t s6, uint32x4_t s7, int32x4_t sr4, int32x4_t sr5, - int32x4_t sr6, int32x4_t sr7, uint32x4_t const_n_val, uint32x4_t s_vec, - uint32x4_t const_val, uint32x4_t one_by_n_minus_1_vec, - uint16x4_t sgrproj_sgr, int32_t *src1, uint16_t *dst_A16, int32_t *src2, - const int buf_stride) { - uint32x4_t q0, q1, q2, q3; - uint32x4_t p0, p1, p2, p3; - uint16x4_t d0, d1, d2, d3; - - s0 = vmulq_u32(s0, const_n_val); - s1 = vmulq_u32(s1, const_n_val); - s2 = vmulq_u32(s2, const_n_val); - s3 = vmulq_u32(s3, const_n_val); - - q0 = vmulq_u32(s4, s4); - q1 = vmulq_u32(s5, s5); - q2 = vmulq_u32(s6, s6); - q3 = vmulq_u32(s7, s7); - - p0 = vcleq_u32(q0, s0); - p1 = vcleq_u32(q1, s1); - p2 = vcleq_u32(q2, s2); - p3 = vcleq_u32(q3, s3); - - q0 = vsubq_u32(s0, q0); - q1 = vsubq_u32(s1, q1); - q2 = vsubq_u32(s2, q2); - q3 = vsubq_u32(s3, q3); - - p0 = vandq_u32(p0, q0); - p1 = vandq_u32(p1, q1); - p2 = vandq_u32(p2, q2); - p3 = vandq_u32(p3, q3); - - p0 = vmulq_u32(p0, s_vec); - p1 = vmulq_u32(p1, s_vec); - p2 = vmulq_u32(p2, s_vec); - p3 = vmulq_u32(p3, s_vec); - - p0 = vrshrq_n_u32(p0, SGRPROJ_MTABLE_BITS); - p1 = vrshrq_n_u32(p1, SGRPROJ_MTABLE_BITS); - p2 = vrshrq_n_u32(p2, SGRPROJ_MTABLE_BITS); - p3 = vrshrq_n_u32(p3, SGRPROJ_MTABLE_BITS); - - p0 = vminq_u32(p0, const_val); - p1 = vminq_u32(p1, const_val); - p2 = vminq_u32(p2, const_val); - p3 = vminq_u32(p3, const_val); - - { - store_u32_4x4((uint32_t *)src1, buf_stride, p0, p1, p2, p3); - - for (int x = 0; x < 4; x++) { - for (int y = 0; y < 4; y++) { - dst_A16[x * buf_stride + y] = x_by_xplus1[src1[x * buf_stride + y]]; - } - } - load_u16_4x4(dst_A16, buf_stride, &d0, &d1, &d2, &d3); - } - p0 = vsubl_u16(sgrproj_sgr, d0); - p1 = vsubl_u16(sgrproj_sgr, d1); - p2 = vsubl_u16(sgrproj_sgr, d2); - p3 = vsubl_u16(sgrproj_sgr, d3); - - s4 = vmulq_u32(vreinterpretq_u32_s32(sr4), one_by_n_minus_1_vec); - s5 = vmulq_u32(vreinterpretq_u32_s32(sr5), one_by_n_minus_1_vec); - s6 = vmulq_u32(vreinterpretq_u32_s32(sr6), one_by_n_minus_1_vec); - s7 = vmulq_u32(vreinterpretq_u32_s32(sr7), one_by_n_minus_1_vec); - - s4 = vmulq_u32(s4, p0); - s5 = vmulq_u32(s5, p1); - s6 = vmulq_u32(s6, p2); - s7 = vmulq_u32(s7, p3); - - p0 = vrshrq_n_u32(s4, SGRPROJ_RECIP_BITS); - p1 = vrshrq_n_u32(s5, SGRPROJ_RECIP_BITS); - p2 = vrshrq_n_u32(s6, SGRPROJ_RECIP_BITS); - p3 = vrshrq_n_u32(s7, SGRPROJ_RECIP_BITS); - - store_s32_4x4(src2, buf_stride, vreinterpretq_s32_u32(p0), - vreinterpretq_s32_u32(p1), vreinterpretq_s32_u32(p2), - vreinterpretq_s32_u32(p3)); -} -static INLINE void calc_ab_internal_common( - uint32x4_t s0, uint32x4_t s1, uint32x4_t s2, uint32x4_t s3, uint32x4_t s4, - uint32x4_t s5, uint32x4_t s6, uint32x4_t s7, uint16x8_t s16_0, - uint16x8_t s16_1, uint16x8_t s16_2, uint16x8_t s16_3, uint16x8_t s16_4, - uint16x8_t s16_5, uint16x8_t s16_6, uint16x8_t s16_7, - uint32x4_t const_n_val, uint32x4_t s_vec, uint32x4_t const_val, - uint16x4_t one_by_n_minus_1_vec, uint16x8_t sgrproj_sgr, int32_t *src1, - uint16_t *dst_A16, int32_t *dst2, const int buf_stride) { - uint16x4_t d0, d1, d2, d3, d4, d5, d6, d7; - uint32x4_t q0, q1, q2, q3, q4, q5, q6, q7; - uint32x4_t p0, p1, p2, p3, p4, p5, p6, p7; - - s0 = vmulq_u32(s0, const_n_val); - s1 = vmulq_u32(s1, const_n_val); - s2 = vmulq_u32(s2, const_n_val); - s3 = vmulq_u32(s3, const_n_val); - s4 = vmulq_u32(s4, const_n_val); - s5 = vmulq_u32(s5, const_n_val); - s6 = vmulq_u32(s6, const_n_val); - s7 = vmulq_u32(s7, const_n_val); - - d0 = vget_low_u16(s16_4); - d1 = vget_low_u16(s16_5); - d2 = vget_low_u16(s16_6); - d3 = vget_low_u16(s16_7); - d4 = vget_high_u16(s16_4); - d5 = vget_high_u16(s16_5); - d6 = vget_high_u16(s16_6); - d7 = vget_high_u16(s16_7); - - q0 = vmull_u16(d0, d0); - q1 = vmull_u16(d1, d1); - q2 = vmull_u16(d2, d2); - q3 = vmull_u16(d3, d3); - q4 = vmull_u16(d4, d4); - q5 = vmull_u16(d5, d5); - q6 = vmull_u16(d6, d6); - q7 = vmull_u16(d7, d7); - - p0 = vcleq_u32(q0, s0); - p1 = vcleq_u32(q1, s1); - p2 = vcleq_u32(q2, s2); - p3 = vcleq_u32(q3, s3); - p4 = vcleq_u32(q4, s4); - p5 = vcleq_u32(q5, s5); - p6 = vcleq_u32(q6, s6); - p7 = vcleq_u32(q7, s7); - - q0 = vsubq_u32(s0, q0); - q1 = vsubq_u32(s1, q1); - q2 = vsubq_u32(s2, q2); - q3 = vsubq_u32(s3, q3); - q4 = vsubq_u32(s4, q4); - q5 = vsubq_u32(s5, q5); - q6 = vsubq_u32(s6, q6); - q7 = vsubq_u32(s7, q7); - - p0 = vandq_u32(p0, q0); - p1 = vandq_u32(p1, q1); - p2 = vandq_u32(p2, q2); - p3 = vandq_u32(p3, q3); - p4 = vandq_u32(p4, q4); - p5 = vandq_u32(p5, q5); - p6 = vandq_u32(p6, q6); - p7 = vandq_u32(p7, q7); - - p0 = vmulq_u32(p0, s_vec); - p1 = vmulq_u32(p1, s_vec); - p2 = vmulq_u32(p2, s_vec); - p3 = vmulq_u32(p3, s_vec); - p4 = vmulq_u32(p4, s_vec); - p5 = vmulq_u32(p5, s_vec); - p6 = vmulq_u32(p6, s_vec); - p7 = vmulq_u32(p7, s_vec); - - p0 = vrshrq_n_u32(p0, SGRPROJ_MTABLE_BITS); - p1 = vrshrq_n_u32(p1, SGRPROJ_MTABLE_BITS); - p2 = vrshrq_n_u32(p2, SGRPROJ_MTABLE_BITS); - p3 = vrshrq_n_u32(p3, SGRPROJ_MTABLE_BITS); - p4 = vrshrq_n_u32(p4, SGRPROJ_MTABLE_BITS); - p5 = vrshrq_n_u32(p5, SGRPROJ_MTABLE_BITS); - p6 = vrshrq_n_u32(p6, SGRPROJ_MTABLE_BITS); - p7 = vrshrq_n_u32(p7, SGRPROJ_MTABLE_BITS); - - p0 = vminq_u32(p0, const_val); - p1 = vminq_u32(p1, const_val); - p2 = vminq_u32(p2, const_val); - p3 = vminq_u32(p3, const_val); - p4 = vminq_u32(p4, const_val); - p5 = vminq_u32(p5, const_val); - p6 = vminq_u32(p6, const_val); - p7 = vminq_u32(p7, const_val); - - { - store_u32_4x4((uint32_t *)src1, buf_stride, p0, p1, p2, p3); - store_u32_4x4((uint32_t *)src1 + 4, buf_stride, p4, p5, p6, p7); - - for (int x = 0; x < 4; x++) { - for (int y = 0; y < 8; y++) { - dst_A16[x * buf_stride + y] = x_by_xplus1[src1[x * buf_stride + y]]; - } - } - load_u16_8x4(dst_A16, buf_stride, &s16_4, &s16_5, &s16_6, &s16_7); - } - - s16_4 = vsubq_u16(sgrproj_sgr, s16_4); - s16_5 = vsubq_u16(sgrproj_sgr, s16_5); - s16_6 = vsubq_u16(sgrproj_sgr, s16_6); - s16_7 = vsubq_u16(sgrproj_sgr, s16_7); - - s0 = vmull_u16(vget_low_u16(s16_0), one_by_n_minus_1_vec); - s1 = vmull_u16(vget_low_u16(s16_1), one_by_n_minus_1_vec); - s2 = vmull_u16(vget_low_u16(s16_2), one_by_n_minus_1_vec); - s3 = vmull_u16(vget_low_u16(s16_3), one_by_n_minus_1_vec); - s4 = vmull_u16(vget_high_u16(s16_0), one_by_n_minus_1_vec); - s5 = vmull_u16(vget_high_u16(s16_1), one_by_n_minus_1_vec); - s6 = vmull_u16(vget_high_u16(s16_2), one_by_n_minus_1_vec); - s7 = vmull_u16(vget_high_u16(s16_3), one_by_n_minus_1_vec); - - s0 = vmulq_u32(s0, vmovl_u16(vget_low_u16(s16_4))); - s1 = vmulq_u32(s1, vmovl_u16(vget_low_u16(s16_5))); - s2 = vmulq_u32(s2, vmovl_u16(vget_low_u16(s16_6))); - s3 = vmulq_u32(s3, vmovl_u16(vget_low_u16(s16_7))); - s4 = vmulq_u32(s4, vmovl_u16(vget_high_u16(s16_4))); - s5 = vmulq_u32(s5, vmovl_u16(vget_high_u16(s16_5))); - s6 = vmulq_u32(s6, vmovl_u16(vget_high_u16(s16_6))); - s7 = vmulq_u32(s7, vmovl_u16(vget_high_u16(s16_7))); - - p0 = vrshrq_n_u32(s0, SGRPROJ_RECIP_BITS); - p1 = vrshrq_n_u32(s1, SGRPROJ_RECIP_BITS); - p2 = vrshrq_n_u32(s2, SGRPROJ_RECIP_BITS); - p3 = vrshrq_n_u32(s3, SGRPROJ_RECIP_BITS); - p4 = vrshrq_n_u32(s4, SGRPROJ_RECIP_BITS); - p5 = vrshrq_n_u32(s5, SGRPROJ_RECIP_BITS); - p6 = vrshrq_n_u32(s6, SGRPROJ_RECIP_BITS); - p7 = vrshrq_n_u32(s7, SGRPROJ_RECIP_BITS); - - store_s32_4x4(dst2, buf_stride, vreinterpretq_s32_u32(p0), - vreinterpretq_s32_u32(p1), vreinterpretq_s32_u32(p2), - vreinterpretq_s32_u32(p3)); - store_s32_4x4(dst2 + 4, buf_stride, vreinterpretq_s32_u32(p4), - vreinterpretq_s32_u32(p5), vreinterpretq_s32_u32(p6), - vreinterpretq_s32_u32(p7)); -} - -static INLINE void boxsum2_square_sum_calc( - int16x4_t t1, int16x4_t t2, int16x4_t t3, int16x4_t t4, int16x4_t t5, - int16x4_t t6, int16x4_t t7, int16x4_t t8, int16x4_t t9, int16x4_t t10, - int16x4_t t11, int32x4_t *r0, int32x4_t *r1, int32x4_t *r2, int32x4_t *r3) { - int32x4_t d1, d2, d3, d4, d5, d6, d7, d8, d9, d10, d11; - int32x4_t r12, r34, r67, r89, r1011; - int32x4_t r345, r6789, r789; - - d1 = vmull_s16(t1, t1); - d2 = vmull_s16(t2, t2); - d3 = vmull_s16(t3, t3); - d4 = vmull_s16(t4, t4); - d5 = vmull_s16(t5, t5); - d6 = vmull_s16(t6, t6); - d7 = vmull_s16(t7, t7); - d8 = vmull_s16(t8, t8); - d9 = vmull_s16(t9, t9); - d10 = vmull_s16(t10, t10); - d11 = vmull_s16(t11, t11); - - r12 = vaddq_s32(d1, d2); - r34 = vaddq_s32(d3, d4); - r67 = vaddq_s32(d6, d7); - r89 = vaddq_s32(d8, d9); - r1011 = vaddq_s32(d10, d11); - r345 = vaddq_s32(r34, d5); - r6789 = vaddq_s32(r67, r89); - r789 = vsubq_s32(r6789, d6); - *r0 = vaddq_s32(r12, r345); - *r1 = vaddq_s32(r67, r345); - *r2 = vaddq_s32(d5, r6789); - *r3 = vaddq_s32(r789, r1011); -} - -static INLINE void boxsum2(int16_t *src, const int src_stride, int16_t *dst16, - int32_t *dst32, int32_t *dst2, const int dst_stride, - const int width, const int height) { - assert(width > 2 * SGRPROJ_BORDER_HORZ); - assert(height > 2 * SGRPROJ_BORDER_VERT); - - int16_t *dst1_16_ptr, *src_ptr; - int32_t *dst2_ptr; - int h, w, count = 0; - const int dst_stride_2 = (dst_stride << 1); - const int dst_stride_8 = (dst_stride << 3); - - dst1_16_ptr = dst16; - dst2_ptr = dst2; - src_ptr = src; - w = width; - { - int16x8_t t1, t2, t3, t4, t5, t6, t7; - int16x8_t t8, t9, t10, t11, t12; - - int16x8_t q12345, q56789, q34567, q7891011; - int16x8_t q12, q34, q67, q89, q1011; - int16x8_t q345, q6789, q789; - - int32x4_t r12345, r56789, r34567, r7891011; - - do { - h = height; - dst1_16_ptr = dst16 + (count << 3); - dst2_ptr = dst2 + (count << 3); - src_ptr = src + (count << 3); - - dst1_16_ptr += dst_stride_2; - dst2_ptr += dst_stride_2; - do { - load_s16_8x4(src_ptr, src_stride, &t1, &t2, &t3, &t4); - src_ptr += 4 * src_stride; - load_s16_8x4(src_ptr, src_stride, &t5, &t6, &t7, &t8); - src_ptr += 4 * src_stride; - load_s16_8x4(src_ptr, src_stride, &t9, &t10, &t11, &t12); - - q12 = vaddq_s16(t1, t2); - q34 = vaddq_s16(t3, t4); - q67 = vaddq_s16(t6, t7); - q89 = vaddq_s16(t8, t9); - q1011 = vaddq_s16(t10, t11); - q345 = vaddq_s16(q34, t5); - q6789 = vaddq_s16(q67, q89); - q789 = vaddq_s16(q89, t7); - q12345 = vaddq_s16(q12, q345); - q34567 = vaddq_s16(q67, q345); - q56789 = vaddq_s16(t5, q6789); - q7891011 = vaddq_s16(q789, q1011); - - store_s16_8x4(dst1_16_ptr, dst_stride_2, q12345, q34567, q56789, - q7891011); - dst1_16_ptr += dst_stride_8; - - boxsum2_square_sum_calc( - vget_low_s16(t1), vget_low_s16(t2), vget_low_s16(t3), - vget_low_s16(t4), vget_low_s16(t5), vget_low_s16(t6), - vget_low_s16(t7), vget_low_s16(t8), vget_low_s16(t9), - vget_low_s16(t10), vget_low_s16(t11), &r12345, &r34567, &r56789, - &r7891011); - - store_s32_4x4(dst2_ptr, dst_stride_2, r12345, r34567, r56789, r7891011); - - boxsum2_square_sum_calc( - vget_high_s16(t1), vget_high_s16(t2), vget_high_s16(t3), - vget_high_s16(t4), vget_high_s16(t5), vget_high_s16(t6), - vget_high_s16(t7), vget_high_s16(t8), vget_high_s16(t9), - vget_high_s16(t10), vget_high_s16(t11), &r12345, &r34567, &r56789, - &r7891011); - - store_s32_4x4(dst2_ptr + 4, dst_stride_2, r12345, r34567, r56789, - r7891011); - dst2_ptr += (dst_stride_8); - h -= 8; - } while (h > 0); - w -= 8; - count++; - } while (w > 0); - } - - { - int16x4_t s1, s2, s3, s4, s5, s6, s7, s8; - int32x4_t d1, d2, d3, d4, d5, d6, d7, d8; - int32x4_t q12345, q34567, q23456, q45678; - int32x4_t q23, q45, q67; - int32x4_t q2345, q4567; - - int32x4_t r12345, r34567, r23456, r45678; - int32x4_t r23, r45, r67; - int32x4_t r2345, r4567; - - int32_t *src2_ptr, *dst1_32_ptr; - int16_t *src1_ptr; - count = 0; - h = height; - do { - dst1_32_ptr = dst32 + count * dst_stride_8 + (dst_stride_2); - dst2_ptr = dst2 + count * dst_stride_8 + (dst_stride_2); - src1_ptr = dst16 + count * dst_stride_8 + (dst_stride_2); - src2_ptr = dst2 + count * dst_stride_8 + (dst_stride_2); - w = width; - - dst1_32_ptr += 2; - dst2_ptr += 2; - load_s16_4x4(src1_ptr, dst_stride_2, &s1, &s2, &s3, &s4); - transpose_s16_4x4d(&s1, &s2, &s3, &s4); - load_s32_4x4(src2_ptr, dst_stride_2, &d1, &d2, &d3, &d4); - transpose_s32_4x4(&d1, &d2, &d3, &d4); - do { - src1_ptr += 4; - src2_ptr += 4; - load_s16_4x4(src1_ptr, dst_stride_2, &s5, &s6, &s7, &s8); - transpose_s16_4x4d(&s5, &s6, &s7, &s8); - load_s32_4x4(src2_ptr, dst_stride_2, &d5, &d6, &d7, &d8); - transpose_s32_4x4(&d5, &d6, &d7, &d8); - q23 = vaddl_s16(s2, s3); - q45 = vaddl_s16(s4, s5); - q67 = vaddl_s16(s6, s7); - q2345 = vaddq_s32(q23, q45); - q4567 = vaddq_s32(q45, q67); - q12345 = vaddq_s32(vmovl_s16(s1), q2345); - q23456 = vaddq_s32(q2345, vmovl_s16(s6)); - q34567 = vaddq_s32(q4567, vmovl_s16(s3)); - q45678 = vaddq_s32(q4567, vmovl_s16(s8)); - - transpose_s32_4x4(&q12345, &q23456, &q34567, &q45678); - store_s32_4x4(dst1_32_ptr, dst_stride_2, q12345, q23456, q34567, - q45678); - dst1_32_ptr += 4; - s1 = s5; - s2 = s6; - s3 = s7; - s4 = s8; - - r23 = vaddq_s32(d2, d3); - r45 = vaddq_s32(d4, d5); - r67 = vaddq_s32(d6, d7); - r2345 = vaddq_s32(r23, r45); - r4567 = vaddq_s32(r45, r67); - r12345 = vaddq_s32(d1, r2345); - r23456 = vaddq_s32(r2345, d6); - r34567 = vaddq_s32(r4567, d3); - r45678 = vaddq_s32(r4567, d8); - - transpose_s32_4x4(&r12345, &r23456, &r34567, &r45678); - store_s32_4x4(dst2_ptr, dst_stride_2, r12345, r23456, r34567, r45678); - dst2_ptr += 4; - d1 = d5; - d2 = d6; - d3 = d7; - d4 = d8; - w -= 4; - } while (w > 0); - h -= 8; - count++; - } while (h > 0); - } -} - -static INLINE void calc_ab_internal_lbd(int32_t *A, uint16_t *A16, - uint16_t *B16, int32_t *B, - const int buf_stride, const int width, - const int height, const int r, - const int s, const int ht_inc) { - int32_t *src1, *dst2, count = 0; - uint16_t *dst_A16, *src2; - const uint32_t n = (2 * r + 1) * (2 * r + 1); - const uint32x4_t const_n_val = vdupq_n_u32(n); - const uint16x8_t sgrproj_sgr = vdupq_n_u16(SGRPROJ_SGR); - const uint16x4_t one_by_n_minus_1_vec = vdup_n_u16(one_by_x[n - 1]); - const uint32x4_t const_val = vdupq_n_u32(255); - - uint16x8_t s16_0, s16_1, s16_2, s16_3, s16_4, s16_5, s16_6, s16_7; - - uint32x4_t s0, s1, s2, s3, s4, s5, s6, s7; - - const uint32x4_t s_vec = vdupq_n_u32(s); - int w, h = height; - - do { - dst_A16 = A16 + (count << 2) * buf_stride; - src1 = A + (count << 2) * buf_stride; - src2 = B16 + (count << 2) * buf_stride; - dst2 = B + (count << 2) * buf_stride; - w = width; - do { - load_u32_4x4((uint32_t *)src1, buf_stride, &s0, &s1, &s2, &s3); - load_u32_4x4((uint32_t *)src1 + 4, buf_stride, &s4, &s5, &s6, &s7); - load_u16_8x4(src2, buf_stride, &s16_0, &s16_1, &s16_2, &s16_3); - - s16_4 = s16_0; - s16_5 = s16_1; - s16_6 = s16_2; - s16_7 = s16_3; - - calc_ab_internal_common( - s0, s1, s2, s3, s4, s5, s6, s7, s16_0, s16_1, s16_2, s16_3, s16_4, - s16_5, s16_6, s16_7, const_n_val, s_vec, const_val, - one_by_n_minus_1_vec, sgrproj_sgr, src1, dst_A16, dst2, buf_stride); - - w -= 8; - dst2 += 8; - src1 += 8; - src2 += 8; - dst_A16 += 8; - } while (w > 0); - count++; - h -= (ht_inc * 4); - } while (h > 0); -} - -static INLINE void calc_ab_internal_hbd(int32_t *A, uint16_t *A16, - uint16_t *B16, int32_t *B, - const int buf_stride, const int width, - const int height, const int bit_depth, - const int r, const int s, - const int ht_inc) { - int32_t *src1, *dst2, count = 0; - uint16_t *dst_A16, *src2; - const uint32_t n = (2 * r + 1) * (2 * r + 1); - const int16x8_t bd_min_2_vec = vdupq_n_s16(-(bit_depth - 8)); - const int32x4_t bd_min_1_vec = vdupq_n_s32(-((bit_depth - 8) << 1)); - const uint32x4_t const_n_val = vdupq_n_u32(n); - const uint16x8_t sgrproj_sgr = vdupq_n_u16(SGRPROJ_SGR); - const uint16x4_t one_by_n_minus_1_vec = vdup_n_u16(one_by_x[n - 1]); - const uint32x4_t const_val = vdupq_n_u32(255); - - int32x4_t sr0, sr1, sr2, sr3, sr4, sr5, sr6, sr7; - uint16x8_t s16_0, s16_1, s16_2, s16_3; - uint16x8_t s16_4, s16_5, s16_6, s16_7; - uint32x4_t s0, s1, s2, s3, s4, s5, s6, s7; - - const uint32x4_t s_vec = vdupq_n_u32(s); - int w, h = height; - - do { - src1 = A + (count << 2) * buf_stride; - src2 = B16 + (count << 2) * buf_stride; - dst2 = B + (count << 2) * buf_stride; - dst_A16 = A16 + (count << 2) * buf_stride; - w = width; - do { - load_s32_4x4(src1, buf_stride, &sr0, &sr1, &sr2, &sr3); - load_s32_4x4(src1 + 4, buf_stride, &sr4, &sr5, &sr6, &sr7); - load_u16_8x4(src2, buf_stride, &s16_0, &s16_1, &s16_2, &s16_3); - - s0 = vrshlq_u32(vreinterpretq_u32_s32(sr0), bd_min_1_vec); - s1 = vrshlq_u32(vreinterpretq_u32_s32(sr1), bd_min_1_vec); - s2 = vrshlq_u32(vreinterpretq_u32_s32(sr2), bd_min_1_vec); - s3 = vrshlq_u32(vreinterpretq_u32_s32(sr3), bd_min_1_vec); - s4 = vrshlq_u32(vreinterpretq_u32_s32(sr4), bd_min_1_vec); - s5 = vrshlq_u32(vreinterpretq_u32_s32(sr5), bd_min_1_vec); - s6 = vrshlq_u32(vreinterpretq_u32_s32(sr6), bd_min_1_vec); - s7 = vrshlq_u32(vreinterpretq_u32_s32(sr7), bd_min_1_vec); - - s16_4 = vrshlq_u16(s16_0, bd_min_2_vec); - s16_5 = vrshlq_u16(s16_1, bd_min_2_vec); - s16_6 = vrshlq_u16(s16_2, bd_min_2_vec); - s16_7 = vrshlq_u16(s16_3, bd_min_2_vec); - - calc_ab_internal_common( - s0, s1, s2, s3, s4, s5, s6, s7, s16_0, s16_1, s16_2, s16_3, s16_4, - s16_5, s16_6, s16_7, const_n_val, s_vec, const_val, - one_by_n_minus_1_vec, sgrproj_sgr, src1, dst_A16, dst2, buf_stride); - - w -= 8; - dst2 += 8; - src1 += 8; - src2 += 8; - dst_A16 += 8; - } while (w > 0); - count++; - h -= (ht_inc * 4); - } while (h > 0); -} - -static INLINE void calc_ab_fast_internal_lbd(int32_t *A, uint16_t *A16, - int32_t *B, const int buf_stride, - const int width, const int height, - const int r, const int s, - const int ht_inc) { - int32_t *src1, *src2, count = 0; - uint16_t *dst_A16; - const uint32_t n = (2 * r + 1) * (2 * r + 1); - const uint32x4_t const_n_val = vdupq_n_u32(n); - const uint16x4_t sgrproj_sgr = vdup_n_u16(SGRPROJ_SGR); - const uint32x4_t one_by_n_minus_1_vec = vdupq_n_u32(one_by_x[n - 1]); - const uint32x4_t const_val = vdupq_n_u32(255); - - int32x4_t sr0, sr1, sr2, sr3, sr4, sr5, sr6, sr7; - uint32x4_t s0, s1, s2, s3, s4, s5, s6, s7; - - const uint32x4_t s_vec = vdupq_n_u32(s); - int w, h = height; - - do { - src1 = A + (count << 2) * buf_stride; - src2 = B + (count << 2) * buf_stride; - dst_A16 = A16 + (count << 2) * buf_stride; - w = width; - do { - load_s32_4x4(src1, buf_stride, &sr0, &sr1, &sr2, &sr3); - load_s32_4x4(src2, buf_stride, &sr4, &sr5, &sr6, &sr7); - - s0 = vreinterpretq_u32_s32(sr0); - s1 = vreinterpretq_u32_s32(sr1); - s2 = vreinterpretq_u32_s32(sr2); - s3 = vreinterpretq_u32_s32(sr3); - s4 = vreinterpretq_u32_s32(sr4); - s5 = vreinterpretq_u32_s32(sr5); - s6 = vreinterpretq_u32_s32(sr6); - s7 = vreinterpretq_u32_s32(sr7); - - calc_ab_fast_internal_common(s0, s1, s2, s3, s4, s5, s6, s7, sr4, sr5, - sr6, sr7, const_n_val, s_vec, const_val, - one_by_n_minus_1_vec, sgrproj_sgr, src1, - dst_A16, src2, buf_stride); - - w -= 4; - src1 += 4; - src2 += 4; - dst_A16 += 4; - } while (w > 0); - count++; - h -= (ht_inc * 4); - } while (h > 0); -} - -static INLINE void calc_ab_fast_internal_hbd(int32_t *A, uint16_t *A16, - int32_t *B, const int buf_stride, - const int width, const int height, - const int bit_depth, const int r, - const int s, const int ht_inc) { - int32_t *src1, *src2, count = 0; - uint16_t *dst_A16; - const uint32_t n = (2 * r + 1) * (2 * r + 1); - const int32x4_t bd_min_2_vec = vdupq_n_s32(-(bit_depth - 8)); - const int32x4_t bd_min_1_vec = vdupq_n_s32(-((bit_depth - 8) << 1)); - const uint32x4_t const_n_val = vdupq_n_u32(n); - const uint16x4_t sgrproj_sgr = vdup_n_u16(SGRPROJ_SGR); - const uint32x4_t one_by_n_minus_1_vec = vdupq_n_u32(one_by_x[n - 1]); - const uint32x4_t const_val = vdupq_n_u32(255); - - int32x4_t sr0, sr1, sr2, sr3, sr4, sr5, sr6, sr7; - uint32x4_t s0, s1, s2, s3, s4, s5, s6, s7; - - const uint32x4_t s_vec = vdupq_n_u32(s); - int w, h = height; - - do { - src1 = A + (count << 2) * buf_stride; - src2 = B + (count << 2) * buf_stride; - dst_A16 = A16 + (count << 2) * buf_stride; - w = width; - do { - load_s32_4x4(src1, buf_stride, &sr0, &sr1, &sr2, &sr3); - load_s32_4x4(src2, buf_stride, &sr4, &sr5, &sr6, &sr7); - - s0 = vrshlq_u32(vreinterpretq_u32_s32(sr0), bd_min_1_vec); - s1 = vrshlq_u32(vreinterpretq_u32_s32(sr1), bd_min_1_vec); - s2 = vrshlq_u32(vreinterpretq_u32_s32(sr2), bd_min_1_vec); - s3 = vrshlq_u32(vreinterpretq_u32_s32(sr3), bd_min_1_vec); - s4 = vrshlq_u32(vreinterpretq_u32_s32(sr4), bd_min_2_vec); - s5 = vrshlq_u32(vreinterpretq_u32_s32(sr5), bd_min_2_vec); - s6 = vrshlq_u32(vreinterpretq_u32_s32(sr6), bd_min_2_vec); - s7 = vrshlq_u32(vreinterpretq_u32_s32(sr7), bd_min_2_vec); - - calc_ab_fast_internal_common(s0, s1, s2, s3, s4, s5, s6, s7, sr4, sr5, - sr6, sr7, const_n_val, s_vec, const_val, - one_by_n_minus_1_vec, sgrproj_sgr, src1, - dst_A16, src2, buf_stride); - - w -= 4; - src1 += 4; - src2 += 4; - dst_A16 += 4; - } while (w > 0); - count++; - h -= (ht_inc * 4); - } while (h > 0); -} - -static INLINE void boxsum1(int16_t *src, const int src_stride, uint16_t *dst1, - int32_t *dst2, const int dst_stride, const int width, - const int height) { - assert(width > 2 * SGRPROJ_BORDER_HORZ); - assert(height > 2 * SGRPROJ_BORDER_VERT); - - int16_t *src_ptr; - int32_t *dst2_ptr; - uint16_t *dst1_ptr; - int h, w, count = 0; - - w = width; - { - int16x8_t s1, s2, s3, s4, s5, s6, s7, s8; - int16x8_t q23, q34, q56, q234, q345, q456, q567; - int32x4_t r23, r56, r345, r456, r567, r78, r678; - int32x4_t r4_low, r4_high, r34_low, r34_high, r234_low, r234_high; - int32x4_t r2, r3, r5, r6, r7, r8; - int16x8_t q678, q78; - - do { - dst1_ptr = dst1 + (count << 3); - dst2_ptr = dst2 + (count << 3); - src_ptr = src + (count << 3); - h = height; - - load_s16_8x4(src_ptr, src_stride, &s1, &s2, &s3, &s4); - src_ptr += 4 * src_stride; - - q23 = vaddq_s16(s2, s3); - q234 = vaddq_s16(q23, s4); - q34 = vaddq_s16(s3, s4); - dst1_ptr += (dst_stride << 1); - - r2 = vmull_s16(vget_low_s16(s2), vget_low_s16(s2)); - r3 = vmull_s16(vget_low_s16(s3), vget_low_s16(s3)); - r4_low = vmull_s16(vget_low_s16(s4), vget_low_s16(s4)); - r23 = vaddq_s32(r2, r3); - r234_low = vaddq_s32(r23, r4_low); - r34_low = vaddq_s32(r3, r4_low); - - r2 = vmull_s16(vget_high_s16(s2), vget_high_s16(s2)); - r3 = vmull_s16(vget_high_s16(s3), vget_high_s16(s3)); - r4_high = vmull_s16(vget_high_s16(s4), vget_high_s16(s4)); - r23 = vaddq_s32(r2, r3); - r234_high = vaddq_s32(r23, r4_high); - r34_high = vaddq_s32(r3, r4_high); - - dst2_ptr += (dst_stride << 1); - - do { - load_s16_8x4(src_ptr, src_stride, &s5, &s6, &s7, &s8); - src_ptr += 4 * src_stride; - - q345 = vaddq_s16(s5, q34); - q56 = vaddq_s16(s5, s6); - q456 = vaddq_s16(s4, q56); - q567 = vaddq_s16(s7, q56); - q78 = vaddq_s16(s7, s8); - q678 = vaddq_s16(s6, q78); - - store_s16_8x4((int16_t *)dst1_ptr, dst_stride, q234, q345, q456, q567); - dst1_ptr += (dst_stride << 2); - - s4 = s8; - q34 = q78; - q234 = q678; - - r5 = vmull_s16(vget_low_s16(s5), vget_low_s16(s5)); - r6 = vmull_s16(vget_low_s16(s6), vget_low_s16(s6)); - r7 = vmull_s16(vget_low_s16(s7), vget_low_s16(s7)); - r8 = vmull_s16(vget_low_s16(s8), vget_low_s16(s8)); - - r345 = vaddq_s32(r5, r34_low); - r56 = vaddq_s32(r5, r6); - r456 = vaddq_s32(r4_low, r56); - r567 = vaddq_s32(r7, r56); - r78 = vaddq_s32(r7, r8); - r678 = vaddq_s32(r6, r78); - store_s32_4x4(dst2_ptr, dst_stride, r234_low, r345, r456, r567); - - r4_low = r8; - r34_low = r78; - r234_low = r678; - - r5 = vmull_s16(vget_high_s16(s5), vget_high_s16(s5)); - r6 = vmull_s16(vget_high_s16(s6), vget_high_s16(s6)); - r7 = vmull_s16(vget_high_s16(s7), vget_high_s16(s7)); - r8 = vmull_s16(vget_high_s16(s8), vget_high_s16(s8)); - - r345 = vaddq_s32(r5, r34_high); - r56 = vaddq_s32(r5, r6); - r456 = vaddq_s32(r4_high, r56); - r567 = vaddq_s32(r7, r56); - r78 = vaddq_s32(r7, r8); - r678 = vaddq_s32(r6, r78); - store_s32_4x4((dst2_ptr + 4), dst_stride, r234_high, r345, r456, r567); - dst2_ptr += (dst_stride << 2); - - r4_high = r8; - r34_high = r78; - r234_high = r678; - - h -= 4; - } while (h > 0); - w -= 8; - count++; - } while (w > 0); - } - - { - int16x4_t d1, d2, d3, d4, d5, d6, d7, d8; - int16x4_t q23, q34, q56, q234, q345, q456, q567; - int32x4_t r23, r56, r234, r345, r456, r567, r34, r78, r678; - int32x4_t r1, r2, r3, r4, r5, r6, r7, r8; - int16x4_t q678, q78; - - int32_t *src2_ptr; - uint16_t *src1_ptr; - count = 0; - h = height; - w = width; - do { - dst1_ptr = dst1 + (count << 2) * dst_stride; - dst2_ptr = dst2 + (count << 2) * dst_stride; - src1_ptr = dst1 + (count << 2) * dst_stride; - src2_ptr = dst2 + (count << 2) * dst_stride; - w = width; - - load_s16_4x4((int16_t *)src1_ptr, dst_stride, &d1, &d2, &d3, &d4); - transpose_s16_4x4d(&d1, &d2, &d3, &d4); - load_s32_4x4(src2_ptr, dst_stride, &r1, &r2, &r3, &r4); - transpose_s32_4x4(&r1, &r2, &r3, &r4); - src1_ptr += 4; - src2_ptr += 4; - - q23 = vadd_s16(d2, d3); - q234 = vadd_s16(q23, d4); - q34 = vadd_s16(d3, d4); - dst1_ptr += 2; - r23 = vaddq_s32(r2, r3); - r234 = vaddq_s32(r23, r4); - r34 = vaddq_s32(r3, r4); - dst2_ptr += 2; - - do { - load_s16_4x4((int16_t *)src1_ptr, dst_stride, &d5, &d6, &d7, &d8); - transpose_s16_4x4d(&d5, &d6, &d7, &d8); - load_s32_4x4(src2_ptr, dst_stride, &r5, &r6, &r7, &r8); - transpose_s32_4x4(&r5, &r6, &r7, &r8); - src1_ptr += 4; - src2_ptr += 4; - - q345 = vadd_s16(d5, q34); - q56 = vadd_s16(d5, d6); - q456 = vadd_s16(d4, q56); - q567 = vadd_s16(d7, q56); - q78 = vadd_s16(d7, d8); - q678 = vadd_s16(d6, q78); - transpose_s16_4x4d(&q234, &q345, &q456, &q567); - store_s16_4x4((int16_t *)dst1_ptr, dst_stride, q234, q345, q456, q567); - dst1_ptr += 4; - - d4 = d8; - q34 = q78; - q234 = q678; - - r345 = vaddq_s32(r5, r34); - r56 = vaddq_s32(r5, r6); - r456 = vaddq_s32(r4, r56); - r567 = vaddq_s32(r7, r56); - r78 = vaddq_s32(r7, r8); - r678 = vaddq_s32(r6, r78); - transpose_s32_4x4(&r234, &r345, &r456, &r567); - store_s32_4x4(dst2_ptr, dst_stride, r234, r345, r456, r567); - dst2_ptr += 4; - - r4 = r8; - r34 = r78; - r234 = r678; - w -= 4; - } while (w > 0); - h -= 4; - count++; - } while (h > 0); - } -} - -static INLINE int32x4_t cross_sum_inp_s32(int32_t *buf, int buf_stride) { - int32x4_t xtr, xt, xtl, xl, x, xr, xbr, xb, xbl; - int32x4_t fours, threes, res; - - xtl = vld1q_s32(buf - buf_stride - 1); - xt = vld1q_s32(buf - buf_stride); - xtr = vld1q_s32(buf - buf_stride + 1); - xl = vld1q_s32(buf - 1); - x = vld1q_s32(buf); - xr = vld1q_s32(buf + 1); - xbl = vld1q_s32(buf + buf_stride - 1); - xb = vld1q_s32(buf + buf_stride); - xbr = vld1q_s32(buf + buf_stride + 1); - - fours = vaddq_s32(xl, vaddq_s32(xt, vaddq_s32(xr, vaddq_s32(xb, x)))); - threes = vaddq_s32(xtl, vaddq_s32(xtr, vaddq_s32(xbr, xbl))); - res = vsubq_s32(vshlq_n_s32(vaddq_s32(fours, threes), 2), threes); - return res; -} - -static INLINE void cross_sum_inp_u16(uint16_t *buf, int buf_stride, - int32x4_t *a0, int32x4_t *a1) { - uint16x8_t xtr, xt, xtl, xl, x, xr, xbr, xb, xbl; - uint16x8_t r0, r1; - - xtl = vld1q_u16(buf - buf_stride - 1); - xt = vld1q_u16(buf - buf_stride); - xtr = vld1q_u16(buf - buf_stride + 1); - xl = vld1q_u16(buf - 1); - x = vld1q_u16(buf); - xr = vld1q_u16(buf + 1); - xbl = vld1q_u16(buf + buf_stride - 1); - xb = vld1q_u16(buf + buf_stride); - xbr = vld1q_u16(buf + buf_stride + 1); - - xb = vaddq_u16(xb, x); - xt = vaddq_u16(xt, xr); - xl = vaddq_u16(xl, xb); - xl = vaddq_u16(xl, xt); - - r0 = vshlq_n_u16(xl, 2); - - xbl = vaddq_u16(xbl, xbr); - xtl = vaddq_u16(xtl, xtr); - xtl = vaddq_u16(xtl, xbl); - - r1 = vshlq_n_u16(xtl, 2); - r1 = vsubq_u16(r1, xtl); - - *a0 = vreinterpretq_s32_u32( - vaddq_u32(vmovl_u16(vget_low_u16(r0)), vmovl_u16(vget_low_u16(r1)))); - *a1 = vreinterpretq_s32_u32( - vaddq_u32(vmovl_u16(vget_high_u16(r0)), vmovl_u16(vget_high_u16(r1)))); -} - -static INLINE int32x4_t cross_sum_fast_even_row(int32_t *buf, int buf_stride) { - int32x4_t xtr, xt, xtl, xbr, xb, xbl; - int32x4_t fives, sixes, fives_plus_sixes; - - xtl = vld1q_s32(buf - buf_stride - 1); - xt = vld1q_s32(buf - buf_stride); - xtr = vld1q_s32(buf - buf_stride + 1); - xbl = vld1q_s32(buf + buf_stride - 1); - xb = vld1q_s32(buf + buf_stride); - xbr = vld1q_s32(buf + buf_stride + 1); - - fives = vaddq_s32(xtl, vaddq_s32(xtr, vaddq_s32(xbr, xbl))); - sixes = vaddq_s32(xt, xb); - fives_plus_sixes = vaddq_s32(fives, sixes); - - return vaddq_s32( - vaddq_s32(vshlq_n_s32(fives_plus_sixes, 2), fives_plus_sixes), sixes); -} - -static INLINE void cross_sum_fast_even_row_inp16(uint16_t *buf, int buf_stride, - int32x4_t *a0, int32x4_t *a1) { - uint16x8_t xtr, xt, xtl, xbr, xb, xbl, xb0; - - xtl = vld1q_u16(buf - buf_stride - 1); - xt = vld1q_u16(buf - buf_stride); - xtr = vld1q_u16(buf - buf_stride + 1); - xbl = vld1q_u16(buf + buf_stride - 1); - xb = vld1q_u16(buf + buf_stride); - xbr = vld1q_u16(buf + buf_stride + 1); - - xbr = vaddq_u16(xbr, xbl); - xtr = vaddq_u16(xtr, xtl); - xbr = vaddq_u16(xbr, xtr); - xtl = vshlq_n_u16(xbr, 2); - xbr = vaddq_u16(xtl, xbr); - - xb = vaddq_u16(xb, xt); - xb0 = vshlq_n_u16(xb, 1); - xb = vshlq_n_u16(xb, 2); - xb = vaddq_u16(xb, xb0); - - *a0 = vreinterpretq_s32_u32( - vaddq_u32(vmovl_u16(vget_low_u16(xbr)), vmovl_u16(vget_low_u16(xb)))); - *a1 = vreinterpretq_s32_u32( - vaddq_u32(vmovl_u16(vget_high_u16(xbr)), vmovl_u16(vget_high_u16(xb)))); -} - -static INLINE int32x4_t cross_sum_fast_odd_row(int32_t *buf) { - int32x4_t xl, x, xr; - int32x4_t fives, sixes, fives_plus_sixes; - - xl = vld1q_s32(buf - 1); - x = vld1q_s32(buf); - xr = vld1q_s32(buf + 1); - fives = vaddq_s32(xl, xr); - sixes = x; - fives_plus_sixes = vaddq_s32(fives, sixes); - - return vaddq_s32( - vaddq_s32(vshlq_n_s32(fives_plus_sixes, 2), fives_plus_sixes), sixes); -} - -static INLINE void cross_sum_fast_odd_row_inp16(uint16_t *buf, int32x4_t *a0, - int32x4_t *a1) { - uint16x8_t xl, x, xr; - uint16x8_t x0; - - xl = vld1q_u16(buf - 1); - x = vld1q_u16(buf); - xr = vld1q_u16(buf + 1); - xl = vaddq_u16(xl, xr); - x0 = vshlq_n_u16(xl, 2); - xl = vaddq_u16(xl, x0); - - x0 = vshlq_n_u16(x, 1); - x = vshlq_n_u16(x, 2); - x = vaddq_u16(x, x0); - - *a0 = vreinterpretq_s32_u32( - vaddq_u32(vmovl_u16(vget_low_u16(xl)), vmovl_u16(vget_low_u16(x)))); - *a1 = vreinterpretq_s32_u32( - vaddq_u32(vmovl_u16(vget_high_u16(xl)), vmovl_u16(vget_high_u16(x)))); -} - -static void final_filter_fast_internal(uint16_t *A, int32_t *B, - const int buf_stride, int16_t *src, - const int src_stride, int32_t *dst, - const int dst_stride, const int width, - const int height) { - int16x8_t s0; - int32_t *B_tmp, *dst_ptr; - uint16_t *A_tmp; - int16_t *src_ptr; - int32x4_t a_res0, a_res1, b_res0, b_res1; - int w, h, count = 0; - assert(SGRPROJ_SGR_BITS == 8); - assert(SGRPROJ_RST_BITS == 4); - - A_tmp = A; - B_tmp = B; - src_ptr = src; - dst_ptr = dst; - h = height; - do { - A_tmp = (A + count * buf_stride); - B_tmp = (B + count * buf_stride); - src_ptr = (src + count * src_stride); - dst_ptr = (dst + count * dst_stride); - w = width; - if (!(count & 1)) { - do { - s0 = vld1q_s16(src_ptr); - cross_sum_fast_even_row_inp16(A_tmp, buf_stride, &a_res0, &a_res1); - a_res0 = vmulq_s32(vmovl_s16(vget_low_s16(s0)), a_res0); - a_res1 = vmulq_s32(vmovl_s16(vget_high_s16(s0)), a_res1); - - b_res0 = cross_sum_fast_even_row(B_tmp, buf_stride); - b_res1 = cross_sum_fast_even_row(B_tmp + 4, buf_stride); - a_res0 = vaddq_s32(a_res0, b_res0); - a_res1 = vaddq_s32(a_res1, b_res1); - - a_res0 = - vrshrq_n_s32(a_res0, SGRPROJ_SGR_BITS + NB_EVEN - SGRPROJ_RST_BITS); - a_res1 = - vrshrq_n_s32(a_res1, SGRPROJ_SGR_BITS + NB_EVEN - SGRPROJ_RST_BITS); - - vst1q_s32(dst_ptr, a_res0); - vst1q_s32(dst_ptr + 4, a_res1); - - A_tmp += 8; - B_tmp += 8; - src_ptr += 8; - dst_ptr += 8; - w -= 8; - } while (w > 0); - } else { - do { - s0 = vld1q_s16(src_ptr); - cross_sum_fast_odd_row_inp16(A_tmp, &a_res0, &a_res1); - a_res0 = vmulq_s32(vmovl_s16(vget_low_s16(s0)), a_res0); - a_res1 = vmulq_s32(vmovl_s16(vget_high_s16(s0)), a_res1); - - b_res0 = cross_sum_fast_odd_row(B_tmp); - b_res1 = cross_sum_fast_odd_row(B_tmp + 4); - a_res0 = vaddq_s32(a_res0, b_res0); - a_res1 = vaddq_s32(a_res1, b_res1); - - a_res0 = - vrshrq_n_s32(a_res0, SGRPROJ_SGR_BITS + NB_ODD - SGRPROJ_RST_BITS); - a_res1 = - vrshrq_n_s32(a_res1, SGRPROJ_SGR_BITS + NB_ODD - SGRPROJ_RST_BITS); - - vst1q_s32(dst_ptr, a_res0); - vst1q_s32(dst_ptr + 4, a_res1); - - A_tmp += 8; - B_tmp += 8; - src_ptr += 8; - dst_ptr += 8; - w -= 8; - } while (w > 0); - } - count++; - h -= 1; - } while (h > 0); -} - -void final_filter_internal(uint16_t *A, int32_t *B, const int buf_stride, - int16_t *src, const int src_stride, int32_t *dst, - const int dst_stride, const int width, - const int height) { - int16x8_t s0; - int32_t *B_tmp, *dst_ptr; - uint16_t *A_tmp; - int16_t *src_ptr; - int32x4_t a_res0, a_res1, b_res0, b_res1; - int w, h, count = 0; - - assert(SGRPROJ_SGR_BITS == 8); - assert(SGRPROJ_RST_BITS == 4); - h = height; - - do { - A_tmp = (A + count * buf_stride); - B_tmp = (B + count * buf_stride); - src_ptr = (src + count * src_stride); - dst_ptr = (dst + count * dst_stride); - w = width; - do { - s0 = vld1q_s16(src_ptr); - cross_sum_inp_u16(A_tmp, buf_stride, &a_res0, &a_res1); - a_res0 = vmulq_s32(vmovl_s16(vget_low_s16(s0)), a_res0); - a_res1 = vmulq_s32(vmovl_s16(vget_high_s16(s0)), a_res1); - - b_res0 = cross_sum_inp_s32(B_tmp, buf_stride); - b_res1 = cross_sum_inp_s32(B_tmp + 4, buf_stride); - a_res0 = vaddq_s32(a_res0, b_res0); - a_res1 = vaddq_s32(a_res1, b_res1); - - a_res0 = - vrshrq_n_s32(a_res0, SGRPROJ_SGR_BITS + NB_EVEN - SGRPROJ_RST_BITS); - a_res1 = - vrshrq_n_s32(a_res1, SGRPROJ_SGR_BITS + NB_EVEN - SGRPROJ_RST_BITS); - vst1q_s32(dst_ptr, a_res0); - vst1q_s32(dst_ptr + 4, a_res1); - - A_tmp += 8; - B_tmp += 8; - src_ptr += 8; - dst_ptr += 8; - w -= 8; - } while (w > 0); - count++; - h -= 1; - } while (h > 0); -} - -static INLINE void restoration_fast_internal(uint16_t *dgd16, int width, - int height, int dgd_stride, - int32_t *dst, int dst_stride, - int bit_depth, int sgr_params_idx, - int radius_idx) { - const sgr_params_type *const params = &sgr_params[sgr_params_idx]; - const int r = params->r[radius_idx]; - const int width_ext = width + 2 * SGRPROJ_BORDER_HORZ; - const int height_ext = height + 2 * SGRPROJ_BORDER_VERT; - - const int buf_stride = ((width_ext + 3) & ~3) + 16; - int32_t A_[RESTORATION_PROC_UNIT_PELS]; - uint16_t A16_[RESTORATION_PROC_UNIT_PELS]; - int32_t B_[RESTORATION_PROC_UNIT_PELS]; - int32_t *square_sum_buf = A_; - int32_t *sum_buf = B_; - uint16_t *tmp16_buf = A16_; - - assert(r <= MAX_RADIUS && "Need MAX_RADIUS >= r"); - assert(r <= SGRPROJ_BORDER_VERT - 1 && r <= SGRPROJ_BORDER_HORZ - 1 && - "Need SGRPROJ_BORDER_* >= r+1"); - - assert(radius_idx == 0); - assert(r == 2); - - // input(dgd16) is 16bit. - // sum of pixels 1st stage output will be in 16bit(tmp16_buf). End output is - // kept in 32bit [sum_buf]. sum of squares output is kept in 32bit - // buffer(square_sum_buf). - boxsum2((int16_t *)(dgd16 - dgd_stride * SGRPROJ_BORDER_VERT - - SGRPROJ_BORDER_HORZ), - dgd_stride, (int16_t *)tmp16_buf, sum_buf, square_sum_buf, buf_stride, - width_ext, height_ext); - - square_sum_buf += SGRPROJ_BORDER_VERT * buf_stride + SGRPROJ_BORDER_HORZ; - sum_buf += SGRPROJ_BORDER_VERT * buf_stride + SGRPROJ_BORDER_HORZ; - tmp16_buf += SGRPROJ_BORDER_VERT * buf_stride + SGRPROJ_BORDER_HORZ; - - // Calculation of a, b. a output is in 16bit tmp_buf which is in range of - // [1, 256] for all bit depths. b output is kept in 32bit buffer. - - if (8 == bit_depth) { - calc_ab_fast_internal_lbd( - (square_sum_buf - buf_stride - 1), (tmp16_buf - buf_stride - 1), - (sum_buf - buf_stride - 1), buf_stride * 2, width + 2, height + 2, r, - params->s[radius_idx], 2); - } else { - calc_ab_fast_internal_hbd( - (square_sum_buf - buf_stride - 1), (tmp16_buf - buf_stride - 1), - (sum_buf - buf_stride - 1), buf_stride * 2, width + 2, height + 2, - bit_depth, r, params->s[radius_idx], 2); - } - final_filter_fast_internal(tmp16_buf, sum_buf, buf_stride, (int16_t *)dgd16, - dgd_stride, dst, dst_stride, width, height); -} - -static INLINE void restoration_internal(uint16_t *dgd16, int width, int height, - int dgd_stride, int32_t *dst, - int dst_stride, int bit_depth, - int sgr_params_idx, int radius_idx) { - const sgr_params_type *const params = &sgr_params[sgr_params_idx]; - const int r = params->r[radius_idx]; - const int width_ext = width + 2 * SGRPROJ_BORDER_HORZ; - const int height_ext = height + 2 * SGRPROJ_BORDER_VERT; - - int buf_stride = ((width_ext + 3) & ~3) + 16; - int32_t A_[RESTORATION_PROC_UNIT_PELS]; - uint16_t A16_[RESTORATION_PROC_UNIT_PELS]; - uint16_t B16_[RESTORATION_PROC_UNIT_PELS]; - int32_t B_[RESTORATION_PROC_UNIT_PELS]; - int32_t *square_sum_buf = A_; - uint16_t *sum_buf = B16_; - uint16_t *A16 = A16_; - int32_t *B = B_; - - assert(r <= MAX_RADIUS && "Need MAX_RADIUS >= r"); - assert(r <= SGRPROJ_BORDER_VERT - 1 && r <= SGRPROJ_BORDER_HORZ - 1 && - "Need SGRPROJ_BORDER_* >= r+1"); - - assert(radius_idx == 1); - assert(r == 1); - - // input(dgd16) is 16bit. - // sum of pixels output will be in 16bit(sum_buf). - // sum of squares output is kept in 32bit buffer(square_sum_buf). - boxsum1((int16_t *)(dgd16 - dgd_stride * SGRPROJ_BORDER_VERT - - SGRPROJ_BORDER_HORZ), - dgd_stride, sum_buf, square_sum_buf, buf_stride, width_ext, - height_ext); - - square_sum_buf += SGRPROJ_BORDER_VERT * buf_stride + SGRPROJ_BORDER_HORZ; - B += SGRPROJ_BORDER_VERT * buf_stride + SGRPROJ_BORDER_HORZ; - A16 += SGRPROJ_BORDER_VERT * buf_stride + SGRPROJ_BORDER_HORZ; - sum_buf += SGRPROJ_BORDER_VERT * buf_stride + SGRPROJ_BORDER_HORZ; - - // Calculation of a, b. a output is in 16bit tmp_buf which is in range of - // [1, 256] for all bit depths. b output is kept in 32bit buffer. - if (8 == bit_depth) { - calc_ab_internal_lbd((square_sum_buf - buf_stride - 1), - (A16 - buf_stride - 1), (sum_buf - buf_stride - 1), - (B - buf_stride - 1), buf_stride, width + 2, - height + 2, r, params->s[radius_idx], 1); - } else { - calc_ab_internal_hbd((square_sum_buf - buf_stride - 1), - (A16 - buf_stride - 1), (sum_buf - buf_stride - 1), - (B - buf_stride - 1), buf_stride, width + 2, - height + 2, bit_depth, r, params->s[radius_idx], 1); - } - final_filter_internal(A16, B, buf_stride, (int16_t *)dgd16, dgd_stride, dst, - dst_stride, width, height); -} - -static INLINE void src_convert_u8_to_u16(const uint8_t *src, - const int src_stride, uint16_t *dst, - const int dst_stride, const int width, - const int height) { - const uint8_t *src_ptr; - uint16_t *dst_ptr; - int h, w, count = 0; - - uint8x8_t t1, t2, t3, t4; - uint16x8_t s1, s2, s3, s4; - h = height; - do { - src_ptr = src + (count << 2) * src_stride; - dst_ptr = dst + (count << 2) * dst_stride; - w = width; - if (w >= 7) { - do { - load_u8_8x4(src_ptr, src_stride, &t1, &t2, &t3, &t4); - s1 = vmovl_u8(t1); - s2 = vmovl_u8(t2); - s3 = vmovl_u8(t3); - s4 = vmovl_u8(t4); - store_u16_8x4(dst_ptr, dst_stride, s1, s2, s3, s4); - - src_ptr += 8; - dst_ptr += 8; - w -= 8; - } while (w > 7); - } - - for (int y = 0; y < w; y++) { - dst_ptr[y] = src_ptr[y]; - dst_ptr[y + 1 * dst_stride] = src_ptr[y + 1 * src_stride]; - dst_ptr[y + 2 * dst_stride] = src_ptr[y + 2 * src_stride]; - dst_ptr[y + 3 * dst_stride] = src_ptr[y + 3 * src_stride]; - } - count++; - h -= 4; - } while (h > 3); - - src_ptr = src + (count << 2) * src_stride; - dst_ptr = dst + (count << 2) * dst_stride; - for (int x = 0; x < h; x++) { - for (int y = 0; y < width; y++) { - dst_ptr[y + x * dst_stride] = src_ptr[y + x * src_stride]; - } - } -} - -static INLINE void src_convert_hbd_copy(const uint16_t *src, int src_stride, - uint16_t *dst, const int dst_stride, - int width, int height) { - const uint16_t *src_ptr; - uint16_t *dst_ptr; - int h, w, count = 0; - uint16x8_t s1, s2, s3, s4; - - h = height; - do { - src_ptr = src + (count << 2) * src_stride; - dst_ptr = dst + (count << 2) * dst_stride; - w = width; - do { - load_u16_8x4(src_ptr, src_stride, &s1, &s2, &s3, &s4); - store_u16_8x4(dst_ptr, dst_stride, s1, s2, s3, s4); - src_ptr += 8; - dst_ptr += 8; - w -= 8; - } while (w > 7); - - for (int y = 0; y < w; y++) { - dst_ptr[y] = src_ptr[y]; - dst_ptr[y + 1 * dst_stride] = src_ptr[y + 1 * src_stride]; - dst_ptr[y + 2 * dst_stride] = src_ptr[y + 2 * src_stride]; - dst_ptr[y + 3 * dst_stride] = src_ptr[y + 3 * src_stride]; - } - count++; - h -= 4; - } while (h > 3); - - src_ptr = src + (count << 2) * src_stride; - dst_ptr = dst + (count << 2) * dst_stride; - - for (int x = 0; x < h; x++) { - memcpy((dst_ptr + x * dst_stride), (src_ptr + x * src_stride), - sizeof(uint16_t) * width); - } -} - -int av1_selfguided_restoration_neon(const uint8_t *dat8, int width, int height, - int stride, int32_t *flt0, int32_t *flt1, - int flt_stride, int sgr_params_idx, - int bit_depth, int highbd) { - const sgr_params_type *const params = &sgr_params[sgr_params_idx]; - assert(!(params->r[0] == 0 && params->r[1] == 0)); - - uint16_t dgd16_[RESTORATION_PROC_UNIT_PELS]; - const int dgd16_stride = width + 2 * SGRPROJ_BORDER_HORZ; - uint16_t *dgd16 = - dgd16_ + dgd16_stride * SGRPROJ_BORDER_VERT + SGRPROJ_BORDER_HORZ; - const int width_ext = width + 2 * SGRPROJ_BORDER_HORZ; - const int height_ext = height + 2 * SGRPROJ_BORDER_VERT; - const int dgd_stride = stride; - - if (highbd) { - const uint16_t *dgd16_tmp = CONVERT_TO_SHORTPTR(dat8); - src_convert_hbd_copy( - dgd16_tmp - SGRPROJ_BORDER_VERT * dgd_stride - SGRPROJ_BORDER_HORZ, - dgd_stride, - dgd16 - SGRPROJ_BORDER_VERT * dgd16_stride - SGRPROJ_BORDER_HORZ, - dgd16_stride, width_ext, height_ext); - } else { - src_convert_u8_to_u16( - dat8 - SGRPROJ_BORDER_VERT * dgd_stride - SGRPROJ_BORDER_HORZ, - dgd_stride, - dgd16 - SGRPROJ_BORDER_VERT * dgd16_stride - SGRPROJ_BORDER_HORZ, - dgd16_stride, width_ext, height_ext); - } - - if (params->r[0] > 0) - restoration_fast_internal(dgd16, width, height, dgd16_stride, flt0, - flt_stride, bit_depth, sgr_params_idx, 0); - if (params->r[1] > 0) - restoration_internal(dgd16, width, height, dgd16_stride, flt1, flt_stride, - bit_depth, sgr_params_idx, 1); - return 0; -} - -void apply_selfguided_restoration_neon(const uint8_t *dat8, int width, - int height, int stride, int eps, - const int *xqd, uint8_t *dst8, - int dst_stride, int32_t *tmpbuf, - int bit_depth, int highbd) { - int32_t *flt0 = tmpbuf; - int32_t *flt1 = flt0 + RESTORATION_UNITPELS_MAX; - assert(width * height <= RESTORATION_UNITPELS_MAX); - uint16_t dgd16_[RESTORATION_PROC_UNIT_PELS]; - const int dgd16_stride = width + 2 * SGRPROJ_BORDER_HORZ; - uint16_t *dgd16 = - dgd16_ + dgd16_stride * SGRPROJ_BORDER_VERT + SGRPROJ_BORDER_HORZ; - const int width_ext = width + 2 * SGRPROJ_BORDER_HORZ; - const int height_ext = height + 2 * SGRPROJ_BORDER_VERT; - const int dgd_stride = stride; - const sgr_params_type *const params = &sgr_params[eps]; - int xq[2]; - - assert(!(params->r[0] == 0 && params->r[1] == 0)); - - if (highbd) { - const uint16_t *dgd16_tmp = CONVERT_TO_SHORTPTR(dat8); - src_convert_hbd_copy( - dgd16_tmp - SGRPROJ_BORDER_VERT * dgd_stride - SGRPROJ_BORDER_HORZ, - dgd_stride, - dgd16 - SGRPROJ_BORDER_VERT * dgd16_stride - SGRPROJ_BORDER_HORZ, - dgd16_stride, width_ext, height_ext); - } else { - src_convert_u8_to_u16( - dat8 - SGRPROJ_BORDER_VERT * dgd_stride - SGRPROJ_BORDER_HORZ, - dgd_stride, - dgd16 - SGRPROJ_BORDER_VERT * dgd16_stride - SGRPROJ_BORDER_HORZ, - dgd16_stride, width_ext, height_ext); - } - - if (params->r[0] > 0) - restoration_fast_internal(dgd16, width, height, dgd16_stride, flt0, width, - bit_depth, eps, 0); - if (params->r[1] > 0) - restoration_internal(dgd16, width, height, dgd16_stride, flt1, width, - bit_depth, eps, 1); - - decode_xq(xqd, xq, params); - - { - int16_t *src_ptr; - uint8_t *dst_ptr; - uint16_t *dst16_ptr; - int16x4_t d0, d4; - int16x8_t r0, s0; - uint16x8_t r4; - int32x4_t u0, u4, v0, v4, f00, f10; - uint8x8_t t0; - int count = 0, w = width, h = height, rc = 0; - - const int32x4_t xq0_vec = vdupq_n_s32(xq[0]); - const int32x4_t xq1_vec = vdupq_n_s32(xq[1]); - const int16x8_t zero = vdupq_n_s16(0); - const uint16x8_t max = vdupq_n_u16((1 << bit_depth) - 1); - uint16_t *dst16 = CONVERT_TO_SHORTPTR(dst8); - dst_ptr = dst8; - src_ptr = (int16_t *)dgd16; - do { - w = width; - count = 0; - dst_ptr = dst8 + rc * dst_stride; - dst16_ptr = dst16 + rc * dst_stride; - do { - s0 = vld1q_s16(src_ptr + count); - - u0 = vshll_n_s16(vget_low_s16(s0), SGRPROJ_RST_BITS); - u4 = vshll_n_s16(vget_high_s16(s0), SGRPROJ_RST_BITS); - - v0 = vshlq_n_s32(u0, SGRPROJ_PRJ_BITS); - v4 = vshlq_n_s32(u4, SGRPROJ_PRJ_BITS); - - if (params->r[0] > 0) { - f00 = vld1q_s32(flt0 + count); - f10 = vld1q_s32(flt0 + count + 4); - - f00 = vsubq_s32(f00, u0); - f10 = vsubq_s32(f10, u4); - - v0 = vmlaq_s32(v0, xq0_vec, f00); - v4 = vmlaq_s32(v4, xq0_vec, f10); - } - - if (params->r[1] > 0) { - f00 = vld1q_s32(flt1 + count); - f10 = vld1q_s32(flt1 + count + 4); - - f00 = vsubq_s32(f00, u0); - f10 = vsubq_s32(f10, u4); - - v0 = vmlaq_s32(v0, xq1_vec, f00); - v4 = vmlaq_s32(v4, xq1_vec, f10); - } - - d0 = vqrshrn_n_s32(v0, SGRPROJ_PRJ_BITS + SGRPROJ_RST_BITS); - d4 = vqrshrn_n_s32(v4, SGRPROJ_PRJ_BITS + SGRPROJ_RST_BITS); - - r0 = vcombine_s16(d0, d4); - - r4 = vreinterpretq_u16_s16(vmaxq_s16(r0, zero)); - - if (highbd) { - r4 = vminq_u16(r4, max); - vst1q_u16(dst16_ptr, r4); - } else { - t0 = vqmovn_u16(r4); - vst1_u8(dst_ptr, t0); - } - w -= 8; - count += 8; - dst_ptr += 8; - dst16_ptr += 8; - } while (w > 0); - - src_ptr += dgd16_stride; - flt1 += width; - flt0 += width; - rc++; - h--; - } while (h > 0); - } -} diff --git a/third_party/aom/av1/common/arm/transpose_neon.h b/third_party/aom/av1/common/arm/transpose_neon.h deleted file mode 100644 index 8a3d9f07f..000000000 --- a/third_party/aom/av1/common/arm/transpose_neon.h +++ /dev/null @@ -1,537 +0,0 @@ -/* - * Copyright (c) 2018, Alliance for Open Media. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#ifndef AOM_AV1_COMMON_ARM_TRANSPOSE_NEON_H_ -#define AOM_AV1_COMMON_ARM_TRANSPOSE_NEON_H_ - -#include <arm_neon.h> - -static INLINE void transpose_u8_8x8(uint8x8_t *a0, uint8x8_t *a1, uint8x8_t *a2, - uint8x8_t *a3, uint8x8_t *a4, uint8x8_t *a5, - uint8x8_t *a6, uint8x8_t *a7) { - // Swap 8 bit elements. Goes from: - // a0: 00 01 02 03 04 05 06 07 - // a1: 10 11 12 13 14 15 16 17 - // a2: 20 21 22 23 24 25 26 27 - // a3: 30 31 32 33 34 35 36 37 - // a4: 40 41 42 43 44 45 46 47 - // a5: 50 51 52 53 54 55 56 57 - // a6: 60 61 62 63 64 65 66 67 - // a7: 70 71 72 73 74 75 76 77 - // to: - // b0.val[0]: 00 10 02 12 04 14 06 16 40 50 42 52 44 54 46 56 - // b0.val[1]: 01 11 03 13 05 15 07 17 41 51 43 53 45 55 47 57 - // b1.val[0]: 20 30 22 32 24 34 26 36 60 70 62 72 64 74 66 76 - // b1.val[1]: 21 31 23 33 25 35 27 37 61 71 63 73 65 75 67 77 - - const uint8x16x2_t b0 = - vtrnq_u8(vcombine_u8(*a0, *a4), vcombine_u8(*a1, *a5)); - const uint8x16x2_t b1 = - vtrnq_u8(vcombine_u8(*a2, *a6), vcombine_u8(*a3, *a7)); - - // Swap 16 bit elements resulting in: - // c0.val[0]: 00 10 20 30 04 14 24 34 40 50 60 70 44 54 64 74 - // c0.val[1]: 02 12 22 32 06 16 26 36 42 52 62 72 46 56 66 76 - // c1.val[0]: 01 11 21 31 05 15 25 35 41 51 61 71 45 55 65 75 - // c1.val[1]: 03 13 23 33 07 17 27 37 43 53 63 73 47 57 67 77 - - const uint16x8x2_t c0 = vtrnq_u16(vreinterpretq_u16_u8(b0.val[0]), - vreinterpretq_u16_u8(b1.val[0])); - const uint16x8x2_t c1 = vtrnq_u16(vreinterpretq_u16_u8(b0.val[1]), - vreinterpretq_u16_u8(b1.val[1])); - - // Unzip 32 bit elements resulting in: - // d0.val[0]: 00 10 20 30 40 50 60 70 01 11 21 31 41 51 61 71 - // d0.val[1]: 04 14 24 34 44 54 64 74 05 15 25 35 45 55 65 75 - // d1.val[0]: 02 12 22 32 42 52 62 72 03 13 23 33 43 53 63 73 - // d1.val[1]: 06 16 26 36 46 56 66 76 07 17 27 37 47 57 67 77 - const uint32x4x2_t d0 = vuzpq_u32(vreinterpretq_u32_u16(c0.val[0]), - vreinterpretq_u32_u16(c1.val[0])); - const uint32x4x2_t d1 = vuzpq_u32(vreinterpretq_u32_u16(c0.val[1]), - vreinterpretq_u32_u16(c1.val[1])); - - *a0 = vreinterpret_u8_u32(vget_low_u32(d0.val[0])); - *a1 = vreinterpret_u8_u32(vget_high_u32(d0.val[0])); - *a2 = vreinterpret_u8_u32(vget_low_u32(d1.val[0])); - *a3 = vreinterpret_u8_u32(vget_high_u32(d1.val[0])); - *a4 = vreinterpret_u8_u32(vget_low_u32(d0.val[1])); - *a5 = vreinterpret_u8_u32(vget_high_u32(d0.val[1])); - *a6 = vreinterpret_u8_u32(vget_low_u32(d1.val[1])); - *a7 = vreinterpret_u8_u32(vget_high_u32(d1.val[1])); -} - -static INLINE void transpose_u8_8x4(uint8x8_t *a0, uint8x8_t *a1, uint8x8_t *a2, - uint8x8_t *a3) { - // Swap 8 bit elements. Goes from: - // a0: 00 01 02 03 04 05 06 07 - // a1: 10 11 12 13 14 15 16 17 - // a2: 20 21 22 23 24 25 26 27 - // a3: 30 31 32 33 34 35 36 37 - // to: - // b0.val[0]: 00 10 02 12 04 14 06 16 - // b0.val[1]: 01 11 03 13 05 15 07 17 - // b1.val[0]: 20 30 22 32 24 34 26 36 - // b1.val[1]: 21 31 23 33 25 35 27 37 - - const uint8x8x2_t b0 = vtrn_u8(*a0, *a1); - const uint8x8x2_t b1 = vtrn_u8(*a2, *a3); - - // Swap 16 bit elements resulting in: - // c0.val[0]: 00 10 20 30 04 14 24 34 - // c0.val[1]: 02 12 22 32 06 16 26 36 - // c1.val[0]: 01 11 21 31 05 15 25 35 - // c1.val[1]: 03 13 23 33 07 17 27 37 - - const uint16x4x2_t c0 = - vtrn_u16(vreinterpret_u16_u8(b0.val[0]), vreinterpret_u16_u8(b1.val[0])); - const uint16x4x2_t c1 = - vtrn_u16(vreinterpret_u16_u8(b0.val[1]), vreinterpret_u16_u8(b1.val[1])); - - *a0 = vreinterpret_u8_u16(c0.val[0]); - *a1 = vreinterpret_u8_u16(c1.val[0]); - *a2 = vreinterpret_u8_u16(c0.val[1]); - *a3 = vreinterpret_u8_u16(c1.val[1]); -} - -static INLINE void transpose_u8_4x4(uint8x8_t *a0, uint8x8_t *a1) { - // Swap 16 bit elements. Goes from: - // a0: 00 01 02 03 10 11 12 13 - // a1: 20 21 22 23 30 31 32 33 - // to: - // b0.val[0]: 00 01 20 21 10 11 30 31 - // b0.val[1]: 02 03 22 23 12 13 32 33 - - const uint16x4x2_t b0 = - vtrn_u16(vreinterpret_u16_u8(*a0), vreinterpret_u16_u8(*a1)); - - // Swap 32 bit elements resulting in: - // c0.val[0]: 00 01 20 21 02 03 22 23 - // c0.val[1]: 10 11 30 31 12 13 32 33 - - const uint32x2x2_t c0 = vtrn_u32(vreinterpret_u32_u16(b0.val[0]), - vreinterpret_u32_u16(b0.val[1])); - - // Swap 8 bit elements resulting in: - // d0.val[0]: 00 10 20 30 02 12 22 32 - // d0.val[1]: 01 11 21 31 03 13 23 33 - - const uint8x8x2_t d0 = - vtrn_u8(vreinterpret_u8_u32(c0.val[0]), vreinterpret_u8_u32(c0.val[1])); - - *a0 = d0.val[0]; - *a1 = d0.val[1]; -} - -static INLINE void transpose_u8_4x8(uint8x8_t *a0, uint8x8_t *a1, uint8x8_t *a2, - uint8x8_t *a3, const uint8x8_t a4, - const uint8x8_t a5, const uint8x8_t a6, - const uint8x8_t a7) { - // Swap 32 bit elements. Goes from: - // a0: 00 01 02 03 XX XX XX XX - // a1: 10 11 12 13 XX XX XX XX - // a2: 20 21 22 23 XX XX XX XX - // a3; 30 31 32 33 XX XX XX XX - // a4: 40 41 42 43 XX XX XX XX - // a5: 50 51 52 53 XX XX XX XX - // a6: 60 61 62 63 XX XX XX XX - // a7: 70 71 72 73 XX XX XX XX - // to: - // b0.val[0]: 00 01 02 03 40 41 42 43 - // b1.val[0]: 10 11 12 13 50 51 52 53 - // b2.val[0]: 20 21 22 23 60 61 62 63 - // b3.val[0]: 30 31 32 33 70 71 72 73 - - const uint32x2x2_t b0 = - vtrn_u32(vreinterpret_u32_u8(*a0), vreinterpret_u32_u8(a4)); - const uint32x2x2_t b1 = - vtrn_u32(vreinterpret_u32_u8(*a1), vreinterpret_u32_u8(a5)); - const uint32x2x2_t b2 = - vtrn_u32(vreinterpret_u32_u8(*a2), vreinterpret_u32_u8(a6)); - const uint32x2x2_t b3 = - vtrn_u32(vreinterpret_u32_u8(*a3), vreinterpret_u32_u8(a7)); - - // Swap 16 bit elements resulting in: - // c0.val[0]: 00 01 20 21 40 41 60 61 - // c0.val[1]: 02 03 22 23 42 43 62 63 - // c1.val[0]: 10 11 30 31 50 51 70 71 - // c1.val[1]: 12 13 32 33 52 53 72 73 - - const uint16x4x2_t c0 = vtrn_u16(vreinterpret_u16_u32(b0.val[0]), - vreinterpret_u16_u32(b2.val[0])); - const uint16x4x2_t c1 = vtrn_u16(vreinterpret_u16_u32(b1.val[0]), - vreinterpret_u16_u32(b3.val[0])); - - // Swap 8 bit elements resulting in: - // d0.val[0]: 00 10 20 30 40 50 60 70 - // d0.val[1]: 01 11 21 31 41 51 61 71 - // d1.val[0]: 02 12 22 32 42 52 62 72 - // d1.val[1]: 03 13 23 33 43 53 63 73 - - const uint8x8x2_t d0 = - vtrn_u8(vreinterpret_u8_u16(c0.val[0]), vreinterpret_u8_u16(c1.val[0])); - const uint8x8x2_t d1 = - vtrn_u8(vreinterpret_u8_u16(c0.val[1]), vreinterpret_u8_u16(c1.val[1])); - - *a0 = d0.val[0]; - *a1 = d0.val[1]; - *a2 = d1.val[0]; - *a3 = d1.val[1]; -} - -static INLINE void transpose_u16_4x8(uint16x4_t *a0, uint16x4_t *a1, - uint16x4_t *a2, uint16x4_t *a3, - uint16x4_t *a4, uint16x4_t *a5, - uint16x4_t *a6, uint16x4_t *a7, - uint16x8_t *o0, uint16x8_t *o1, - uint16x8_t *o2, uint16x8_t *o3) { - // Swap 16 bit elements. Goes from: - // a0: 00 01 02 03 - // a1: 10 11 12 13 - // a2: 20 21 22 23 - // a3: 30 31 32 33 - // a4: 40 41 42 43 - // a5: 50 51 52 53 - // a6: 60 61 62 63 - // a7: 70 71 72 73 - // to: - // b0.val[0]: 00 10 02 12 - // b0.val[1]: 01 11 03 13 - // b1.val[0]: 20 30 22 32 - // b1.val[1]: 21 31 23 33 - // b2.val[0]: 40 50 42 52 - // b2.val[1]: 41 51 43 53 - // b3.val[0]: 60 70 62 72 - // b3.val[1]: 61 71 63 73 - - uint16x4x2_t b0 = vtrn_u16(*a0, *a1); - uint16x4x2_t b1 = vtrn_u16(*a2, *a3); - uint16x4x2_t b2 = vtrn_u16(*a4, *a5); - uint16x4x2_t b3 = vtrn_u16(*a6, *a7); - - // Swap 32 bit elements resulting in: - // c0.val[0]: 00 10 20 30 - // c0.val[1]: 02 12 22 32 - // c1.val[0]: 01 11 21 31 - // c1.val[1]: 03 13 23 33 - // c2.val[0]: 40 50 60 70 - // c2.val[1]: 42 52 62 72 - // c3.val[0]: 41 51 61 71 - // c3.val[1]: 43 53 63 73 - - uint32x2x2_t c0 = vtrn_u32(vreinterpret_u32_u16(b0.val[0]), - vreinterpret_u32_u16(b1.val[0])); - uint32x2x2_t c1 = vtrn_u32(vreinterpret_u32_u16(b0.val[1]), - vreinterpret_u32_u16(b1.val[1])); - uint32x2x2_t c2 = vtrn_u32(vreinterpret_u32_u16(b2.val[0]), - vreinterpret_u32_u16(b3.val[0])); - uint32x2x2_t c3 = vtrn_u32(vreinterpret_u32_u16(b2.val[1]), - vreinterpret_u32_u16(b3.val[1])); - - // Swap 64 bit elements resulting in: - // o0: 00 10 20 30 40 50 60 70 - // o1: 01 11 21 31 41 51 61 71 - // o2: 02 12 22 32 42 52 62 72 - // o3: 03 13 23 33 43 53 63 73 - - *o0 = vcombine_u16(vreinterpret_u16_u32(c0.val[0]), - vreinterpret_u16_u32(c2.val[0])); - *o1 = vcombine_u16(vreinterpret_u16_u32(c1.val[0]), - vreinterpret_u16_u32(c3.val[0])); - *o2 = vcombine_u16(vreinterpret_u16_u32(c0.val[1]), - vreinterpret_u16_u32(c2.val[1])); - *o3 = vcombine_u16(vreinterpret_u16_u32(c1.val[1]), - vreinterpret_u16_u32(c3.val[1])); -} - -static INLINE void transpose_u16_8x8(uint16x8_t *a0, uint16x8_t *a1, - uint16x8_t *a2, uint16x8_t *a3, - uint16x8_t *a4, uint16x8_t *a5, - uint16x8_t *a6, uint16x8_t *a7) { - // Swap 16 bit elements. Goes from: - // a0: 00 01 02 03 04 05 06 07 - // a1: 10 11 12 13 14 15 16 17 - // a2: 20 21 22 23 24 25 26 27 - // a3: 30 31 32 33 34 35 36 37 - // a4: 40 41 42 43 44 45 46 47 - // a5: 50 51 52 53 54 55 56 57 - // a6: 60 61 62 63 64 65 66 67 - // a7: 70 71 72 73 74 75 76 77 - // to: - // b0.val[0]: 00 10 02 12 04 14 06 16 - // b0.val[1]: 01 11 03 13 05 15 07 17 - // b1.val[0]: 20 30 22 32 24 34 26 36 - // b1.val[1]: 21 31 23 33 25 35 27 37 - // b2.val[0]: 40 50 42 52 44 54 46 56 - // b2.val[1]: 41 51 43 53 45 55 47 57 - // b3.val[0]: 60 70 62 72 64 74 66 76 - // b3.val[1]: 61 71 63 73 65 75 67 77 - - const uint16x8x2_t b0 = vtrnq_u16(*a0, *a1); - const uint16x8x2_t b1 = vtrnq_u16(*a2, *a3); - const uint16x8x2_t b2 = vtrnq_u16(*a4, *a5); - const uint16x8x2_t b3 = vtrnq_u16(*a6, *a7); - - // Swap 32 bit elements resulting in: - // c0.val[0]: 00 10 20 30 04 14 24 34 - // c0.val[1]: 02 12 22 32 06 16 26 36 - // c1.val[0]: 01 11 21 31 05 15 25 35 - // c1.val[1]: 03 13 23 33 07 17 27 37 - // c2.val[0]: 40 50 60 70 44 54 64 74 - // c2.val[1]: 42 52 62 72 46 56 66 76 - // c3.val[0]: 41 51 61 71 45 55 65 75 - // c3.val[1]: 43 53 63 73 47 57 67 77 - - const uint32x4x2_t c0 = vtrnq_u32(vreinterpretq_u32_u16(b0.val[0]), - vreinterpretq_u32_u16(b1.val[0])); - const uint32x4x2_t c1 = vtrnq_u32(vreinterpretq_u32_u16(b0.val[1]), - vreinterpretq_u32_u16(b1.val[1])); - const uint32x4x2_t c2 = vtrnq_u32(vreinterpretq_u32_u16(b2.val[0]), - vreinterpretq_u32_u16(b3.val[0])); - const uint32x4x2_t c3 = vtrnq_u32(vreinterpretq_u32_u16(b2.val[1]), - vreinterpretq_u32_u16(b3.val[1])); - - *a0 = vcombine_u16(vget_low_u16(vreinterpretq_u16_u32(c0.val[0])), - vget_low_u16(vreinterpretq_u16_u32(c2.val[0]))); - *a4 = vcombine_u16(vget_high_u16(vreinterpretq_u16_u32(c0.val[0])), - vget_high_u16(vreinterpretq_u16_u32(c2.val[0]))); - - *a2 = vcombine_u16(vget_low_u16(vreinterpretq_u16_u32(c0.val[1])), - vget_low_u16(vreinterpretq_u16_u32(c2.val[1]))); - *a6 = vcombine_u16(vget_high_u16(vreinterpretq_u16_u32(c0.val[1])), - vget_high_u16(vreinterpretq_u16_u32(c2.val[1]))); - - *a1 = vcombine_u16(vget_low_u16(vreinterpretq_u16_u32(c1.val[0])), - vget_low_u16(vreinterpretq_u16_u32(c3.val[0]))); - *a5 = vcombine_u16(vget_high_u16(vreinterpretq_u16_u32(c1.val[0])), - vget_high_u16(vreinterpretq_u16_u32(c3.val[0]))); - - *a3 = vcombine_u16(vget_low_u16(vreinterpretq_u16_u32(c1.val[1])), - vget_low_u16(vreinterpretq_u16_u32(c3.val[1]))); - *a7 = vcombine_u16(vget_high_u16(vreinterpretq_u16_u32(c1.val[1])), - vget_high_u16(vreinterpretq_u16_u32(c3.val[1]))); -} - -static INLINE void transpose_s16_8x8(int16x8_t *a0, int16x8_t *a1, - int16x8_t *a2, int16x8_t *a3, - int16x8_t *a4, int16x8_t *a5, - int16x8_t *a6, int16x8_t *a7) { - // Swap 16 bit elements. Goes from: - // a0: 00 01 02 03 04 05 06 07 - // a1: 10 11 12 13 14 15 16 17 - // a2: 20 21 22 23 24 25 26 27 - // a3: 30 31 32 33 34 35 36 37 - // a4: 40 41 42 43 44 45 46 47 - // a5: 50 51 52 53 54 55 56 57 - // a6: 60 61 62 63 64 65 66 67 - // a7: 70 71 72 73 74 75 76 77 - // to: - // b0.val[0]: 00 10 02 12 04 14 06 16 - // b0.val[1]: 01 11 03 13 05 15 07 17 - // b1.val[0]: 20 30 22 32 24 34 26 36 - // b1.val[1]: 21 31 23 33 25 35 27 37 - // b2.val[0]: 40 50 42 52 44 54 46 56 - // b2.val[1]: 41 51 43 53 45 55 47 57 - // b3.val[0]: 60 70 62 72 64 74 66 76 - // b3.val[1]: 61 71 63 73 65 75 67 77 - - const int16x8x2_t b0 = vtrnq_s16(*a0, *a1); - const int16x8x2_t b1 = vtrnq_s16(*a2, *a3); - const int16x8x2_t b2 = vtrnq_s16(*a4, *a5); - const int16x8x2_t b3 = vtrnq_s16(*a6, *a7); - - // Swap 32 bit elements resulting in: - // c0.val[0]: 00 10 20 30 04 14 24 34 - // c0.val[1]: 02 12 22 32 06 16 26 36 - // c1.val[0]: 01 11 21 31 05 15 25 35 - // c1.val[1]: 03 13 23 33 07 17 27 37 - // c2.val[0]: 40 50 60 70 44 54 64 74 - // c2.val[1]: 42 52 62 72 46 56 66 76 - // c3.val[0]: 41 51 61 71 45 55 65 75 - // c3.val[1]: 43 53 63 73 47 57 67 77 - - const int32x4x2_t c0 = vtrnq_s32(vreinterpretq_s32_s16(b0.val[0]), - vreinterpretq_s32_s16(b1.val[0])); - const int32x4x2_t c1 = vtrnq_s32(vreinterpretq_s32_s16(b0.val[1]), - vreinterpretq_s32_s16(b1.val[1])); - const int32x4x2_t c2 = vtrnq_s32(vreinterpretq_s32_s16(b2.val[0]), - vreinterpretq_s32_s16(b3.val[0])); - const int32x4x2_t c3 = vtrnq_s32(vreinterpretq_s32_s16(b2.val[1]), - vreinterpretq_s32_s16(b3.val[1])); - - *a0 = vcombine_s16(vget_low_s16(vreinterpretq_s16_s32(c0.val[0])), - vget_low_s16(vreinterpretq_s16_s32(c2.val[0]))); - *a4 = vcombine_s16(vget_high_s16(vreinterpretq_s16_s32(c0.val[0])), - vget_high_s16(vreinterpretq_s16_s32(c2.val[0]))); - - *a2 = vcombine_s16(vget_low_s16(vreinterpretq_s16_s32(c0.val[1])), - vget_low_s16(vreinterpretq_s16_s32(c2.val[1]))); - *a6 = vcombine_s16(vget_high_s16(vreinterpretq_s16_s32(c0.val[1])), - vget_high_s16(vreinterpretq_s16_s32(c2.val[1]))); - - *a1 = vcombine_s16(vget_low_s16(vreinterpretq_s16_s32(c1.val[0])), - vget_low_s16(vreinterpretq_s16_s32(c3.val[0]))); - *a5 = vcombine_s16(vget_high_s16(vreinterpretq_s16_s32(c1.val[0])), - vget_high_s16(vreinterpretq_s16_s32(c3.val[0]))); - - *a3 = vcombine_s16(vget_low_s16(vreinterpretq_s16_s32(c1.val[1])), - vget_low_s16(vreinterpretq_s16_s32(c3.val[1]))); - *a7 = vcombine_s16(vget_high_s16(vreinterpretq_s16_s32(c1.val[1])), - vget_high_s16(vreinterpretq_s16_s32(c3.val[1]))); -} - -static INLINE int16x8x2_t vpx_vtrnq_s64_to_s16(int32x4_t a0, int32x4_t a1) { - int16x8x2_t b0; - b0.val[0] = vcombine_s16(vreinterpret_s16_s32(vget_low_s32(a0)), - vreinterpret_s16_s32(vget_low_s32(a1))); - b0.val[1] = vcombine_s16(vreinterpret_s16_s32(vget_high_s32(a0)), - vreinterpret_s16_s32(vget_high_s32(a1))); - return b0; -} - -static INLINE void transpose_s16_8x8q(int16x8_t *a0, int16x8_t *out) { - // Swap 16 bit elements. Goes from: - // a0: 00 01 02 03 04 05 06 07 - // a1: 10 11 12 13 14 15 16 17 - // a2: 20 21 22 23 24 25 26 27 - // a3: 30 31 32 33 34 35 36 37 - // a4: 40 41 42 43 44 45 46 47 - // a5: 50 51 52 53 54 55 56 57 - // a6: 60 61 62 63 64 65 66 67 - // a7: 70 71 72 73 74 75 76 77 - // to: - // b0.val[0]: 00 10 02 12 04 14 06 16 - // b0.val[1]: 01 11 03 13 05 15 07 17 - // b1.val[0]: 20 30 22 32 24 34 26 36 - // b1.val[1]: 21 31 23 33 25 35 27 37 - // b2.val[0]: 40 50 42 52 44 54 46 56 - // b2.val[1]: 41 51 43 53 45 55 47 57 - // b3.val[0]: 60 70 62 72 64 74 66 76 - // b3.val[1]: 61 71 63 73 65 75 67 77 - - const int16x8x2_t b0 = vtrnq_s16(*a0, *(a0 + 1)); - const int16x8x2_t b1 = vtrnq_s16(*(a0 + 2), *(a0 + 3)); - const int16x8x2_t b2 = vtrnq_s16(*(a0 + 4), *(a0 + 5)); - const int16x8x2_t b3 = vtrnq_s16(*(a0 + 6), *(a0 + 7)); - - // Swap 32 bit elements resulting in: - // c0.val[0]: 00 10 20 30 04 14 24 34 - // c0.val[1]: 02 12 22 32 06 16 26 36 - // c1.val[0]: 01 11 21 31 05 15 25 35 - // c1.val[1]: 03 13 23 33 07 17 27 37 - // c2.val[0]: 40 50 60 70 44 54 64 74 - // c2.val[1]: 42 52 62 72 46 56 66 76 - // c3.val[0]: 41 51 61 71 45 55 65 75 - // c3.val[1]: 43 53 63 73 47 57 67 77 - - const int32x4x2_t c0 = vtrnq_s32(vreinterpretq_s32_s16(b0.val[0]), - vreinterpretq_s32_s16(b1.val[0])); - const int32x4x2_t c1 = vtrnq_s32(vreinterpretq_s32_s16(b0.val[1]), - vreinterpretq_s32_s16(b1.val[1])); - const int32x4x2_t c2 = vtrnq_s32(vreinterpretq_s32_s16(b2.val[0]), - vreinterpretq_s32_s16(b3.val[0])); - const int32x4x2_t c3 = vtrnq_s32(vreinterpretq_s32_s16(b2.val[1]), - vreinterpretq_s32_s16(b3.val[1])); - - // Swap 64 bit elements resulting in: - // d0.val[0]: 00 10 20 30 40 50 60 70 - // d0.val[1]: 04 14 24 34 44 54 64 74 - // d1.val[0]: 01 11 21 31 41 51 61 71 - // d1.val[1]: 05 15 25 35 45 55 65 75 - // d2.val[0]: 02 12 22 32 42 52 62 72 - // d2.val[1]: 06 16 26 36 46 56 66 76 - // d3.val[0]: 03 13 23 33 43 53 63 73 - // d3.val[1]: 07 17 27 37 47 57 67 77 - const int16x8x2_t d0 = vpx_vtrnq_s64_to_s16(c0.val[0], c2.val[0]); - const int16x8x2_t d1 = vpx_vtrnq_s64_to_s16(c1.val[0], c3.val[0]); - const int16x8x2_t d2 = vpx_vtrnq_s64_to_s16(c0.val[1], c2.val[1]); - const int16x8x2_t d3 = vpx_vtrnq_s64_to_s16(c1.val[1], c3.val[1]); - - *out = d0.val[0]; - *(out + 1) = d1.val[0]; - *(out + 2) = d2.val[0]; - *(out + 3) = d3.val[0]; - *(out + 4) = d0.val[1]; - *(out + 5) = d1.val[1]; - *(out + 6) = d2.val[1]; - *(out + 7) = d3.val[1]; -} - -static INLINE void transpose_s16_4x4d(int16x4_t *a0, int16x4_t *a1, - int16x4_t *a2, int16x4_t *a3) { - // Swap 16 bit elements. Goes from: - // a0: 00 01 02 03 - // a1: 10 11 12 13 - // a2: 20 21 22 23 - // a3: 30 31 32 33 - // to: - // b0.val[0]: 00 10 02 12 - // b0.val[1]: 01 11 03 13 - // b1.val[0]: 20 30 22 32 - // b1.val[1]: 21 31 23 33 - - const int16x4x2_t b0 = vtrn_s16(*a0, *a1); - const int16x4x2_t b1 = vtrn_s16(*a2, *a3); - - // Swap 32 bit elements resulting in: - // c0.val[0]: 00 10 20 30 - // c0.val[1]: 02 12 22 32 - // c1.val[0]: 01 11 21 31 - // c1.val[1]: 03 13 23 33 - - const int32x2x2_t c0 = vtrn_s32(vreinterpret_s32_s16(b0.val[0]), - vreinterpret_s32_s16(b1.val[0])); - const int32x2x2_t c1 = vtrn_s32(vreinterpret_s32_s16(b0.val[1]), - vreinterpret_s32_s16(b1.val[1])); - - *a0 = vreinterpret_s16_s32(c0.val[0]); - *a1 = vreinterpret_s16_s32(c1.val[0]); - *a2 = vreinterpret_s16_s32(c0.val[1]); - *a3 = vreinterpret_s16_s32(c1.val[1]); -} - -static INLINE int32x4x2_t aom_vtrnq_s64_to_s32(int32x4_t a0, int32x4_t a1) { - int32x4x2_t b0; - b0.val[0] = vcombine_s32(vget_low_s32(a0), vget_low_s32(a1)); - b0.val[1] = vcombine_s32(vget_high_s32(a0), vget_high_s32(a1)); - return b0; -} - -static INLINE void transpose_s32_4x4(int32x4_t *a0, int32x4_t *a1, - int32x4_t *a2, int32x4_t *a3) { - // Swap 32 bit elements. Goes from: - // a0: 00 01 02 03 - // a1: 10 11 12 13 - // a2: 20 21 22 23 - // a3: 30 31 32 33 - // to: - // b0.val[0]: 00 10 02 12 - // b0.val[1]: 01 11 03 13 - // b1.val[0]: 20 30 22 32 - // b1.val[1]: 21 31 23 33 - - const int32x4x2_t b0 = vtrnq_s32(*a0, *a1); - const int32x4x2_t b1 = vtrnq_s32(*a2, *a3); - - // Swap 64 bit elements resulting in: - // c0.val[0]: 00 10 20 30 - // c0.val[1]: 02 12 22 32 - // c1.val[0]: 01 11 21 31 - // c1.val[1]: 03 13 23 33 - - const int32x4x2_t c0 = aom_vtrnq_s64_to_s32(b0.val[0], b1.val[0]); - const int32x4x2_t c1 = aom_vtrnq_s64_to_s32(b0.val[1], b1.val[1]); - - *a0 = c0.val[0]; - *a1 = c1.val[0]; - *a2 = c0.val[1]; - *a3 = c1.val[1]; -} - -#endif // AOM_AV1_COMMON_ARM_TRANSPOSE_NEON_H_ diff --git a/third_party/aom/av1/common/arm/warp_plane_neon.c b/third_party/aom/av1/common/arm/warp_plane_neon.c deleted file mode 100644 index 7f02d42a7..000000000 --- a/third_party/aom/av1/common/arm/warp_plane_neon.c +++ /dev/null @@ -1,714 +0,0 @@ -/* - * Copyright (c) 2018, Alliance for Open Media. All rights reserved - * - * This source code is subject to the terms of the BSD 2 Clause License and - * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License - * was not distributed with this source code in the LICENSE file, you can - * obtain it at www.aomedia.org/license/software. If the Alliance for Open - * Media Patent License 1.0 was not distributed with this source code in the - * PATENTS file, you can obtain it at www.aomedia.org/license/patent. - */ - -#include <assert.h> -#include <arm_neon.h> -#include <memory.h> -#include <math.h> - -#include "aom_dsp/aom_dsp_common.h" -#include "aom_ports/mem.h" -#include "config/av1_rtcd.h" -#include "av1/common/warped_motion.h" -#include "av1/common/scale.h" - -/* This is a modified version of 'warped_filter' from warped_motion.c: - * Each coefficient is stored in 8 bits instead of 16 bits - * The coefficients are rearranged in the column order 0, 2, 4, 6, 1, 3, 5, 7 - - This is done in order to avoid overflow: Since the tap with the largest - coefficient could be any of taps 2, 3, 4 or 5, we can't use the summation - order ((0 + 1) + (4 + 5)) + ((2 + 3) + (6 + 7)) used in the regular - convolve functions. - - Instead, we use the summation order - ((0 + 2) + (4 + 6)) + ((1 + 3) + (5 + 7)). - The rearrangement of coefficients in this table is so that we can get the - coefficients into the correct order more quickly. -*/ -/* clang-format off */ -DECLARE_ALIGNED(8, static const int8_t, - filter_8bit_neon[WARPEDPIXEL_PREC_SHIFTS * 3 + 1][8]) = { -#if WARPEDPIXEL_PREC_BITS == 6 - // [-1, 0) - { 0, 127, 0, 0, 0, 1, 0, 0}, { 0, 127, 0, 0, -1, 2, 0, 0}, - { 1, 127, -1, 0, -3, 4, 0, 0}, { 1, 126, -2, 0, -4, 6, 1, 0}, - { 1, 126, -3, 0, -5, 8, 1, 0}, { 1, 125, -4, 0, -6, 11, 1, 0}, - { 1, 124, -4, 0, -7, 13, 1, 0}, { 2, 123, -5, 0, -8, 15, 1, 0}, - { 2, 122, -6, 0, -9, 18, 1, 0}, { 2, 121, -6, 0, -10, 20, 1, 0}, - { 2, 120, -7, 0, -11, 22, 2, 0}, { 2, 119, -8, 0, -12, 25, 2, 0}, - { 3, 117, -8, 0, -13, 27, 2, 0}, { 3, 116, -9, 0, -13, 29, 2, 0}, - { 3, 114, -10, 0, -14, 32, 3, 0}, { 3, 113, -10, 0, -15, 35, 2, 0}, - { 3, 111, -11, 0, -15, 37, 3, 0}, { 3, 109, -11, 0, -16, 40, 3, 0}, - { 3, 108, -12, 0, -16, 42, 3, 0}, { 4, 106, -13, 0, -17, 45, 3, 0}, - { 4, 104, -13, 0, -17, 47, 3, 0}, { 4, 102, -14, 0, -17, 50, 3, 0}, - { 4, 100, -14, 0, -17, 52, 3, 0}, { 4, 98, -15, 0, -18, 55, 4, 0}, - { 4, 96, -15, 0, -18, 58, 3, 0}, { 4, 94, -16, 0, -18, 60, 4, 0}, - { 4, 91, -16, 0, -18, 63, 4, 0}, { 4, 89, -16, 0, -18, 65, 4, 0}, - { 4, 87, -17, 0, -18, 68, 4, 0}, { 4, 85, -17, 0, -18, 70, 4, 0}, - { 4, 82, -17, 0, -18, 73, 4, 0}, { 4, 80, -17, 0, -18, 75, 4, 0}, - { 4, 78, -18, 0, -18, 78, 4, 0}, { 4, 75, -18, 0, -17, 80, 4, 0}, - { 4, 73, -18, 0, -17, 82, 4, 0}, { 4, 70, -18, 0, -17, 85, 4, 0}, - { 4, 68, -18, 0, -17, 87, 4, 0}, { 4, 65, -18, 0, -16, 89, 4, 0}, - { 4, 63, -18, 0, -16, 91, 4, 0}, { 4, 60, -18, 0, -16, 94, 4, 0}, - { 3, 58, -18, 0, -15, 96, 4, 0}, { 4, 55, -18, 0, -15, 98, 4, 0}, - { 3, 52, -17, 0, -14, 100, 4, 0}, { 3, 50, -17, 0, -14, 102, 4, 0}, - { 3, 47, -17, 0, -13, 104, 4, 0}, { 3, 45, -17, 0, -13, 106, 4, 0}, - { 3, 42, -16, 0, -12, 108, 3, 0}, { 3, 40, -16, 0, -11, 109, 3, 0}, - { 3, 37, -15, 0, -11, 111, 3, 0}, { 2, 35, -15, 0, -10, 113, 3, 0}, - { 3, 32, -14, 0, -10, 114, 3, 0}, { 2, 29, -13, 0, -9, 116, 3, 0}, - { 2, 27, -13, 0, -8, 117, 3, 0}, { 2, 25, -12, 0, -8, 119, 2, 0}, - { 2, 22, -11, 0, -7, 120, 2, 0}, { 1, 20, -10, 0, -6, 121, 2, 0}, - { 1, 18, -9, 0, -6, 122, 2, 0}, { 1, 15, -8, 0, -5, 123, 2, 0}, - { 1, 13, -7, 0, -4, 124, 1, 0}, { 1, 11, -6, 0, -4, 125, 1, 0}, - { 1, 8, -5, 0, -3, 126, 1, 0}, { 1, 6, -4, 0, -2, 126, 1, 0}, - { 0, 4, -3, 0, -1, 127, 1, 0}, { 0, 2, -1, 0, 0, 127, 0, 0}, - // [0, 1) - { 0, 0, 1, 0, 0, 127, 0, 0}, { 0, -1, 2, 0, 0, 127, 0, 0}, - { 0, -3, 4, 1, 1, 127, -2, 0}, { 0, -5, 6, 1, 1, 127, -2, 0}, - { 0, -6, 8, 1, 2, 126, -3, 0}, {-1, -7, 11, 2, 2, 126, -4, -1}, - {-1, -8, 13, 2, 3, 125, -5, -1}, {-1, -10, 16, 3, 3, 124, -6, -1}, - {-1, -11, 18, 3, 4, 123, -7, -1}, {-1, -12, 20, 3, 4, 122, -7, -1}, - {-1, -13, 23, 3, 4, 121, -8, -1}, {-2, -14, 25, 4, 5, 120, -9, -1}, - {-1, -15, 27, 4, 5, 119, -10, -1}, {-1, -16, 30, 4, 5, 118, -11, -1}, - {-2, -17, 33, 5, 6, 116, -12, -1}, {-2, -17, 35, 5, 6, 114, -12, -1}, - {-2, -18, 38, 5, 6, 113, -13, -1}, {-2, -19, 41, 6, 7, 111, -14, -2}, - {-2, -19, 43, 6, 7, 110, -15, -2}, {-2, -20, 46, 6, 7, 108, -15, -2}, - {-2, -20, 49, 6, 7, 106, -16, -2}, {-2, -21, 51, 7, 7, 104, -16, -2}, - {-2, -21, 54, 7, 7, 102, -17, -2}, {-2, -21, 56, 7, 8, 100, -18, -2}, - {-2, -22, 59, 7, 8, 98, -18, -2}, {-2, -22, 62, 7, 8, 96, -19, -2}, - {-2, -22, 64, 7, 8, 94, -19, -2}, {-2, -22, 67, 8, 8, 91, -20, -2}, - {-2, -22, 69, 8, 8, 89, -20, -2}, {-2, -22, 72, 8, 8, 87, -21, -2}, - {-2, -21, 74, 8, 8, 84, -21, -2}, {-2, -22, 77, 8, 8, 82, -21, -2}, - {-2, -21, 79, 8, 8, 79, -21, -2}, {-2, -21, 82, 8, 8, 77, -22, -2}, - {-2, -21, 84, 8, 8, 74, -21, -2}, {-2, -21, 87, 8, 8, 72, -22, -2}, - {-2, -20, 89, 8, 8, 69, -22, -2}, {-2, -20, 91, 8, 8, 67, -22, -2}, - {-2, -19, 94, 8, 7, 64, -22, -2}, {-2, -19, 96, 8, 7, 62, -22, -2}, - {-2, -18, 98, 8, 7, 59, -22, -2}, {-2, -18, 100, 8, 7, 56, -21, -2}, - {-2, -17, 102, 7, 7, 54, -21, -2}, {-2, -16, 104, 7, 7, 51, -21, -2}, - {-2, -16, 106, 7, 6, 49, -20, -2}, {-2, -15, 108, 7, 6, 46, -20, -2}, - {-2, -15, 110, 7, 6, 43, -19, -2}, {-2, -14, 111, 7, 6, 41, -19, -2}, - {-1, -13, 113, 6, 5, 38, -18, -2}, {-1, -12, 114, 6, 5, 35, -17, -2}, - {-1, -12, 116, 6, 5, 33, -17, -2}, {-1, -11, 118, 5, 4, 30, -16, -1}, - {-1, -10, 119, 5, 4, 27, -15, -1}, {-1, -9, 120, 5, 4, 25, -14, -2}, - {-1, -8, 121, 4, 3, 23, -13, -1}, {-1, -7, 122, 4, 3, 20, -12, -1}, - {-1, -7, 123, 4, 3, 18, -11, -1}, {-1, -6, 124, 3, 3, 16, -10, -1}, - {-1, -5, 125, 3, 2, 13, -8, -1}, {-1, -4, 126, 2, 2, 11, -7, -1}, - { 0, -3, 126, 2, 1, 8, -6, 0}, { 0, -2, 127, 1, 1, 6, -5, 0}, - { 0, -2, 127, 1, 1, 4, -3, 0}, { 0, 0, 127, 0, 0, 2, -1, 0}, - // [1, 2) - { 0, 0, 127, 0, 0, 1, 0, 0}, { 0, 0, 127, 0, 0, -1, 2, 0}, - { 0, 1, 127, -1, 0, -3, 4, 0}, { 0, 1, 126, -2, 0, -4, 6, 1}, - { 0, 1, 126, -3, 0, -5, 8, 1}, { 0, 1, 125, -4, 0, -6, 11, 1}, - { 0, 1, 124, -4, 0, -7, 13, 1}, { 0, 2, 123, -5, 0, -8, 15, 1}, - { 0, 2, 122, -6, 0, -9, 18, 1}, { 0, 2, 121, -6, 0, -10, 20, 1}, - { 0, 2, 120, -7, 0, -11, 22, 2}, { 0, 2, 119, -8, 0, -12, 25, 2}, - { 0, 3, 117, -8, 0, -13, 27, 2}, { 0, 3, 116, -9, 0, -13, 29, 2}, - { 0, 3, 114, -10, 0, -14, 32, 3}, { 0, 3, 113, -10, 0, -15, 35, 2}, - { 0, 3, 111, -11, 0, -15, 37, 3}, { 0, 3, 109, -11, 0, -16, 40, 3}, - { 0, 3, 108, -12, 0, -16, 42, 3}, { 0, 4, 106, -13, 0, -17, 45, 3}, - { 0, 4, 104, -13, 0, -17, 47, 3}, { 0, 4, 102, -14, 0, -17, 50, 3}, - { 0, 4, 100, -14, 0, -17, 52, 3}, { 0, 4, 98, -15, 0, -18, 55, 4}, - { 0, 4, 96, -15, 0, -18, 58, 3}, { 0, 4, 94, -16, 0, -18, 60, 4}, - { 0, 4, 91, -16, 0, -18, 63, 4}, { 0, 4, 89, -16, 0, -18, 65, 4}, - { 0, 4, 87, -17, 0, -18, 68, 4}, { 0, 4, 85, -17, 0, -18, 70, 4}, - { 0, 4, 82, -17, 0, -18, 73, 4}, { 0, 4, 80, -17, 0, -18, 75, 4}, - { 0, 4, 78, -18, 0, -18, 78, 4}, { 0, 4, 75, -18, 0, -17, 80, 4}, - { 0, 4, 73, -18, 0, -17, 82, 4}, { 0, 4, 70, -18, 0, -17, 85, 4}, - { 0, 4, 68, -18, 0, -17, 87, 4}, { 0, 4, 65, -18, 0, -16, 89, 4}, - { 0, 4, 63, -18, 0, -16, 91, 4}, { 0, 4, 60, -18, 0, -16, 94, 4}, - { 0, 3, 58, -18, 0, -15, 96, 4}, { 0, 4, 55, -18, 0, -15, 98, 4}, - { 0, 3, 52, -17, 0, -14, 100, 4}, { 0, 3, 50, -17, 0, -14, 102, 4}, - { 0, 3, 47, -17, 0, -13, 104, 4}, { 0, 3, 45, -17, 0, -13, 106, 4}, - { 0, 3, 42, -16, 0, -12, 108, 3}, { 0, 3, 40, -16, 0, -11, 109, 3}, - { 0, 3, 37, -15, 0, -11, 111, 3}, { 0, 2, 35, -15, 0, -10, 113, 3}, - { 0, 3, 32, -14, 0, -10, 114, 3}, { 0, 2, 29, -13, 0, -9, 116, 3}, - { 0, 2, 27, -13, 0, -8, 117, 3}, { 0, 2, 25, -12, 0, -8, 119, 2}, - { 0, 2, 22, -11, 0, -7, 120, 2}, { 0, 1, 20, -10, 0, -6, 121, 2}, - { 0, 1, 18, -9, 0, -6, 122, 2}, { 0, 1, 15, -8, 0, -5, 123, 2}, - { 0, 1, 13, -7, 0, -4, 124, 1}, { 0, 1, 11, -6, 0, -4, 125, 1}, - { 0, 1, 8, -5, 0, -3, 126, 1}, { 0, 1, 6, -4, 0, -2, 126, 1}, - { 0, 0, 4, -3, 0, -1, 127, 1}, { 0, 0, 2, -1, 0, 0, 127, 0}, - // dummy (replicate row index 191) - { 0, 0, 2, -1, 0, 0, 127, 0}, - -#else - // [-1, 0) - { 0, 127, 0, 0, 0, 1, 0, 0}, { 1, 127, -1, 0, -3, 4, 0, 0}, - { 1, 126, -3, 0, -5, 8, 1, 0}, { 1, 124, -4, 0, -7, 13, 1, 0}, - { 2, 122, -6, 0, -9, 18, 1, 0}, { 2, 120, -7, 0, -11, 22, 2, 0}, - { 3, 117, -8, 0, -13, 27, 2, 0}, { 3, 114, -10, 0, -14, 32, 3, 0}, - { 3, 111, -11, 0, -15, 37, 3, 0}, { 3, 108, -12, 0, -16, 42, 3, 0}, - { 4, 104, -13, 0, -17, 47, 3, 0}, { 4, 100, -14, 0, -17, 52, 3, 0}, - { 4, 96, -15, 0, -18, 58, 3, 0}, { 4, 91, -16, 0, -18, 63, 4, 0}, - { 4, 87, -17, 0, -18, 68, 4, 0}, { 4, 82, -17, 0, -18, 73, 4, 0}, - { 4, 78, -18, 0, -18, 78, 4, 0}, { 4, 73, -18, 0, -17, 82, 4, 0}, - { 4, 68, -18, 0, -17, 87, 4, 0}, { 4, 63, -18, 0, -16, 91, 4, 0}, - { 3, 58, -18, 0, -15, 96, 4, 0}, { 3, 52, -17, 0, -14, 100, 4, 0}, - { 3, 47, -17, 0, -13, 104, 4, 0}, { 3, 42, -16, 0, -12, 108, 3, 0}, - { 3, 37, -15, 0, -11, 111, 3, 0}, { 3, 32, -14, 0, -10, 114, 3, 0}, - { 2, 27, -13, 0, -8, 117, 3, 0}, { 2, 22, -11, 0, -7, 120, 2, 0}, - { 1, 18, -9, 0, -6, 122, 2, 0}, { 1, 13, -7, 0, -4, 124, 1, 0}, - { 1, 8, -5, 0, -3, 126, 1, 0}, { 0, 4, -3, 0, -1, 127, 1, 0}, - // [0, 1) - { 0, 0, 1, 0, 0, 127, 0, 0}, { 0, -3, 4, 1, 1, 127, -2, 0}, - { 0, -6, 8, 1, 2, 126, -3, 0}, {-1, -8, 13, 2, 3, 125, -5, -1}, - {-1, -11, 18, 3, 4, 123, -7, -1}, {-1, -13, 23, 3, 4, 121, -8, -1}, - {-1, -15, 27, 4, 5, 119, -10, -1}, {-2, -17, 33, 5, 6, 116, -12, -1}, - {-2, -18, 38, 5, 6, 113, -13, -1}, {-2, -19, 43, 6, 7, 110, -15, -2}, - {-2, -20, 49, 6, 7, 106, -16, -2}, {-2, -21, 54, 7, 7, 102, -17, -2}, - {-2, -22, 59, 7, 8, 98, -18, -2}, {-2, -22, 64, 7, 8, 94, -19, -2}, - {-2, -22, 69, 8, 8, 89, -20, -2}, {-2, -21, 74, 8, 8, 84, -21, -2}, - {-2, -21, 79, 8, 8, 79, -21, -2}, {-2, -21, 84, 8, 8, 74, -21, -2}, - {-2, -20, 89, 8, 8, 69, -22, -2}, {-2, -19, 94, 8, 7, 64, -22, -2}, - {-2, -18, 98, 8, 7, 59, -22, -2}, {-2, -17, 102, 7, 7, 54, -21, -2}, - {-2, -16, 106, 7, 6, 49, -20, -2}, {-2, -15, 110, 7, 6, 43, -19, -2}, - {-1, -13, 113, 6, 5, 38, -18, -2}, {-1, -12, 116, 6, 5, 33, -17, -2}, - {-1, -10, 119, 5, 4, 27, -15, -1}, {-1, -8, 121, 4, 3, 23, -13, -1}, - {-1, -7, 123, 4, 3, 18, -11, -1}, {-1, -5, 125, 3, 2, 13, -8, -1}, - { 0, -3, 126, 2, 1, 8, -6, 0}, { 0, -2, 127, 1, 1, 4, -3, 0}, - // [1, 2) - { 0, 0, 127, 0, 0, 1, 0, 0}, { 0, 1, 127, -1, 0, -3, 4, 0}, - { 0, 1, 126, -3, 0, -5, 8, 1}, { 0, 1, 124, -4, 0, -7, 13, 1}, - { 0, 2, 122, -6, 0, -9, 18, 1}, { 0, 2, 120, -7, 0, -11, 22, 2}, - { 0, 3, 117, -8, 0, -13, 27, 2}, { 0, 3, 114, -10, 0, -14, 32, 3}, - { 0, 3, 111, -11, 0, -15, 37, 3}, { 0, 3, 108, -12, 0, -16, 42, 3}, - { 0, 4, 104, -13, 0, -17, 47, 3}, { 0, 4, 100, -14, 0, -17, 52, 3}, - { 0, 4, 96, -15, 0, -18, 58, 3}, { 0, 4, 91, -16, 0, -18, 63, 4}, - { 0, 4, 87, -17, 0, -18, 68, 4}, { 0, 4, 82, -17, 0, -18, 73, 4}, - { 0, 4, 78, -18, 0, -18, 78, 4}, { 0, 4, 73, -18, 0, -17, 82, 4}, - { 0, 4, 68, -18, 0, -17, 87, 4}, { 0, 4, 63, -18, 0, -16, 91, 4}, - { 0, 3, 58, -18, 0, -15, 96, 4}, { 0, 3, 52, -17, 0, -14, 100, 4}, - { 0, 3, 47, -17, 0, -13, 104, 4}, { 0, 3, 42, -16, 0, -12, 108, 3}, - { 0, 3, 37, -15, 0, -11, 111, 3}, { 0, 3, 32, -14, 0, -10, 114, 3}, - { 0, 2, 27, -13, 0, -8, 117, 3}, { 0, 2, 22, -11, 0, -7, 120, 2}, - { 0, 1, 18, -9, 0, -6, 122, 2}, { 0, 1, 13, -7, 0, -4, 124, 1}, - { 0, 1, 8, -5, 0, -3, 126, 1}, { 0, 0, 4, -3, 0, -1, 127, 1}, - // dummy (replicate row index 95) - { 0, 0, 4, -3, 0, -1, 127, 1}, -#endif // WARPEDPIXEL_PREC_BITS == 6 -}; -/* clang-format on */ - -static INLINE void convolve(int32x2x2_t x0, int32x2x2_t x1, uint8x8_t src_0, - uint8x8_t src_1, int16x4_t *res) { - int16x8_t coeff_0, coeff_1; - int16x8_t pix_0, pix_1; - - coeff_0 = vcombine_s16(vreinterpret_s16_s32(x0.val[0]), - vreinterpret_s16_s32(x1.val[0])); - coeff_1 = vcombine_s16(vreinterpret_s16_s32(x0.val[1]), - vreinterpret_s16_s32(x1.val[1])); - - pix_0 = vreinterpretq_s16_u16(vmovl_u8(src_0)); - pix_0 = vmulq_s16(coeff_0, pix_0); - - pix_1 = vreinterpretq_s16_u16(vmovl_u8(src_1)); - pix_0 = vmlaq_s16(pix_0, coeff_1, pix_1); - - *res = vpadd_s16(vget_low_s16(pix_0), vget_high_s16(pix_0)); -} - -static INLINE void horizontal_filter_neon(uint8x16_t src_1, uint8x16_t src_2, - uint8x16_t src_3, uint8x16_t src_4, - int16x8_t *tmp_dst, int sx, int alpha, - int k, const int offset_bits_horiz, - const int reduce_bits_horiz) { - const uint8x16_t mask = { 255, 0, 255, 0, 255, 0, 255, 0, - 255, 0, 255, 0, 255, 0, 255, 0 }; - const int32x4_t add_const = vdupq_n_s32((int32_t)(1 << offset_bits_horiz)); - const int16x8_t shift = vdupq_n_s16(-(int16_t)reduce_bits_horiz); - - int16x8_t f0, f1, f2, f3, f4, f5, f6, f7; - int32x2x2_t b0, b1; - uint8x8_t src_1_low, src_2_low, src_3_low, src_4_low, src_5_low, src_6_low; - int32x4_t tmp_res_low, tmp_res_high; - uint16x8_t res; - int16x4_t res_0246_even, res_0246_odd, res_1357_even, res_1357_odd; - - uint8x16_t tmp_0 = vandq_u8(src_1, mask); - uint8x16_t tmp_1 = vandq_u8(src_2, mask); - uint8x16_t tmp_2 = vandq_u8(src_3, mask); - uint8x16_t tmp_3 = vandq_u8(src_4, mask); - - tmp_2 = vextq_u8(tmp_0, tmp_0, 1); - tmp_3 = vextq_u8(tmp_1, tmp_1, 1); - - src_1 = vaddq_u8(tmp_0, tmp_2); - src_2 = vaddq_u8(tmp_1, tmp_3); - - src_1_low = vget_low_u8(src_1); - src_2_low = vget_low_u8(src_2); - src_3_low = vget_low_u8(vextq_u8(src_1, src_1, 4)); - src_4_low = vget_low_u8(vextq_u8(src_2, src_2, 4)); - src_5_low = vget_low_u8(vextq_u8(src_1, src_1, 2)); - src_6_low = vget_low_u8(vextq_u8(src_1, src_1, 6)); - - // Loading the 8 filter taps - f0 = vmovl_s8( - vld1_s8(filter_8bit_neon[(sx + 0 * alpha) >> WARPEDDIFF_PREC_BITS])); - f1 = vmovl_s8( - vld1_s8(filter_8bit_neon[(sx + 1 * alpha) >> WARPEDDIFF_PREC_BITS])); - f2 = vmovl_s8( - vld1_s8(filter_8bit_neon[(sx + 2 * alpha) >> WARPEDDIFF_PREC_BITS])); - f3 = vmovl_s8( - vld1_s8(filter_8bit_neon[(sx + 3 * alpha) >> WARPEDDIFF_PREC_BITS])); - f4 = vmovl_s8( - vld1_s8(filter_8bit_neon[(sx + 4 * alpha) >> WARPEDDIFF_PREC_BITS])); - f5 = vmovl_s8( - vld1_s8(filter_8bit_neon[(sx + 5 * alpha) >> WARPEDDIFF_PREC_BITS])); - f6 = vmovl_s8( - vld1_s8(filter_8bit_neon[(sx + 6 * alpha) >> WARPEDDIFF_PREC_BITS])); - f7 = vmovl_s8( - vld1_s8(filter_8bit_neon[(sx + 7 * alpha) >> WARPEDDIFF_PREC_BITS])); - - b0 = vtrn_s32(vreinterpret_s32_s16(vget_low_s16(f0)), - vreinterpret_s32_s16(vget_low_s16(f2))); - b1 = vtrn_s32(vreinterpret_s32_s16(vget_low_s16(f4)), - vreinterpret_s32_s16(vget_low_s16(f6))); - convolve(b0, b1, src_1_low, src_3_low, &res_0246_even); - - b0 = vtrn_s32(vreinterpret_s32_s16(vget_low_s16(f1)), - vreinterpret_s32_s16(vget_low_s16(f3))); - b1 = vtrn_s32(vreinterpret_s32_s16(vget_low_s16(f5)), - vreinterpret_s32_s16(vget_low_s16(f7))); - convolve(b0, b1, src_2_low, src_4_low, &res_0246_odd); - - b0 = vtrn_s32(vreinterpret_s32_s16(vget_high_s16(f0)), - vreinterpret_s32_s16(vget_high_s16(f2))); - b1 = vtrn_s32(vreinterpret_s32_s16(vget_high_s16(f4)), - vreinterpret_s32_s16(vget_high_s16(f6))); - convolve(b0, b1, src_2_low, src_4_low, &res_1357_even); - - b0 = vtrn_s32(vreinterpret_s32_s16(vget_high_s16(f1)), - vreinterpret_s32_s16(vget_high_s16(f3))); - b1 = vtrn_s32(vreinterpret_s32_s16(vget_high_s16(f5)), - vreinterpret_s32_s16(vget_high_s16(f7))); - convolve(b0, b1, src_5_low, src_6_low, &res_1357_odd); - - tmp_res_low = vaddl_s16(res_0246_even, res_1357_even); - tmp_res_high = vaddl_s16(res_0246_odd, res_1357_odd); - - tmp_res_low = vaddq_s32(tmp_res_low, add_const); - tmp_res_high = vaddq_s32(tmp_res_high, add_const); - - res = vcombine_u16(vqmovun_s32(tmp_res_low), vqmovun_s32(tmp_res_high)); - res = vqrshlq_u16(res, shift); - - tmp_dst[k + 7] = vreinterpretq_s16_u16(res); -} - -static INLINE void vertical_filter_neon(const int16x8_t *src, - int32x4_t *res_low, int32x4_t *res_high, - int sy, int gamma) { - int16x4_t src_0, src_1, fltr_0, fltr_1; - int32x4_t res_0, res_1; - int32x2_t res_0_im, res_1_im; - int32x4_t res_even, res_odd, im_res_0, im_res_1; - - int16x8_t f0, f1, f2, f3, f4, f5, f6, f7; - int16x8x2_t b0, b1, b2, b3; - int32x4x2_t c0, c1, c2, c3; - int32x4x2_t d0, d1, d2, d3; - - b0 = vtrnq_s16(src[0], src[1]); - b1 = vtrnq_s16(src[2], src[3]); - b2 = vtrnq_s16(src[4], src[5]); - b3 = vtrnq_s16(src[6], src[7]); - - c0 = vtrnq_s32(vreinterpretq_s32_s16(b0.val[0]), - vreinterpretq_s32_s16(b0.val[1])); - c1 = vtrnq_s32(vreinterpretq_s32_s16(b1.val[0]), - vreinterpretq_s32_s16(b1.val[1])); - c2 = vtrnq_s32(vreinterpretq_s32_s16(b2.val[0]), - vreinterpretq_s32_s16(b2.val[1])); - c3 = vtrnq_s32(vreinterpretq_s32_s16(b3.val[0]), - vreinterpretq_s32_s16(b3.val[1])); - - f0 = vld1q_s16( - (int16_t *)(warped_filter + ((sy + 0 * gamma) >> WARPEDDIFF_PREC_BITS))); - f1 = vld1q_s16( - (int16_t *)(warped_filter + ((sy + 1 * gamma) >> WARPEDDIFF_PREC_BITS))); - f2 = vld1q_s16( - (int16_t *)(warped_filter + ((sy + 2 * gamma) >> WARPEDDIFF_PREC_BITS))); - f3 = vld1q_s16( - (int16_t *)(warped_filter + ((sy + 3 * gamma) >> WARPEDDIFF_PREC_BITS))); - f4 = vld1q_s16( - (int16_t *)(warped_filter + ((sy + 4 * gamma) >> WARPEDDIFF_PREC_BITS))); - f5 = vld1q_s16( - (int16_t *)(warped_filter + ((sy + 5 * gamma) >> WARPEDDIFF_PREC_BITS))); - f6 = vld1q_s16( - (int16_t *)(warped_filter + ((sy + 6 * gamma) >> WARPEDDIFF_PREC_BITS))); - f7 = vld1q_s16( - (int16_t *)(warped_filter + ((sy + 7 * gamma) >> WARPEDDIFF_PREC_BITS))); - - d0 = vtrnq_s32(vreinterpretq_s32_s16(f0), vreinterpretq_s32_s16(f2)); - d1 = vtrnq_s32(vreinterpretq_s32_s16(f4), vreinterpretq_s32_s16(f6)); - d2 = vtrnq_s32(vreinterpretq_s32_s16(f1), vreinterpretq_s32_s16(f3)); - d3 = vtrnq_s32(vreinterpretq_s32_s16(f5), vreinterpretq_s32_s16(f7)); - - // row:0,1 even_col:0,2 - src_0 = vget_low_s16(vreinterpretq_s16_s32(c0.val[0])); - fltr_0 = vget_low_s16(vreinterpretq_s16_s32(d0.val[0])); - res_0 = vmull_s16(src_0, fltr_0); - - // row:0,1,2,3 even_col:0,2 - src_0 = vget_low_s16(vreinterpretq_s16_s32(c1.val[0])); - fltr_0 = vget_low_s16(vreinterpretq_s16_s32(d0.val[1])); - res_0 = vmlal_s16(res_0, src_0, fltr_0); - res_0_im = vpadd_s32(vget_low_s32(res_0), vget_high_s32(res_0)); - - // row:0,1 even_col:4,6 - src_1 = vget_low_s16(vreinterpretq_s16_s32(c0.val[1])); - fltr_1 = vget_low_s16(vreinterpretq_s16_s32(d1.val[0])); - res_1 = vmull_s16(src_1, fltr_1); - - // row:0,1,2,3 even_col:4,6 - src_1 = vget_low_s16(vreinterpretq_s16_s32(c1.val[1])); - fltr_1 = vget_low_s16(vreinterpretq_s16_s32(d1.val[1])); - res_1 = vmlal_s16(res_1, src_1, fltr_1); - res_1_im = vpadd_s32(vget_low_s32(res_1), vget_high_s32(res_1)); - - // row:0,1,2,3 even_col:0,2,4,6 - im_res_0 = vcombine_s32(res_0_im, res_1_im); - - // row:4,5 even_col:0,2 - src_0 = vget_low_s16(vreinterpretq_s16_s32(c2.val[0])); - fltr_0 = vget_high_s16(vreinterpretq_s16_s32(d0.val[0])); - res_0 = vmull_s16(src_0, fltr_0); - - // row:4,5,6,7 even_col:0,2 - src_0 = vget_low_s16(vreinterpretq_s16_s32(c3.val[0])); - fltr_0 = vget_high_s16(vreinterpretq_s16_s32(d0.val[1])); - res_0 = vmlal_s16(res_0, src_0, fltr_0); - res_0_im = vpadd_s32(vget_low_s32(res_0), vget_high_s32(res_0)); - - // row:4,5 even_col:4,6 - src_1 = vget_low_s16(vreinterpretq_s16_s32(c2.val[1])); - fltr_1 = vget_high_s16(vreinterpretq_s16_s32(d1.val[0])); - res_1 = vmull_s16(src_1, fltr_1); - - // row:4,5,6,7 even_col:4,6 - src_1 = vget_low_s16(vreinterpretq_s16_s32(c3.val[1])); - fltr_1 = vget_high_s16(vreinterpretq_s16_s32(d1.val[1])); - res_1 = vmlal_s16(res_1, src_1, fltr_1); - res_1_im = vpadd_s32(vget_low_s32(res_1), vget_high_s32(res_1)); - - // row:4,5,6,7 even_col:0,2,4,6 - im_res_1 = vcombine_s32(res_0_im, res_1_im); - - // row:0-7 even_col:0,2,4,6 - res_even = vaddq_s32(im_res_0, im_res_1); - - // row:0,1 odd_col:1,3 - src_0 = vget_high_s16(vreinterpretq_s16_s32(c0.val[0])); - fltr_0 = vget_low_s16(vreinterpretq_s16_s32(d2.val[0])); - res_0 = vmull_s16(src_0, fltr_0); - - // row:0,1,2,3 odd_col:1,3 - src_0 = vget_high_s16(vreinterpretq_s16_s32(c1.val[0])); - fltr_0 = vget_low_s16(vreinterpretq_s16_s32(d2.val[1])); - res_0 = vmlal_s16(res_0, src_0, fltr_0); - res_0_im = vpadd_s32(vget_low_s32(res_0), vget_high_s32(res_0)); - - // row:0,1 odd_col:5,7 - src_1 = vget_high_s16(vreinterpretq_s16_s32(c0.val[1])); - fltr_1 = vget_low_s16(vreinterpretq_s16_s32(d3.val[0])); - res_1 = vmull_s16(src_1, fltr_1); - - // row:0,1,2,3 odd_col:5,7 - src_1 = vget_high_s16(vreinterpretq_s16_s32(c1.val[1])); - fltr_1 = vget_low_s16(vreinterpretq_s16_s32(d3.val[1])); - res_1 = vmlal_s16(res_1, src_1, fltr_1); - res_1_im = vpadd_s32(vget_low_s32(res_1), vget_high_s32(res_1)); - - // row:0,1,2,3 odd_col:1,3,5,7 - im_res_0 = vcombine_s32(res_0_im, res_1_im); - - // row:4,5 odd_col:1,3 - src_0 = vget_high_s16(vreinterpretq_s16_s32(c2.val[0])); - fltr_0 = vget_high_s16(vreinterpretq_s16_s32(d2.val[0])); - res_0 = vmull_s16(src_0, fltr_0); - - // row:4,5,6,7 odd_col:1,3 - src_0 = vget_high_s16(vreinterpretq_s16_s32(c3.val[0])); - fltr_0 = vget_high_s16(vreinterpretq_s16_s32(d2.val[1])); - res_0 = vmlal_s16(res_0, src_0, fltr_0); - res_0_im = vpadd_s32(vget_low_s32(res_0), vget_high_s32(res_0)); - - // row:4,5 odd_col:5,7 - src_1 = vget_high_s16(vreinterpretq_s16_s32(c2.val[1])); - fltr_1 = vget_high_s16(vreinterpretq_s16_s32(d3.val[0])); - res_1 = vmull_s16(src_1, fltr_1); - - // row:4,5,6,7 odd_col:5,7 - src_1 = vget_high_s16(vreinterpretq_s16_s32(c3.val[1])); - fltr_1 = vget_high_s16(vreinterpretq_s16_s32(d3.val[1])); - res_1 = vmlal_s16(res_1, src_1, fltr_1); - res_1_im = vpadd_s32(vget_low_s32(res_1), vget_high_s32(res_1)); - - // row:4,5,6,7 odd_col:1,3,5,7 - im_res_1 = vcombine_s32(res_0_im, res_1_im); - - // row:0-7 odd_col:1,3,5,7 - res_odd = vaddq_s32(im_res_0, im_res_1); - - // reordering as 0 1 2 3 | 4 5 6 7 - c0 = vtrnq_s32(res_even, res_odd); - - // Final store - *res_low = vcombine_s32(vget_low_s32(c0.val[0]), vget_low_s32(c0.val[1])); - *res_high = vcombine_s32(vget_high_s32(c0.val[0]), vget_high_s32(c0.val[1])); -} - -void av1_warp_affine_neon(const int32_t *mat, const uint8_t *ref, int width, - int height, int stride, uint8_t *pred, int p_col, - int p_row, int p_width, int p_height, int p_stride, - int subsampling_x, int subsampling_y, - ConvolveParams *conv_params, int16_t alpha, - int16_t beta, int16_t gamma, int16_t delta) { - int16x8_t tmp[15]; - const int bd = 8; - const int w0 = conv_params->fwd_offset; - const int w1 = conv_params->bck_offset; - const int32x4_t fwd = vdupq_n_s32((int32_t)w0); - const int32x4_t bwd = vdupq_n_s32((int32_t)w1); - const int16x8_t sub_constant = vdupq_n_s16((1 << (bd - 1)) + (1 << bd)); - - int limit = 0; - uint8x16_t vec_dup, mask_val; - int32x4_t res_lo, res_hi; - int16x8_t result_final; - uint8x16_t src_1, src_2, src_3, src_4; - uint8x16_t indx_vec = { - 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 - }; - uint8x16_t cmp_vec; - - const int reduce_bits_horiz = conv_params->round_0; - const int reduce_bits_vert = conv_params->is_compound - ? conv_params->round_1 - : 2 * FILTER_BITS - reduce_bits_horiz; - const int32x4_t shift_vert = vdupq_n_s32(-(int32_t)reduce_bits_vert); - const int offset_bits_horiz = bd + FILTER_BITS - 1; - - assert(IMPLIES(conv_params->is_compound, conv_params->dst != NULL)); - - const int offset_bits_vert = bd + 2 * FILTER_BITS - reduce_bits_horiz; - int32x4_t add_const_vert = vdupq_n_s32((int32_t)(1 << offset_bits_vert)); - const int round_bits = - 2 * FILTER_BITS - conv_params->round_0 - conv_params->round_1; - const int16x4_t round_bits_vec = vdup_n_s16(-(int16_t)round_bits); - const int offset_bits = bd + 2 * FILTER_BITS - conv_params->round_0; - const int16x4_t res_sub_const = - vdup_n_s16(-((1 << (offset_bits - conv_params->round_1)) + - (1 << (offset_bits - conv_params->round_1 - 1)))); - int k; - - assert(IMPLIES(conv_params->do_average, conv_params->is_compound)); - - for (int i = 0; i < p_height; i += 8) { - for (int j = 0; j < p_width; j += 8) { - const int32_t src_x = (p_col + j + 4) << subsampling_x; - const int32_t src_y = (p_row + i + 4) << subsampling_y; - const int32_t dst_x = mat[2] * src_x + mat[3] * src_y + mat[0]; - const int32_t dst_y = mat[4] * src_x + mat[5] * src_y + mat[1]; - const int32_t x4 = dst_x >> subsampling_x; - const int32_t y4 = dst_y >> subsampling_y; - - int32_t ix4 = x4 >> WARPEDMODEL_PREC_BITS; - int32_t sx4 = x4 & ((1 << WARPEDMODEL_PREC_BITS) - 1); - int32_t iy4 = y4 >> WARPEDMODEL_PREC_BITS; - int32_t sy4 = y4 & ((1 << WARPEDMODEL_PREC_BITS) - 1); - - sx4 += alpha * (-4) + beta * (-4) + (1 << (WARPEDDIFF_PREC_BITS - 1)) + - (WARPEDPIXEL_PREC_SHIFTS << WARPEDDIFF_PREC_BITS); - sy4 += gamma * (-4) + delta * (-4) + (1 << (WARPEDDIFF_PREC_BITS - 1)) + - (WARPEDPIXEL_PREC_SHIFTS << WARPEDDIFF_PREC_BITS); - - sx4 &= ~((1 << WARP_PARAM_REDUCE_BITS) - 1); - sy4 &= ~((1 << WARP_PARAM_REDUCE_BITS) - 1); - // horizontal - if (ix4 <= -7) { - for (k = -7; k < AOMMIN(8, p_height - i); ++k) { - int iy = iy4 + k; - if (iy < 0) - iy = 0; - else if (iy > height - 1) - iy = height - 1; - int16_t dup_val = - (1 << (bd + FILTER_BITS - reduce_bits_horiz - 1)) + - ref[iy * stride] * (1 << (FILTER_BITS - reduce_bits_horiz)); - - tmp[k + 7] = vdupq_n_s16(dup_val); - } - } else if (ix4 >= width + 6) { - for (k = -7; k < AOMMIN(8, p_height - i); ++k) { - int iy = iy4 + k; - if (iy < 0) - iy = 0; - else if (iy > height - 1) - iy = height - 1; - int16_t dup_val = (1 << (bd + FILTER_BITS - reduce_bits_horiz - 1)) + - ref[iy * stride + (width - 1)] * - (1 << (FILTER_BITS - reduce_bits_horiz)); - tmp[k + 7] = vdupq_n_s16(dup_val); - } - } else if (((ix4 - 7) < 0) || ((ix4 + 9) > width)) { - const int out_of_boundary_left = -(ix4 - 6); - const int out_of_boundary_right = (ix4 + 8) - width; - - for (k = -7; k < AOMMIN(8, p_height - i); ++k) { - int iy = iy4 + k; - if (iy < 0) - iy = 0; - else if (iy > height - 1) - iy = height - 1; - int sx = sx4 + beta * (k + 4); - - const uint8_t *src = ref + iy * stride + ix4 - 7; - src_1 = vld1q_u8(src); - - if (out_of_boundary_left >= 0) { - limit = out_of_boundary_left + 1; - cmp_vec = vdupq_n_u8(out_of_boundary_left); - vec_dup = vdupq_n_u8(*(src + limit)); - mask_val = vcleq_u8(indx_vec, cmp_vec); - src_1 = vbslq_u8(mask_val, vec_dup, src_1); - } - if (out_of_boundary_right >= 0) { - limit = 15 - (out_of_boundary_right + 1); - cmp_vec = vdupq_n_u8(15 - out_of_boundary_right); - vec_dup = vdupq_n_u8(*(src + limit)); - mask_val = vcgeq_u8(indx_vec, cmp_vec); - src_1 = vbslq_u8(mask_val, vec_dup, src_1); - } - src_2 = vextq_u8(src_1, src_1, 1); - src_3 = vextq_u8(src_2, src_2, 1); - src_4 = vextq_u8(src_3, src_3, 1); - - horizontal_filter_neon(src_1, src_2, src_3, src_4, tmp, sx, alpha, k, - offset_bits_horiz, reduce_bits_horiz); - } - } else { - for (k = -7; k < AOMMIN(8, p_height - i); ++k) { - int iy = iy4 + k; - if (iy < 0) - iy = 0; - else if (iy > height - 1) - iy = height - 1; - int sx = sx4 + beta * (k + 4); - - const uint8_t *src = ref + iy * stride + ix4 - 7; - src_1 = vld1q_u8(src); - src_2 = vextq_u8(src_1, src_1, 1); - src_3 = vextq_u8(src_2, src_2, 1); - src_4 = vextq_u8(src_3, src_3, 1); - - horizontal_filter_neon(src_1, src_2, src_3, src_4, tmp, sx, alpha, k, - offset_bits_horiz, reduce_bits_horiz); - } - } - - // vertical - for (k = -4; k < AOMMIN(4, p_height - i - 4); ++k) { - int sy = sy4 + delta * (k + 4); - - const int16x8_t *v_src = tmp + (k + 4); - - vertical_filter_neon(v_src, &res_lo, &res_hi, sy, gamma); - - res_lo = vaddq_s32(res_lo, add_const_vert); - res_hi = vaddq_s32(res_hi, add_const_vert); - - if (conv_params->is_compound) { - uint16_t *const p = - (uint16_t *)&conv_params - ->dst[(i + k + 4) * conv_params->dst_stride + j]; - - res_lo = vrshlq_s32(res_lo, shift_vert); - if (conv_params->do_average) { - uint8_t *const dst8 = &pred[(i + k + 4) * p_stride + j]; - uint16x4_t tmp16_lo = vld1_u16(p); - int32x4_t tmp32_lo = vreinterpretq_s32_u32(vmovl_u16(tmp16_lo)); - int16x4_t tmp16_low; - if (conv_params->use_jnt_comp_avg) { - res_lo = vmulq_s32(res_lo, bwd); - tmp32_lo = vmulq_s32(tmp32_lo, fwd); - tmp32_lo = vaddq_s32(tmp32_lo, res_lo); - tmp16_low = vshrn_n_s32(tmp32_lo, DIST_PRECISION_BITS); - } else { - tmp32_lo = vaddq_s32(tmp32_lo, res_lo); - tmp16_low = vshrn_n_s32(tmp32_lo, 1); - } - int16x4_t res_low = vadd_s16(tmp16_low, res_sub_const); - res_low = vqrshl_s16(res_low, round_bits_vec); - int16x8_t final_res_low = vcombine_s16(res_low, res_low); - uint8x8_t res_8_low = vqmovun_s16(final_res_low); - - vst1_lane_u32((uint32_t *)dst8, vreinterpret_u32_u8(res_8_low), 0); - } else { - uint16x4_t res_u16_low = vqmovun_s32(res_lo); - vst1_u16(p, res_u16_low); - } - if (p_width > 4) { - uint16_t *const p4 = - (uint16_t *)&conv_params - ->dst[(i + k + 4) * conv_params->dst_stride + j + 4]; - - res_hi = vrshlq_s32(res_hi, shift_vert); - if (conv_params->do_average) { - uint8_t *const dst8_4 = &pred[(i + k + 4) * p_stride + j + 4]; - - uint16x4_t tmp16_hi = vld1_u16(p4); - int32x4_t tmp32_hi = vreinterpretq_s32_u32(vmovl_u16(tmp16_hi)); - int16x4_t tmp16_high; - if (conv_params->use_jnt_comp_avg) { - res_hi = vmulq_s32(res_hi, bwd); - tmp32_hi = vmulq_s32(tmp32_hi, fwd); - tmp32_hi = vaddq_s32(tmp32_hi, res_hi); - tmp16_high = vshrn_n_s32(tmp32_hi, DIST_PRECISION_BITS); - } else { - tmp32_hi = vaddq_s32(tmp32_hi, res_hi); - tmp16_high = vshrn_n_s32(tmp32_hi, 1); - } - int16x4_t res_high = vadd_s16(tmp16_high, res_sub_const); - res_high = vqrshl_s16(res_high, round_bits_vec); - int16x8_t final_res_high = vcombine_s16(res_high, res_high); - uint8x8_t res_8_high = vqmovun_s16(final_res_high); - - vst1_lane_u32((uint32_t *)dst8_4, vreinterpret_u32_u8(res_8_high), - 0); - } else { - uint16x4_t res_u16_high = vqmovun_s32(res_hi); - vst1_u16(p4, res_u16_high); - } - } - } else { - res_lo = vrshlq_s32(res_lo, shift_vert); - res_hi = vrshlq_s32(res_hi, shift_vert); - - result_final = vcombine_s16(vmovn_s32(res_lo), vmovn_s32(res_hi)); - result_final = vsubq_s16(result_final, sub_constant); - - uint8_t *const p = (uint8_t *)&pred[(i + k + 4) * p_stride + j]; - uint8x8_t val = vqmovun_s16(result_final); - - if (p_width == 4) { - vst1_lane_u32((uint32_t *)p, vreinterpret_u32_u8(val), 0); - } else { - vst1_u8(p, val); - } - } - } - } - } -} diff --git a/third_party/aom/av1/common/arm/wiener_convolve_neon.c b/third_party/aom/av1/common/arm/wiener_convolve_neon.c deleted file mode 100644 index a9bb5bcf0..000000000 --- a/third_party/aom/av1/common/arm/wiener_convolve_neon.c +++ /dev/null @@ -1,530 +0,0 @@ -/* - * Copyright (c) 2018, Alliance for Open Media. All rights reserved - * - * This source code is subject to the terms of the BSD 2 Clause License and - * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License - * was not distributed with this source code in the LICENSE file, you can - * obtain it at www.aomedia.org/license/software. If the Alliance for Open - * Media Patent License 1.0 was not distributed with this source code in the - * PATENTS file, you can obtain it at www.aomedia.org/license/patent. - */ - -#include <arm_neon.h> -#include <assert.h> - -#include "config/aom_config.h" -#include "config/av1_rtcd.h" - -#include "aom_dsp/txfm_common.h" -#include "aom_ports/mem.h" -#include "av1/common/common.h" -#include "av1/common/arm/convolve_neon.h" -#include "av1/common/arm/mem_neon.h" -#include "av1/common/arm/transpose_neon.h" - -/* Wiener filter 2D - Apply horizontal filter and store in a temporary buffer. When applying - vertical filter, overwrite the original pixel values. - */ -void av1_wiener_convolve_add_src_neon(const uint8_t *src, ptrdiff_t src_stride, - uint8_t *dst, ptrdiff_t dst_stride, - const int16_t *filter_x, int x_step_q4, - const int16_t *filter_y, int y_step_q4, - int w, int h, - const ConvolveParams *conv_params) { - uint16_t *d_tmp; - uint8_t *d; - const uint8_t *src_ptr, *s_tmp; - uint16_t *dst_ptr; - (void)x_step_q4; - (void)y_step_q4; - - int width, height; - const int bd = 8; - const int intermediate_height = h + SUBPEL_TAPS - 1; - const int center_tap = ((SUBPEL_TAPS - 1) / 2); - int16_t filter_x_tmp[7], filter_y_tmp[7]; - - DECLARE_ALIGNED(16, uint16_t, - temp[(MAX_SB_SIZE + HORIZ_EXTRA_ROWS) * MAX_SB_SIZE]); - - assert(x_step_q4 == 16 && y_step_q4 == 16); - assert(!(w % 8)); - - assert(w <= MAX_SB_SIZE); - assert(h <= MAX_SB_SIZE); - - assert(filter_x[7] == 0); - assert(filter_y[7] == 0); - - /* assumption of horizontal filtering output will not exceed 15 bit. - ((bd) + 1 + FILTER_BITS - conv_params->round_0) <= 15 - 16 - conv_params->round_0 <= 15 -- (conv_params->round_0) >= 1 - */ - assert((conv_params->round_0) >= 1); - - memcpy(&filter_x_tmp[0], filter_x, sizeof(*filter_x) * FILTER_BITS); - memcpy(&filter_y_tmp[0], filter_y, sizeof(*filter_y) * FILTER_BITS); - - filter_x_tmp[3] += (1 << FILTER_BITS); - filter_y_tmp[3] += (1 << FILTER_BITS); - - s_tmp = src - center_tap * src_stride - center_tap; - dst_ptr = temp; - src_ptr = s_tmp; - height = intermediate_height; - - /* if height is a multiple of 8 */ - if (!(h & 7)) { - int16x8_t res0, res1, res2, res3; - uint16x8_t res4; - uint8x8_t t0, t1, t2, t3, t4, t5, t6, t7; -#if defined(__aarch64__) - uint16x8_t res5, res6, res7, res8, res9, res10, res11; - uint8x8_t t8, t9, t10, t11, t12, t13, t14; - - do { - const uint8_t *s; - - __builtin_prefetch(src_ptr + 0 * src_stride); - __builtin_prefetch(src_ptr + 1 * src_stride); - __builtin_prefetch(src_ptr + 2 * src_stride); - __builtin_prefetch(src_ptr + 3 * src_stride); - __builtin_prefetch(src_ptr + 4 * src_stride); - __builtin_prefetch(src_ptr + 5 * src_stride); - __builtin_prefetch(src_ptr + 6 * src_stride); - __builtin_prefetch(src_ptr + 7 * src_stride); - - load_u8_8x8(src_ptr, src_stride, &t0, &t1, &t2, &t3, &t4, &t5, &t6, &t7); - transpose_u8_8x8(&t0, &t1, &t2, &t3, &t4, &t5, &t6, &t7); - - s = src_ptr + 7; - d_tmp = dst_ptr; - width = w; - - __builtin_prefetch(dst_ptr + 0 * dst_stride); - __builtin_prefetch(dst_ptr + 1 * dst_stride); - __builtin_prefetch(dst_ptr + 2 * dst_stride); - __builtin_prefetch(dst_ptr + 3 * dst_stride); - __builtin_prefetch(dst_ptr + 4 * dst_stride); - __builtin_prefetch(dst_ptr + 5 * dst_stride); - __builtin_prefetch(dst_ptr + 6 * dst_stride); - __builtin_prefetch(dst_ptr + 7 * dst_stride); - - do { - load_u8_8x8(s, src_stride, &t7, &t8, &t9, &t10, &t11, &t12, &t13, &t14); - transpose_u8_8x8(&t7, &t8, &t9, &t10, &t11, &t12, &t13, &t14); - - res0 = vreinterpretq_s16_u16(vaddl_u8(t0, t6)); - res1 = vreinterpretq_s16_u16(vaddl_u8(t1, t5)); - res2 = vreinterpretq_s16_u16(vaddl_u8(t2, t4)); - res3 = vreinterpretq_s16_u16(vmovl_u8(t3)); - res4 = wiener_convolve8_horiz_8x8(res0, res1, res2, res3, filter_x_tmp, - bd, conv_params->round_0); - - res0 = vreinterpretq_s16_u16(vaddl_u8(t1, t7)); - res1 = vreinterpretq_s16_u16(vaddl_u8(t2, t6)); - res2 = vreinterpretq_s16_u16(vaddl_u8(t3, t5)); - res3 = vreinterpretq_s16_u16(vmovl_u8(t4)); - res5 = wiener_convolve8_horiz_8x8(res0, res1, res2, res3, filter_x_tmp, - bd, conv_params->round_0); - - res0 = vreinterpretq_s16_u16(vaddl_u8(t2, t8)); - res1 = vreinterpretq_s16_u16(vaddl_u8(t3, t7)); - res2 = vreinterpretq_s16_u16(vaddl_u8(t4, t6)); - res3 = vreinterpretq_s16_u16(vmovl_u8(t5)); - res6 = wiener_convolve8_horiz_8x8(res0, res1, res2, res3, filter_x_tmp, - bd, conv_params->round_0); - - res0 = vreinterpretq_s16_u16(vaddl_u8(t3, t9)); - res1 = vreinterpretq_s16_u16(vaddl_u8(t4, t8)); - res2 = vreinterpretq_s16_u16(vaddl_u8(t5, t7)); - res3 = vreinterpretq_s16_u16(vmovl_u8(t6)); - res7 = wiener_convolve8_horiz_8x8(res0, res1, res2, res3, filter_x_tmp, - bd, conv_params->round_0); - - res0 = vreinterpretq_s16_u16(vaddl_u8(t4, t10)); - res1 = vreinterpretq_s16_u16(vaddl_u8(t5, t9)); - res2 = vreinterpretq_s16_u16(vaddl_u8(t6, t8)); - res3 = vreinterpretq_s16_u16(vmovl_u8(t7)); - res8 = wiener_convolve8_horiz_8x8(res0, res1, res2, res3, filter_x_tmp, - bd, conv_params->round_0); - - res0 = vreinterpretq_s16_u16(vaddl_u8(t5, t11)); - res1 = vreinterpretq_s16_u16(vaddl_u8(t6, t10)); - res2 = vreinterpretq_s16_u16(vaddl_u8(t7, t9)); - res3 = vreinterpretq_s16_u16(vmovl_u8(t8)); - res9 = wiener_convolve8_horiz_8x8(res0, res1, res2, res3, filter_x_tmp, - bd, conv_params->round_0); - - res0 = vreinterpretq_s16_u16(vaddl_u8(t6, t12)); - res1 = vreinterpretq_s16_u16(vaddl_u8(t7, t11)); - res2 = vreinterpretq_s16_u16(vaddl_u8(t8, t10)); - res3 = vreinterpretq_s16_u16(vmovl_u8(t9)); - res10 = wiener_convolve8_horiz_8x8(res0, res1, res2, res3, filter_x_tmp, - bd, conv_params->round_0); - - res0 = vreinterpretq_s16_u16(vaddl_u8(t7, t13)); - res1 = vreinterpretq_s16_u16(vaddl_u8(t8, t12)); - res2 = vreinterpretq_s16_u16(vaddl_u8(t9, t11)); - res3 = vreinterpretq_s16_u16(vmovl_u8(t10)); - res11 = wiener_convolve8_horiz_8x8(res0, res1, res2, res3, filter_x_tmp, - bd, conv_params->round_0); - - transpose_u16_8x8(&res4, &res5, &res6, &res7, &res8, &res9, &res10, - &res11); - store_u16_8x8(d_tmp, MAX_SB_SIZE, res4, res5, res6, res7, res8, res9, - res10, res11); - - t0 = t8; - t1 = t9; - t2 = t10; - t3 = t11; - t4 = t12; - t5 = t13; - t6 = t14; - s += 8; - d_tmp += 8; - width -= 8; - } while (width > 0); - src_ptr += 8 * src_stride; - dst_ptr += 8 * MAX_SB_SIZE; - height -= 8; - } while (height > 0); -#else - uint8x8_t temp_0; - - do { - const uint8_t *s; - - __builtin_prefetch(src_ptr); - - t0 = vld1_u8(src_ptr); // a0 a1 a2 a3 a4 a5 a6 a7 - s = src_ptr + 8; - d_tmp = dst_ptr; - width = w; - - __builtin_prefetch(dst_ptr); - - do { - t7 = vld1_u8(s); // a8 a9 a10 a11 a12 a13 a14 a15 - temp_0 = t0; - t0 = t7; - - t1 = vext_u8(temp_0, t7, 1); // a1 a2 a3 a4 a5 a6 a7 a8 - t2 = vext_u8(temp_0, t7, 2); // a2 a3 a4 a5 a6 a7 a8 a9 - t3 = vext_u8(temp_0, t7, 3); // a3 a4 a5 a6 a7 a8 a9 a10 - t4 = vext_u8(temp_0, t7, 4); // a4 a5 a6 a7 a8 a9 a10 a11 - t5 = vext_u8(temp_0, t7, 5); // a5 a6 a7 a8 a9 a10 a11 a12 - t6 = vext_u8(temp_0, t7, 6); // a6 a7 a8 a9 a10 a11 a12 a13 - t7 = vext_u8(temp_0, t7, 7); // a7 a8 a9 a10 a11 a12 a13 a14 - - res0 = vreinterpretq_s16_u16(vaddl_u8(temp_0, t6)); - res1 = vreinterpretq_s16_u16(vaddl_u8(t1, t5)); - res2 = vreinterpretq_s16_u16(vaddl_u8(t2, t4)); - res3 = vreinterpretq_s16_u16(vmovl_u8(t3)); - res4 = wiener_convolve8_horiz_8x8(res0, res1, res2, res3, filter_x_tmp, - bd, conv_params->round_0); - - vst1q_u16(d_tmp, res4); - - s += 8; - d_tmp += 8; - width -= 8; - } while (width > 0); - src_ptr += src_stride; - dst_ptr += MAX_SB_SIZE; - height--; - } while (height > 0); -#endif - } else { - /*if height is a multiple of 4*/ - const uint8_t *s; - int16x8_t tt0, tt1, tt2, tt3; - uint16x8_t d0; - uint8x8_t t0, t1, t2, t3; - -#if defined(__aarch64__) - uint16x4_t res0, res1, res2, res3, res4, res5, res6, res7; - uint16x8_t d1, d2, d3; - int16x4_t s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10; - int16x4_t s11, s12, s13, s14; - do { - __builtin_prefetch(src_ptr + 0 * src_stride); - __builtin_prefetch(src_ptr + 1 * src_stride); - __builtin_prefetch(src_ptr + 2 * src_stride); - __builtin_prefetch(src_ptr + 3 * src_stride); - - load_u8_8x4(src_ptr, src_stride, &t0, &t1, &t2, &t3); /*8x4*/ - transpose_u8_8x4(&t0, &t1, &t2, - &t3); /*first 8 pixels of 4 rows transposed-- 4x8*/ - - tt0 = vreinterpretq_s16_u16(vmovl_u8(t0)); - tt1 = vreinterpretq_s16_u16(vmovl_u8(t1)); - tt2 = vreinterpretq_s16_u16(vmovl_u8(t2)); - tt3 = vreinterpretq_s16_u16(vmovl_u8(t3)); - - s0 = vget_low_s16(tt0); /*pa0 pb0 pc0 pd0 -- pixel_a0*/ - s1 = vget_low_s16(tt1); /*pa1 pb1 pc1 pd1 */ - s2 = vget_low_s16(tt2); /*pa2 pb2 pc2 pd2 */ - s3 = vget_low_s16(tt3); /*pa3 pb3 pc3 pd3 */ - s4 = vget_high_s16(tt0); /*pa4 pb4 pc4 pd4 */ - s5 = vget_high_s16(tt1); /*pa5 pb5 pc5 pd5 */ - s6 = vget_high_s16(tt2); /*pa6 pb6 pc6 pd6 */ - - __builtin_prefetch(dst_ptr + 0 * dst_stride); - __builtin_prefetch(dst_ptr + 1 * dst_stride); - __builtin_prefetch(dst_ptr + 2 * dst_stride); - __builtin_prefetch(dst_ptr + 3 * dst_stride); - - s = src_ptr + 7; - d_tmp = dst_ptr; - width = w; - - do { - load_u8_8x4(s, src_stride, &t0, &t1, &t2, &t3); /*8x4*/ - transpose_u8_8x4(&t0, &t1, &t2, &t3); - - tt0 = vreinterpretq_s16_u16(vmovl_u8(t0)); - tt1 = vreinterpretq_s16_u16(vmovl_u8(t1)); - tt2 = vreinterpretq_s16_u16(vmovl_u8(t2)); - tt3 = vreinterpretq_s16_u16(vmovl_u8(t3)); - - s7 = vget_low_s16(tt0); /*pa7 pb7 pc7 pd7 */ /*4x8*/ - s8 = vget_low_s16(tt1); /*pa8 pb8 pc8 pd8 */ - s9 = vget_low_s16(tt2); /*pa9 pb9 pc9 pd9 */ - s10 = vget_low_s16(tt3); /*pa10 pb10 pc10 pd10 */ - s11 = vget_high_s16(tt0); /*pa11 pb11 pc11 pd11 */ - s12 = vget_high_s16(tt1); /*pa12 pb12 pc12 pd12 */ - s13 = vget_high_s16(tt2); /*pa13 pb13 pc13 pd13 */ - s14 = vget_high_s16(tt3); /*pa14 pb14 pc14 pd14 */ - - res0 = wiener_convolve8_horiz_4x8( - s0, s1, s2, s3, s4, s5, s6, filter_x_tmp, bd, conv_params->round_0); - res1 = wiener_convolve8_horiz_4x8( - s1, s2, s3, s4, s5, s6, s7, filter_x_tmp, bd, conv_params->round_0); - res2 = wiener_convolve8_horiz_4x8( - s2, s3, s4, s5, s6, s7, s8, filter_x_tmp, bd, conv_params->round_0); - res3 = wiener_convolve8_horiz_4x8( - s3, s4, s5, s6, s7, s8, s9, filter_x_tmp, bd, conv_params->round_0); - res4 = - wiener_convolve8_horiz_4x8(s4, s5, s6, s7, s8, s9, s10, - filter_x_tmp, bd, conv_params->round_0); - res5 = - wiener_convolve8_horiz_4x8(s5, s6, s7, s8, s9, s10, s11, - filter_x_tmp, bd, conv_params->round_0); - res6 = - wiener_convolve8_horiz_4x8(s6, s7, s8, s9, s10, s11, s12, - filter_x_tmp, bd, conv_params->round_0); - res7 = - wiener_convolve8_horiz_4x8(s7, s8, s9, s10, s11, s12, s13, - filter_x_tmp, bd, conv_params->round_0); - - transpose_u16_4x8(&res0, &res1, &res2, &res3, &res4, &res5, &res6, - &res7, &d0, &d1, &d2, &d3); - - store_u16_8x4(d_tmp, MAX_SB_SIZE, d0, d1, d2, d3); - - s0 = s8; - s1 = s9; - s2 = s10; - s3 = s11; - s4 = s12; - s5 = s13; - s6 = s14; - s += 8; - d_tmp += 8; - width -= 8; - } while (width > 0); - - src_ptr += 4 * src_stride; - dst_ptr += 4 * MAX_SB_SIZE; - height -= 4; - } while (height > 0); -#else - uint8x8_t temp_0, t4, t5, t6, t7; - - do { - __builtin_prefetch(src_ptr); - - t0 = vld1_u8(src_ptr); // a0 a1 a2 a3 a4 a5 a6 a7 - - __builtin_prefetch(dst_ptr); - - s = src_ptr + 8; - d_tmp = dst_ptr; - width = w; - - do { - t7 = vld1_u8(s); // a8 a9 a10 a11 a12 a13 a14 a15 - temp_0 = t0; - t0 = t7; - - t1 = vext_u8(temp_0, t7, 1); // a1 a2 a3 a4 a5 a6 a7 a8 - t2 = vext_u8(temp_0, t7, 2); // a2 a3 a4 a5 a6 a7 a8 a9 - t3 = vext_u8(temp_0, t7, 3); // a3 a4 a5 a6 a7 a8 a9 a10 - t4 = vext_u8(temp_0, t7, 4); // a4 a5 a6 a7 a8 a9 a10 a11 - t5 = vext_u8(temp_0, t7, 5); // a5 a6 a7 a8 a9 a10 a11 a12 - t6 = vext_u8(temp_0, t7, 6); // a6 a7 a8 a9 a10 a11 a12 a13 - t7 = vext_u8(temp_0, t7, 7); // a7 a8 a9 a10 a11 a12 a13 a14 - - tt0 = vreinterpretq_s16_u16(vaddl_u8(temp_0, t6)); - tt1 = vreinterpretq_s16_u16(vaddl_u8(t1, t5)); - tt2 = vreinterpretq_s16_u16(vaddl_u8(t2, t4)); - tt3 = vreinterpretq_s16_u16(vmovl_u8(t3)); - d0 = wiener_convolve8_horiz_8x8(tt0, tt1, tt2, tt3, filter_x_tmp, bd, - conv_params->round_0); - - vst1q_u16(d_tmp, d0); - - s += 8; - d_tmp += 8; - width -= 8; - } while (width > 0); - - src_ptr += src_stride; - dst_ptr += MAX_SB_SIZE; - height -= 1; - } while (height > 0); -#endif - } - - { - int16x8_t s0, s1, s2, s3, s4, s5, s6, s7; - uint8x8_t t0; -#if defined(__aarch64__) - int16x8_t s8, s9, s10; - uint8x8_t t1, t2, t3; -#endif - int16_t *src_tmp_ptr, *s; - uint8_t *dst_tmp_ptr; - height = h; - width = w; - src_tmp_ptr = (int16_t *)temp; - dst_tmp_ptr = dst; - src_stride = MAX_SB_SIZE; - - do { - s = src_tmp_ptr; - s0 = vld1q_s16(s); - s += src_stride; - s1 = vld1q_s16(s); - s += src_stride; - s2 = vld1q_s16(s); - s += src_stride; - s3 = vld1q_s16(s); - s += src_stride; - s4 = vld1q_s16(s); - s += src_stride; - s5 = vld1q_s16(s); - s += src_stride; - s6 = vld1q_s16(s); - s += src_stride; - d = dst_tmp_ptr; - height = h; - -#if defined(__aarch64__) - do { - __builtin_prefetch(dst_tmp_ptr + 0 * dst_stride); - __builtin_prefetch(dst_tmp_ptr + 1 * dst_stride); - __builtin_prefetch(dst_tmp_ptr + 2 * dst_stride); - __builtin_prefetch(dst_tmp_ptr + 3 * dst_stride); - - s7 = vld1q_s16(s); - s += src_stride; - s8 = vld1q_s16(s); - s += src_stride; - s9 = vld1q_s16(s); - s += src_stride; - s10 = vld1q_s16(s); - s += src_stride; - - t0 = wiener_convolve8_vert_4x8(s0, s1, s2, s3, s4, s5, s6, filter_y_tmp, - bd, conv_params->round_1); - t1 = wiener_convolve8_vert_4x8(s1, s2, s3, s4, s5, s6, s7, filter_y_tmp, - bd, conv_params->round_1); - t2 = wiener_convolve8_vert_4x8(s2, s3, s4, s5, s6, s7, s8, filter_y_tmp, - bd, conv_params->round_1); - t3 = wiener_convolve8_vert_4x8(s3, s4, s5, s6, s7, s8, s9, filter_y_tmp, - bd, conv_params->round_1); - - vst1_u8(d, t0); - d += dst_stride; - vst1_u8(d, t1); - d += dst_stride; - vst1_u8(d, t2); - d += dst_stride; - vst1_u8(d, t3); - d += dst_stride; - - s0 = s4; - s1 = s5; - s2 = s6; - s3 = s7; - s4 = s8; - s5 = s9; - s6 = s10; - height -= 4; - } while (height > 3); - - if (height != 0) { - __builtin_prefetch(dst_tmp_ptr + 0 * dst_stride); - __builtin_prefetch(dst_tmp_ptr + 1 * dst_stride); - - do { - s7 = vld1q_s16(s); - s += src_stride; - - t0 = - wiener_convolve8_vert_4x8(s0, s1, s2, s3, s4, s5, s6, - filter_y_tmp, bd, conv_params->round_1); - vst1_u8(d, t0); - d += dst_stride; - - s0 = s1; - s1 = s2; - s2 = s3; - s3 = s4; - s4 = s5; - s5 = s6; - s6 = s7; - height -= 1; - } while (height > 0); - } - - src_tmp_ptr += 8; - dst_tmp_ptr += 8; - - w -= 8; - } while (w > 0); -#else - do { - __builtin_prefetch(dst_tmp_ptr + 0 * dst_stride); - - s7 = vld1q_s16(s); - s += src_stride; - - t0 = wiener_convolve8_vert_4x8(s0, s1, s2, s3, s4, s5, s6, filter_y_tmp, - bd, conv_params->round_1); - - vst1_u8(d, t0); - d += dst_stride; - - s0 = s1; - s1 = s2; - s2 = s3; - s3 = s4; - s4 = s5; - s5 = s6; - s6 = s7; - height -= 1; - } while (height > 0); - - src_tmp_ptr += 8; - dst_tmp_ptr += 8; - - w -= 8; - } while (w > 0); -#endif - } -} |