path: root/third_party/aom/av1/common/x86/hybrid_inv_txfm_avx2.c

/*
 * Copyright (c) 2016, Alliance for Open Media. All rights reserved
 *
 * This source code is subject to the terms of the BSD 2 Clause License and
 * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
 * was not distributed with this source code in the LICENSE file, you can
 * obtain it at www.aomedia.org/license/software. If the Alliance for Open
 * Media Patent License 1.0 was not distributed with this source code in the
 * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
 */

#include <immintrin.h>  // avx2

#include "./aom_config.h"
#include "./av1_rtcd.h"

#include "aom_dsp/x86/txfm_common_avx2.h"

static INLINE void load_coeff(const tran_low_t *coeff, __m256i *in) {
#if CONFIG_HIGHBITDEPTH
  *in = _mm256_setr_epi16(
      (int16_t)coeff[0], (int16_t)coeff[1], (int16_t)coeff[2],
      (int16_t)coeff[3], (int16_t)coeff[4], (int16_t)coeff[5],
      (int16_t)coeff[6], (int16_t)coeff[7], (int16_t)coeff[8],
      (int16_t)coeff[9], (int16_t)coeff[10], (int16_t)coeff[11],
      (int16_t)coeff[12], (int16_t)coeff[13], (int16_t)coeff[14],
      (int16_t)coeff[15]);
#else
  *in = _mm256_loadu_si256((const __m256i *)coeff);
#endif
}

static void load_buffer_16x16(const tran_low_t *coeff, __m256i *in) {
  int i = 0;
  while (i < 16) {
    load_coeff(coeff + (i << 4), &in[i]);
    i += 1;
  }
}

static void recon_and_store(const __m256i *res, uint8_t *output) {
  const __m128i zero = _mm_setzero_si128();
  __m128i x = _mm_loadu_si128((__m128i const *)output);
  __m128i p0 = _mm_unpacklo_epi8(x, zero);
  __m128i p1 = _mm_unpackhi_epi8(x, zero);

  p0 = _mm_add_epi16(p0, _mm256_castsi256_si128(*res));
  p1 = _mm_add_epi16(p1, _mm256_extractf128_si256(*res, 1));
  x = _mm_packus_epi16(p0, p1);
  _mm_storeu_si128((__m128i *)output, x);
}

#define IDCT_ROUNDING_POS (6)

static void write_buffer_16x16(__m256i *in, const int stride, uint8_t *output) {
  const __m256i rounding = _mm256_set1_epi16(1 << (IDCT_ROUNDING_POS - 1));
  int i = 0;

  while (i < 16) {
    in[i] = _mm256_add_epi16(in[i], rounding);
    in[i] = _mm256_srai_epi16(in[i], IDCT_ROUNDING_POS);
    recon_and_store(&in[i], output + i * stride);
    i += 1;
  }
}

static INLINE void unpack_butter_fly(const __m256i *a0, const __m256i *a1,
                                     const __m256i *c0, const __m256i *c1,
                                     __m256i *b0, __m256i *b1) {
  __m256i x0, x1;
  x0 = _mm256_unpacklo_epi16(*a0, *a1);
  x1 = _mm256_unpackhi_epi16(*a0, *a1);
  *b0 = butter_fly(x0, x1, *c0);
  *b1 = butter_fly(x0, x1, *c1);
}

static void idct16_avx2(__m256i *in) {
  const __m256i cospi_p30_m02 = pair256_set_epi16(cospi_30_64, -cospi_2_64);
  const __m256i cospi_p02_p30 = pair256_set_epi16(cospi_2_64, cospi_30_64);
  const __m256i cospi_p14_m18 = pair256_set_epi16(cospi_14_64, -cospi_18_64);
  const __m256i cospi_p18_p14 = pair256_set_epi16(cospi_18_64, cospi_14_64);
  const __m256i cospi_p22_m10 = pair256_set_epi16(cospi_22_64, -cospi_10_64);
  const __m256i cospi_p10_p22 = pair256_set_epi16(cospi_10_64, cospi_22_64);
  const __m256i cospi_p06_m26 = pair256_set_epi16(cospi_6_64, -cospi_26_64);
  const __m256i cospi_p26_p06 = pair256_set_epi16(cospi_26_64, cospi_6_64);
  const __m256i cospi_p28_m04 = pair256_set_epi16(cospi_28_64, -cospi_4_64);
  const __m256i cospi_p04_p28 = pair256_set_epi16(cospi_4_64, cospi_28_64);
  const __m256i cospi_p12_m20 = pair256_set_epi16(cospi_12_64, -cospi_20_64);
  const __m256i cospi_p20_p12 = pair256_set_epi16(cospi_20_64, cospi_12_64);
  const __m256i cospi_p16_p16 = _mm256_set1_epi16((int16_t)cospi_16_64);
  const __m256i cospi_p16_m16 = pair256_set_epi16(cospi_16_64, -cospi_16_64);
  const __m256i cospi_p24_m08 = pair256_set_epi16(cospi_24_64, -cospi_8_64);
  const __m256i cospi_p08_p24 = pair256_set_epi16(cospi_8_64, cospi_24_64);
  const __m256i cospi_m08_p24 = pair256_set_epi16(-cospi_8_64, cospi_24_64);
  const __m256i cospi_p24_p08 = pair256_set_epi16(cospi_24_64, cospi_8_64);
  const __m256i cospi_m24_m08 = pair256_set_epi16(-cospi_24_64, -cospi_8_64);
  __m256i u0, u1, u2, u3, u4, u5, u6, u7;
  __m256i v0, v1, v2, v3, v4, v5, v6, v7;
  __m256i t0, t1, t2, t3, t4, t5, t6, t7;

  // stage 1, (0-7)
  u0 = in[0];
  u1 = in[8];
  u2 = in[4];
  u3 = in[12];
  u4 = in[2];
  u5 = in[10];
  u6 = in[6];
  u7 = in[14];

  // stage 2, (0-7)
  // stage 3, (0-7)
  t0 = u0;
  t1 = u1;
  t2 = u2;
  t3 = u3;
  unpack_butter_fly(&u4, &u7, &cospi_p28_m04, &cospi_p04_p28, &t4, &t7);
  unpack_butter_fly(&u5, &u6, &cospi_p12_m20, &cospi_p20_p12, &t5, &t6);

  // stage 4, (0-7)
  unpack_butter_fly(&t0, &t1, &cospi_p16_p16, &cospi_p16_m16, &u0, &u1);
  unpack_butter_fly(&t2, &t3, &cospi_p24_m08, &cospi_p08_p24, &u2, &u3);
  u4 = _mm256_add_epi16(t4, t5);
  u5 = _mm256_sub_epi16(t4, t5);
  u6 = _mm256_sub_epi16(t7, t6);
  u7 = _mm256_add_epi16(t7, t6);

  // stage 5, (0-7)
  t0 = _mm256_add_epi16(u0, u3);
  t1 = _mm256_add_epi16(u1, u2);
  t2 = _mm256_sub_epi16(u1, u2);
  t3 = _mm256_sub_epi16(u0, u3);
  t4 = u4;
  t7 = u7;
  unpack_butter_fly(&u6, &u5, &cospi_p16_m16, &cospi_p16_p16, &t5, &t6);

  // stage 6, (0-7)
  u0 = _mm256_add_epi16(t0, t7);
  u1 = _mm256_add_epi16(t1, t6);
  u2 = _mm256_add_epi16(t2, t5);
  u3 = _mm256_add_epi16(t3, t4);
  u4 = _mm256_sub_epi16(t3, t4);
  u5 = _mm256_sub_epi16(t2, t5);
  u6 = _mm256_sub_epi16(t1, t6);
  u7 = _mm256_sub_epi16(t0, t7);

  // stage 1, (8-15)
  v0 = in[1];
  v1 = in[9];
  v2 = in[5];
  v3 = in[13];
  v4 = in[3];
  v5 = in[11];
  v6 = in[7];
  v7 = in[15];

  // stage 2, (8-15)
  unpack_butter_fly(&v0, &v7, &cospi_p30_m02, &cospi_p02_p30, &t0, &t7);
  unpack_butter_fly(&v1, &v6, &cospi_p14_m18, &cospi_p18_p14, &t1, &t6);
  unpack_butter_fly(&v2, &v5, &cospi_p22_m10, &cospi_p10_p22, &t2, &t5);
  unpack_butter_fly(&v3, &v4, &cospi_p06_m26, &cospi_p26_p06, &t3, &t4);

  // stage 3, (8-15)
  v0 = _mm256_add_epi16(t0, t1);
  v1 = _mm256_sub_epi16(t0, t1);
  v2 = _mm256_sub_epi16(t3, t2);
  v3 = _mm256_add_epi16(t2, t3);
  v4 = _mm256_add_epi16(t4, t5);
  v5 = _mm256_sub_epi16(t4, t5);
  v6 = _mm256_sub_epi16(t7, t6);
  v7 = _mm256_add_epi16(t6, t7);

  // stage 4, (8-15)
  t0 = v0;
  t7 = v7;
  t3 = v3;
  t4 = v4;
  unpack_butter_fly(&v1, &v6, &cospi_m08_p24, &cospi_p24_p08, &t1, &t6);
  unpack_butter_fly(&v2, &v5, &cospi_m24_m08, &cospi_m08_p24, &t2, &t5);

  // stage 5, (8-15)
  v0 = _mm256_add_epi16(t0, t3);
  v1 = _mm256_add_epi16(t1, t2);
  v2 = _mm256_sub_epi16(t1, t2);
  v3 = _mm256_sub_epi16(t0, t3);
  v4 = _mm256_sub_epi16(t7, t4);
  v5 = _mm256_sub_epi16(t6, t5);
  v6 = _mm256_add_epi16(t6, t5);
  v7 = _mm256_add_epi16(t7, t4);

  // stage 6, (8-15)
  t0 = v0;
  t1 = v1;
  t6 = v6;
  t7 = v7;
  unpack_butter_fly(&v5, &v2, &cospi_p16_m16, &cospi_p16_p16, &t2, &t5);
  unpack_butter_fly(&v4, &v3, &cospi_p16_m16, &cospi_p16_p16, &t3, &t4);

  // stage 7
  in[0] = _mm256_add_epi16(u0, t7);
  in[1] = _mm256_add_epi16(u1, t6);
  in[2] = _mm256_add_epi16(u2, t5);
  in[3] = _mm256_add_epi16(u3, t4);
  in[4] = _mm256_add_epi16(u4, t3);
  in[5] = _mm256_add_epi16(u5, t2);
  in[6] = _mm256_add_epi16(u6, t1);
  in[7] = _mm256_add_epi16(u7, t0);
  in[8] = _mm256_sub_epi16(u7, t0);
  in[9] = _mm256_sub_epi16(u6, t1);
  in[10] = _mm256_sub_epi16(u5, t2);
  in[11] = _mm256_sub_epi16(u4, t3);
  in[12] = _mm256_sub_epi16(u3, t4);
  in[13] = _mm256_sub_epi16(u2, t5);
  in[14] = _mm256_sub_epi16(u1, t6);
  in[15] = _mm256_sub_epi16(u0, t7);
}

static void idct16(__m256i *in) {
  mm256_transpose_16x16(in);
  idct16_avx2(in);
}

static INLINE void butterfly_32b(const __m256i *a0, const __m256i *a1,
                                 const __m256i *c0, const __m256i *c1,
                                 __m256i *b) {
  __m256i x0, x1;
  x0 = _mm256_unpacklo_epi16(*a0, *a1);
  x1 = _mm256_unpackhi_epi16(*a0, *a1);
  b[0] = _mm256_madd_epi16(x0, *c0);
  b[1] = _mm256_madd_epi16(x1, *c0);
  b[2] = _mm256_madd_epi16(x0, *c1);
  b[3] = _mm256_madd_epi16(x1, *c1);
}

static INLINE void group_rounding(__m256i *a, int num) {
  const __m256i dct_rounding = _mm256_set1_epi32(DCT_CONST_ROUNDING);
  int i;
  for (i = 0; i < num; ++i) {
    a[i] = _mm256_add_epi32(a[i], dct_rounding);
    a[i] = _mm256_srai_epi32(a[i], DCT_CONST_BITS);
  }
}

static INLINE void add_rnd(const __m256i *a, const __m256i *b, __m256i *out) {
  __m256i x[4];
  x[0] = _mm256_add_epi32(a[0], b[0]);
  x[1] = _mm256_add_epi32(a[1], b[1]);
  x[2] = _mm256_add_epi32(a[2], b[2]);
  x[3] = _mm256_add_epi32(a[3], b[3]);

  group_rounding(x, 4);

  out[0] = _mm256_packs_epi32(x[0], x[1]);
  out[1] = _mm256_packs_epi32(x[2], x[3]);
}

static INLINE void sub_rnd(const __m256i *a, const __m256i *b, __m256i *out) {
  __m256i x[4];
  x[0] = _mm256_sub_epi32(a[0], b[0]);
  x[1] = _mm256_sub_epi32(a[1], b[1]);
  x[2] = _mm256_sub_epi32(a[2], b[2]);
  x[3] = _mm256_sub_epi32(a[3], b[3]);

  group_rounding(x, 4);

  out[0] = _mm256_packs_epi32(x[0], x[1]);
  out[1] = _mm256_packs_epi32(x[2], x[3]);
}

static INLINE void butterfly_rnd(__m256i *a, __m256i *out) {
  group_rounding(a, 4);
  out[0] = _mm256_packs_epi32(a[0], a[1]);
  out[1] = _mm256_packs_epi32(a[2], a[3]);
}

static void iadst16_avx2(__m256i *in) {
  const __m256i cospi_p01_p31 = pair256_set_epi16(cospi_1_64, cospi_31_64);
  const __m256i cospi_p31_m01 = pair256_set_epi16(cospi_31_64, -cospi_1_64);
  const __m256i cospi_p05_p27 = pair256_set_epi16(cospi_5_64, cospi_27_64);
  const __m256i cospi_p27_m05 = pair256_set_epi16(cospi_27_64, -cospi_5_64);
  const __m256i cospi_p09_p23 = pair256_set_epi16(cospi_9_64, cospi_23_64);
  const __m256i cospi_p23_m09 = pair256_set_epi16(cospi_23_64, -cospi_9_64);
  const __m256i cospi_p13_p19 = pair256_set_epi16(cospi_13_64, cospi_19_64);
  const __m256i cospi_p19_m13 = pair256_set_epi16(cospi_19_64, -cospi_13_64);
  const __m256i cospi_p17_p15 = pair256_set_epi16(cospi_17_64, cospi_15_64);
  const __m256i cospi_p15_m17 = pair256_set_epi16(cospi_15_64, -cospi_17_64);
  const __m256i cospi_p21_p11 = pair256_set_epi16(cospi_21_64, cospi_11_64);
  const __m256i cospi_p11_m21 = pair256_set_epi16(cospi_11_64, -cospi_21_64);
  const __m256i cospi_p25_p07 = pair256_set_epi16(cospi_25_64, cospi_7_64);
  const __m256i cospi_p07_m25 = pair256_set_epi16(cospi_7_64, -cospi_25_64);
  const __m256i cospi_p29_p03 = pair256_set_epi16(cospi_29_64, cospi_3_64);
  const __m256i cospi_p03_m29 = pair256_set_epi16(cospi_3_64, -cospi_29_64);
  const __m256i cospi_p04_p28 = pair256_set_epi16(cospi_4_64, cospi_28_64);
  const __m256i cospi_p28_m04 = pair256_set_epi16(cospi_28_64, -cospi_4_64);
  const __m256i cospi_p20_p12 = pair256_set_epi16(cospi_20_64, cospi_12_64);
  const __m256i cospi_p12_m20 = pair256_set_epi16(cospi_12_64, -cospi_20_64);
  const __m256i cospi_m28_p04 = pair256_set_epi16(-cospi_28_64, cospi_4_64);
  const __m256i cospi_m12_p20 = pair256_set_epi16(-cospi_12_64, cospi_20_64);
  const __m256i cospi_p08_p24 = pair256_set_epi16(cospi_8_64, cospi_24_64);
  const __m256i cospi_p24_m08 = pair256_set_epi16(cospi_24_64, -cospi_8_64);
  const __m256i cospi_m24_p08 = pair256_set_epi16(-cospi_24_64, cospi_8_64);
  const __m256i cospi_m16_m16 = _mm256_set1_epi16((int16_t)-cospi_16_64);
  const __m256i cospi_p16_p16 = _mm256_set1_epi16((int16_t)cospi_16_64);
  const __m256i cospi_p16_m16 = pair256_set_epi16(cospi_16_64, -cospi_16_64);
  const __m256i cospi_m16_p16 = pair256_set_epi16(-cospi_16_64, cospi_16_64);
  const __m256i zero = _mm256_setzero_si256();
  __m256i x[16], s[16];
  __m256i u[4], v[4];

  // stage 1
  butterfly_32b(&in[15], &in[0], &cospi_p01_p31, &cospi_p31_m01, u);
  butterfly_32b(&in[7], &in[8], &cospi_p17_p15, &cospi_p15_m17, v);
  add_rnd(u, v, &x[0]);
  sub_rnd(u, v, &x[8]);

  butterfly_32b(&in[13], &in[2], &cospi_p05_p27, &cospi_p27_m05, u);
  butterfly_32b(&in[5], &in[10], &cospi_p21_p11, &cospi_p11_m21, v);
  add_rnd(u, v, &x[2]);
  sub_rnd(u, v, &x[10]);

  butterfly_32b(&in[11], &in[4], &cospi_p09_p23, &cospi_p23_m09, u);
  butterfly_32b(&in[3], &in[12], &cospi_p25_p07, &cospi_p07_m25, v);
  add_rnd(u, v, &x[4]);
  sub_rnd(u, v, &x[12]);

  butterfly_32b(&in[9], &in[6], &cospi_p13_p19, &cospi_p19_m13, u);
  butterfly_32b(&in[1], &in[14], &cospi_p29_p03, &cospi_p03_m29, v);
  add_rnd(u, v, &x[6]);
  sub_rnd(u, v, &x[14]);

  // stage 2
  s[0] = _mm256_add_epi16(x[0], x[4]);
  s[1] = _mm256_add_epi16(x[1], x[5]);
  s[2] = _mm256_add_epi16(x[2], x[6]);
  s[3] = _mm256_add_epi16(x[3], x[7]);
  s[4] = _mm256_sub_epi16(x[0], x[4]);
  s[5] = _mm256_sub_epi16(x[1], x[5]);
  s[6] = _mm256_sub_epi16(x[2], x[6]);
  s[7] = _mm256_sub_epi16(x[3], x[7]);
  butterfly_32b(&x[8], &x[9], &cospi_p04_p28, &cospi_p28_m04, u);
  butterfly_32b(&x[12], &x[13], &cospi_m28_p04, &cospi_p04_p28, v);
  add_rnd(u, v, &s[8]);
  sub_rnd(u, v, &s[12]);

  butterfly_32b(&x[10], &x[11], &cospi_p20_p12, &cospi_p12_m20, u);
  butterfly_32b(&x[14], &x[15], &cospi_m12_p20, &cospi_p20_p12, v);
  add_rnd(u, v, &s[10]);
  sub_rnd(u, v, &s[14]);

  // stage 3
  x[0] = _mm256_add_epi16(s[0], s[2]);
  x[1] = _mm256_add_epi16(s[1], s[3]);
  x[2] = _mm256_sub_epi16(s[0], s[2]);
  x[3] = _mm256_sub_epi16(s[1], s[3]);

  x[8] = _mm256_add_epi16(s[8], s[10]);
  x[9] = _mm256_add_epi16(s[9], s[11]);
  x[10] = _mm256_sub_epi16(s[8], s[10]);
  x[11] = _mm256_sub_epi16(s[9], s[11]);

  butterfly_32b(&s[4], &s[5], &cospi_p08_p24, &cospi_p24_m08, u);
  butterfly_32b(&s[6], &s[7], &cospi_m24_p08, &cospi_p08_p24, v);
  add_rnd(u, v, &x[4]);
  sub_rnd(u, v, &x[6]);

  butterfly_32b(&s[12], &s[13], &cospi_p08_p24, &cospi_p24_m08, u);
  butterfly_32b(&s[14], &s[15], &cospi_m24_p08, &cospi_p08_p24, v);
  add_rnd(u, v, &x[12]);
  sub_rnd(u, v, &x[14]);

  // stage 4
  butterfly_32b(&x[2], &x[3], &cospi_m16_m16, &cospi_p16_m16, u);
  butterfly_32b(&x[6], &x[7], &cospi_p16_p16, &cospi_m16_p16, v);
  butterfly_rnd(u, &x[2]);
  butterfly_rnd(v, &x[6]);

  butterfly_32b(&x[10], &x[11], &cospi_p16_p16, &cospi_m16_p16, u);
  butterfly_32b(&x[14], &x[15], &cospi_m16_m16, &cospi_p16_m16, v);
  butterfly_rnd(u, &x[10]);
  butterfly_rnd(v, &x[14]);

  in[0] = x[0];
  in[1] = _mm256_sub_epi16(zero, x[8]);
  in[2] = x[12];
  in[3] = _mm256_sub_epi16(zero, x[4]);
  in[4] = x[6];
  in[5] = x[14];
  in[6] = x[10];
  in[7] = x[2];
  in[8] = x[3];
  in[9] = x[11];
  in[10] = x[15];
  in[11] = x[7];
  in[12] = x[5];
  in[13] = _mm256_sub_epi16(zero, x[13]);
  in[14] = x[9];
  in[15] = _mm256_sub_epi16(zero, x[1]);
}

static void iadst16(__m256i *in) {
  mm256_transpose_16x16(in);
  iadst16_avx2(in);
}

#if CONFIG_EXT_TX
static void flip_row(__m256i *in, int rows) {
  int i;
  for (i = 0; i < rows; ++i) {
    mm256_reverse_epi16(&in[i]);
  }
}

static void flip_col(uint8_t **dest, int *stride, int rows) {
  *dest = *dest + (rows - 1) * (*stride);
  *stride = -*stride;
}

static void iidtx16(__m256i *in) {
  mm256_transpose_16x16(in);
  txfm_scaling16_avx2(Sqrt2, in);
}
#endif

void av1_iht16x16_256_add_avx2(const tran_low_t *input, uint8_t *dest,
                               int stride, int tx_type) {
  __m256i in[16];

  load_buffer_16x16(input, in);
  switch (tx_type) {
    case DCT_DCT:
      idct16(in);
      idct16(in);
      break;
    case ADST_DCT:
      idct16(in);
      iadst16(in);
      break;
    case DCT_ADST:
      iadst16(in);
      idct16(in);
      break;
    case ADST_ADST:
      iadst16(in);
      iadst16(in);
      break;
#if CONFIG_EXT_TX
    case FLIPADST_DCT:
      idct16(in);
      iadst16(in);
      flip_col(&dest, &stride, 16);
      break;
    case DCT_FLIPADST:
      iadst16(in);
      idct16(in);
      flip_row(in, 16);
      break;
    case FLIPADST_FLIPADST:
      iadst16(in);
      iadst16(in);
      flip_row(in, 16);
      flip_col(&dest, &stride, 16);
      break;
    case ADST_FLIPADST:
      iadst16(in);
      iadst16(in);
      flip_row(in, 16);
      break;
    case FLIPADST_ADST:
      iadst16(in);
      iadst16(in);
      flip_col(&dest, &stride, 16);
      break;
    case IDTX:
      iidtx16(in);
      iidtx16(in);
      break;
    case V_DCT:
      iidtx16(in);
      idct16(in);
      break;
    case H_DCT:
      idct16(in);
      iidtx16(in);
      break;
    case V_ADST:
      iidtx16(in);
      iadst16(in);
      break;
    case H_ADST:
      iadst16(in);
      iidtx16(in);
      break;
    case V_FLIPADST:
      iidtx16(in);
      iadst16(in);
      flip_col(&dest, &stride, 16);
      break;
    case H_FLIPADST:
      iadst16(in);
      iidtx16(in);
      flip_row(in, 16);
      break;
#endif  // CONFIG_EXT_TX
    default: assert(0); break;
  }
  write_buffer_16x16(in, stride, dest);
}