diff options
Diffstat (limited to 'third_party/aom/aom_dsp/simd')
| -rw-r--r-- | third_party/aom/aom_dsp/simd/v128_intrinsics.h | 344 | ||||
| -rw-r--r-- | third_party/aom/aom_dsp/simd/v128_intrinsics_arm.h | 958 | ||||
| -rw-r--r-- | third_party/aom/aom_dsp/simd/v128_intrinsics_c.h | 888 | ||||
| -rw-r--r-- | third_party/aom/aom_dsp/simd/v128_intrinsics_x86.h | 656 | ||||
| -rw-r--r-- | third_party/aom/aom_dsp/simd/v256_intrinsics.h | 376 | ||||
| -rw-r--r-- | third_party/aom/aom_dsp/simd/v256_intrinsics_arm.h | 17 | ||||
| -rw-r--r-- | third_party/aom/aom_dsp/simd/v256_intrinsics_c.h | 953 | ||||
| -rw-r--r-- | third_party/aom/aom_dsp/simd/v256_intrinsics_v128.h | 873 | ||||
| -rw-r--r-- | third_party/aom/aom_dsp/simd/v256_intrinsics_x86.h | 750 | ||||
| -rw-r--r-- | third_party/aom/aom_dsp/simd/v64_intrinsics.h | 232 | ||||
| -rw-r--r-- | third_party/aom/aom_dsp/simd/v64_intrinsics_arm.h | 680 | ||||
| -rw-r--r-- | third_party/aom/aom_dsp/simd/v64_intrinsics_c.h | 968 | ||||
| -rw-r--r-- | third_party/aom/aom_dsp/simd/v64_intrinsics_x86.h | 491 |
13 files changed, 0 insertions, 8186 deletions
diff --git a/third_party/aom/aom_dsp/simd/v128_intrinsics.h b/third_party/aom/aom_dsp/simd/v128_intrinsics.h deleted file mode 100644 index 01dbb8fd2..000000000 --- a/third_party/aom/aom_dsp/simd/v128_intrinsics.h +++ /dev/null @@ -1,344 +0,0 @@ -/* - * Copyright (c) 2016, Alliance for Open Media. All rights reserved - * - * This source code is subject to the terms of the BSD 2 Clause License and - * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License - * was not distributed with this source code in the LICENSE file, you can - * obtain it at www.aomedia.org/license/software. If the Alliance for Open - * Media Patent License 1.0 was not distributed with this source code in the - * PATENTS file, you can obtain it at www.aomedia.org/license/patent. - */ - -#ifndef AOM_AOM_DSP_SIMD_V128_INTRINSICS_H_ -#define AOM_AOM_DSP_SIMD_V128_INTRINSICS_H_ - -#include <stdio.h> -#include <stdlib.h> -#include <string.h> - -#include "aom_dsp/simd/v128_intrinsics_c.h" -#include "aom_dsp/simd/v64_intrinsics.h" - -/* Fallback to plain, unoptimised C. */ - -typedef c_v128 v128; - -SIMD_INLINE uint32_t v128_low_u32(v128 a) { return c_v128_low_u32(a); } -SIMD_INLINE v64 v128_low_v64(v128 a) { return c_v128_low_v64(a); } -SIMD_INLINE v64 v128_high_v64(v128 a) { return c_v128_high_v64(a); } -SIMD_INLINE v128 v128_from_64(uint64_t hi, uint64_t lo) { - return c_v128_from_64(hi, lo); -} -SIMD_INLINE v128 v128_from_v64(v64 hi, v64 lo) { - return c_v128_from_v64(hi, lo); -} -SIMD_INLINE v128 v128_from_32(uint32_t a, uint32_t b, uint32_t c, uint32_t d) { - return c_v128_from_32(a, b, c, d); -} - -SIMD_INLINE v128 v128_load_unaligned(const void *p) { - return c_v128_load_unaligned(p); -} -SIMD_INLINE v128 v128_load_aligned(const void *p) { - return c_v128_load_aligned(p); -} - -SIMD_INLINE void v128_store_unaligned(void *p, v128 a) { - c_v128_store_unaligned(p, a); -} -SIMD_INLINE void v128_store_aligned(void *p, v128 a) { - c_v128_store_aligned(p, a); -} - -SIMD_INLINE v128 v128_align(v128 a, v128 b, unsigned int c) { - return c_v128_align(a, b, c); -} - -SIMD_INLINE v128 v128_zero() { return c_v128_zero(); } -SIMD_INLINE v128 v128_dup_8(uint8_t x) { return c_v128_dup_8(x); } -SIMD_INLINE v128 v128_dup_16(uint16_t x) { return c_v128_dup_16(x); } -SIMD_INLINE v128 v128_dup_32(uint32_t x) { return c_v128_dup_32(x); } -SIMD_INLINE v128 v128_dup_64(uint64_t x) { return c_v128_dup_64(x); } - -typedef uint32_t sad128_internal; -SIMD_INLINE sad128_internal v128_sad_u8_init() { return c_v128_sad_u8_init(); } -SIMD_INLINE sad128_internal v128_sad_u8(sad128_internal s, v128 a, v128 b) { - return c_v128_sad_u8(s, a, b); -} -SIMD_INLINE uint32_t v128_sad_u8_sum(sad128_internal s) { - return c_v128_sad_u8_sum(s); -} -typedef uint32_t ssd128_internal; -SIMD_INLINE ssd128_internal v128_ssd_u8_init() { return c_v128_ssd_u8_init(); } -SIMD_INLINE ssd128_internal v128_ssd_u8(ssd128_internal s, v128 a, v128 b) { - return c_v128_ssd_u8(s, a, b); -} -SIMD_INLINE uint32_t v128_ssd_u8_sum(ssd128_internal s) { - return c_v128_ssd_u8_sum(s); -} -SIMD_INLINE int64_t v128_dotp_su8(v128 a, v128 b) { - return c_v128_dotp_su8(a, b); -} -SIMD_INLINE int64_t v128_dotp_s16(v128 a, v128 b) { - return c_v128_dotp_s16(a, b); -} -SIMD_INLINE int64_t v128_dotp_s32(v128 a, v128 b) { - return c_v128_dotp_s32(a, b); -} -SIMD_INLINE uint64_t v128_hadd_u8(v128 a) { return c_v128_hadd_u8(a); } - -SIMD_INLINE v128 v128_or(v128 a, v128 b) { return c_v128_or(a, b); } -SIMD_INLINE v128 v128_xor(v128 a, v128 b) { return c_v128_xor(a, b); } -SIMD_INLINE v128 v128_and(v128 a, v128 b) { return c_v128_and(a, b); } -SIMD_INLINE v128 v128_andn(v128 a, v128 b) { return c_v128_andn(a, b); } - -SIMD_INLINE v128 v128_add_8(v128 a, v128 b) { return c_v128_add_8(a, b); } -SIMD_INLINE v128 v128_add_16(v128 a, v128 b) { return c_v128_add_16(a, b); } -SIMD_INLINE v128 v128_sadd_u8(v128 a, v128 b) { return c_v128_sadd_u8(a, b); } -SIMD_INLINE v128 v128_sadd_s8(v128 a, v128 b) { return c_v128_sadd_s8(a, b); } -SIMD_INLINE v128 v128_sadd_s16(v128 a, v128 b) { return c_v128_sadd_s16(a, b); } -SIMD_INLINE v128 v128_add_32(v128 a, v128 b) { return c_v128_add_32(a, b); } -SIMD_INLINE v128 v128_add_64(v128 a, v128 b) { return c_v128_add_64(a, b); } -SIMD_INLINE v128 v128_padd_u8(v128 a) { return c_v128_padd_u8(a); } -SIMD_INLINE v128 v128_padd_s16(v128 a) { return c_v128_padd_s16(a); } -SIMD_INLINE v128 v128_sub_8(v128 a, v128 b) { return c_v128_sub_8(a, b); } -SIMD_INLINE v128 v128_ssub_u8(v128 a, v128 b) { return c_v128_ssub_u8(a, b); } -SIMD_INLINE v128 v128_ssub_s8(v128 a, v128 b) { return c_v128_ssub_s8(a, b); } -SIMD_INLINE v128 v128_sub_16(v128 a, v128 b) { return c_v128_sub_16(a, b); } -SIMD_INLINE v128 v128_ssub_s16(v128 a, v128 b) { return c_v128_ssub_s16(a, b); } -SIMD_INLINE v128 v128_ssub_u16(v128 a, v128 b) { return c_v128_ssub_u16(a, b); } -SIMD_INLINE v128 v128_sub_32(v128 a, v128 b) { return c_v128_sub_32(a, b); } -SIMD_INLINE v128 v128_sub_64(v128 a, v128 b) { return c_v128_sub_64(a, b); } -SIMD_INLINE v128 v128_abs_s16(v128 a) { return c_v128_abs_s16(a); } -SIMD_INLINE v128 v128_abs_s8(v128 a) { return c_v128_abs_s8(a); } - -SIMD_INLINE v128 v128_mul_s16(v64 a, v64 b) { return c_v128_mul_s16(a, b); } -SIMD_INLINE v128 v128_mullo_s16(v128 a, v128 b) { - return c_v128_mullo_s16(a, b); -} -SIMD_INLINE v128 v128_mulhi_s16(v128 a, v128 b) { - return c_v128_mulhi_s16(a, b); -} -SIMD_INLINE v128 v128_mullo_s32(v128 a, v128 b) { - return c_v128_mullo_s32(a, b); -} -SIMD_INLINE v128 v128_madd_s16(v128 a, v128 b) { return c_v128_madd_s16(a, b); } -SIMD_INLINE v128 v128_madd_us8(v128 a, v128 b) { return c_v128_madd_us8(a, b); } - -SIMD_INLINE uint32_t v128_movemask_8(v128 a) { return c_v128_movemask_8(a); } -SIMD_INLINE v128 v128_blend_8(v128 a, v128 b, v128 c) { - return c_v128_blend_8(a, b, c); -} - -SIMD_INLINE v128 v128_avg_u8(v128 a, v128 b) { return c_v128_avg_u8(a, b); } -SIMD_INLINE v128 v128_rdavg_u8(v128 a, v128 b) { return c_v128_rdavg_u8(a, b); } -SIMD_INLINE v128 v128_rdavg_u16(v128 a, v128 b) { - return c_v128_rdavg_u16(a, b); -} -SIMD_INLINE v128 v128_avg_u16(v128 a, v128 b) { return c_v128_avg_u16(a, b); } -SIMD_INLINE v128 v128_min_u8(v128 a, v128 b) { return c_v128_min_u8(a, b); } -SIMD_INLINE v128 v128_max_u8(v128 a, v128 b) { return c_v128_max_u8(a, b); } -SIMD_INLINE v128 v128_min_s8(v128 a, v128 b) { return c_v128_min_s8(a, b); } -SIMD_INLINE v128 v128_max_s8(v128 a, v128 b) { return c_v128_max_s8(a, b); } -SIMD_INLINE v128 v128_min_s16(v128 a, v128 b) { return c_v128_min_s16(a, b); } -SIMD_INLINE v128 v128_max_s16(v128 a, v128 b) { return c_v128_max_s16(a, b); } -SIMD_INLINE v128 v128_min_s32(v128 a, v128 b) { return c_v128_min_s32(a, b); } -SIMD_INLINE v128 v128_max_s32(v128 a, v128 b) { return c_v128_max_s32(a, b); } - -SIMD_INLINE v128 v128_ziplo_8(v128 a, v128 b) { return c_v128_ziplo_8(a, b); } -SIMD_INLINE v128 v128_ziphi_8(v128 a, v128 b) { return c_v128_ziphi_8(a, b); } -SIMD_INLINE v128 v128_ziplo_16(v128 a, v128 b) { return c_v128_ziplo_16(a, b); } -SIMD_INLINE v128 v128_ziphi_16(v128 a, v128 b) { return c_v128_ziphi_16(a, b); } -SIMD_INLINE v128 v128_ziplo_32(v128 a, v128 b) { return c_v128_ziplo_32(a, b); } -SIMD_INLINE v128 v128_ziphi_32(v128 a, v128 b) { return c_v128_ziphi_32(a, b); } -SIMD_INLINE v128 v128_ziplo_64(v128 a, v128 b) { return c_v128_ziplo_64(a, b); } -SIMD_INLINE v128 v128_ziphi_64(v128 a, v128 b) { return c_v128_ziphi_64(a, b); } -SIMD_INLINE v128 v128_zip_8(v64 a, v64 b) { return c_v128_zip_8(a, b); } -SIMD_INLINE v128 v128_zip_16(v64 a, v64 b) { return c_v128_zip_16(a, b); } -SIMD_INLINE v128 v128_zip_32(v64 a, v64 b) { return c_v128_zip_32(a, b); } -SIMD_INLINE v128 v128_unziplo_8(v128 a, v128 b) { - return c_v128_unziplo_8(a, b); -} -SIMD_INLINE v128 v128_unziphi_8(v128 a, v128 b) { - return c_v128_unziphi_8(a, b); -} -SIMD_INLINE v128 v128_unziplo_16(v128 a, v128 b) { - return c_v128_unziplo_16(a, b); -} -SIMD_INLINE v128 v128_unziphi_16(v128 a, v128 b) { - return c_v128_unziphi_16(a, b); -} -SIMD_INLINE v128 v128_unziplo_32(v128 a, v128 b) { - return c_v128_unziplo_32(a, b); -} -SIMD_INLINE v128 v128_unziphi_32(v128 a, v128 b) { - return c_v128_unziphi_32(a, b); -} -SIMD_INLINE v128 v128_unpack_u8_s16(v64 a) { return c_v128_unpack_u8_s16(a); } -SIMD_INLINE v128 v128_unpacklo_u8_s16(v128 a) { - return c_v128_unpacklo_u8_s16(a); -} -SIMD_INLINE v128 v128_unpackhi_u8_s16(v128 a) { - return c_v128_unpackhi_u8_s16(a); -} -SIMD_INLINE v128 v128_unpack_s8_s16(v64 a) { return c_v128_unpack_s8_s16(a); } -SIMD_INLINE v128 v128_unpacklo_s8_s16(v128 a) { - return c_v128_unpacklo_s8_s16(a); -} -SIMD_INLINE v128 v128_unpackhi_s8_s16(v128 a) { - return c_v128_unpackhi_s8_s16(a); -} -SIMD_INLINE v128 v128_pack_s32_s16(v128 a, v128 b) { - return c_v128_pack_s32_s16(a, b); -} -SIMD_INLINE v128 v128_pack_s32_u16(v128 a, v128 b) { - return c_v128_pack_s32_u16(a, b); -} -SIMD_INLINE v128 v128_pack_s16_u8(v128 a, v128 b) { - return c_v128_pack_s16_u8(a, b); -} -SIMD_INLINE v128 v128_pack_s16_s8(v128 a, v128 b) { - return c_v128_pack_s16_s8(a, b); -} -SIMD_INLINE v128 v128_unpack_u16_s32(v64 a) { return c_v128_unpack_u16_s32(a); } -SIMD_INLINE v128 v128_unpack_s16_s32(v64 a) { return c_v128_unpack_s16_s32(a); } -SIMD_INLINE v128 v128_unpacklo_u16_s32(v128 a) { - return c_v128_unpacklo_u16_s32(a); -} -SIMD_INLINE v128 v128_unpacklo_s16_s32(v128 a) { - return c_v128_unpacklo_s16_s32(a); -} -SIMD_INLINE v128 v128_unpackhi_u16_s32(v128 a) { - return c_v128_unpackhi_u16_s32(a); -} -SIMD_INLINE v128 v128_unpackhi_s16_s32(v128 a) { - return c_v128_unpackhi_s16_s32(a); -} -SIMD_INLINE v128 v128_shuffle_8(v128 a, v128 pattern) { - return c_v128_shuffle_8(a, pattern); -} - -SIMD_INLINE v128 v128_cmpgt_s8(v128 a, v128 b) { return c_v128_cmpgt_s8(a, b); } -SIMD_INLINE v128 v128_cmplt_s8(v128 a, v128 b) { return c_v128_cmplt_s8(a, b); } -SIMD_INLINE v128 v128_cmpeq_8(v128 a, v128 b) { return c_v128_cmpeq_8(a, b); } -SIMD_INLINE v128 v128_cmpgt_s16(v128 a, v128 b) { - return c_v128_cmpgt_s16(a, b); -} -SIMD_INLINE v128 v128_cmplt_s16(v128 a, v128 b) { - return c_v128_cmplt_s16(a, b); -} -SIMD_INLINE v128 v128_cmpeq_16(v128 a, v128 b) { return c_v128_cmpeq_16(a, b); } - -SIMD_INLINE v128 v128_cmpgt_s32(v128 a, v128 b) { - return c_v128_cmpgt_s32(a, b); -} -SIMD_INLINE v128 v128_cmplt_s32(v128 a, v128 b) { - return c_v128_cmplt_s32(a, b); -} -SIMD_INLINE v128 v128_cmpeq_32(v128 a, v128 b) { return c_v128_cmpeq_32(a, b); } - -SIMD_INLINE v128 v128_shl_8(v128 a, unsigned int c) { - return c_v128_shl_8(a, c); -} -SIMD_INLINE v128 v128_shr_u8(v128 a, unsigned int c) { - return c_v128_shr_u8(a, c); -} -SIMD_INLINE v128 v128_shr_s8(v128 a, unsigned int c) { - return c_v128_shr_s8(a, c); -} -SIMD_INLINE v128 v128_shl_16(v128 a, unsigned int c) { - return c_v128_shl_16(a, c); -} -SIMD_INLINE v128 v128_shr_u16(v128 a, unsigned int c) { - return c_v128_shr_u16(a, c); -} -SIMD_INLINE v128 v128_shr_s16(v128 a, unsigned int c) { - return c_v128_shr_s16(a, c); -} -SIMD_INLINE v128 v128_shl_32(v128 a, unsigned int c) { - return c_v128_shl_32(a, c); -} -SIMD_INLINE v128 v128_shr_u32(v128 a, unsigned int c) { - return c_v128_shr_u32(a, c); -} -SIMD_INLINE v128 v128_shr_s32(v128 a, unsigned int c) { - return c_v128_shr_s32(a, c); -} -SIMD_INLINE v128 v128_shl_64(v128 a, unsigned int c) { - return c_v128_shl_64(a, c); -} -SIMD_INLINE v128 v128_shr_u64(v128 a, unsigned int c) { - return c_v128_shr_u64(a, c); -} -SIMD_INLINE v128 v128_shr_s64(v128 a, unsigned int c) { - return c_v128_shr_s64(a, c); -} - -SIMD_INLINE v128 v128_shr_n_byte(v128 a, unsigned int n) { - return c_v128_shr_n_byte(a, n); -} -SIMD_INLINE v128 v128_shl_n_byte(v128 a, unsigned int n) { - return c_v128_shl_n_byte(a, n); -} -SIMD_INLINE v128 v128_shl_n_8(v128 a, unsigned int n) { - return c_v128_shl_n_8(a, n); -} -SIMD_INLINE v128 v128_shl_n_16(v128 a, unsigned int n) { - return c_v128_shl_n_16(a, n); -} -SIMD_INLINE v128 v128_shl_n_32(v128 a, unsigned int n) { - return c_v128_shl_n_32(a, n); -} -SIMD_INLINE v128 v128_shl_n_64(v128 a, unsigned int n) { - return c_v128_shl_n_64(a, n); -} -SIMD_INLINE v128 v128_shr_n_u8(v128 a, unsigned int n) { - return c_v128_shr_n_u8(a, n); -} -SIMD_INLINE v128 v128_shr_n_u16(v128 a, unsigned int n) { - return c_v128_shr_n_u16(a, n); -} -SIMD_INLINE v128 v128_shr_n_u32(v128 a, unsigned int n) { - return c_v128_shr_n_u32(a, n); -} -SIMD_INLINE v128 v128_shr_n_u64(v128 a, unsigned int n) { - return c_v128_shr_n_u64(a, n); -} -SIMD_INLINE v128 v128_shr_n_s8(v128 a, unsigned int n) { - return c_v128_shr_n_s8(a, n); -} -SIMD_INLINE v128 v128_shr_n_s16(v128 a, unsigned int n) { - return c_v128_shr_n_s16(a, n); -} -SIMD_INLINE v128 v128_shr_n_s32(v128 a, unsigned int n) { - return c_v128_shr_n_s32(a, n); -} -SIMD_INLINE v128 v128_shr_n_s64(v128 a, unsigned int n) { - return c_v128_shr_n_s64(a, n); -} - -typedef uint32_t sad128_internal_u16; -SIMD_INLINE sad128_internal_u16 v128_sad_u16_init() { - return c_v128_sad_u16_init(); -} -SIMD_INLINE sad128_internal_u16 v128_sad_u16(sad128_internal_u16 s, v128 a, - v128 b) { - return c_v128_sad_u16(s, a, b); -} -SIMD_INLINE uint32_t v128_sad_u16_sum(sad128_internal_u16 s) { - return c_v128_sad_u16_sum(s); -} - -typedef uint64_t ssd128_internal_s16; -SIMD_INLINE ssd128_internal_s16 v128_ssd_s16_init() { - return c_v128_ssd_s16_init(); -} -SIMD_INLINE ssd128_internal_s16 v128_ssd_s16(ssd128_internal_s16 s, v128 a, - v128 b) { - return c_v128_ssd_s16(s, a, b); -} -SIMD_INLINE uint64_t v128_ssd_s16_sum(ssd128_internal_s16 s) { - return c_v128_ssd_s16_sum(s); -} - -#endif // AOM_AOM_DSP_SIMD_V128_INTRINSICS_H_ diff --git a/third_party/aom/aom_dsp/simd/v128_intrinsics_arm.h b/third_party/aom/aom_dsp/simd/v128_intrinsics_arm.h deleted file mode 100644 index 3c669d579..000000000 --- a/third_party/aom/aom_dsp/simd/v128_intrinsics_arm.h +++ /dev/null @@ -1,958 +0,0 @@ -/* - * Copyright (c) 2016, Alliance for Open Media. All rights reserved - * - * This source code is subject to the terms of the BSD 2 Clause License and - * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License - * was not distributed with this source code in the LICENSE file, you can - * obtain it at www.aomedia.org/license/software. If the Alliance for Open - * Media Patent License 1.0 was not distributed with this source code in the - * PATENTS file, you can obtain it at www.aomedia.org/license/patent. - */ - -#ifndef AOM_AOM_DSP_SIMD_V128_INTRINSICS_ARM_H_ -#define AOM_AOM_DSP_SIMD_V128_INTRINSICS_ARM_H_ - -#include <arm_neon.h> - -#include "aom_dsp/simd/v64_intrinsics_arm.h" - -typedef int64x2_t v128; - -SIMD_INLINE uint32_t v128_low_u32(v128 a) { - return v64_low_u32(vget_low_s64(a)); -} - -SIMD_INLINE v64 v128_low_v64(v128 a) { return vget_low_s64(a); } - -SIMD_INLINE v64 v128_high_v64(v128 a) { return vget_high_s64(a); } - -SIMD_INLINE v128 v128_from_v64(v64 a, v64 b) { return vcombine_s64(b, a); } - -SIMD_INLINE v128 v128_from_64(uint64_t a, uint64_t b) { - return vcombine_s64((int64x1_t)b, (int64x1_t)a); -} - -SIMD_INLINE v128 v128_from_32(uint32_t a, uint32_t b, uint32_t c, uint32_t d) { - return vcombine_s64(v64_from_32(c, d), v64_from_32(a, b)); -} - -SIMD_INLINE v128 v128_load_aligned(const void *p) { - return vreinterpretq_s64_u8(vld1q_u8((const uint8_t *)p)); -} - -SIMD_INLINE v128 v128_load_unaligned(const void *p) { - return v128_load_aligned(p); -} - -SIMD_INLINE void v128_store_aligned(void *p, v128 r) { - vst1q_u8((uint8_t *)p, vreinterpretq_u8_s64(r)); -} - -SIMD_INLINE void v128_store_unaligned(void *p, v128 r) { - vst1q_u8((uint8_t *)p, vreinterpretq_u8_s64(r)); -} - -SIMD_INLINE v128 v128_align(v128 a, v128 b, unsigned int c) { -// The following functions require an immediate. -// Some compilers will check this during optimisation, others wont. -#if defined(__OPTIMIZE__) && __OPTIMIZE__ && !defined(__clang__) - return c ? vreinterpretq_s64_s8( - vextq_s8(vreinterpretq_s8_s64(b), vreinterpretq_s8_s64(a), c)) - : b; -#else - return c < 8 ? v128_from_v64(v64_align(v128_low_v64(a), v128_high_v64(b), c), - v64_align(v128_high_v64(b), v128_low_v64(b), c)) - : v128_from_v64( - v64_align(v128_high_v64(a), v128_low_v64(a), c - 8), - v64_align(v128_low_v64(a), v128_high_v64(b), c - 8)); -#endif -} - -SIMD_INLINE v128 v128_zero() { return vreinterpretq_s64_u8(vdupq_n_u8(0)); } - -SIMD_INLINE v128 v128_ones() { return vreinterpretq_s64_u8(vdupq_n_u8(-1)); } - -SIMD_INLINE v128 v128_dup_8(uint8_t x) { - return vreinterpretq_s64_u8(vdupq_n_u8(x)); -} - -SIMD_INLINE v128 v128_dup_16(uint16_t x) { - return vreinterpretq_s64_u16(vdupq_n_u16(x)); -} - -SIMD_INLINE v128 v128_dup_32(uint32_t x) { - return vreinterpretq_s64_u32(vdupq_n_u32(x)); -} - -SIMD_INLINE v128 v128_dup_64(uint64_t x) { - return vreinterpretq_s64_u64(vdupq_n_u64(x)); -} - -SIMD_INLINE int64_t v128_dotp_su8(v128 a, v128 b) { - int16x8_t t1 = vmulq_s16( - vmovl_s8(vreinterpret_s8_s64(vget_low_s64(a))), - vreinterpretq_s16_u16(vmovl_u8(vreinterpret_u8_s64(vget_low_s64(b))))); - int16x8_t t2 = vmulq_s16( - vmovl_s8(vreinterpret_s8_s64(vget_high_s64(a))), - vreinterpretq_s16_u16(vmovl_u8(vreinterpret_u8_s64(vget_high_s64(b))))); -#if defined(__aarch64__) - return vaddlvq_s16(t1) + vaddlvq_s16(t2); -#else - int64x2_t t = vpaddlq_s32(vaddq_s32(vpaddlq_s16(t1), vpaddlq_s16(t2))); - return (int64_t)vget_high_s64(t) + (int64_t)vget_low_s64(t); -#endif -} - -SIMD_INLINE int64_t v128_dotp_s16(v128 a, v128 b) { - return v64_dotp_s16(vget_high_s64(a), vget_high_s64(b)) + - v64_dotp_s16(vget_low_s64(a), vget_low_s64(b)); -} - -SIMD_INLINE int64_t v128_dotp_s32(v128 a, v128 b) { - int64x2_t t = vpaddlq_s32( - vmulq_s32(vreinterpretq_s32_s64(a), vreinterpretq_s32_s64(b))); - return (int64_t)vget_high_s64(t) + (int64_t)vget_low_s64(t); -} - -SIMD_INLINE uint64_t v128_hadd_u8(v128 x) { -#if defined(__aarch64__) - return vaddlvq_u8(vreinterpretq_u8_s64(x)); -#else - uint64x2_t t = vpaddlq_u32(vpaddlq_u16(vpaddlq_u8(vreinterpretq_u8_s64(x)))); - return vget_lane_s32( - vreinterpret_s32_u64(vadd_u64(vget_high_u64(t), vget_low_u64(t))), 0); -#endif -} - -SIMD_INLINE v128 v128_padd_s16(v128 a) { - return vreinterpretq_s64_s32(vpaddlq_s16(vreinterpretq_s16_s64(a))); -} - -SIMD_INLINE v128 v128_padd_u8(v128 a) { - return vreinterpretq_s64_u16(vpaddlq_u8(vreinterpretq_u8_s64(a))); -} - -typedef struct { - sad64_internal hi, lo; -} sad128_internal; - -SIMD_INLINE sad128_internal v128_sad_u8_init() { - sad128_internal s; - s.hi = s.lo = vdupq_n_u16(0); - return s; -} - -/* Implementation dependent return value. Result must be finalised with - v128_sad_u8_sum(). - The result for more than 32 v128_sad_u8() calls is undefined. */ -SIMD_INLINE sad128_internal v128_sad_u8(sad128_internal s, v128 a, v128 b) { - sad128_internal r; - r.hi = v64_sad_u8(s.hi, vget_high_s64(a), vget_high_s64(b)); - r.lo = v64_sad_u8(s.lo, vget_low_s64(a), vget_low_s64(b)); - return r; -} - -SIMD_INLINE uint32_t v128_sad_u8_sum(sad128_internal s) { -#if defined(__aarch64__) - return vaddlvq_u16(s.hi) + vaddlvq_u16(s.lo); -#else - uint64x2_t t = vpaddlq_u32(vpaddlq_u16(vaddq_u16(s.hi, s.lo))); - return (uint32_t)(uint64_t)(vget_high_u64(t) + vget_low_u64(t)); -#endif -} - -typedef struct { - ssd64_internal hi, lo; -} ssd128_internal; - -SIMD_INLINE ssd128_internal v128_ssd_u8_init() { - ssd128_internal s; - s.hi = s.lo = v64_ssd_u8_init(); - return s; -} - -/* Implementation dependent return value. Result must be finalised with - * v128_ssd_u8_sum(). */ -SIMD_INLINE ssd128_internal v128_ssd_u8(ssd128_internal s, v128 a, v128 b) { - ssd128_internal r; - r.hi = v64_ssd_u8(s.hi, vget_high_s64(a), vget_high_s64(b)); - r.lo = v64_ssd_u8(s.lo, vget_low_s64(a), vget_low_s64(b)); - return r; -} - -SIMD_INLINE uint32_t v128_ssd_u8_sum(ssd128_internal s) { - return (uint32_t)(v64_ssd_u8_sum(s.hi) + v64_ssd_u8_sum(s.lo)); -} - -SIMD_INLINE v128 v128_or(v128 x, v128 y) { return vorrq_s64(x, y); } - -SIMD_INLINE v128 v128_xor(v128 x, v128 y) { return veorq_s64(x, y); } - -SIMD_INLINE v128 v128_and(v128 x, v128 y) { return vandq_s64(x, y); } - -SIMD_INLINE v128 v128_andn(v128 x, v128 y) { return vbicq_s64(x, y); } - -SIMD_INLINE v128 v128_add_8(v128 x, v128 y) { - return vreinterpretq_s64_u8( - vaddq_u8(vreinterpretq_u8_s64(x), vreinterpretq_u8_s64(y))); -} - -SIMD_INLINE v128 v128_sadd_u8(v128 x, v128 y) { - return vreinterpretq_s64_u8( - vqaddq_u8(vreinterpretq_u8_s64(x), vreinterpretq_u8_s64(y))); -} - -SIMD_INLINE v128 v128_sadd_s8(v128 x, v128 y) { - return vreinterpretq_s64_s8( - vqaddq_s8(vreinterpretq_s8_s64(x), vreinterpretq_s8_s64(y))); -} - -SIMD_INLINE v128 v128_add_16(v128 x, v128 y) { - return vreinterpretq_s64_s16( - vaddq_s16(vreinterpretq_s16_s64(x), vreinterpretq_s16_s64(y))); -} - -SIMD_INLINE v128 v128_sadd_s16(v128 x, v128 y) { - return vreinterpretq_s64_s16( - vqaddq_s16(vreinterpretq_s16_s64(x), vreinterpretq_s16_s64(y))); -} - -SIMD_INLINE v128 v128_add_32(v128 x, v128 y) { - return vreinterpretq_s64_u32( - vaddq_u32(vreinterpretq_u32_s64(x), vreinterpretq_u32_s64(y))); -} - -SIMD_INLINE v128 v128_add_64(v128 x, v128 y) { - return vreinterpretq_s64_u64( - vaddq_u64(vreinterpretq_u64_s64(x), vreinterpretq_u64_s64(y))); -} - -SIMD_INLINE v128 v128_sub_8(v128 x, v128 y) { - return vreinterpretq_s64_u8( - vsubq_u8(vreinterpretq_u8_s64(x), vreinterpretq_u8_s64(y))); -} - -SIMD_INLINE v128 v128_sub_16(v128 x, v128 y) { - return vreinterpretq_s64_s16( - vsubq_s16(vreinterpretq_s16_s64(x), vreinterpretq_s16_s64(y))); -} - -SIMD_INLINE v128 v128_ssub_s16(v128 x, v128 y) { - return vreinterpretq_s64_s16( - vqsubq_s16(vreinterpretq_s16_s64(x), vreinterpretq_s16_s64(y))); -} - -SIMD_INLINE v128 v128_ssub_u16(v128 x, v128 y) { - return vreinterpretq_s64_u16( - vqsubq_u16(vreinterpretq_u16_s64(x), vreinterpretq_u16_s64(y))); -} - -SIMD_INLINE v128 v128_ssub_u8(v128 x, v128 y) { - return vreinterpretq_s64_u8( - vqsubq_u8(vreinterpretq_u8_s64(x), vreinterpretq_u8_s64(y))); -} - -SIMD_INLINE v128 v128_ssub_s8(v128 x, v128 y) { - return vreinterpretq_s64_s8( - vqsubq_s8(vreinterpretq_s8_s64(x), vreinterpretq_s8_s64(y))); -} - -SIMD_INLINE v128 v128_sub_32(v128 x, v128 y) { - return vreinterpretq_s64_s32( - vsubq_s32(vreinterpretq_s32_s64(x), vreinterpretq_s32_s64(y))); -} - -SIMD_INLINE v128 v128_sub_64(v128 x, v128 y) { return vsubq_s64(x, y); } - -SIMD_INLINE v128 v128_abs_s16(v128 x) { - return vreinterpretq_s64_s16(vabsq_s16(vreinterpretq_s16_s64(x))); -} - -SIMD_INLINE v128 v128_abs_s8(v128 x) { - return vreinterpretq_s64_s8(vabsq_s8(vreinterpretq_s8_s64(x))); -} - -SIMD_INLINE v128 v128_mul_s16(v64 a, v64 b) { - return vreinterpretq_s64_s32( - vmull_s16(vreinterpret_s16_s64(a), vreinterpret_s16_s64(b))); -} - -SIMD_INLINE v128 v128_mullo_s16(v128 a, v128 b) { - return vreinterpretq_s64_s16( - vmulq_s16(vreinterpretq_s16_s64(a), vreinterpretq_s16_s64(b))); -} - -SIMD_INLINE v128 v128_mulhi_s16(v128 a, v128 b) { -#if defined(__aarch64__) - return vreinterpretq_s64_s16(vuzp2q_s16( - vreinterpretq_s16_s32(vmull_s16(vreinterpret_s16_s64(vget_low_s64(a)), - vreinterpret_s16_s64(vget_low_s64(b)))), - vreinterpretq_s16_s32( - vmull_high_s16(vreinterpretq_s16_s64(a), vreinterpretq_s16_s64(b))))); -#else - return v128_from_v64(v64_mulhi_s16(vget_high_s64(a), vget_high_s64(b)), - v64_mulhi_s16(vget_low_s64(a), vget_low_s64(b))); -#endif -} - -SIMD_INLINE v128 v128_mullo_s32(v128 a, v128 b) { - return vreinterpretq_s64_s32( - vmulq_s32(vreinterpretq_s32_s64(a), vreinterpretq_s32_s64(b))); -} - -SIMD_INLINE v128 v128_madd_s16(v128 a, v128 b) { -#if defined(__aarch64__) - int32x4_t t1 = vmull_s16(vreinterpret_s16_s64(vget_low_s64(a)), - vreinterpret_s16_s64(vget_low_s64(b))); - int32x4_t t2 = - vmull_high_s16(vreinterpretq_s16_s64(a), vreinterpretq_s16_s64(b)); - return vreinterpretq_s64_s32(vpaddq_s32(t1, t2)); -#else - return v128_from_v64(v64_madd_s16(vget_high_s64(a), vget_high_s64(b)), - v64_madd_s16(vget_low_s64(a), vget_low_s64(b))); -#endif -} - -SIMD_INLINE v128 v128_madd_us8(v128 a, v128 b) { -#if defined(__aarch64__) - int16x8_t t1 = vmulq_s16( - vreinterpretq_s16_u16(vmovl_u8(vreinterpret_u8_s64(vget_low_s64(a)))), - vmovl_s8(vreinterpret_s8_s64(vget_low_s64(b)))); - int16x8_t t2 = vmulq_s16( - vreinterpretq_s16_u16(vmovl_u8(vreinterpret_u8_s64(vget_high_s64(a)))), - vmovl_s8(vreinterpret_s8_s64(vget_high_s64(b)))); - return vreinterpretq_s64_s16( - vqaddq_s16(vuzp1q_s16(t1, t2), vuzp2q_s16(t1, t2))); -#else - return v128_from_v64(v64_madd_us8(vget_high_s64(a), vget_high_s64(b)), - v64_madd_us8(vget_low_s64(a), vget_low_s64(b))); -#endif -} - -SIMD_INLINE v128 v128_avg_u8(v128 x, v128 y) { - return vreinterpretq_s64_u8( - vrhaddq_u8(vreinterpretq_u8_s64(x), vreinterpretq_u8_s64(y))); -} - -SIMD_INLINE v128 v128_rdavg_u8(v128 x, v128 y) { - return vreinterpretq_s64_u8( - vhaddq_u8(vreinterpretq_u8_s64(x), vreinterpretq_u8_s64(y))); -} - -SIMD_INLINE v128 v128_rdavg_u16(v128 x, v128 y) { - return vreinterpretq_s64_u16( - vhaddq_u16(vreinterpretq_u16_s64(x), vreinterpretq_u16_s64(y))); -} - -SIMD_INLINE v128 v128_avg_u16(v128 x, v128 y) { - return vreinterpretq_s64_u16( - vrhaddq_u16(vreinterpretq_u16_s64(x), vreinterpretq_u16_s64(y))); -} - -SIMD_INLINE v128 v128_min_u8(v128 x, v128 y) { - return vreinterpretq_s64_u8( - vminq_u8(vreinterpretq_u8_s64(x), vreinterpretq_u8_s64(y))); -} - -SIMD_INLINE v128 v128_max_u8(v128 x, v128 y) { - return vreinterpretq_s64_u8( - vmaxq_u8(vreinterpretq_u8_s64(x), vreinterpretq_u8_s64(y))); -} - -SIMD_INLINE v128 v128_min_s8(v128 x, v128 y) { - return vreinterpretq_s64_s8( - vminq_s8(vreinterpretq_s8_s64(x), vreinterpretq_s8_s64(y))); -} - -SIMD_INLINE uint32_t v128_movemask_8(v128 a) { - a = vreinterpretq_s64_u8(vcltq_s8(vreinterpretq_s8_s64(a), vdupq_n_s8(0))); -#if defined(__aarch64__) - uint8x16_t m = - vandq_u8(vreinterpretq_u8_s64(a), - vreinterpretq_u8_u64(vdupq_n_u64(0x8040201008040201ULL))); - return vaddv_u8(vget_low_u8(m)) + (vaddv_u8(vget_high_u8(m)) << 8); -#else - uint64x2_t m = vpaddlq_u32(vpaddlq_u16(vpaddlq_u8( - vandq_u8(vreinterpretq_u8_s64(a), - vreinterpretq_u8_u64(vdupq_n_u64(0x8040201008040201ULL)))))); - return v64_low_u32( - v64_ziplo_8(v128_high_v64((v128)m), v128_low_v64((v128)m))); -#endif -} - -SIMD_INLINE v128 v128_blend_8(v128 a, v128 b, v128 c) { - c = vreinterpretq_s64_u8(vcltq_s8(vreinterpretq_s8_s64(c), vdupq_n_s8(0))); - return v128_or(v128_and(b, c), v128_andn(a, c)); -} - -SIMD_INLINE v128 v128_max_s8(v128 x, v128 y) { - return vreinterpretq_s64_s8( - vmaxq_s8(vreinterpretq_s8_s64(x), vreinterpretq_s8_s64(y))); -} - -SIMD_INLINE v128 v128_min_s16(v128 x, v128 y) { - return vreinterpretq_s64_s16( - vminq_s16(vreinterpretq_s16_s64(x), vreinterpretq_s16_s64(y))); -} - -SIMD_INLINE v128 v128_max_s16(v128 x, v128 y) { - return vreinterpretq_s64_s16( - vmaxq_s16(vreinterpretq_s16_s64(x), vreinterpretq_s16_s64(y))); -} - -SIMD_INLINE v128 v128_min_s32(v128 x, v128 y) { - return vreinterpretq_s64_s32( - vminq_s32(vreinterpretq_s32_s64(x), vreinterpretq_s32_s64(y))); -} - -SIMD_INLINE v128 v128_max_s32(v128 x, v128 y) { - return vreinterpretq_s64_s32( - vmaxq_s32(vreinterpretq_s32_s64(x), vreinterpretq_s32_s64(y))); -} - -SIMD_INLINE v128 v128_ziplo_8(v128 x, v128 y) { -#if defined(__aarch64__) - return vreinterpretq_s64_u8( - vzip1q_u8(vreinterpretq_u8_s64(y), vreinterpretq_u8_s64(x))); -#else - uint8x16x2_t r = vzipq_u8(vreinterpretq_u8_s64(y), vreinterpretq_u8_s64(x)); - return vreinterpretq_s64_u8(r.val[0]); -#endif -} - -SIMD_INLINE v128 v128_ziphi_8(v128 x, v128 y) { -#if defined(__aarch64__) - return vreinterpretq_s64_u8( - vzip2q_u8(vreinterpretq_u8_s64(y), vreinterpretq_u8_s64(x))); -#else - uint8x16x2_t r = vzipq_u8(vreinterpretq_u8_s64(y), vreinterpretq_u8_s64(x)); - return vreinterpretq_s64_u8(r.val[1]); -#endif -} - -SIMD_INLINE v128 v128_zip_8(v64 x, v64 y) { - uint8x8x2_t r = vzip_u8(vreinterpret_u8_s64(y), vreinterpret_u8_s64(x)); - return vreinterpretq_s64_u8(vcombine_u8(r.val[0], r.val[1])); -} - -SIMD_INLINE v128 v128_ziplo_16(v128 x, v128 y) { -#if defined(__aarch64__) - return vreinterpretq_s64_u16( - vzip1q_u16(vreinterpretq_u16_s64(y), vreinterpretq_u16_s64(x))); -#else - int16x8x2_t r = vzipq_s16(vreinterpretq_s16_s64(y), vreinterpretq_s16_s64(x)); - return vreinterpretq_s64_s16(r.val[0]); -#endif -} - -SIMD_INLINE v128 v128_ziphi_16(v128 x, v128 y) { -#if defined(__aarch64__) - return vreinterpretq_s64_u16( - vzip2q_u16(vreinterpretq_u16_s64(y), vreinterpretq_u16_s64(x))); -#else - int16x8x2_t r = vzipq_s16(vreinterpretq_s16_s64(y), vreinterpretq_s16_s64(x)); - return vreinterpretq_s64_s16(r.val[1]); -#endif -} - -SIMD_INLINE v128 v128_zip_16(v64 x, v64 y) { - uint16x4x2_t r = vzip_u16(vreinterpret_u16_s64(y), vreinterpret_u16_s64(x)); - return vreinterpretq_s64_u16(vcombine_u16(r.val[0], r.val[1])); -} - -SIMD_INLINE v128 v128_ziplo_32(v128 x, v128 y) { -#if defined(__aarch64__) - return vreinterpretq_s64_u32( - vzip1q_u32(vreinterpretq_u32_s64(y), vreinterpretq_u32_s64(x))); -#else - int32x4x2_t r = vzipq_s32(vreinterpretq_s32_s64(y), vreinterpretq_s32_s64(x)); - return vreinterpretq_s64_s32(r.val[0]); -#endif -} - -SIMD_INLINE v128 v128_ziphi_32(v128 x, v128 y) { -#if defined(__aarch64__) - return vreinterpretq_s64_u32( - vzip2q_u32(vreinterpretq_u32_s64(y), vreinterpretq_u32_s64(x))); -#else - int32x4x2_t r = vzipq_s32(vreinterpretq_s32_s64(y), vreinterpretq_s32_s64(x)); - return vreinterpretq_s64_s32(r.val[1]); -#endif -} - -SIMD_INLINE v128 v128_zip_32(v64 x, v64 y) { - uint32x2x2_t r = vzip_u32(vreinterpret_u32_s64(y), vreinterpret_u32_s64(x)); - return vreinterpretq_s64_u32(vcombine_u32(r.val[0], r.val[1])); -} - -SIMD_INLINE v128 v128_ziplo_64(v128 a, v128 b) { - return v128_from_v64(vget_low_s64((int64x2_t)a), vget_low_s64((int64x2_t)b)); -} - -SIMD_INLINE v128 v128_ziphi_64(v128 a, v128 b) { - return v128_from_v64(vget_high_s64((int64x2_t)a), - vget_high_s64((int64x2_t)b)); -} - -SIMD_INLINE v128 v128_unziplo_8(v128 x, v128 y) { -#if defined(__aarch64__) - return vreinterpretq_s64_u8( - vuzp1q_u8(vreinterpretq_u8_s64(y), vreinterpretq_u8_s64(x))); -#else - uint8x16x2_t r = vuzpq_u8(vreinterpretq_u8_s64(y), vreinterpretq_u8_s64(x)); - return vreinterpretq_s64_u8(r.val[0]); -#endif -} - -SIMD_INLINE v128 v128_unziphi_8(v128 x, v128 y) { -#if defined(__aarch64__) - return vreinterpretq_s64_u8( - vuzp2q_u8(vreinterpretq_u8_s64(y), vreinterpretq_u8_s64(x))); -#else - uint8x16x2_t r = vuzpq_u8(vreinterpretq_u8_s64(y), vreinterpretq_u8_s64(x)); - return vreinterpretq_s64_u8(r.val[1]); -#endif -} - -SIMD_INLINE v128 v128_unziplo_16(v128 x, v128 y) { -#if defined(__aarch64__) - return vreinterpretq_s64_u16( - vuzp1q_u16(vreinterpretq_u16_s64(y), vreinterpretq_u16_s64(x))); -#else - uint16x8x2_t r = - vuzpq_u16(vreinterpretq_u16_s64(y), vreinterpretq_u16_s64(x)); - return vreinterpretq_s64_u16(r.val[0]); -#endif -} - -SIMD_INLINE v128 v128_unziphi_16(v128 x, v128 y) { -#if defined(__aarch64__) - return vreinterpretq_s64_u16( - vuzp2q_u16(vreinterpretq_u16_s64(y), vreinterpretq_u16_s64(x))); -#else - uint16x8x2_t r = - vuzpq_u16(vreinterpretq_u16_s64(y), vreinterpretq_u16_s64(x)); - return vreinterpretq_s64_u16(r.val[1]); -#endif -} - -SIMD_INLINE v128 v128_unziplo_32(v128 x, v128 y) { -#if defined(__aarch64__) - return vreinterpretq_s64_u32( - vuzp1q_u32(vreinterpretq_u32_s64(y), vreinterpretq_u32_s64(x))); -#else - uint32x4x2_t r = - vuzpq_u32(vreinterpretq_u32_s64(y), vreinterpretq_u32_s64(x)); - return vreinterpretq_s64_u32(r.val[0]); -#endif -} - -SIMD_INLINE v128 v128_unziphi_32(v128 x, v128 y) { -#if defined(__aarch64__) - return vreinterpretq_s64_u32( - vuzp2q_u32(vreinterpretq_u32_s64(y), vreinterpretq_u32_s64(x))); -#else - uint32x4x2_t r = - vuzpq_u32(vreinterpretq_u32_s64(y), vreinterpretq_u32_s64(x)); - return vreinterpretq_s64_u32(r.val[1]); -#endif -} - -SIMD_INLINE v128 v128_unpack_u8_s16(v64 a) { - return vreinterpretq_s64_u16(vmovl_u8(vreinterpret_u8_s64(a))); -} - -SIMD_INLINE v128 v128_unpacklo_u8_s16(v128 a) { - return vreinterpretq_s64_u16(vmovl_u8(vreinterpret_u8_s64(vget_low_s64(a)))); -} - -SIMD_INLINE v128 v128_unpackhi_u8_s16(v128 a) { - return vreinterpretq_s64_u16(vmovl_u8(vreinterpret_u8_s64(vget_high_s64(a)))); -} - -SIMD_INLINE v128 v128_unpack_s8_s16(v64 a) { - return vreinterpretq_s64_s16(vmovl_s8(vreinterpret_s8_s64(a))); -} - -SIMD_INLINE v128 v128_unpacklo_s8_s16(v128 a) { - return vreinterpretq_s64_s16(vmovl_s8(vreinterpret_s8_s64(vget_low_s64(a)))); -} - -SIMD_INLINE v128 v128_unpackhi_s8_s16(v128 a) { - return vreinterpretq_s64_s16(vmovl_s8(vreinterpret_s8_s64(vget_high_s64(a)))); -} - -SIMD_INLINE v128 v128_pack_s32_s16(v128 a, v128 b) { - return v128_from_v64( - vreinterpret_s64_s16(vqmovn_s32(vreinterpretq_s32_s64(a))), - vreinterpret_s64_s16(vqmovn_s32(vreinterpretq_s32_s64(b)))); -} - -SIMD_INLINE v128 v128_pack_s32_u16(v128 a, v128 b) { - return v128_from_v64( - vreinterpret_s64_u16(vqmovun_s32(vreinterpretq_s32_s64(a))), - vreinterpret_s64_u16(vqmovun_s32(vreinterpretq_s32_s64(b)))); -} - -SIMD_INLINE v128 v128_pack_s16_u8(v128 a, v128 b) { - return v128_from_v64( - vreinterpret_s64_u8(vqmovun_s16(vreinterpretq_s16_s64(a))), - vreinterpret_s64_u8(vqmovun_s16(vreinterpretq_s16_s64(b)))); -} - -SIMD_INLINE v128 v128_pack_s16_s8(v128 a, v128 b) { - return v128_from_v64( - vreinterpret_s64_s8(vqmovn_s16(vreinterpretq_s16_s64(a))), - vreinterpret_s64_s8(vqmovn_s16(vreinterpretq_s16_s64(b)))); -} - -SIMD_INLINE v128 v128_unpack_u16_s32(v64 a) { - return vreinterpretq_s64_u32(vmovl_u16(vreinterpret_u16_s64(a))); -} - -SIMD_INLINE v128 v128_unpack_s16_s32(v64 a) { - return vreinterpretq_s64_s32(vmovl_s16(vreinterpret_s16_s64(a))); -} - -SIMD_INLINE v128 v128_unpacklo_u16_s32(v128 a) { - return vreinterpretq_s64_u32( - vmovl_u16(vreinterpret_u16_s64(vget_low_s64(a)))); -} - -SIMD_INLINE v128 v128_unpacklo_s16_s32(v128 a) { - return vreinterpretq_s64_s32( - vmovl_s16(vreinterpret_s16_s64(vget_low_s64(a)))); -} - -SIMD_INLINE v128 v128_unpackhi_u16_s32(v128 a) { - return vreinterpretq_s64_u32( - vmovl_u16(vreinterpret_u16_s64(vget_high_s64(a)))); -} - -SIMD_INLINE v128 v128_unpackhi_s16_s32(v128 a) { - return vreinterpretq_s64_s32( - vmovl_s16(vreinterpret_s16_s64(vget_high_s64(a)))); -} - -SIMD_INLINE v128 v128_shuffle_8(v128 x, v128 pattern) { -#if defined(__aarch64__) - return vreinterpretq_s64_u8( - vqtbl1q_u8(vreinterpretq_u8_s64(x), vreinterpretq_u8_s64(pattern))); -#else - uint8x8x2_t p = { { vget_low_u8(vreinterpretq_u8_s64(x)), - vget_high_u8(vreinterpretq_u8_s64(x)) } }; - return v128_from_64((uint64_t)vreinterpret_s64_u8(vtbl2_u8( - p, vreinterpret_u8_s64(vget_high_s64(pattern)))), - (uint64_t)vreinterpret_s64_u8(vtbl2_u8( - p, vreinterpret_u8_s64(vget_low_s64(pattern))))); -#endif -} - -SIMD_INLINE v128 v128_cmpgt_s8(v128 x, v128 y) { - return vreinterpretq_s64_u8( - vcgtq_s8(vreinterpretq_s8_s64(x), vreinterpretq_s8_s64(y))); -} - -SIMD_INLINE v128 v128_cmplt_s8(v128 x, v128 y) { - return vreinterpretq_s64_u8( - vcltq_s8(vreinterpretq_s8_s64(x), vreinterpretq_s8_s64(y))); -} - -SIMD_INLINE v128 v128_cmpeq_8(v128 x, v128 y) { - return vreinterpretq_s64_u8( - vceqq_u8(vreinterpretq_u8_s64(x), vreinterpretq_u8_s64(y))); -} - -SIMD_INLINE v128 v128_cmpgt_s16(v128 x, v128 y) { - return vreinterpretq_s64_u16( - vcgtq_s16(vreinterpretq_s16_s64(x), vreinterpretq_s16_s64(y))); -} - -SIMD_INLINE v128 v128_cmplt_s16(v128 x, v128 y) { - return vreinterpretq_s64_u16( - vcltq_s16(vreinterpretq_s16_s64(x), vreinterpretq_s16_s64(y))); -} - -SIMD_INLINE v128 v128_cmpeq_16(v128 x, v128 y) { - return vreinterpretq_s64_u16( - vceqq_s16(vreinterpretq_s16_s64(x), vreinterpretq_s16_s64(y))); -} - -SIMD_INLINE v128 v128_cmpgt_s32(v128 x, v128 y) { - return vreinterpretq_s64_u32( - vcgtq_s32(vreinterpretq_s32_s64(x), vreinterpretq_s32_s64(y))); -} - -SIMD_INLINE v128 v128_cmplt_s32(v128 x, v128 y) { - return vreinterpretq_s64_u32( - vcltq_s32(vreinterpretq_s32_s64(x), vreinterpretq_s32_s64(y))); -} - -SIMD_INLINE v128 v128_cmpeq_32(v128 x, v128 y) { - return vreinterpretq_s64_u32( - vceqq_s32(vreinterpretq_s32_s64(x), vreinterpretq_s32_s64(y))); -} - -SIMD_INLINE v128 v128_shl_8(v128 a, unsigned int c) { - return (c > 7) ? v128_zero() - : vreinterpretq_s64_u8( - vshlq_u8(vreinterpretq_u8_s64(a), vdupq_n_s8(c))); -} - -SIMD_INLINE v128 v128_shr_u8(v128 a, unsigned int c) { - return (c > 7) ? v128_zero() - : vreinterpretq_s64_u8( - vshlq_u8(vreinterpretq_u8_s64(a), vdupq_n_s8(-c))); -} - -SIMD_INLINE v128 v128_shr_s8(v128 a, unsigned int c) { - return (c > 7) ? v128_ones() - : vreinterpretq_s64_s8( - vshlq_s8(vreinterpretq_s8_s64(a), vdupq_n_s8(-c))); -} - -SIMD_INLINE v128 v128_shl_16(v128 a, unsigned int c) { - return (c > 15) ? v128_zero() - : vreinterpretq_s64_u16( - vshlq_u16(vreinterpretq_u16_s64(a), vdupq_n_s16(c))); -} - -SIMD_INLINE v128 v128_shr_u16(v128 a, unsigned int c) { - return (c > 15) ? v128_zero() - : vreinterpretq_s64_u16( - vshlq_u16(vreinterpretq_u16_s64(a), vdupq_n_s16(-c))); -} - -SIMD_INLINE v128 v128_shr_s16(v128 a, unsigned int c) { - return (c > 15) ? v128_ones() - : vreinterpretq_s64_s16( - vshlq_s16(vreinterpretq_s16_s64(a), vdupq_n_s16(-c))); -} - -SIMD_INLINE v128 v128_shl_32(v128 a, unsigned int c) { - return (c > 31) ? v128_zero() - : vreinterpretq_s64_u32( - vshlq_u32(vreinterpretq_u32_s64(a), vdupq_n_s32(c))); -} - -SIMD_INLINE v128 v128_shr_u32(v128 a, unsigned int c) { - return (c > 31) ? v128_zero() - : vreinterpretq_s64_u32( - vshlq_u32(vreinterpretq_u32_s64(a), vdupq_n_s32(-c))); -} - -SIMD_INLINE v128 v128_shr_s32(v128 a, unsigned int c) { - return (c > 31) ? v128_ones() - : vreinterpretq_s64_s32( - vshlq_s32(vreinterpretq_s32_s64(a), vdupq_n_s32(-c))); -} - -SIMD_INLINE v128 v128_shl_64(v128 a, unsigned int c) { - return (c > 63) ? v128_zero() - : vreinterpretq_s64_u64( - vshlq_u64(vreinterpretq_u64_s64(a), vdupq_n_s64(c))); -} - -SIMD_INLINE v128 v128_shr_u64(v128 a, unsigned int c) { - return (c > 63) ? v128_zero() - : vreinterpretq_s64_u64( - vshlq_u64(vreinterpretq_u64_s64(a), vdupq_n_s64(-c))); -} - -SIMD_INLINE v128 v128_shr_s64(v128 a, unsigned int c) { - return (c > 63) ? v128_ones() : vshlq_s64(a, vdupq_n_s64(-c)); -} - -#if defined(__OPTIMIZE__) && __OPTIMIZE__ && !defined(__clang__) - -SIMD_INLINE v128 v128_shl_n_byte(v128 a, unsigned int n) { - return n < 8 - ? v128_from_64( - (uint64_t)vorr_u64( - vshl_n_u64(vreinterpret_u64_s64(vget_high_s64(a)), - n * 8), - vshr_n_u64(vreinterpret_u64_s64(vget_low_s64(a)), - (8 - n) * 8)), - (uint64_t)vshl_n_u64(vreinterpret_u64_s64(vget_low_s64(a)), - n * 8)) - : (n == 8 ? v128_from_64( - (uint64_t)vreinterpret_u64_s64(vget_low_s64(a)), 0) - : v128_from_64((uint64_t)vshl_n_u64( - vreinterpret_u64_s64(vget_low_s64(a)), - (n - 8) * 8), - 0)); -} - -SIMD_INLINE v128 v128_shr_n_byte(v128 a, unsigned int n) { - return n < 8 - ? v128_from_64( - (uint64_t)vshr_n_u64(vreinterpret_u64_s64(vget_high_s64(a)), - n * 8), - (uint64_t)vorr_u64( - vshr_n_u64(vreinterpret_u64_s64(vget_low_s64(a)), n * 8), - vshl_n_u64(vreinterpret_u64_s64(vget_high_s64(a)), - (8 - n) * 8))) - : (n == 8 ? v128_from_64(0, (uint64_t)vreinterpret_u64_s64( - vget_high_s64(a))) - : v128_from_64( - 0, (uint64_t)vshr_n_u64( - vreinterpret_u64_s64(vget_high_s64(a)), - (n - 8) * 8))); -} - -SIMD_INLINE v128 v128_shl_n_8(v128 a, unsigned int c) { - return vreinterpretq_s64_u8(vshlq_n_u8(vreinterpretq_u8_s64(a), c)); -} - -SIMD_INLINE v128 v128_shr_n_u8(v128 a, unsigned int c) { - return vreinterpretq_s64_u8(vshrq_n_u8(vreinterpretq_u8_s64(a), c)); -} - -SIMD_INLINE v128 v128_shr_n_s8(v128 a, unsigned int c) { - return vreinterpretq_s64_s8(vshrq_n_s8(vreinterpretq_s8_s64(a), c)); -} - -SIMD_INLINE v128 v128_shl_n_16(v128 a, unsigned int c) { - return vreinterpretq_s64_u16(vshlq_n_u16(vreinterpretq_u16_s64(a), c)); -} - -SIMD_INLINE v128 v128_shr_n_u16(v128 a, unsigned int c) { - return vreinterpretq_s64_u16(vshrq_n_u16(vreinterpretq_u16_s64(a), c)); -} - -SIMD_INLINE v128 v128_shr_n_s16(v128 a, unsigned int c) { - return vreinterpretq_s64_s16(vshrq_n_s16(vreinterpretq_s16_s64(a), c)); -} - -SIMD_INLINE v128 v128_shl_n_32(v128 a, unsigned int c) { - return vreinterpretq_s64_u32(vshlq_n_u32(vreinterpretq_u32_s64(a), c)); -} - -SIMD_INLINE v128 v128_shr_n_u32(v128 a, unsigned int c) { - return vreinterpretq_s64_u32(vshrq_n_u32(vreinterpretq_u32_s64(a), c)); -} - -SIMD_INLINE v128 v128_shr_n_s32(v128 a, unsigned int c) { - return vreinterpretq_s64_s32(vshrq_n_s32(vreinterpretq_s32_s64(a), c)); -} - -SIMD_INLINE v128 v128_shl_n_64(v128 a, unsigned int c) { - return vreinterpretq_s64_u64(vshlq_n_u64(vreinterpretq_u64_s64(a), c)); -} - -SIMD_INLINE v128 v128_shr_n_u64(v128 a, unsigned int c) { - return vreinterpretq_s64_u64(vshrq_n_u64(vreinterpretq_u64_s64(a), c)); -} - -SIMD_INLINE v128 v128_shr_n_s64(v128 a, unsigned int c) { - return vshrq_n_s64(a, c); -} - -#else - -SIMD_INLINE v128 v128_shl_n_byte(v128 a, unsigned int n) { - if (n < 8) - return v128_from_v64(v64_or(v64_shl_n_byte(v128_high_v64(a), n), - v64_shr_n_byte(v128_low_v64(a), 8 - n)), - v64_shl_n_byte(v128_low_v64(a), n)); - else - return v128_from_v64(v64_shl_n_byte(v128_low_v64(a), n - 8), v64_zero()); -} - -SIMD_INLINE v128 v128_shr_n_byte(v128 a, unsigned int n) { - if (n < 8) - return v128_from_v64(v64_shr_n_byte(v128_high_v64(a), n), - v64_or(v64_shr_n_byte(v128_low_v64(a), n), - v64_shl_n_byte(v128_high_v64(a), 8 - n))); - else - return v128_from_v64(v64_zero(), v64_shr_n_byte(v128_high_v64(a), n - 8)); -} - -SIMD_INLINE v128 v128_shl_n_8(v128 a, unsigned int c) { - return v128_shl_8(a, c); -} - -SIMD_INLINE v128 v128_shr_n_u8(v128 a, unsigned int c) { - return v128_shr_u8(a, c); -} - -SIMD_INLINE v128 v128_shr_n_s8(v128 a, unsigned int c) { - return v128_shr_s8(a, c); -} - -SIMD_INLINE v128 v128_shl_n_16(v128 a, unsigned int c) { - return v128_shl_16(a, c); -} - -SIMD_INLINE v128 v128_shr_n_u16(v128 a, unsigned int c) { - return v128_shr_u16(a, c); -} - -SIMD_INLINE v128 v128_shr_n_s16(v128 a, unsigned int c) { - return v128_shr_s16(a, c); -} - -SIMD_INLINE v128 v128_shl_n_32(v128 a, unsigned int c) { - return v128_shl_32(a, c); -} - -SIMD_INLINE v128 v128_shr_n_u32(v128 a, unsigned int c) { - return v128_shr_u32(a, c); -} - -SIMD_INLINE v128 v128_shr_n_s32(v128 a, unsigned int c) { - return v128_shr_s32(a, c); -} - -SIMD_INLINE v128 v128_shl_n_64(v128 a, unsigned int c) { - return v128_shl_64(a, c); -} - -SIMD_INLINE v128 v128_shr_n_u64(v128 a, unsigned int c) { - return v128_shr_u64(a, c); -} - -SIMD_INLINE v128 v128_shr_n_s64(v128 a, unsigned int c) { - return v128_shr_s64(a, c); -} - -#endif - -typedef uint32x4_t sad128_internal_u16; - -SIMD_INLINE sad128_internal_u16 v128_sad_u16_init() { return vdupq_n_u32(0); } - -/* Implementation dependent return value. Result must be finalised with - * v128_sad_u16_sum(). */ -SIMD_INLINE sad128_internal_u16 v128_sad_u16(sad128_internal_u16 s, v128 a, - v128 b) { - return vaddq_u32( - s, vpaddlq_u16(vsubq_u16( - vmaxq_u16(vreinterpretq_u16_s64(a), vreinterpretq_u16_s64(b)), - vminq_u16(vreinterpretq_u16_s64(a), vreinterpretq_u16_s64(b))))); -} - -SIMD_INLINE uint32_t v128_sad_u16_sum(sad128_internal_u16 s) { - uint64x2_t t = vpaddlq_u32(s); - return (uint32_t)(uint64_t)vget_high_u64(t) + - (uint32_t)(uint64_t)vget_low_u64(t); -} - -typedef v128 ssd128_internal_s16; -SIMD_INLINE ssd128_internal_s16 v128_ssd_s16_init() { return v128_zero(); } - -/* Implementation dependent return value. Result must be finalised with - * v128_ssd_s16_sum(). */ -SIMD_INLINE ssd128_internal_s16 v128_ssd_s16(ssd128_internal_s16 s, v128 a, - v128 b) { - v128 d = v128_sub_16(a, b); - d = v128_madd_s16(d, d); - return v128_add_64( - s, vreinterpretq_s64_u64(vpaddlq_u32(vreinterpretq_u32_s64(d)))); -} - -SIMD_INLINE uint64_t v128_ssd_s16_sum(ssd128_internal_s16 s) { - return v64_u64(v128_low_v64(s)) + v64_u64(v128_high_v64(s)); -} - -#endif // AOM_AOM_DSP_SIMD_V128_INTRINSICS_ARM_H_ diff --git a/third_party/aom/aom_dsp/simd/v128_intrinsics_c.h b/third_party/aom/aom_dsp/simd/v128_intrinsics_c.h deleted file mode 100644 index bbe9a9d28..000000000 --- a/third_party/aom/aom_dsp/simd/v128_intrinsics_c.h +++ /dev/null @@ -1,888 +0,0 @@ -/* - * Copyright (c) 2016, Alliance for Open Media. All rights reserved - * - * This source code is subject to the terms of the BSD 2 Clause License and - * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License - * was not distributed with this source code in the LICENSE file, you can - * obtain it at www.aomedia.org/license/software. If the Alliance for Open - * Media Patent License 1.0 was not distributed with this source code in the - * PATENTS file, you can obtain it at www.aomedia.org/license/patent. - */ - -#ifndef AOM_AOM_DSP_SIMD_V128_INTRINSICS_C_H_ -#define AOM_AOM_DSP_SIMD_V128_INTRINSICS_C_H_ - -#include <stdio.h> -#include <stdlib.h> - -#include "config/aom_config.h" - -#include "aom_dsp/simd/v64_intrinsics_c.h" - -typedef union { - uint8_t u8[16]; - uint16_t u16[8]; - uint32_t u32[4]; - uint64_t u64[2]; - int8_t s8[16]; - int16_t s16[8]; - int32_t s32[4]; - int64_t s64[2]; - c_v64 v64[2]; -} c_v128; - -SIMD_INLINE uint32_t c_v128_low_u32(c_v128 a) { return a.u32[0]; } - -SIMD_INLINE c_v64 c_v128_low_v64(c_v128 a) { return a.v64[0]; } - -SIMD_INLINE c_v64 c_v128_high_v64(c_v128 a) { return a.v64[1]; } - -SIMD_INLINE c_v128 c_v128_from_64(uint64_t hi, uint64_t lo) { - c_v128 t; - t.u64[1] = hi; - t.u64[0] = lo; - return t; -} - -SIMD_INLINE c_v128 c_v128_from_v64(c_v64 hi, c_v64 lo) { - c_v128 t; - t.v64[1] = hi; - t.v64[0] = lo; - return t; -} - -SIMD_INLINE c_v128 c_v128_from_32(uint32_t a, uint32_t b, uint32_t c, - uint32_t d) { - c_v128 t; - t.u32[3] = a; - t.u32[2] = b; - t.u32[1] = c; - t.u32[0] = d; - return t; -} - -SIMD_INLINE c_v128 c_v128_load_unaligned(const void *p) { - c_v128 t; - uint8_t *pp = (uint8_t *)p; - uint8_t *q = (uint8_t *)&t; - int c; - for (c = 0; c < 16; c++) q[c] = pp[c]; - return t; -} - -SIMD_INLINE c_v128 c_v128_load_aligned(const void *p) { - if (SIMD_CHECK && (uintptr_t)p & 15) { - fprintf(stderr, "Error: unaligned v128 load at %p\n", p); - abort(); - } - return c_v128_load_unaligned(p); -} - -SIMD_INLINE void c_v128_store_unaligned(void *p, c_v128 a) { - uint8_t *pp = (uint8_t *)p; - uint8_t *q = (uint8_t *)&a; - int c; - for (c = 0; c < 16; c++) pp[c] = q[c]; -} - -SIMD_INLINE void c_v128_store_aligned(void *p, c_v128 a) { - if (SIMD_CHECK && (uintptr_t)p & 15) { - fprintf(stderr, "Error: unaligned v128 store at %p\n", p); - abort(); - } - c_v128_store_unaligned(p, a); -} - -SIMD_INLINE c_v128 c_v128_zero() { - c_v128 t; - t.u64[1] = t.u64[0] = 0; - return t; -} - -SIMD_INLINE c_v128 c_v128_dup_8(uint8_t x) { - c_v128 t; - t.v64[1] = t.v64[0] = c_v64_dup_8(x); - return t; -} - -SIMD_INLINE c_v128 c_v128_dup_16(uint16_t x) { - c_v128 t; - t.v64[1] = t.v64[0] = c_v64_dup_16(x); - return t; -} - -SIMD_INLINE c_v128 c_v128_dup_32(uint32_t x) { - c_v128 t; - t.v64[1] = t.v64[0] = c_v64_dup_32(x); - return t; -} - -SIMD_INLINE c_v128 c_v128_dup_64(uint64_t x) { - c_v128 t; - t.u64[1] = t.u64[0] = x; - return t; -} - -SIMD_INLINE int64_t c_v128_dotp_su8(c_v128 a, c_v128 b) { - return c_v64_dotp_su8(a.v64[1], b.v64[1]) + - c_v64_dotp_su8(a.v64[0], b.v64[0]); -} - -SIMD_INLINE int64_t c_v128_dotp_s16(c_v128 a, c_v128 b) { - return c_v64_dotp_s16(a.v64[1], b.v64[1]) + - c_v64_dotp_s16(a.v64[0], b.v64[0]); -} - -SIMD_INLINE int64_t c_v128_dotp_s32(c_v128 a, c_v128 b) { - // 32 bit products, 64 bit sum - return (int64_t)(int32_t)((int64_t)a.s32[3] * b.s32[3]) + - (int64_t)(int32_t)((int64_t)a.s32[2] * b.s32[2]) + - (int64_t)(int32_t)((int64_t)a.s32[1] * b.s32[1]) + - (int64_t)(int32_t)((int64_t)a.s32[0] * b.s32[0]); -} - -SIMD_INLINE uint64_t c_v128_hadd_u8(c_v128 a) { - return c_v64_hadd_u8(a.v64[1]) + c_v64_hadd_u8(a.v64[0]); -} - -typedef uint32_t c_sad128_internal; - -SIMD_INLINE c_sad128_internal c_v128_sad_u8_init() { return 0; } - -/* Implementation dependent return value. Result must be finalised with - v128_sad_u8_sum(). - The result for more than 32 v128_sad_u8() calls is undefined. */ -SIMD_INLINE c_sad128_internal c_v128_sad_u8(c_sad128_internal s, c_v128 a, - c_v128 b) { - int c; - for (c = 0; c < 16; c++) - s += a.u8[c] > b.u8[c] ? a.u8[c] - b.u8[c] : b.u8[c] - a.u8[c]; - return s; -} - -SIMD_INLINE uint32_t c_v128_sad_u8_sum(c_sad128_internal s) { return s; } - -typedef uint32_t c_ssd128_internal; - -SIMD_INLINE c_ssd128_internal c_v128_ssd_u8_init() { return 0; } - -/* Implementation dependent return value. Result must be finalised with - * v128_ssd_u8_sum(). */ -SIMD_INLINE c_ssd128_internal c_v128_ssd_u8(c_ssd128_internal s, c_v128 a, - c_v128 b) { - int c; - for (c = 0; c < 16; c++) s += (a.u8[c] - b.u8[c]) * (a.u8[c] - b.u8[c]); - return s; -} - -SIMD_INLINE uint32_t c_v128_ssd_u8_sum(c_ssd128_internal s) { return s; } - -SIMD_INLINE c_v128 c_v128_or(c_v128 a, c_v128 b) { - return c_v128_from_v64(c_v64_or(a.v64[1], b.v64[1]), - c_v64_or(a.v64[0], b.v64[0])); -} - -SIMD_INLINE c_v128 c_v128_xor(c_v128 a, c_v128 b) { - return c_v128_from_v64(c_v64_xor(a.v64[1], b.v64[1]), - c_v64_xor(a.v64[0], b.v64[0])); -} - -SIMD_INLINE c_v128 c_v128_and(c_v128 a, c_v128 b) { - return c_v128_from_v64(c_v64_and(a.v64[1], b.v64[1]), - c_v64_and(a.v64[0], b.v64[0])); -} - -SIMD_INLINE c_v128 c_v128_andn(c_v128 a, c_v128 b) { - return c_v128_from_v64(c_v64_andn(a.v64[1], b.v64[1]), - c_v64_andn(a.v64[0], b.v64[0])); -} - -SIMD_INLINE c_v128 c_v128_add_8(c_v128 a, c_v128 b) { - return c_v128_from_v64(c_v64_add_8(a.v64[1], b.v64[1]), - c_v64_add_8(a.v64[0], b.v64[0])); -} - -SIMD_INLINE c_v128 c_v128_add_16(c_v128 a, c_v128 b) { - return c_v128_from_v64(c_v64_add_16(a.v64[1], b.v64[1]), - c_v64_add_16(a.v64[0], b.v64[0])); -} - -SIMD_INLINE c_v128 c_v128_sadd_u8(c_v128 a, c_v128 b) { - return c_v128_from_v64(c_v64_sadd_u8(a.v64[1], b.v64[1]), - c_v64_sadd_u8(a.v64[0], b.v64[0])); -} - -SIMD_INLINE c_v128 c_v128_sadd_s8(c_v128 a, c_v128 b) { - return c_v128_from_v64(c_v64_sadd_s8(a.v64[1], b.v64[1]), - c_v64_sadd_s8(a.v64[0], b.v64[0])); -} - -SIMD_INLINE c_v128 c_v128_sadd_s16(c_v128 a, c_v128 b) { - return c_v128_from_v64(c_v64_sadd_s16(a.v64[1], b.v64[1]), - c_v64_sadd_s16(a.v64[0], b.v64[0])); -} - -SIMD_INLINE c_v128 c_v128_add_32(c_v128 a, c_v128 b) { - return c_v128_from_v64(c_v64_add_32(a.v64[1], b.v64[1]), - c_v64_add_32(a.v64[0], b.v64[0])); -} - -SIMD_INLINE c_v128 c_v128_add_64(c_v128 a, c_v128 b) { - // Two complement overflow (silences sanitizers) - return c_v128_from_64( - a.v64[1].u64 > ~b.v64[1].u64 ? a.v64[1].u64 - ~b.v64[1].u64 - 1 - : a.v64[1].u64 + b.v64[1].u64, - a.v64[0].u64 > ~b.v64[0].u64 ? a.v64[0].u64 - ~b.v64[0].u64 - 1 - : a.v64[0].u64 + b.v64[0].u64); -} - -SIMD_INLINE c_v128 c_v128_padd_s16(c_v128 a) { - c_v128 t; - t.s32[0] = (int32_t)a.s16[0] + (int32_t)a.s16[1]; - t.s32[1] = (int32_t)a.s16[2] + (int32_t)a.s16[3]; - t.s32[2] = (int32_t)a.s16[4] + (int32_t)a.s16[5]; - t.s32[3] = (int32_t)a.s16[6] + (int32_t)a.s16[7]; - return t; -} - -SIMD_INLINE c_v128 c_v128_padd_u8(c_v128 a) { - c_v128 t; - t.u16[0] = (uint16_t)a.u8[0] + (uint16_t)a.u8[1]; - t.u16[1] = (uint16_t)a.u8[2] + (uint16_t)a.u8[3]; - t.u16[2] = (uint16_t)a.u8[4] + (uint16_t)a.u8[5]; - t.u16[3] = (uint16_t)a.u8[6] + (uint16_t)a.u8[7]; - t.u16[4] = (uint16_t)a.u8[8] + (uint16_t)a.u8[9]; - t.u16[5] = (uint16_t)a.u8[10] + (uint16_t)a.u8[11]; - t.u16[6] = (uint16_t)a.u8[12] + (uint16_t)a.u8[13]; - t.u16[7] = (uint16_t)a.u8[14] + (uint16_t)a.u8[15]; - return t; -} - -SIMD_INLINE c_v128 c_v128_sub_8(c_v128 a, c_v128 b) { - return c_v128_from_v64(c_v64_sub_8(a.v64[1], b.v64[1]), - c_v64_sub_8(a.v64[0], b.v64[0])); -} - -SIMD_INLINE c_v128 c_v128_ssub_u8(c_v128 a, c_v128 b) { - return c_v128_from_v64(c_v64_ssub_u8(a.v64[1], b.v64[1]), - c_v64_ssub_u8(a.v64[0], b.v64[0])); -} - -SIMD_INLINE c_v128 c_v128_ssub_s8(c_v128 a, c_v128 b) { - return c_v128_from_v64(c_v64_ssub_s8(a.v64[1], b.v64[1]), - c_v64_ssub_s8(a.v64[0], b.v64[0])); -} - -SIMD_INLINE c_v128 c_v128_sub_16(c_v128 a, c_v128 b) { - return c_v128_from_v64(c_v64_sub_16(a.v64[1], b.v64[1]), - c_v64_sub_16(a.v64[0], b.v64[0])); -} - -SIMD_INLINE c_v128 c_v128_ssub_s16(c_v128 a, c_v128 b) { - return c_v128_from_v64(c_v64_ssub_s16(a.v64[1], b.v64[1]), - c_v64_ssub_s16(a.v64[0], b.v64[0])); -} - -SIMD_INLINE c_v128 c_v128_ssub_u16(c_v128 a, c_v128 b) { - return c_v128_from_v64(c_v64_ssub_u16(a.v64[1], b.v64[1]), - c_v64_ssub_u16(a.v64[0], b.v64[0])); -} - -SIMD_INLINE c_v128 c_v128_sub_32(c_v128 a, c_v128 b) { - return c_v128_from_v64(c_v64_sub_32(a.v64[1], b.v64[1]), - c_v64_sub_32(a.v64[0], b.v64[0])); -} - -SIMD_INLINE c_v128 c_v128_sub_64(c_v128 a, c_v128 b) { - // Two complement underflow (silences sanitizers) - return c_v128_from_64( - a.v64[1].u64 < b.v64[1].u64 ? a.v64[1].u64 + ~b.v64[1].u64 + 1 - : a.v64[1].u64 - b.v64[1].u64, - a.v64[0].u64 < b.v64[0].u64 ? a.v64[0].u64 + ~b.v64[0].u64 + 1 - : a.v64[0].u64 - b.v64[0].u64); -} - -SIMD_INLINE c_v128 c_v128_abs_s16(c_v128 a) { - return c_v128_from_v64(c_v64_abs_s16(a.v64[1]), c_v64_abs_s16(a.v64[0])); -} - -SIMD_INLINE c_v128 c_v128_abs_s8(c_v128 a) { - return c_v128_from_v64(c_v64_abs_s8(a.v64[1]), c_v64_abs_s8(a.v64[0])); -} - -SIMD_INLINE c_v128 c_v128_mul_s16(c_v64 a, c_v64 b) { - c_v64 lo_bits = c_v64_mullo_s16(a, b); - c_v64 hi_bits = c_v64_mulhi_s16(a, b); - return c_v128_from_v64(c_v64_ziphi_16(hi_bits, lo_bits), - c_v64_ziplo_16(hi_bits, lo_bits)); -} - -SIMD_INLINE c_v128 c_v128_mullo_s16(c_v128 a, c_v128 b) { - return c_v128_from_v64(c_v64_mullo_s16(a.v64[1], b.v64[1]), - c_v64_mullo_s16(a.v64[0], b.v64[0])); -} - -SIMD_INLINE c_v128 c_v128_mulhi_s16(c_v128 a, c_v128 b) { - return c_v128_from_v64(c_v64_mulhi_s16(a.v64[1], b.v64[1]), - c_v64_mulhi_s16(a.v64[0], b.v64[0])); -} - -SIMD_INLINE c_v128 c_v128_mullo_s32(c_v128 a, c_v128 b) { - return c_v128_from_v64(c_v64_mullo_s32(a.v64[1], b.v64[1]), - c_v64_mullo_s32(a.v64[0], b.v64[0])); -} - -SIMD_INLINE c_v128 c_v128_madd_s16(c_v128 a, c_v128 b) { - return c_v128_from_v64(c_v64_madd_s16(a.v64[1], b.v64[1]), - c_v64_madd_s16(a.v64[0], b.v64[0])); -} - -SIMD_INLINE c_v128 c_v128_madd_us8(c_v128 a, c_v128 b) { - return c_v128_from_v64(c_v64_madd_us8(a.v64[1], b.v64[1]), - c_v64_madd_us8(a.v64[0], b.v64[0])); -} - -SIMD_INLINE c_v128 c_v128_avg_u8(c_v128 a, c_v128 b) { - return c_v128_from_v64(c_v64_avg_u8(a.v64[1], b.v64[1]), - c_v64_avg_u8(a.v64[0], b.v64[0])); -} - -SIMD_INLINE c_v128 c_v128_rdavg_u8(c_v128 a, c_v128 b) { - return c_v128_from_v64(c_v64_rdavg_u8(a.v64[1], b.v64[1]), - c_v64_rdavg_u8(a.v64[0], b.v64[0])); -} - -SIMD_INLINE c_v128 c_v128_rdavg_u16(c_v128 a, c_v128 b) { - return c_v128_from_v64(c_v64_rdavg_u16(a.v64[1], b.v64[1]), - c_v64_rdavg_u16(a.v64[0], b.v64[0])); -} - -SIMD_INLINE c_v128 c_v128_avg_u16(c_v128 a, c_v128 b) { - return c_v128_from_v64(c_v64_avg_u16(a.v64[1], b.v64[1]), - c_v64_avg_u16(a.v64[0], b.v64[0])); -} - -SIMD_INLINE c_v128 c_v128_min_u8(c_v128 a, c_v128 b) { - return c_v128_from_v64(c_v64_min_u8(a.v64[1], b.v64[1]), - c_v64_min_u8(a.v64[0], b.v64[0])); -} - -SIMD_INLINE c_v128 c_v128_max_u8(c_v128 a, c_v128 b) { - return c_v128_from_v64(c_v64_max_u8(a.v64[1], b.v64[1]), - c_v64_max_u8(a.v64[0], b.v64[0])); -} - -SIMD_INLINE c_v128 c_v128_min_s8(c_v128 a, c_v128 b) { - return c_v128_from_v64(c_v64_min_s8(a.v64[1], b.v64[1]), - c_v64_min_s8(a.v64[0], b.v64[0])); -} - -SIMD_INLINE uint32_t c_v128_movemask_8(c_v128 a) { - return ((a.s8[15] < 0) << 15) | ((a.s8[14] < 0) << 14) | - ((a.s8[13] < 0) << 13) | ((a.s8[12] < 0) << 12) | - ((a.s8[11] < 0) << 11) | ((a.s8[10] < 0) << 10) | - ((a.s8[9] < 0) << 9) | ((a.s8[8] < 0) << 8) | ((a.s8[7] < 0) << 7) | - ((a.s8[6] < 0) << 6) | ((a.s8[5] < 0) << 5) | ((a.s8[4] < 0) << 4) | - ((a.s8[3] < 0) << 3) | ((a.s8[2] < 0) << 2) | ((a.s8[1] < 0) << 1) | - ((a.s8[0] < 0) << 0); -} - -SIMD_INLINE c_v128 c_v128_blend_8(c_v128 a, c_v128 b, c_v128 c) { - c_v128 t; - for (int i = 0; i < 16; i++) t.u8[i] = c.s8[i] < 0 ? b.u8[i] : a.u8[i]; - return t; -} - -SIMD_INLINE c_v128 c_v128_max_s8(c_v128 a, c_v128 b) { - return c_v128_from_v64(c_v64_max_s8(a.v64[1], b.v64[1]), - c_v64_max_s8(a.v64[0], b.v64[0])); -} - -SIMD_INLINE c_v128 c_v128_min_s16(c_v128 a, c_v128 b) { - return c_v128_from_v64(c_v64_min_s16(a.v64[1], b.v64[1]), - c_v64_min_s16(a.v64[0], b.v64[0])); -} - -SIMD_INLINE c_v128 c_v128_max_s16(c_v128 a, c_v128 b) { - return c_v128_from_v64(c_v64_max_s16(a.v64[1], b.v64[1]), - c_v64_max_s16(a.v64[0], b.v64[0])); -} - -SIMD_INLINE c_v128 c_v128_max_s32(c_v128 a, c_v128 b) { - c_v128 t; - int c; - for (c = 0; c < 4; c++) t.s32[c] = a.s32[c] > b.s32[c] ? a.s32[c] : b.s32[c]; - return t; -} - -SIMD_INLINE c_v128 c_v128_min_s32(c_v128 a, c_v128 b) { - c_v128 t; - int c; - for (c = 0; c < 4; c++) t.s32[c] = a.s32[c] > b.s32[c] ? b.s32[c] : a.s32[c]; - return t; -} - -SIMD_INLINE c_v128 c_v128_ziplo_8(c_v128 a, c_v128 b) { - return c_v128_from_v64(c_v64_ziphi_8(a.v64[0], b.v64[0]), - c_v64_ziplo_8(a.v64[0], b.v64[0])); -} - -SIMD_INLINE c_v128 c_v128_ziphi_8(c_v128 a, c_v128 b) { - return c_v128_from_v64(c_v64_ziphi_8(a.v64[1], b.v64[1]), - c_v64_ziplo_8(a.v64[1], b.v64[1])); -} - -SIMD_INLINE c_v128 c_v128_ziplo_16(c_v128 a, c_v128 b) { - return c_v128_from_v64(c_v64_ziphi_16(a.v64[0], b.v64[0]), - c_v64_ziplo_16(a.v64[0], b.v64[0])); -} - -SIMD_INLINE c_v128 c_v128_ziphi_16(c_v128 a, c_v128 b) { - return c_v128_from_v64(c_v64_ziphi_16(a.v64[1], b.v64[1]), - c_v64_ziplo_16(a.v64[1], b.v64[1])); -} - -SIMD_INLINE c_v128 c_v128_ziplo_32(c_v128 a, c_v128 b) { - return c_v128_from_v64(c_v64_ziphi_32(a.v64[0], b.v64[0]), - c_v64_ziplo_32(a.v64[0], b.v64[0])); -} - -SIMD_INLINE c_v128 c_v128_ziphi_32(c_v128 a, c_v128 b) { - return c_v128_from_v64(c_v64_ziphi_32(a.v64[1], b.v64[1]), - c_v64_ziplo_32(a.v64[1], b.v64[1])); -} - -SIMD_INLINE c_v128 c_v128_ziplo_64(c_v128 a, c_v128 b) { - return c_v128_from_v64(a.v64[0], b.v64[0]); -} - -SIMD_INLINE c_v128 c_v128_ziphi_64(c_v128 a, c_v128 b) { - return c_v128_from_v64(a.v64[1], b.v64[1]); -} - -SIMD_INLINE c_v128 c_v128_zip_8(c_v64 a, c_v64 b) { - return c_v128_from_v64(c_v64_ziphi_8(a, b), c_v64_ziplo_8(a, b)); -} - -SIMD_INLINE c_v128 c_v128_zip_16(c_v64 a, c_v64 b) { - return c_v128_from_v64(c_v64_ziphi_16(a, b), c_v64_ziplo_16(a, b)); -} - -SIMD_INLINE c_v128 c_v128_zip_32(c_v64 a, c_v64 b) { - return c_v128_from_v64(c_v64_ziphi_32(a, b), c_v64_ziplo_32(a, b)); -} - -SIMD_INLINE c_v128 _c_v128_unzip_8(c_v128 a, c_v128 b, int mode) { - c_v128 t; - if (mode) { - t.u8[15] = b.u8[15]; - t.u8[14] = b.u8[13]; - t.u8[13] = b.u8[11]; - t.u8[12] = b.u8[9]; - t.u8[11] = b.u8[7]; - t.u8[10] = b.u8[5]; - t.u8[9] = b.u8[3]; - t.u8[8] = b.u8[1]; - t.u8[7] = a.u8[15]; - t.u8[6] = a.u8[13]; - t.u8[5] = a.u8[11]; - t.u8[4] = a.u8[9]; - t.u8[3] = a.u8[7]; - t.u8[2] = a.u8[5]; - t.u8[1] = a.u8[3]; - t.u8[0] = a.u8[1]; - } else { - t.u8[15] = a.u8[14]; - t.u8[14] = a.u8[12]; - t.u8[13] = a.u8[10]; - t.u8[12] = a.u8[8]; - t.u8[11] = a.u8[6]; - t.u8[10] = a.u8[4]; - t.u8[9] = a.u8[2]; - t.u8[8] = a.u8[0]; - t.u8[7] = b.u8[14]; - t.u8[6] = b.u8[12]; - t.u8[5] = b.u8[10]; - t.u8[4] = b.u8[8]; - t.u8[3] = b.u8[6]; - t.u8[2] = b.u8[4]; - t.u8[1] = b.u8[2]; - t.u8[0] = b.u8[0]; - } - return t; -} - -SIMD_INLINE c_v128 c_v128_unziplo_8(c_v128 a, c_v128 b) { - return CONFIG_BIG_ENDIAN ? _c_v128_unzip_8(a, b, 1) - : _c_v128_unzip_8(a, b, 0); -} - -SIMD_INLINE c_v128 c_v128_unziphi_8(c_v128 a, c_v128 b) { - return CONFIG_BIG_ENDIAN ? _c_v128_unzip_8(b, a, 0) - : _c_v128_unzip_8(b, a, 1); -} - -SIMD_INLINE c_v128 _c_v128_unzip_16(c_v128 a, c_v128 b, int mode) { - c_v128 t; - if (mode) { - t.u16[7] = b.u16[7]; - t.u16[6] = b.u16[5]; - t.u16[5] = b.u16[3]; - t.u16[4] = b.u16[1]; - t.u16[3] = a.u16[7]; - t.u16[2] = a.u16[5]; - t.u16[1] = a.u16[3]; - t.u16[0] = a.u16[1]; - } else { - t.u16[7] = a.u16[6]; - t.u16[6] = a.u16[4]; - t.u16[5] = a.u16[2]; - t.u16[4] = a.u16[0]; - t.u16[3] = b.u16[6]; - t.u16[2] = b.u16[4]; - t.u16[1] = b.u16[2]; - t.u16[0] = b.u16[0]; - } - return t; -} - -SIMD_INLINE c_v128 c_v128_unziplo_16(c_v128 a, c_v128 b) { - return CONFIG_BIG_ENDIAN ? _c_v128_unzip_16(a, b, 1) - : _c_v128_unzip_16(a, b, 0); -} - -SIMD_INLINE c_v128 c_v128_unziphi_16(c_v128 a, c_v128 b) { - return CONFIG_BIG_ENDIAN ? _c_v128_unzip_16(b, a, 0) - : _c_v128_unzip_16(b, a, 1); -} - -SIMD_INLINE c_v128 _c_v128_unzip_32(c_v128 a, c_v128 b, int mode) { - c_v128 t; - if (mode) { - t.u32[3] = b.u32[3]; - t.u32[2] = b.u32[1]; - t.u32[1] = a.u32[3]; - t.u32[0] = a.u32[1]; - } else { - t.u32[3] = a.u32[2]; - t.u32[2] = a.u32[0]; - t.u32[1] = b.u32[2]; - t.u32[0] = b.u32[0]; - } - return t; -} - -SIMD_INLINE c_v128 c_v128_unziplo_32(c_v128 a, c_v128 b) { - return CONFIG_BIG_ENDIAN ? _c_v128_unzip_32(a, b, 1) - : _c_v128_unzip_32(a, b, 0); -} - -SIMD_INLINE c_v128 c_v128_unziphi_32(c_v128 a, c_v128 b) { - return CONFIG_BIG_ENDIAN ? _c_v128_unzip_32(b, a, 0) - : _c_v128_unzip_32(b, a, 1); -} - -SIMD_INLINE c_v128 c_v128_unpack_u8_s16(c_v64 a) { - return c_v128_from_v64(c_v64_unpackhi_u8_s16(a), c_v64_unpacklo_u8_s16(a)); -} - -SIMD_INLINE c_v128 c_v128_unpacklo_u8_s16(c_v128 a) { - return c_v128_from_v64(c_v64_unpackhi_u8_s16(a.v64[0]), - c_v64_unpacklo_u8_s16(a.v64[0])); -} - -SIMD_INLINE c_v128 c_v128_unpackhi_u8_s16(c_v128 a) { - return c_v128_from_v64(c_v64_unpackhi_u8_s16(a.v64[1]), - c_v64_unpacklo_u8_s16(a.v64[1])); -} - -SIMD_INLINE c_v128 c_v128_unpack_s8_s16(c_v64 a) { - return c_v128_from_v64(c_v64_unpackhi_s8_s16(a), c_v64_unpacklo_s8_s16(a)); -} - -SIMD_INLINE c_v128 c_v128_unpacklo_s8_s16(c_v128 a) { - return c_v128_from_v64(c_v64_unpackhi_s8_s16(a.v64[0]), - c_v64_unpacklo_s8_s16(a.v64[0])); -} - -SIMD_INLINE c_v128 c_v128_unpackhi_s8_s16(c_v128 a) { - return c_v128_from_v64(c_v64_unpackhi_s8_s16(a.v64[1]), - c_v64_unpacklo_s8_s16(a.v64[1])); -} - -SIMD_INLINE c_v128 c_v128_pack_s32_s16(c_v128 a, c_v128 b) { - return c_v128_from_v64(c_v64_pack_s32_s16(a.v64[1], a.v64[0]), - c_v64_pack_s32_s16(b.v64[1], b.v64[0])); -} - -SIMD_INLINE c_v128 c_v128_pack_s32_u16(c_v128 a, c_v128 b) { - return c_v128_from_v64(c_v64_pack_s32_u16(a.v64[1], a.v64[0]), - c_v64_pack_s32_u16(b.v64[1], b.v64[0])); -} - -SIMD_INLINE c_v128 c_v128_pack_s16_u8(c_v128 a, c_v128 b) { - return c_v128_from_v64(c_v64_pack_s16_u8(a.v64[1], a.v64[0]), - c_v64_pack_s16_u8(b.v64[1], b.v64[0])); -} - -SIMD_INLINE c_v128 c_v128_pack_s16_s8(c_v128 a, c_v128 b) { - return c_v128_from_v64(c_v64_pack_s16_s8(a.v64[1], a.v64[0]), - c_v64_pack_s16_s8(b.v64[1], b.v64[0])); -} - -SIMD_INLINE c_v128 c_v128_unpack_u16_s32(c_v64 a) { - return c_v128_from_v64(c_v64_unpackhi_u16_s32(a), c_v64_unpacklo_u16_s32(a)); -} - -SIMD_INLINE c_v128 c_v128_unpack_s16_s32(c_v64 a) { - return c_v128_from_v64(c_v64_unpackhi_s16_s32(a), c_v64_unpacklo_s16_s32(a)); -} - -SIMD_INLINE c_v128 c_v128_unpacklo_u16_s32(c_v128 a) { - return c_v128_from_v64(c_v64_unpackhi_u16_s32(a.v64[0]), - c_v64_unpacklo_u16_s32(a.v64[0])); -} - -SIMD_INLINE c_v128 c_v128_unpacklo_s16_s32(c_v128 a) { - return c_v128_from_v64(c_v64_unpackhi_s16_s32(a.v64[0]), - c_v64_unpacklo_s16_s32(a.v64[0])); -} - -SIMD_INLINE c_v128 c_v128_unpackhi_u16_s32(c_v128 a) { - return c_v128_from_v64(c_v64_unpackhi_u16_s32(a.v64[1]), - c_v64_unpacklo_u16_s32(a.v64[1])); -} - -SIMD_INLINE c_v128 c_v128_unpackhi_s16_s32(c_v128 a) { - return c_v128_from_v64(c_v64_unpackhi_s16_s32(a.v64[1]), - c_v64_unpacklo_s16_s32(a.v64[1])); -} - -SIMD_INLINE c_v128 c_v128_shuffle_8(c_v128 a, c_v128 pattern) { - c_v128 t; - int c; - for (c = 0; c < 16; c++) - t.u8[c] = a.u8[CONFIG_BIG_ENDIAN ? 15 - (pattern.u8[c] & 15) - : pattern.u8[c] & 15]; - - return t; -} - -SIMD_INLINE c_v128 c_v128_cmpgt_s8(c_v128 a, c_v128 b) { - return c_v128_from_v64(c_v64_cmpgt_s8(a.v64[1], b.v64[1]), - c_v64_cmpgt_s8(a.v64[0], b.v64[0])); -} - -SIMD_INLINE c_v128 c_v128_cmplt_s8(c_v128 a, c_v128 b) { - return c_v128_from_v64(c_v64_cmplt_s8(a.v64[1], b.v64[1]), - c_v64_cmplt_s8(a.v64[0], b.v64[0])); -} - -SIMD_INLINE c_v128 c_v128_cmpeq_8(c_v128 a, c_v128 b) { - return c_v128_from_v64(c_v64_cmpeq_8(a.v64[1], b.v64[1]), - c_v64_cmpeq_8(a.v64[0], b.v64[0])); -} - -SIMD_INLINE c_v128 c_v128_cmpgt_s16(c_v128 a, c_v128 b) { - return c_v128_from_v64(c_v64_cmpgt_s16(a.v64[1], b.v64[1]), - c_v64_cmpgt_s16(a.v64[0], b.v64[0])); -} - -SIMD_INLINE c_v128 c_v128_cmplt_s16(c_v128 a, c_v128 b) { - return c_v128_from_v64(c_v64_cmplt_s16(a.v64[1], b.v64[1]), - c_v64_cmplt_s16(a.v64[0], b.v64[0])); -} - -SIMD_INLINE c_v128 c_v128_cmpeq_16(c_v128 a, c_v128 b) { - return c_v128_from_v64(c_v64_cmpeq_16(a.v64[1], b.v64[1]), - c_v64_cmpeq_16(a.v64[0], b.v64[0])); -} - -SIMD_INLINE c_v128 c_v128_cmpgt_s32(c_v128 a, c_v128 b) { - c_v128 t; - int c; - for (c = 0; c < 4; c++) t.s32[c] = -(a.s32[c] > b.s32[c]); - return t; -} - -SIMD_INLINE c_v128 c_v128_cmplt_s32(c_v128 a, c_v128 b) { - c_v128 t; - int c; - for (c = 0; c < 4; c++) t.s32[c] = -(a.s32[c] < b.s32[c]); - return t; -} - -SIMD_INLINE c_v128 c_v128_cmpeq_32(c_v128 a, c_v128 b) { - c_v128 t; - int c; - for (c = 0; c < 4; c++) t.s32[c] = -(a.s32[c] == b.s32[c]); - return t; -} - -SIMD_INLINE c_v128 c_v128_shl_n_byte(c_v128 a, const unsigned int n) { - if (n < 8) - return c_v128_from_v64(c_v64_or(c_v64_shl_n_byte(a.v64[1], n), - c_v64_shr_n_byte(a.v64[0], 8 - n)), - c_v64_shl_n_byte(a.v64[0], n)); - else - return c_v128_from_v64(c_v64_shl_n_byte(a.v64[0], n - 8), c_v64_zero()); -} - -SIMD_INLINE c_v128 c_v128_shr_n_byte(c_v128 a, const unsigned int n) { - if (n < 8) - return c_v128_from_v64(c_v64_shr_n_byte(a.v64[1], n), - c_v64_or(c_v64_shr_n_byte(a.v64[0], n), - c_v64_shl_n_byte(a.v64[1], 8 - n))); - else - return c_v128_from_v64(c_v64_zero(), c_v64_shr_n_byte(a.v64[1], n - 8)); -} - -SIMD_INLINE c_v128 c_v128_align(c_v128 a, c_v128 b, const unsigned int c) { - if (SIMD_CHECK && c > 15) { - fprintf(stderr, "Error: undefined alignment %d\n", c); - abort(); - } - return c ? c_v128_or(c_v128_shr_n_byte(b, c), c_v128_shl_n_byte(a, 16 - c)) - : b; -} - -SIMD_INLINE c_v128 c_v128_shl_8(c_v128 a, const unsigned int c) { - return c_v128_from_v64(c_v64_shl_8(a.v64[1], c), c_v64_shl_8(a.v64[0], c)); -} - -SIMD_INLINE c_v128 c_v128_shr_u8(c_v128 a, const unsigned int c) { - return c_v128_from_v64(c_v64_shr_u8(a.v64[1], c), c_v64_shr_u8(a.v64[0], c)); -} - -SIMD_INLINE c_v128 c_v128_shr_s8(c_v128 a, const unsigned int c) { - return c_v128_from_v64(c_v64_shr_s8(a.v64[1], c), c_v64_shr_s8(a.v64[0], c)); -} - -SIMD_INLINE c_v128 c_v128_shl_16(c_v128 a, const unsigned int c) { - return c_v128_from_v64(c_v64_shl_16(a.v64[1], c), c_v64_shl_16(a.v64[0], c)); -} - -SIMD_INLINE c_v128 c_v128_shr_u16(c_v128 a, const unsigned int c) { - return c_v128_from_v64(c_v64_shr_u16(a.v64[1], c), - c_v64_shr_u16(a.v64[0], c)); -} - -SIMD_INLINE c_v128 c_v128_shr_s16(c_v128 a, const unsigned int c) { - return c_v128_from_v64(c_v64_shr_s16(a.v64[1], c), - c_v64_shr_s16(a.v64[0], c)); -} - -SIMD_INLINE c_v128 c_v128_shl_32(c_v128 a, const unsigned int c) { - return c_v128_from_v64(c_v64_shl_32(a.v64[1], c), c_v64_shl_32(a.v64[0], c)); -} - -SIMD_INLINE c_v128 c_v128_shr_u32(c_v128 a, const unsigned int c) { - return c_v128_from_v64(c_v64_shr_u32(a.v64[1], c), - c_v64_shr_u32(a.v64[0], c)); -} - -SIMD_INLINE c_v128 c_v128_shr_s32(c_v128 a, const unsigned int c) { - return c_v128_from_v64(c_v64_shr_s32(a.v64[1], c), - c_v64_shr_s32(a.v64[0], c)); -} - -SIMD_INLINE c_v128 c_v128_shl_64(c_v128 a, const unsigned int c) { - a.v64[1].u64 <<= c; - a.v64[0].u64 <<= c; - return c_v128_from_v64(a.v64[1], a.v64[0]); -} - -SIMD_INLINE c_v128 c_v128_shr_u64(c_v128 a, const unsigned int c) { - a.v64[1].u64 >>= c; - a.v64[0].u64 >>= c; - return c_v128_from_v64(a.v64[1], a.v64[0]); -} - -SIMD_INLINE c_v128 c_v128_shr_s64(c_v128 a, const unsigned int c) { - a.v64[1].s64 >>= c; - a.v64[0].s64 >>= c; - return c_v128_from_v64(a.v64[1], a.v64[0]); -} - -SIMD_INLINE c_v128 c_v128_shl_n_8(c_v128 a, const unsigned int n) { - return c_v128_shl_8(a, n); -} - -SIMD_INLINE c_v128 c_v128_shl_n_16(c_v128 a, const unsigned int n) { - return c_v128_shl_16(a, n); -} - -SIMD_INLINE c_v128 c_v128_shl_n_32(c_v128 a, const unsigned int n) { - return c_v128_shl_32(a, n); -} - -SIMD_INLINE c_v128 c_v128_shl_n_64(c_v128 a, const unsigned int n) { - return c_v128_shl_64(a, n); -} - -SIMD_INLINE c_v128 c_v128_shr_n_u8(c_v128 a, const unsigned int n) { - return c_v128_shr_u8(a, n); -} - -SIMD_INLINE c_v128 c_v128_shr_n_u16(c_v128 a, const unsigned int n) { - return c_v128_shr_u16(a, n); -} - -SIMD_INLINE c_v128 c_v128_shr_n_u32(c_v128 a, const unsigned int n) { - return c_v128_shr_u32(a, n); -} - -SIMD_INLINE c_v128 c_v128_shr_n_u64(c_v128 a, const unsigned int n) { - return c_v128_shr_u64(a, n); -} - -SIMD_INLINE c_v128 c_v128_shr_n_s8(c_v128 a, const unsigned int n) { - return c_v128_shr_s8(a, n); -} - -SIMD_INLINE c_v128 c_v128_shr_n_s16(c_v128 a, const unsigned int n) { - return c_v128_shr_s16(a, n); -} - -SIMD_INLINE c_v128 c_v128_shr_n_s32(c_v128 a, const unsigned int n) { - return c_v128_shr_s32(a, n); -} - -SIMD_INLINE c_v128 c_v128_shr_n_s64(c_v128 a, const unsigned int n) { - return c_v128_shr_s64(a, n); -} - -typedef uint32_t c_sad128_internal_u16; - -SIMD_INLINE c_sad128_internal_u16 c_v128_sad_u16_init() { return 0; } - -/* Implementation dependent return value. Result must be finalised with - * v128_sad_u16_sum(). */ -SIMD_INLINE c_sad128_internal_u16 c_v128_sad_u16(c_sad128_internal_u16 s, - c_v128 a, c_v128 b) { - int c; - for (c = 0; c < 8; c++) - s += a.u16[c] > b.u16[c] ? a.u16[c] - b.u16[c] : b.u16[c] - a.u16[c]; - return s; -} - -SIMD_INLINE uint32_t c_v128_sad_u16_sum(c_sad128_internal_u16 s) { return s; } - -typedef uint64_t c_ssd128_internal_s16; - -SIMD_INLINE c_ssd128_internal_s16 c_v128_ssd_s16_init() { return 0; } - -/* Implementation dependent return value. Result must be finalised with - * v128_ssd_s16_sum(). */ -SIMD_INLINE c_ssd128_internal_s16 c_v128_ssd_s16(c_ssd128_internal_s16 s, - c_v128 a, c_v128 b) { - int c; - for (c = 0; c < 8; c++) - s += (int32_t)(int16_t)(a.s16[c] - b.s16[c]) * - (int32_t)(int16_t)(a.s16[c] - b.s16[c]); - return s; -} - -SIMD_INLINE uint64_t c_v128_ssd_s16_sum(c_ssd128_internal_s16 s) { return s; } - -#endif // AOM_AOM_DSP_SIMD_V128_INTRINSICS_C_H_ diff --git a/third_party/aom/aom_dsp/simd/v128_intrinsics_x86.h b/third_party/aom/aom_dsp/simd/v128_intrinsics_x86.h deleted file mode 100644 index 6c7241ff4..000000000 --- a/third_party/aom/aom_dsp/simd/v128_intrinsics_x86.h +++ /dev/null @@ -1,656 +0,0 @@ -/* - * Copyright (c) 2016, Alliance for Open Media. All rights reserved - * - * This source code is subject to the terms of the BSD 2 Clause License and - * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License - * was not distributed with this source code in the LICENSE file, you can - * obtain it at www.aomedia.org/license/software. If the Alliance for Open - * Media Patent License 1.0 was not distributed with this source code in the - * PATENTS file, you can obtain it at www.aomedia.org/license/patent. - */ - -#ifndef AOM_AOM_DSP_SIMD_V128_INTRINSICS_X86_H_ -#define AOM_AOM_DSP_SIMD_V128_INTRINSICS_X86_H_ - -#include <stdint.h> -#include "aom_dsp/simd/v64_intrinsics_x86.h" - -typedef __m128i v128; - -SIMD_INLINE uint32_t v128_low_u32(v128 a) { - return (uint32_t)_mm_cvtsi128_si32(a); -} - -SIMD_INLINE v64 v128_low_v64(v128 a) { - return _mm_unpacklo_epi64(a, v64_zero()); -} - -SIMD_INLINE v64 v128_high_v64(v128 a) { return _mm_srli_si128(a, 8); } - -SIMD_INLINE v128 v128_from_v64(v64 a, v64 b) { - return _mm_unpacklo_epi64(b, a); -} - -SIMD_INLINE v128 v128_from_64(uint64_t a, uint64_t b) { - return v128_from_v64(v64_from_64(a), v64_from_64(b)); -} - -SIMD_INLINE v128 v128_from_32(uint32_t a, uint32_t b, uint32_t c, uint32_t d) { - return _mm_set_epi32(a, b, c, d); -} - -SIMD_INLINE v128 v128_load_aligned(const void *p) { - return _mm_load_si128((__m128i *)p); -} - -SIMD_INLINE v128 v128_load_unaligned(const void *p) { -#if defined(__SSSE3__) - return (__m128i)_mm_lddqu_si128((__m128i *)p); -#else - return _mm_loadu_si128((__m128i *)p); -#endif -} - -SIMD_INLINE void v128_store_aligned(void *p, v128 a) { - _mm_store_si128((__m128i *)p, a); -} - -SIMD_INLINE void v128_store_unaligned(void *p, v128 a) { - _mm_storeu_si128((__m128i *)p, a); -} - -// The following function requires an immediate. -// Some compilers will check this during optimisation, others wont. -#if defined(__OPTIMIZE__) && __OPTIMIZE__ && !defined(__clang__) -#if defined(__SSSE3__) -SIMD_INLINE v128 v128_align(v128 a, v128 b, const unsigned int c) { - return c ? _mm_alignr_epi8(a, b, c) : b; -} -#else -#define v128_align(a, b, c) \ - ((c) ? _mm_or_si128(_mm_srli_si128(b, c), _mm_slli_si128(a, 16 - (c))) : (b)) -#endif -#else -#if defined(__SSSE3__) -#define v128_align(a, b, c) ((c) ? _mm_alignr_epi8(a, b, (uint8_t)(c)) : (b)) -#else -#define v128_align(a, b, c) \ - ((c) ? _mm_or_si128(_mm_srli_si128(b, c), _mm_slli_si128(a, 16 - (c))) : (b)) -#endif -#endif - -SIMD_INLINE v128 v128_zero() { return _mm_setzero_si128(); } - -SIMD_INLINE v128 v128_dup_8(uint8_t x) { return _mm_set1_epi8(x); } - -SIMD_INLINE v128 v128_dup_16(uint16_t x) { return _mm_set1_epi16(x); } - -SIMD_INLINE v128 v128_dup_32(uint32_t x) { return _mm_set1_epi32(x); } - -SIMD_INLINE v128 v128_dup_64(uint64_t x) { - // _mm_set_pi64x and _mm_cvtsi64x_si64 missing in some compilers - return _mm_set_epi32(x >> 32, (uint32_t)x, x >> 32, (uint32_t)x); -} - -SIMD_INLINE v128 v128_add_8(v128 a, v128 b) { return _mm_add_epi8(a, b); } - -SIMD_INLINE v128 v128_add_16(v128 a, v128 b) { return _mm_add_epi16(a, b); } - -SIMD_INLINE v128 v128_sadd_u8(v128 a, v128 b) { return _mm_adds_epu8(a, b); } - -SIMD_INLINE v128 v128_sadd_s8(v128 a, v128 b) { return _mm_adds_epi8(a, b); } - -SIMD_INLINE v128 v128_sadd_s16(v128 a, v128 b) { return _mm_adds_epi16(a, b); } - -SIMD_INLINE v128 v128_add_32(v128 a, v128 b) { return _mm_add_epi32(a, b); } - -SIMD_INLINE v128 v128_add_64(v128 a, v128 b) { return _mm_add_epi64(a, b); } - -SIMD_INLINE v128 v128_padd_s16(v128 a) { - return _mm_madd_epi16(a, _mm_set1_epi16(1)); -} - -SIMD_INLINE v128 v128_sub_8(v128 a, v128 b) { return _mm_sub_epi8(a, b); } - -SIMD_INLINE v128 v128_ssub_u8(v128 a, v128 b) { return _mm_subs_epu8(a, b); } - -SIMD_INLINE v128 v128_ssub_s8(v128 a, v128 b) { return _mm_subs_epi8(a, b); } - -SIMD_INLINE v128 v128_sub_16(v128 a, v128 b) { return _mm_sub_epi16(a, b); } - -SIMD_INLINE v128 v128_ssub_s16(v128 a, v128 b) { return _mm_subs_epi16(a, b); } - -SIMD_INLINE v128 v128_ssub_u16(v128 a, v128 b) { return _mm_subs_epu16(a, b); } - -SIMD_INLINE v128 v128_sub_32(v128 a, v128 b) { return _mm_sub_epi32(a, b); } - -SIMD_INLINE v128 v128_sub_64(v128 a, v128 b) { return _mm_sub_epi64(a, b); } - -SIMD_INLINE v128 v128_abs_s16(v128 a) { -#if defined(__SSSE3__) - return _mm_abs_epi16(a); -#else - return _mm_max_epi16(a, _mm_sub_epi16(_mm_setzero_si128(), a)); -#endif -} - -SIMD_INLINE v128 v128_abs_s8(v128 a) { -#if defined(__SSSE3__) - return _mm_abs_epi8(a); -#else - v128 sign = _mm_cmplt_epi8(a, _mm_setzero_si128()); - return _mm_xor_si128(sign, _mm_add_epi8(a, sign)); -#endif -} - -SIMD_INLINE v128 v128_ziplo_8(v128 a, v128 b) { - return _mm_unpacklo_epi8(b, a); -} - -SIMD_INLINE v128 v128_ziphi_8(v128 a, v128 b) { - return _mm_unpackhi_epi8(b, a); -} - -SIMD_INLINE v128 v128_ziplo_16(v128 a, v128 b) { - return _mm_unpacklo_epi16(b, a); -} - -SIMD_INLINE v128 v128_ziphi_16(v128 a, v128 b) { - return _mm_unpackhi_epi16(b, a); -} - -SIMD_INLINE v128 v128_ziplo_32(v128 a, v128 b) { - return _mm_unpacklo_epi32(b, a); -} - -SIMD_INLINE v128 v128_ziphi_32(v128 a, v128 b) { - return _mm_unpackhi_epi32(b, a); -} - -SIMD_INLINE v128 v128_ziplo_64(v128 a, v128 b) { - return _mm_unpacklo_epi64(b, a); -} - -SIMD_INLINE v128 v128_ziphi_64(v128 a, v128 b) { - return _mm_unpackhi_epi64(b, a); -} - -SIMD_INLINE v128 v128_zip_8(v64 a, v64 b) { return _mm_unpacklo_epi8(b, a); } - -SIMD_INLINE v128 v128_zip_16(v64 a, v64 b) { return _mm_unpacklo_epi16(b, a); } - -SIMD_INLINE v128 v128_zip_32(v64 a, v64 b) { return _mm_unpacklo_epi32(b, a); } - -SIMD_INLINE v128 v128_unziphi_8(v128 a, v128 b) { - return _mm_packs_epi16(_mm_srai_epi16(b, 8), _mm_srai_epi16(a, 8)); -} - -SIMD_INLINE v128 v128_unziplo_8(v128 a, v128 b) { -#if defined(__SSSE3__) -#ifdef __x86_64__ - v128 order = _mm_cvtsi64_si128(0x0e0c0a0806040200LL); -#else - v128 order = _mm_set_epi32(0, 0, 0x0e0c0a08, 0x06040200); -#endif - return _mm_unpacklo_epi64(_mm_shuffle_epi8(b, order), - _mm_shuffle_epi8(a, order)); -#else - return v128_unziphi_8(_mm_slli_si128(a, 1), _mm_slli_si128(b, 1)); -#endif -} - -SIMD_INLINE v128 v128_unziphi_16(v128 a, v128 b) { - return _mm_packs_epi32(_mm_srai_epi32(b, 16), _mm_srai_epi32(a, 16)); -} - -SIMD_INLINE v128 v128_unziplo_16(v128 a, v128 b) { -#if defined(__SSSE3__) -#ifdef __x86_64__ - v128 order = _mm_cvtsi64_si128(0x0d0c090805040100LL); -#else - v128 order = _mm_set_epi32(0, 0, 0x0d0c0908, 0x05040100); -#endif - return _mm_unpacklo_epi64(_mm_shuffle_epi8(b, order), - _mm_shuffle_epi8(a, order)); -#else - return v128_unziphi_16(_mm_slli_si128(a, 2), _mm_slli_si128(b, 2)); -#endif -} - -SIMD_INLINE v128 v128_unziphi_32(v128 a, v128 b) { - return _mm_castps_si128(_mm_shuffle_ps( - _mm_castsi128_ps(b), _mm_castsi128_ps(a), _MM_SHUFFLE(3, 1, 3, 1))); -} - -SIMD_INLINE v128 v128_unziplo_32(v128 a, v128 b) { - return _mm_castps_si128(_mm_shuffle_ps( - _mm_castsi128_ps(b), _mm_castsi128_ps(a), _MM_SHUFFLE(2, 0, 2, 0))); -} - -SIMD_INLINE v128 v128_unpack_u8_s16(v64 a) { - return _mm_unpacklo_epi8(a, _mm_setzero_si128()); -} - -SIMD_INLINE v128 v128_unpacklo_u8_s16(v128 a) { - return _mm_unpacklo_epi8(a, _mm_setzero_si128()); -} - -SIMD_INLINE v128 v128_unpackhi_u8_s16(v128 a) { - return _mm_unpackhi_epi8(a, _mm_setzero_si128()); -} - -SIMD_INLINE v128 v128_unpack_s8_s16(v64 a) { - return _mm_srai_epi16(_mm_unpacklo_epi8(a, a), 8); -} - -SIMD_INLINE v128 v128_unpacklo_s8_s16(v128 a) { - return _mm_srai_epi16(_mm_unpacklo_epi8(a, a), 8); -} - -SIMD_INLINE v128 v128_unpackhi_s8_s16(v128 a) { - return _mm_srai_epi16(_mm_unpackhi_epi8(a, a), 8); -} - -SIMD_INLINE v128 v128_pack_s32_s16(v128 a, v128 b) { - return _mm_packs_epi32(b, a); -} - -SIMD_INLINE v128 v128_pack_s32_u16(v128 a, v128 b) { -#if defined(__SSE4_1__) - return _mm_packus_epi32(b, a); -#else - return v128_from_v64(v64_pack_s32_u16(v128_high_v64(a), v128_low_v64(a)), - v64_pack_s32_u16(v128_high_v64(b), v128_low_v64(b))); -#endif -} - -SIMD_INLINE v128 v128_pack_s16_u8(v128 a, v128 b) { - return _mm_packus_epi16(b, a); -} - -SIMD_INLINE v128 v128_pack_s16_s8(v128 a, v128 b) { - return _mm_packs_epi16(b, a); -} - -SIMD_INLINE v128 v128_unpack_u16_s32(v64 a) { - return _mm_unpacklo_epi16(a, _mm_setzero_si128()); -} - -SIMD_INLINE v128 v128_unpack_s16_s32(v64 a) { - return _mm_srai_epi32(_mm_unpacklo_epi16(a, a), 16); -} - -SIMD_INLINE v128 v128_unpacklo_u16_s32(v128 a) { - return _mm_unpacklo_epi16(a, _mm_setzero_si128()); -} - -SIMD_INLINE v128 v128_unpacklo_s16_s32(v128 a) { - return _mm_srai_epi32(_mm_unpacklo_epi16(a, a), 16); -} - -SIMD_INLINE v128 v128_unpackhi_u16_s32(v128 a) { - return _mm_unpackhi_epi16(a, _mm_setzero_si128()); -} - -SIMD_INLINE v128 v128_unpackhi_s16_s32(v128 a) { - return _mm_srai_epi32(_mm_unpackhi_epi16(a, a), 16); -} - -SIMD_INLINE v128 v128_shuffle_8(v128 x, v128 pattern) { -#if defined(__SSSE3__) - return _mm_shuffle_epi8(x, pattern); -#else - v128 output; - unsigned char *input = (unsigned char *)&x; - unsigned char *index = (unsigned char *)&pattern; - char *selected = (char *)&output; - int counter; - - for (counter = 0; counter < 16; counter++) { - selected[counter] = input[index[counter] & 15]; - } - - return output; -#endif -} - -SIMD_INLINE int64_t v128_dotp_su8(v128 a, v128 b) { - v128 t1 = _mm_madd_epi16(v128_unpackhi_s8_s16(a), v128_unpackhi_u8_s16(b)); - v128 t2 = _mm_madd_epi16(v128_unpacklo_s8_s16(a), v128_unpacklo_u8_s16(b)); - v128 t = v128_add_32(t1, t2); - t = v128_add_32(t, _mm_srli_si128(t, 8)); - t = v128_add_32(t, _mm_srli_si128(t, 4)); - return (int32_t)v128_low_u32(t); -} - -SIMD_INLINE int64_t v128_dotp_s16(v128 a, v128 b) { - v128 r = _mm_madd_epi16(a, b); -#if defined(__SSE4_1__) && defined(__x86_64__) - v128 c = _mm_add_epi64(_mm_cvtepi32_epi64(r), - _mm_cvtepi32_epi64(_mm_srli_si128(r, 8))); - return _mm_cvtsi128_si64(_mm_add_epi64(c, _mm_srli_si128(c, 8))); -#else - return (int64_t)_mm_cvtsi128_si32(r) + - (int64_t)_mm_cvtsi128_si32(_mm_srli_si128(r, 4)) + - (int64_t)_mm_cvtsi128_si32(_mm_srli_si128(r, 8)) + - (int64_t)_mm_cvtsi128_si32(_mm_srli_si128(r, 12)); -#endif -} - -SIMD_INLINE uint64_t v128_hadd_u8(v128 a) { - v128 t = _mm_sad_epu8(a, _mm_setzero_si128()); - return v64_low_u32(v128_low_v64(t)) + v64_low_u32(v128_high_v64(t)); -} - -typedef v128 sad128_internal; - -SIMD_INLINE sad128_internal v128_sad_u8_init() { return _mm_setzero_si128(); } - -/* Implementation dependent return value. Result must be finalised with - v128_sad_sum(). - The result for more than 32 v128_sad_u8() calls is undefined. */ -SIMD_INLINE sad128_internal v128_sad_u8(sad128_internal s, v128 a, v128 b) { - return _mm_add_epi64(s, _mm_sad_epu8(a, b)); -} - -SIMD_INLINE uint32_t v128_sad_u8_sum(sad128_internal s) { - return v128_low_u32(_mm_add_epi32(s, _mm_unpackhi_epi64(s, s))); -} - -typedef int32_t ssd128_internal; - -SIMD_INLINE ssd128_internal v128_ssd_u8_init() { return 0; } - -/* Implementation dependent return value. Result must be finalised with - * v128_ssd_sum(). */ -SIMD_INLINE ssd128_internal v128_ssd_u8(ssd128_internal s, v128 a, v128 b) { - v128 z = _mm_setzero_si128(); - v128 l = _mm_sub_epi16(_mm_unpacklo_epi8(a, z), _mm_unpacklo_epi8(b, z)); - v128 h = _mm_sub_epi16(_mm_unpackhi_epi8(a, z), _mm_unpackhi_epi8(b, z)); - v128 rl = _mm_madd_epi16(l, l); - v128 rh = _mm_madd_epi16(h, h); - v128 r = _mm_add_epi32(rl, rh); - r = _mm_add_epi32(r, _mm_srli_si128(r, 8)); - r = _mm_add_epi32(r, _mm_srli_si128(r, 4)); - return s + _mm_cvtsi128_si32(r); -} - -SIMD_INLINE int32_t v128_ssd_u8_sum(ssd128_internal s) { return s; } - -SIMD_INLINE v128 v128_or(v128 a, v128 b) { return _mm_or_si128(a, b); } - -SIMD_INLINE v128 v128_xor(v128 a, v128 b) { return _mm_xor_si128(a, b); } - -SIMD_INLINE v128 v128_and(v128 a, v128 b) { return _mm_and_si128(a, b); } - -SIMD_INLINE v128 v128_andn(v128 a, v128 b) { return _mm_andnot_si128(b, a); } - -SIMD_INLINE v128 v128_mul_s16(v64 a, v64 b) { - v64 lo_bits = v64_mullo_s16(a, b); - v64 hi_bits = v64_mulhi_s16(a, b); - return v128_from_v64(v64_ziphi_16(hi_bits, lo_bits), - v64_ziplo_16(hi_bits, lo_bits)); -} - -SIMD_INLINE v128 v128_mullo_s16(v128 a, v128 b) { - return _mm_mullo_epi16(a, b); -} - -SIMD_INLINE v128 v128_mulhi_s16(v128 a, v128 b) { - return _mm_mulhi_epi16(a, b); -} - -SIMD_INLINE v128 v128_mullo_s32(v128 a, v128 b) { -#if defined(__SSE4_1__) - return _mm_mullo_epi32(a, b); -#else - return _mm_unpacklo_epi32( - _mm_shuffle_epi32(_mm_mul_epu32(a, b), 8), - _mm_shuffle_epi32( - _mm_mul_epu32(_mm_srli_si128(a, 4), _mm_srli_si128(b, 4)), 8)); -#endif -} - -SIMD_INLINE int64_t v128_dotp_s32(v128 a, v128 b) { - v128 r = v128_mullo_s32(a, b); - return (int64_t)_mm_cvtsi128_si32(r) + - (int64_t)_mm_cvtsi128_si32(_mm_srli_si128(r, 4)) + - (int64_t)_mm_cvtsi128_si32(_mm_srli_si128(r, 8)) + - (int64_t)_mm_cvtsi128_si32(_mm_srli_si128(r, 12)); -} - -SIMD_INLINE v128 v128_madd_s16(v128 a, v128 b) { return _mm_madd_epi16(a, b); } - -SIMD_INLINE v128 v128_madd_us8(v128 a, v128 b) { -#if defined(__SSSE3__) - return _mm_maddubs_epi16(a, b); -#else - return _mm_packs_epi32( - _mm_madd_epi16(_mm_unpacklo_epi8(a, _mm_setzero_si128()), - _mm_srai_epi16(_mm_unpacklo_epi8(b, b), 8)), - _mm_madd_epi16(_mm_unpackhi_epi8(a, _mm_setzero_si128()), - _mm_srai_epi16(_mm_unpackhi_epi8(b, b), 8))); -#endif -} - -SIMD_INLINE v128 v128_padd_u8(v128 a) { - return v128_madd_us8(a, _mm_set1_epi8(1)); -} - -SIMD_INLINE v128 v128_avg_u8(v128 a, v128 b) { return _mm_avg_epu8(a, b); } - -SIMD_INLINE v128 v128_rdavg_u8(v128 a, v128 b) { - return _mm_sub_epi8(_mm_avg_epu8(a, b), - _mm_and_si128(_mm_xor_si128(a, b), v128_dup_8(1))); -} - -SIMD_INLINE v128 v128_rdavg_u16(v128 a, v128 b) { - return _mm_sub_epi16(_mm_avg_epu16(a, b), - _mm_and_si128(_mm_xor_si128(a, b), v128_dup_16(1))); -} - -SIMD_INLINE v128 v128_avg_u16(v128 a, v128 b) { return _mm_avg_epu16(a, b); } - -SIMD_INLINE v128 v128_min_u8(v128 a, v128 b) { return _mm_min_epu8(a, b); } - -SIMD_INLINE v128 v128_max_u8(v128 a, v128 b) { return _mm_max_epu8(a, b); } - -SIMD_INLINE v128 v128_min_s8(v128 a, v128 b) { -#if defined(__SSE4_1__) - return _mm_min_epi8(a, b); -#else - v128 mask = _mm_cmplt_epi8(a, b); - return _mm_or_si128(_mm_andnot_si128(mask, b), _mm_and_si128(mask, a)); -#endif -} - -SIMD_INLINE uint32_t v128_movemask_8(v128 a) { return _mm_movemask_epi8(a); } - -SIMD_INLINE v128 v128_blend_8(v128 a, v128 b, v128 c) { -#if defined(__SSE4_1__) - return _mm_blendv_epi8(a, b, c); -#else - c = _mm_cmplt_epi8(c, v128_zero()); - return v128_or(v128_and(b, c), v128_andn(a, c)); -#endif -} - -SIMD_INLINE v128 v128_max_s8(v128 a, v128 b) { -#if defined(__SSE4_1__) - return _mm_max_epi8(a, b); -#else - v128 mask = _mm_cmplt_epi8(b, a); - return _mm_or_si128(_mm_andnot_si128(mask, b), _mm_and_si128(mask, a)); -#endif -} - -SIMD_INLINE v128 v128_min_s16(v128 a, v128 b) { return _mm_min_epi16(a, b); } - -SIMD_INLINE v128 v128_max_s16(v128 a, v128 b) { return _mm_max_epi16(a, b); } - -SIMD_INLINE v128 v128_min_s32(v128 a, v128 b) { -#if defined(__SSE4_1__) - return _mm_min_epi32(a, b); -#else - v128 mask = _mm_cmplt_epi32(a, b); - return _mm_or_si128(_mm_andnot_si128(mask, b), _mm_and_si128(mask, a)); -#endif -} - -SIMD_INLINE v128 v128_max_s32(v128 a, v128 b) { -#if defined(__SSE4_1__) - return _mm_max_epi32(a, b); -#else - v128 mask = _mm_cmplt_epi32(b, a); - return _mm_or_si128(_mm_andnot_si128(mask, b), _mm_and_si128(mask, a)); -#endif -} - -SIMD_INLINE v128 v128_cmpgt_s8(v128 a, v128 b) { return _mm_cmpgt_epi8(a, b); } - -SIMD_INLINE v128 v128_cmplt_s8(v128 a, v128 b) { return _mm_cmplt_epi8(a, b); } - -SIMD_INLINE v128 v128_cmpeq_8(v128 a, v128 b) { return _mm_cmpeq_epi8(a, b); } - -SIMD_INLINE v128 v128_cmpgt_s16(v128 a, v128 b) { - return _mm_cmpgt_epi16(a, b); -} - -SIMD_INLINE v128 v128_cmplt_s16(v128 a, v128 b) { - return _mm_cmplt_epi16(a, b); -} - -SIMD_INLINE v128 v128_cmpeq_32(v128 a, v128 b) { return _mm_cmpeq_epi32(a, b); } - -SIMD_INLINE v128 v128_cmpgt_s32(v128 a, v128 b) { - return _mm_cmpgt_epi32(a, b); -} - -SIMD_INLINE v128 v128_cmplt_s32(v128 a, v128 b) { - return _mm_cmplt_epi32(a, b); -} - -SIMD_INLINE v128 v128_cmpeq_16(v128 a, v128 b) { return _mm_cmpeq_epi16(a, b); } - -SIMD_INLINE v128 v128_shl_8(v128 a, unsigned int c) { - return _mm_and_si128(_mm_set1_epi8((uint8_t)(0xff << c)), - _mm_sll_epi16(a, _mm_cvtsi32_si128(c))); -} - -SIMD_INLINE v128 v128_shr_u8(v128 a, unsigned int c) { - return _mm_and_si128(_mm_set1_epi8(0xff >> c), - _mm_srl_epi16(a, _mm_cvtsi32_si128(c))); -} - -SIMD_INLINE v128 v128_shr_s8(v128 a, unsigned int c) { - __m128i x = _mm_cvtsi32_si128(c + 8); - return _mm_packs_epi16(_mm_sra_epi16(_mm_unpacklo_epi8(a, a), x), - _mm_sra_epi16(_mm_unpackhi_epi8(a, a), x)); -} - -SIMD_INLINE v128 v128_shl_16(v128 a, unsigned int c) { - return _mm_sll_epi16(a, _mm_cvtsi32_si128(c)); -} - -SIMD_INLINE v128 v128_shr_u16(v128 a, unsigned int c) { - return _mm_srl_epi16(a, _mm_cvtsi32_si128(c)); -} - -SIMD_INLINE v128 v128_shr_s16(v128 a, unsigned int c) { - return _mm_sra_epi16(a, _mm_cvtsi32_si128(c)); -} - -SIMD_INLINE v128 v128_shl_32(v128 a, unsigned int c) { - return _mm_sll_epi32(a, _mm_cvtsi32_si128(c)); -} - -SIMD_INLINE v128 v128_shr_u32(v128 a, unsigned int c) { - return _mm_srl_epi32(a, _mm_cvtsi32_si128(c)); -} - -SIMD_INLINE v128 v128_shr_s32(v128 a, unsigned int c) { - return _mm_sra_epi32(a, _mm_cvtsi32_si128(c)); -} - -SIMD_INLINE v128 v128_shl_64(v128 a, unsigned int c) { - return _mm_sll_epi64(a, _mm_cvtsi32_si128(c)); -} - -SIMD_INLINE v128 v128_shr_u64(v128 a, unsigned int c) { - return _mm_srl_epi64(a, _mm_cvtsi32_si128(c)); -} - -SIMD_INLINE v128 v128_shr_s64(v128 a, unsigned int c) { - // _mm_sra_epi64 is missing in gcc? - return v128_from_64((int64_t)v64_u64(v128_high_v64(a)) >> c, - (int64_t)v64_u64(v128_low_v64(a)) >> c); - // return _mm_sra_epi64(a, _mm_cvtsi32_si128(c)); -} - -/* These intrinsics require immediate values, so we must use #defines - to enforce that. */ -#define v128_shl_n_byte(a, c) _mm_slli_si128(a, (c)&127) -#define v128_shr_n_byte(a, c) _mm_srli_si128(a, (c)&127) -#define v128_shl_n_8(a, c) \ - _mm_and_si128(_mm_set1_epi8((uint8_t)(0xff << (c))), _mm_slli_epi16(a, c)) -#define v128_shr_n_u8(a, c) \ - _mm_and_si128(_mm_set1_epi8(0xff >> (c)), _mm_srli_epi16(a, c)) -#define v128_shr_n_s8(a, c) \ - _mm_packs_epi16(_mm_srai_epi16(_mm_unpacklo_epi8(a, a), (c) + 8), \ - _mm_srai_epi16(_mm_unpackhi_epi8(a, a), (c) + 8)) -#define v128_shl_n_16(a, c) _mm_slli_epi16(a, c) -#define v128_shr_n_u16(a, c) _mm_srli_epi16(a, c) -#define v128_shr_n_s16(a, c) _mm_srai_epi16(a, c) -#define v128_shl_n_32(a, c) _mm_slli_epi32(a, c) -#define v128_shr_n_u32(a, c) _mm_srli_epi32(a, c) -#define v128_shr_n_s32(a, c) _mm_srai_epi32(a, c) -#define v128_shl_n_64(a, c) _mm_slli_epi64(a, c) -#define v128_shr_n_u64(a, c) _mm_srli_epi64(a, c) -#define v128_shr_n_s64(a, c) \ - v128_shr_s64(a, c) // _mm_srai_epi64 missing in gcc? - -typedef v128 sad128_internal_u16; - -SIMD_INLINE sad128_internal_u16 v128_sad_u16_init() { return v128_zero(); } - -/* Implementation dependent return value. Result must be finalised with - * v128_sad_u16_sum(). */ -SIMD_INLINE sad128_internal_u16 v128_sad_u16(sad128_internal_u16 s, v128 a, - v128 b) { -#if defined(__SSE4_1__) - v128 t = v128_sub_16(_mm_max_epu16(a, b), _mm_min_epu16(a, b)); -#else - v128 t = v128_cmplt_s16(v128_xor(a, v128_dup_16(32768)), - v128_xor(b, v128_dup_16(32768))); - t = v128_sub_16(v128_or(v128_and(b, t), v128_andn(a, t)), - v128_or(v128_and(a, t), v128_andn(b, t))); -#endif - return v128_add_32( - s, v128_add_32(v128_unpackhi_u16_s32(t), v128_unpacklo_u16_s32(t))); -} - -SIMD_INLINE uint32_t v128_sad_u16_sum(sad128_internal_u16 s) { - return v128_low_u32(s) + v128_low_u32(v128_shr_n_byte(s, 4)) + - v128_low_u32(v128_shr_n_byte(s, 8)) + - v128_low_u32(v128_shr_n_byte(s, 12)); -} - -typedef v128 ssd128_internal_s16; - -SIMD_INLINE ssd128_internal_s16 v128_ssd_s16_init() { return v128_zero(); } - -/* Implementation dependent return value. Result must be finalised with - * v128_ssd_s16_sum(). */ -SIMD_INLINE ssd128_internal_s16 v128_ssd_s16(ssd128_internal_s16 s, v128 a, - v128 b) { - v128 d = v128_sub_16(a, b); - d = v128_madd_s16(d, d); - return v128_add_64(s, v128_add_64(_mm_unpackhi_epi32(d, v128_zero()), - _mm_unpacklo_epi32(d, v128_zero()))); -} - -SIMD_INLINE uint64_t v128_ssd_s16_sum(ssd128_internal_s16 s) { - return v64_u64(v128_low_v64(s)) + v64_u64(v128_high_v64(s)); -} - -#endif // AOM_AOM_DSP_SIMD_V128_INTRINSICS_X86_H_ diff --git a/third_party/aom/aom_dsp/simd/v256_intrinsics.h b/third_party/aom/aom_dsp/simd/v256_intrinsics.h deleted file mode 100644 index cb99d35b7..000000000 --- a/third_party/aom/aom_dsp/simd/v256_intrinsics.h +++ /dev/null @@ -1,376 +0,0 @@ -/* - * Copyright (c) 2016, Alliance for Open Media. All rights reserved - * - * This source code is subject to the terms of the BSD 2 Clause License and - * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License - * was not distributed with this source code in the LICENSE file, you can - * obtain it at www.aomedia.org/license/software. If the Alliance for Open - * Media Patent License 1.0 was not distributed with this source code in the - * PATENTS file, you can obtain it at www.aomedia.org/license/patent. - */ - -#ifndef AOM_AOM_DSP_SIMD_V256_INTRINSICS_H_ -#define AOM_AOM_DSP_SIMD_V256_INTRINSICS_H_ - -#include <stdio.h> -#include <stdlib.h> -#include <string.h> - -#include "aom_dsp/simd/v256_intrinsics_c.h" -#include "aom_dsp/simd/v128_intrinsics.h" -#include "aom_dsp/simd/v64_intrinsics.h" - -/* Fallback to plain, unoptimised C. */ - -typedef c_v256 v256; - -SIMD_INLINE uint32_t v256_low_u32(v256 a) { return c_v256_low_u32(a); } -SIMD_INLINE v64 v256_low_v64(v256 a) { return c_v256_low_v64(a); } -SIMD_INLINE uint64_t v256_low_u64(v256 a) { return c_v256_low_u64(a); } -SIMD_INLINE v128 v256_low_v128(v256 a) { return c_v256_low_v128(a); } -SIMD_INLINE v128 v256_high_v128(v256 a) { return c_v256_high_v128(a); } -SIMD_INLINE v256 v256_from_v128(v128 hi, v128 lo) { - return c_v256_from_v128(hi, lo); -} -SIMD_INLINE v256 v256_from_64(uint64_t a, uint64_t b, uint64_t c, uint64_t d) { - return c_v256_from_64(a, b, c, d); -} -SIMD_INLINE v256 v256_from_v64(v64 a, v64 b, v64 c, v64 d) { - return c_v256_from_v64(a, b, c, d); -} - -SIMD_INLINE v256 v256_load_unaligned(const void *p) { - return c_v256_load_unaligned(p); -} -SIMD_INLINE v256 v256_load_aligned(const void *p) { - return c_v256_load_aligned(p); -} - -SIMD_INLINE void v256_store_unaligned(void *p, v256 a) { - c_v256_store_unaligned(p, a); -} -SIMD_INLINE void v256_store_aligned(void *p, v256 a) { - c_v256_store_aligned(p, a); -} - -SIMD_INLINE v256 v256_align(v256 a, v256 b, unsigned int c) { - return c_v256_align(a, b, c); -} - -SIMD_INLINE v256 v256_zero() { return c_v256_zero(); } -SIMD_INLINE v256 v256_dup_8(uint8_t x) { return c_v256_dup_8(x); } -SIMD_INLINE v256 v256_dup_16(uint16_t x) { return c_v256_dup_16(x); } -SIMD_INLINE v256 v256_dup_32(uint32_t x) { return c_v256_dup_32(x); } -SIMD_INLINE v256 v256_dup_64(uint64_t x) { return c_v256_dup_64(x); } - -typedef uint32_t sad256_internal; -SIMD_INLINE sad256_internal v256_sad_u8_init() { return c_v256_sad_u8_init(); } -SIMD_INLINE sad256_internal v256_sad_u8(sad256_internal s, v256 a, v256 b) { - return c_v256_sad_u8(s, a, b); -} -SIMD_INLINE uint32_t v256_sad_u8_sum(sad256_internal s) { - return c_v256_sad_u8_sum(s); -} -typedef uint32_t ssd256_internal; -SIMD_INLINE ssd256_internal v256_ssd_u8_init() { return c_v256_ssd_u8_init(); } -SIMD_INLINE ssd256_internal v256_ssd_u8(ssd256_internal s, v256 a, v256 b) { - return c_v256_ssd_u8(s, a, b); -} -SIMD_INLINE uint32_t v256_ssd_u8_sum(ssd256_internal s) { - return c_v256_ssd_u8_sum(s); -} - -SIMD_INLINE int64_t v256_dotp_su8(v256 a, v256 b) { - return c_v256_dotp_su8(a, b); -} -SIMD_INLINE int64_t v256_dotp_s16(v256 a, v256 b) { - return c_v256_dotp_s16(a, b); -} -SIMD_INLINE int64_t v256_dotp_s32(v256 a, v256 b) { - return c_v256_dotp_s32(a, b); -} -SIMD_INLINE uint64_t v256_hadd_u8(v256 a) { return c_v256_hadd_u8(a); } - -SIMD_INLINE v256 v256_or(v256 a, v256 b) { return c_v256_or(a, b); } -SIMD_INLINE v256 v256_xor(v256 a, v256 b) { return c_v256_xor(a, b); } -SIMD_INLINE v256 v256_and(v256 a, v256 b) { return c_v256_and(a, b); } -SIMD_INLINE v256 v256_andn(v256 a, v256 b) { return c_v256_andn(a, b); } - -SIMD_INLINE v256 v256_add_8(v256 a, v256 b) { return c_v256_add_8(a, b); } -SIMD_INLINE v256 v256_add_16(v256 a, v256 b) { return c_v256_add_16(a, b); } -SIMD_INLINE v256 v256_sadd_s8(v256 a, v256 b) { return c_v256_sadd_s8(a, b); } -SIMD_INLINE v256 v256_sadd_u8(v256 a, v256 b) { return c_v256_sadd_u8(a, b); } -SIMD_INLINE v256 v256_sadd_s16(v256 a, v256 b) { return c_v256_sadd_s16(a, b); } -SIMD_INLINE v256 v256_add_32(v256 a, v256 b) { return c_v256_add_32(a, b); } -SIMD_INLINE v256 v256_add_64(v256 a, v256 b) { return c_v256_add_64(a, b); } -SIMD_INLINE v256 v256_sub_64(v256 a, v256 b) { return c_v256_sub_64(a, b); } -SIMD_INLINE v256 v256_padd_u8(v256 a) { return c_v256_padd_u8(a); } -SIMD_INLINE v256 v256_padd_s16(v256 a) { return c_v256_padd_s16(a); } -SIMD_INLINE v256 v256_sub_8(v256 a, v256 b) { return c_v256_sub_8(a, b); } -SIMD_INLINE v256 v256_ssub_u8(v256 a, v256 b) { return c_v256_ssub_u8(a, b); } -SIMD_INLINE v256 v256_ssub_s8(v256 a, v256 b) { return c_v256_ssub_s8(a, b); } -SIMD_INLINE v256 v256_sub_16(v256 a, v256 b) { return c_v256_sub_16(a, b); } -SIMD_INLINE v256 v256_ssub_s16(v256 a, v256 b) { return c_v256_ssub_s16(a, b); } -SIMD_INLINE v256 v256_ssub_u16(v256 a, v256 b) { return c_v256_ssub_u16(a, b); } -SIMD_INLINE v256 v256_sub_32(v256 a, v256 b) { return c_v256_sub_32(a, b); } -SIMD_INLINE v256 v256_abs_s16(v256 a) { return c_v256_abs_s16(a); } -SIMD_INLINE v256 v256_abs_s8(v256 a) { return c_v256_abs_s8(a); } - -SIMD_INLINE v256 v256_mul_s16(v128 a, v128 b) { return c_v256_mul_s16(a, b); } -SIMD_INLINE v256 v256_mullo_s16(v256 a, v256 b) { - return c_v256_mullo_s16(a, b); -} -SIMD_INLINE v256 v256_mulhi_s16(v256 a, v256 b) { - return c_v256_mulhi_s16(a, b); -} -SIMD_INLINE v256 v256_mullo_s32(v256 a, v256 b) { - return c_v256_mullo_s32(a, b); -} -SIMD_INLINE v256 v256_madd_s16(v256 a, v256 b) { return c_v256_madd_s16(a, b); } -SIMD_INLINE v256 v256_madd_us8(v256 a, v256 b) { return c_v256_madd_us8(a, b); } - -SIMD_INLINE uint32_t v256_movemask_8(v256 a) { return c_v256_movemask_8(a); } -SIMD_INLINE v256 v256_blend_8(v256 a, v256 b, v256 c) { - return c_v256_blend_8(a, b, c); -} - -SIMD_INLINE v256 v256_avg_u8(v256 a, v256 b) { return c_v256_avg_u8(a, b); } -SIMD_INLINE v256 v256_rdavg_u8(v256 a, v256 b) { return c_v256_rdavg_u8(a, b); } -SIMD_INLINE v256 v256_rdavg_u16(v256 a, v256 b) { - return c_v256_rdavg_u16(a, b); -} -SIMD_INLINE v256 v256_avg_u16(v256 a, v256 b) { return c_v256_avg_u16(a, b); } -SIMD_INLINE v256 v256_min_u8(v256 a, v256 b) { return c_v256_min_u8(a, b); } -SIMD_INLINE v256 v256_max_u8(v256 a, v256 b) { return c_v256_max_u8(a, b); } -SIMD_INLINE v256 v256_min_s8(v256 a, v256 b) { return c_v256_min_s8(a, b); } -SIMD_INLINE v256 v256_max_s8(v256 a, v256 b) { return c_v256_max_s8(a, b); } -SIMD_INLINE v256 v256_min_s16(v256 a, v256 b) { return c_v256_min_s16(a, b); } -SIMD_INLINE v256 v256_max_s16(v256 a, v256 b) { return c_v256_max_s16(a, b); } -SIMD_INLINE v256 v256_min_s32(v256 a, v256 b) { return c_v256_min_s32(a, b); } -SIMD_INLINE v256 v256_max_s32(v256 a, v256 b) { return c_v256_max_s32(a, b); } - -SIMD_INLINE v256 v256_ziplo_8(v256 a, v256 b) { return c_v256_ziplo_8(a, b); } -SIMD_INLINE v256 v256_ziphi_8(v256 a, v256 b) { return c_v256_ziphi_8(a, b); } -SIMD_INLINE v256 v256_ziplo_16(v256 a, v256 b) { return c_v256_ziplo_16(a, b); } -SIMD_INLINE v256 v256_ziphi_16(v256 a, v256 b) { return c_v256_ziphi_16(a, b); } -SIMD_INLINE v256 v256_ziplo_32(v256 a, v256 b) { return c_v256_ziplo_32(a, b); } -SIMD_INLINE v256 v256_ziphi_32(v256 a, v256 b) { return c_v256_ziphi_32(a, b); } -SIMD_INLINE v256 v256_ziplo_64(v256 a, v256 b) { return c_v256_ziplo_64(a, b); } -SIMD_INLINE v256 v256_ziphi_64(v256 a, v256 b) { return c_v256_ziphi_64(a, b); } -SIMD_INLINE v256 v256_ziplo_128(v256 a, v256 b) { - return c_v256_ziplo_128(a, b); -} -SIMD_INLINE v256 v256_ziphi_128(v256 a, v256 b) { - return c_v256_ziphi_128(a, b); -} -SIMD_INLINE v256 v256_zip_8(v128 a, v128 b) { return c_v256_zip_8(a, b); } -SIMD_INLINE v256 v256_zip_16(v128 a, v128 b) { return c_v256_zip_16(a, b); } -SIMD_INLINE v256 v256_zip_32(v128 a, v128 b) { return c_v256_zip_32(a, b); } -SIMD_INLINE v256 v256_unziplo_8(v256 a, v256 b) { - return c_v256_unziplo_8(a, b); -} -SIMD_INLINE v256 v256_unziphi_8(v256 a, v256 b) { - return c_v256_unziphi_8(a, b); -} -SIMD_INLINE v256 v256_unziplo_16(v256 a, v256 b) { - return c_v256_unziplo_16(a, b); -} -SIMD_INLINE v256 v256_unziphi_16(v256 a, v256 b) { - return c_v256_unziphi_16(a, b); -} -SIMD_INLINE v256 v256_unziplo_32(v256 a, v256 b) { - return c_v256_unziplo_32(a, b); -} -SIMD_INLINE v256 v256_unziphi_32(v256 a, v256 b) { - return c_v256_unziphi_32(a, b); -} -SIMD_INLINE v256 v256_unziplo_64(v256 a, v256 b) { - return c_v256_unziplo_64(a, b); -} -SIMD_INLINE v256 v256_unziphi_64(v256 a, v256 b) { - return c_v256_unziphi_64(a, b); -} -SIMD_INLINE v256 v256_unpack_u8_s16(v128 a) { return c_v256_unpack_u8_s16(a); } -SIMD_INLINE v256 v256_unpacklo_u8_s16(v256 a) { - return c_v256_unpacklo_u8_s16(a); -} -SIMD_INLINE v256 v256_unpackhi_u8_s16(v256 a) { - return c_v256_unpackhi_u8_s16(a); -} -SIMD_INLINE v256 v256_unpack_s8_s16(v128 a) { return c_v256_unpack_s8_s16(a); } -SIMD_INLINE v256 v256_unpacklo_s8_s16(v256 a) { - return c_v256_unpacklo_s8_s16(a); -} -SIMD_INLINE v256 v256_unpackhi_s8_s16(v256 a) { - return c_v256_unpackhi_s8_s16(a); -} -SIMD_INLINE v256 v256_pack_s32_s16(v256 a, v256 b) { - return c_v256_pack_s32_s16(a, b); -} -SIMD_INLINE v256 v256_pack_s32_u16(v256 a, v256 b) { - return c_v256_pack_s32_u16(a, b); -} -SIMD_INLINE v256 v256_pack_s16_u8(v256 a, v256 b) { - return c_v256_pack_s16_u8(a, b); -} -SIMD_INLINE v256 v256_pack_s16_s8(v256 a, v256 b) { - return c_v256_pack_s16_s8(a, b); -} -SIMD_INLINE v256 v256_unpack_u16_s32(v128 a) { - return c_v256_unpack_u16_s32(a); -} -SIMD_INLINE v256 v256_unpack_s16_s32(v128 a) { - return c_v256_unpack_s16_s32(a); -} -SIMD_INLINE v256 v256_unpacklo_u16_s32(v256 a) { - return c_v256_unpacklo_u16_s32(a); -} -SIMD_INLINE v256 v256_unpacklo_s16_s32(v256 a) { - return c_v256_unpacklo_s16_s32(a); -} -SIMD_INLINE v256 v256_unpackhi_u16_s32(v256 a) { - return c_v256_unpackhi_u16_s32(a); -} -SIMD_INLINE v256 v256_unpackhi_s16_s32(v256 a) { - return c_v256_unpackhi_s16_s32(a); -} -SIMD_INLINE v256 v256_shuffle_8(v256 a, v256 pattern) { - return c_v256_shuffle_8(a, pattern); -} -SIMD_INLINE v256 v256_wideshuffle_8(v256 a, v256 b, v256 pattern) { - return c_v256_wideshuffle_8(a, b, pattern); -} -SIMD_INLINE v256 v256_pshuffle_8(v256 a, v256 pattern) { - return c_v256_pshuffle_8(a, pattern); -} - -SIMD_INLINE v256 v256_cmpgt_s8(v256 a, v256 b) { return c_v256_cmpgt_s8(a, b); } -SIMD_INLINE v256 v256_cmplt_s8(v256 a, v256 b) { return c_v256_cmplt_s8(a, b); } -SIMD_INLINE v256 v256_cmpeq_8(v256 a, v256 b) { return c_v256_cmpeq_8(a, b); } -SIMD_INLINE v256 v256_cmpgt_s16(v256 a, v256 b) { - return c_v256_cmpgt_s16(a, b); -} -SIMD_INLINE v256 v256_cmplt_s16(v256 a, v256 b) { - return c_v256_cmplt_s16(a, b); -} -SIMD_INLINE v256 v256_cmpeq_16(v256 a, v256 b) { return c_v256_cmpeq_16(a, b); } -SIMD_INLINE v256 v256_cmpeq_32(v256 a, v256 b) { return c_v256_cmpeq_32(a, b); } - -SIMD_INLINE v256 v256_cmpgt_s32(v256 a, v256 b) { - return c_v256_cmpgt_s32(a, b); -} -SIMD_INLINE v256 v256_cmplt_s32(v256 a, v256 b) { - return c_v256_cmplt_s32(a, b); -} -SIMD_INLINE v256 v256_shl_8(v256 a, unsigned int c) { - return c_v256_shl_8(a, c); -} -SIMD_INLINE v256 v256_shr_u8(v256 a, unsigned int c) { - return c_v256_shr_u8(a, c); -} -SIMD_INLINE v256 v256_shr_s8(v256 a, unsigned int c) { - return c_v256_shr_s8(a, c); -} -SIMD_INLINE v256 v256_shl_16(v256 a, unsigned int c) { - return c_v256_shl_16(a, c); -} -SIMD_INLINE v256 v256_shr_u16(v256 a, unsigned int c) { - return c_v256_shr_u16(a, c); -} -SIMD_INLINE v256 v256_shr_s16(v256 a, unsigned int c) { - return c_v256_shr_s16(a, c); -} -SIMD_INLINE v256 v256_shl_32(v256 a, unsigned int c) { - return c_v256_shl_32(a, c); -} -SIMD_INLINE v256 v256_shr_u32(v256 a, unsigned int c) { - return c_v256_shr_u32(a, c); -} -SIMD_INLINE v256 v256_shr_s32(v256 a, unsigned int c) { - return c_v256_shr_s32(a, c); -} -SIMD_INLINE v256 v256_shl_64(v256 a, unsigned int c) { - return c_v256_shl_64(a, c); -} -SIMD_INLINE v256 v256_shr_u64(v256 a, unsigned int c) { - return c_v256_shr_u64(a, c); -} -SIMD_INLINE v256 v256_shr_s64(v256 a, unsigned int c) { - return c_v256_shr_s64(a, c); -} - -SIMD_INLINE v256 v256_shr_n_byte(v256 a, unsigned int n) { - return c_v256_shr_n_byte(a, n); -} -SIMD_INLINE v256 v256_shl_n_byte(v256 a, unsigned int n) { - return c_v256_shl_n_byte(a, n); -} -SIMD_INLINE v256 v256_shl_n_8(v256 a, unsigned int n) { - return c_v256_shl_n_8(a, n); -} -SIMD_INLINE v256 v256_shl_n_16(v256 a, unsigned int n) { - return c_v256_shl_n_16(a, n); -} -SIMD_INLINE v256 v256_shl_n_32(v256 a, unsigned int n) { - return c_v256_shl_n_32(a, n); -} -SIMD_INLINE v256 v256_shl_n_64(v256 a, unsigned int n) { - return c_v256_shl_n_64(a, n); -} -SIMD_INLINE v256 v256_shr_n_u8(v256 a, unsigned int n) { - return c_v256_shr_n_u8(a, n); -} -SIMD_INLINE v256 v256_shr_n_u16(v256 a, unsigned int n) { - return c_v256_shr_n_u16(a, n); -} -SIMD_INLINE v256 v256_shr_n_u32(v256 a, unsigned int n) { - return c_v256_shr_n_u32(a, n); -} -SIMD_INLINE v256 v256_shr_n_u64(v256 a, unsigned int n) { - return c_v256_shr_n_u64(a, n); -} -SIMD_INLINE v256 v256_shr_n_s8(v256 a, unsigned int n) { - return c_v256_shr_n_s8(a, n); -} -SIMD_INLINE v256 v256_shr_n_s16(v256 a, unsigned int n) { - return c_v256_shr_n_s16(a, n); -} -SIMD_INLINE v256 v256_shr_n_s32(v256 a, unsigned int n) { - return c_v256_shr_n_s32(a, n); -} -SIMD_INLINE v256 v256_shr_n_s64(v256 a, unsigned int n) { - return c_v256_shr_n_s64(a, n); -} - -SIMD_INLINE v256 v256_shr_n_word(v256 a, unsigned int n) { - return c_v256_shr_n_word(a, n); -} -SIMD_INLINE v256 v256_shl_n_word(v256 a, unsigned int n) { - return c_v256_shl_n_word(a, n); -} - -typedef uint32_t sad256_internal_u16; -SIMD_INLINE sad256_internal_u16 v256_sad_u16_init() { - return c_v256_sad_u16_init(); -} -SIMD_INLINE sad256_internal_u16 v256_sad_u16(sad256_internal_u16 s, v256 a, - v256 b) { - return c_v256_sad_u16(s, a, b); -} -SIMD_INLINE uint32_t v256_sad_u16_sum(sad256_internal_u16 s) { - return c_v256_sad_u16_sum(s); -} - -typedef uint64_t ssd256_internal_s16; -SIMD_INLINE ssd256_internal_s16 v256_ssd_s16_init() { - return c_v256_ssd_s16_init(); -} -SIMD_INLINE ssd256_internal_s16 v256_ssd_s16(ssd256_internal_s16 s, v256 a, - v256 b) { - return c_v256_ssd_s16(s, a, b); -} -SIMD_INLINE uint64_t v256_ssd_s16_sum(ssd256_internal_s16 s) { - return c_v256_ssd_s16_sum(s); -} - -#endif // AOM_AOM_DSP_SIMD_V256_INTRINSICS_H_ diff --git a/third_party/aom/aom_dsp/simd/v256_intrinsics_arm.h b/third_party/aom/aom_dsp/simd/v256_intrinsics_arm.h deleted file mode 100644 index bd86ea172..000000000 --- a/third_party/aom/aom_dsp/simd/v256_intrinsics_arm.h +++ /dev/null @@ -1,17 +0,0 @@ -/* - * Copyright (c) 2016, Alliance for Open Media. All rights reserved - * - * This source code is subject to the terms of the BSD 2 Clause License and - * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License - * was not distributed with this source code in the LICENSE file, you can - * obtain it at www.aomedia.org/license/software. If the Alliance for Open - * Media Patent License 1.0 was not distributed with this source code in the - * PATENTS file, you can obtain it at www.aomedia.org/license/patent. - */ - -#ifndef AOM_AOM_DSP_SIMD_V256_INTRINSICS_ARM_H_ -#define AOM_AOM_DSP_SIMD_V256_INTRINSICS_ARM_H_ - -#include "aom_dsp/simd/v256_intrinsics_v128.h" - -#endif // AOM_AOM_DSP_SIMD_V256_INTRINSICS_ARM_H_ diff --git a/third_party/aom/aom_dsp/simd/v256_intrinsics_c.h b/third_party/aom/aom_dsp/simd/v256_intrinsics_c.h deleted file mode 100644 index a1c08e95a..000000000 --- a/third_party/aom/aom_dsp/simd/v256_intrinsics_c.h +++ /dev/null @@ -1,953 +0,0 @@ -/* - * Copyright (c) 2016, Alliance for Open Media. All rights reserved - * - * This source code is subject to the terms of the BSD 2 Clause License and - * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License - * was not distributed with this source code in the LICENSE file, you can - * obtain it at www.aomedia.org/license/software. If the Alliance for Open - * Media Patent License 1.0 was not distributed with this source code in the - * PATENTS file, you can obtain it at www.aomedia.org/license/patent. - */ - -#ifndef AOM_AOM_DSP_SIMD_V256_INTRINSICS_C_H_ -#define AOM_AOM_DSP_SIMD_V256_INTRINSICS_C_H_ - -#include <stdio.h> -#include <stdlib.h> - -#include "config/aom_config.h" - -#include "aom_dsp/simd/v128_intrinsics_c.h" - -typedef union { - uint8_t u8[32]; - uint16_t u16[16]; - uint32_t u32[8]; - uint64_t u64[4]; - int8_t s8[32]; - int16_t s16[16]; - int32_t s32[8]; - int64_t s64[4]; - c_v64 v64[4]; - c_v128 v128[2]; -} c_v256; - -SIMD_INLINE uint32_t c_v256_low_u32(c_v256 a) { return a.u32[0]; } - -SIMD_INLINE c_v64 c_v256_low_v64(c_v256 a) { return a.v64[0]; } - -SIMD_INLINE uint64_t c_v256_low_u64(c_v256 a) { return a.u64[0]; } - -SIMD_INLINE c_v128 c_v256_low_v128(c_v256 a) { return a.v128[0]; } - -SIMD_INLINE c_v128 c_v256_high_v128(c_v256 a) { return a.v128[1]; } - -SIMD_INLINE c_v256 c_v256_from_v128(c_v128 hi, c_v128 lo) { - c_v256 t; - t.v128[1] = hi; - t.v128[0] = lo; - return t; -} - -SIMD_INLINE c_v256 c_v256_from_64(uint64_t a, uint64_t b, uint64_t c, - uint64_t d) { - c_v256 t; - t.u64[3] = a; - t.u64[2] = b; - t.u64[1] = c; - t.u64[0] = d; - return t; -} - -SIMD_INLINE c_v256 c_v256_from_v64(c_v64 a, c_v64 b, c_v64 c, c_v64 d) { - c_v256 t; - t.u64[3] = a.u64; - t.u64[2] = b.u64; - t.u64[1] = c.u64; - t.u64[0] = d.u64; - return t; -} - -SIMD_INLINE c_v256 c_v256_load_unaligned(const void *p) { - c_v256 t; - uint8_t *pp = (uint8_t *)p; - uint8_t *q = (uint8_t *)&t; - int c; - for (c = 0; c < 32; c++) q[c] = pp[c]; - return t; -} - -SIMD_INLINE c_v256 c_v256_load_aligned(const void *p) { - if (SIMD_CHECK && (uintptr_t)p & 31) { - fprintf(stderr, "Error: unaligned v256 load at %p\n", p); - abort(); - } - return c_v256_load_unaligned(p); -} - -SIMD_INLINE void c_v256_store_unaligned(void *p, c_v256 a) { - uint8_t *pp = (uint8_t *)p; - uint8_t *q = (uint8_t *)&a; - int c; - for (c = 0; c < 32; c++) pp[c] = q[c]; -} - -SIMD_INLINE void c_v256_store_aligned(void *p, c_v256 a) { - if (SIMD_CHECK && (uintptr_t)p & 31) { - fprintf(stderr, "Error: unaligned v256 store at %p\n", p); - abort(); - } - c_v256_store_unaligned(p, a); -} - -SIMD_INLINE c_v256 c_v256_zero() { - c_v256 t; - t.u64[3] = t.u64[2] = t.u64[1] = t.u64[0] = 0; - return t; -} - -SIMD_INLINE c_v256 c_v256_dup_8(uint8_t x) { - c_v256 t; - t.v64[3] = t.v64[2] = t.v64[1] = t.v64[0] = c_v64_dup_8(x); - return t; -} - -SIMD_INLINE c_v256 c_v256_dup_16(uint16_t x) { - c_v256 t; - t.v64[3] = t.v64[2] = t.v64[1] = t.v64[0] = c_v64_dup_16(x); - return t; -} - -SIMD_INLINE c_v256 c_v256_dup_32(uint32_t x) { - c_v256 t; - t.v64[3] = t.v64[2] = t.v64[1] = t.v64[0] = c_v64_dup_32(x); - return t; -} - -SIMD_INLINE c_v256 c_v256_dup_64(uint64_t x) { - c_v256 t; - t.u64[3] = t.u64[2] = t.u64[1] = t.u64[0] = x; - return t; -} - -SIMD_INLINE int64_t c_v256_dotp_su8(c_v256 a, c_v256 b) { - return c_v128_dotp_su8(a.v128[1], b.v128[1]) + - c_v128_dotp_su8(a.v128[0], b.v128[0]); -} - -SIMD_INLINE int64_t c_v256_dotp_s16(c_v256 a, c_v256 b) { - return c_v128_dotp_s16(a.v128[1], b.v128[1]) + - c_v128_dotp_s16(a.v128[0], b.v128[0]); -} - -SIMD_INLINE int64_t c_v256_dotp_s32(c_v256 a, c_v256 b) { - return c_v128_dotp_s32(a.v128[1], b.v128[1]) + - c_v128_dotp_s32(a.v128[0], b.v128[0]); -} - -SIMD_INLINE uint64_t c_v256_hadd_u8(c_v256 a) { - return c_v128_hadd_u8(a.v128[1]) + c_v128_hadd_u8(a.v128[0]); -} - -typedef uint32_t c_sad256_internal; - -SIMD_INLINE c_sad256_internal c_v256_sad_u8_init() { return 0; } - -/* Implementation dependent return value. Result must be finalised with - v256_sad_u8_sum(). - The result for more than 16 v256_sad_u8() calls is undefined. */ -SIMD_INLINE c_sad256_internal c_v256_sad_u8(c_sad256_internal s, c_v256 a, - c_v256 b) { - int c; - for (c = 0; c < 32; c++) - s += a.u8[c] > b.u8[c] ? a.u8[c] - b.u8[c] : b.u8[c] - a.u8[c]; - return s; -} - -SIMD_INLINE uint32_t c_v256_sad_u8_sum(c_sad256_internal s) { return s; } - -typedef uint32_t c_ssd256_internal; - -SIMD_INLINE c_ssd256_internal c_v256_ssd_u8_init() { return 0; } - -/* Implementation dependent return value. Result must be finalised with - * v256_ssd_u8_sum(). */ -SIMD_INLINE c_ssd256_internal c_v256_ssd_u8(c_ssd256_internal s, c_v256 a, - c_v256 b) { - int c; - for (c = 0; c < 32; c++) s += (a.u8[c] - b.u8[c]) * (a.u8[c] - b.u8[c]); - return s; -} - -SIMD_INLINE uint32_t c_v256_ssd_u8_sum(c_ssd256_internal s) { return s; } - -SIMD_INLINE c_v256 c_v256_or(c_v256 a, c_v256 b) { - return c_v256_from_v128(c_v128_or(a.v128[1], b.v128[1]), - c_v128_or(a.v128[0], b.v128[0])); -} - -SIMD_INLINE c_v256 c_v256_xor(c_v256 a, c_v256 b) { - return c_v256_from_v128(c_v128_xor(a.v128[1], b.v128[1]), - c_v128_xor(a.v128[0], b.v128[0])); -} - -SIMD_INLINE c_v256 c_v256_and(c_v256 a, c_v256 b) { - return c_v256_from_v128(c_v128_and(a.v128[1], b.v128[1]), - c_v128_and(a.v128[0], b.v128[0])); -} - -SIMD_INLINE c_v256 c_v256_andn(c_v256 a, c_v256 b) { - return c_v256_from_v128(c_v128_andn(a.v128[1], b.v128[1]), - c_v128_andn(a.v128[0], b.v128[0])); -} - -SIMD_INLINE c_v256 c_v256_add_8(c_v256 a, c_v256 b) { - return c_v256_from_v128(c_v128_add_8(a.v128[1], b.v128[1]), - c_v128_add_8(a.v128[0], b.v128[0])); -} - -SIMD_INLINE c_v256 c_v256_add_16(c_v256 a, c_v256 b) { - return c_v256_from_v128(c_v128_add_16(a.v128[1], b.v128[1]), - c_v128_add_16(a.v128[0], b.v128[0])); -} - -SIMD_INLINE c_v256 c_v256_sadd_s8(c_v256 a, c_v256 b) { - return c_v256_from_v128(c_v128_sadd_s8(a.v128[1], b.v128[1]), - c_v128_sadd_s8(a.v128[0], b.v128[0])); -} - -SIMD_INLINE c_v256 c_v256_sadd_u8(c_v256 a, c_v256 b) { - return c_v256_from_v128(c_v128_sadd_u8(a.v128[1], b.v128[1]), - c_v128_sadd_u8(a.v128[0], b.v128[0])); -} - -SIMD_INLINE c_v256 c_v256_sadd_s16(c_v256 a, c_v256 b) { - return c_v256_from_v128(c_v128_sadd_s16(a.v128[1], b.v128[1]), - c_v128_sadd_s16(a.v128[0], b.v128[0])); -} - -SIMD_INLINE c_v256 c_v256_add_32(c_v256 a, c_v256 b) { - return c_v256_from_v128(c_v128_add_32(a.v128[1], b.v128[1]), - c_v128_add_32(a.v128[0], b.v128[0])); -} - -SIMD_INLINE c_v256 c_v256_add_64(c_v256 a, c_v256 b) { - return c_v256_from_v128(c_v128_add_64(a.v128[1], b.v128[1]), - c_v128_add_64(a.v128[0], b.v128[0])); -} - -SIMD_INLINE c_v256 c_v256_sub_64(c_v256 a, c_v256 b) { - return c_v256_from_v128(c_v128_sub_64(a.v128[1], b.v128[1]), - c_v128_sub_64(a.v128[0], b.v128[0])); -} - -SIMD_INLINE c_v256 c_v256_padd_u8(c_v256 a) { - c_v256 t; - for (int i = 0; i < 16; i++) - t.u16[i] = (uint16_t)a.u8[i * 2] + (uint16_t)a.u8[i * 2 + 1]; - return t; -} - -SIMD_INLINE c_v256 c_v256_padd_s16(c_v256 a) { - c_v256 t; - t.s32[0] = (int32_t)a.s16[0] + (int32_t)a.s16[1]; - t.s32[1] = (int32_t)a.s16[2] + (int32_t)a.s16[3]; - t.s32[2] = (int32_t)a.s16[4] + (int32_t)a.s16[5]; - t.s32[3] = (int32_t)a.s16[6] + (int32_t)a.s16[7]; - t.s32[4] = (int32_t)a.s16[8] + (int32_t)a.s16[9]; - t.s32[5] = (int32_t)a.s16[10] + (int32_t)a.s16[11]; - t.s32[6] = (int32_t)a.s16[12] + (int32_t)a.s16[13]; - t.s32[7] = (int32_t)a.s16[14] + (int32_t)a.s16[15]; - return t; -} - -SIMD_INLINE c_v256 c_v256_sub_8(c_v256 a, c_v256 b) { - return c_v256_from_v128(c_v128_sub_8(a.v128[1], b.v128[1]), - c_v128_sub_8(a.v128[0], b.v128[0])); -} - -SIMD_INLINE c_v256 c_v256_ssub_u8(c_v256 a, c_v256 b) { - return c_v256_from_v128(c_v128_ssub_u8(a.v128[1], b.v128[1]), - c_v128_ssub_u8(a.v128[0], b.v128[0])); -} - -SIMD_INLINE c_v256 c_v256_ssub_s8(c_v256 a, c_v256 b) { - return c_v256_from_v128(c_v128_ssub_s8(a.v128[1], b.v128[1]), - c_v128_ssub_s8(a.v128[0], b.v128[0])); -} - -SIMD_INLINE c_v256 c_v256_sub_16(c_v256 a, c_v256 b) { - return c_v256_from_v128(c_v128_sub_16(a.v128[1], b.v128[1]), - c_v128_sub_16(a.v128[0], b.v128[0])); -} - -SIMD_INLINE c_v256 c_v256_ssub_s16(c_v256 a, c_v256 b) { - return c_v256_from_v128(c_v128_ssub_s16(a.v128[1], b.v128[1]), - c_v128_ssub_s16(a.v128[0], b.v128[0])); -} - -SIMD_INLINE c_v256 c_v256_ssub_u16(c_v256 a, c_v256 b) { - return c_v256_from_v128(c_v128_ssub_u16(a.v128[1], b.v128[1]), - c_v128_ssub_u16(a.v128[0], b.v128[0])); -} - -SIMD_INLINE c_v256 c_v256_sub_32(c_v256 a, c_v256 b) { - return c_v256_from_v128(c_v128_sub_32(a.v128[1], b.v128[1]), - c_v128_sub_32(a.v128[0], b.v128[0])); -} - -SIMD_INLINE c_v256 c_v256_abs_s16(c_v256 a) { - return c_v256_from_v128(c_v128_abs_s16(a.v128[1]), c_v128_abs_s16(a.v128[0])); -} - -SIMD_INLINE c_v256 c_v256_abs_s8(c_v256 a) { - return c_v256_from_v128(c_v128_abs_s8(a.v128[1]), c_v128_abs_s8(a.v128[0])); -} - -SIMD_INLINE c_v256 c_v256_mul_s16(c_v128 a, c_v128 b) { - c_v128 lo_bits = c_v128_mullo_s16(a, b); - c_v128 hi_bits = c_v128_mulhi_s16(a, b); - return c_v256_from_v128(c_v128_ziphi_16(hi_bits, lo_bits), - c_v128_ziplo_16(hi_bits, lo_bits)); -} - -SIMD_INLINE c_v256 c_v256_mullo_s16(c_v256 a, c_v256 b) { - return c_v256_from_v128(c_v128_mullo_s16(a.v128[1], b.v128[1]), - c_v128_mullo_s16(a.v128[0], b.v128[0])); -} - -SIMD_INLINE c_v256 c_v256_mulhi_s16(c_v256 a, c_v256 b) { - return c_v256_from_v128(c_v128_mulhi_s16(a.v128[1], b.v128[1]), - c_v128_mulhi_s16(a.v128[0], b.v128[0])); -} - -SIMD_INLINE c_v256 c_v256_mullo_s32(c_v256 a, c_v256 b) { - return c_v256_from_v128(c_v128_mullo_s32(a.v128[1], b.v128[1]), - c_v128_mullo_s32(a.v128[0], b.v128[0])); -} - -SIMD_INLINE c_v256 c_v256_madd_s16(c_v256 a, c_v256 b) { - return c_v256_from_v128(c_v128_madd_s16(a.v128[1], b.v128[1]), - c_v128_madd_s16(a.v128[0], b.v128[0])); -} - -SIMD_INLINE c_v256 c_v256_madd_us8(c_v256 a, c_v256 b) { - return c_v256_from_v128(c_v128_madd_us8(a.v128[1], b.v128[1]), - c_v128_madd_us8(a.v128[0], b.v128[0])); -} - -SIMD_INLINE c_v256 c_v256_avg_u8(c_v256 a, c_v256 b) { - return c_v256_from_v128(c_v128_avg_u8(a.v128[1], b.v128[1]), - c_v128_avg_u8(a.v128[0], b.v128[0])); -} - -SIMD_INLINE c_v256 c_v256_rdavg_u8(c_v256 a, c_v256 b) { - return c_v256_from_v128(c_v128_rdavg_u8(a.v128[1], b.v128[1]), - c_v128_rdavg_u8(a.v128[0], b.v128[0])); -} - -SIMD_INLINE c_v256 c_v256_rdavg_u16(c_v256 a, c_v256 b) { - return c_v256_from_v128(c_v128_rdavg_u16(a.v128[1], b.v128[1]), - c_v128_rdavg_u16(a.v128[0], b.v128[0])); -} - -SIMD_INLINE c_v256 c_v256_avg_u16(c_v256 a, c_v256 b) { - return c_v256_from_v128(c_v128_avg_u16(a.v128[1], b.v128[1]), - c_v128_avg_u16(a.v128[0], b.v128[0])); -} - -SIMD_INLINE c_v256 c_v256_min_u8(c_v256 a, c_v256 b) { - return c_v256_from_v128(c_v128_min_u8(a.v128[1], b.v128[1]), - c_v128_min_u8(a.v128[0], b.v128[0])); -} - -SIMD_INLINE c_v256 c_v256_max_u8(c_v256 a, c_v256 b) { - return c_v256_from_v128(c_v128_max_u8(a.v128[1], b.v128[1]), - c_v128_max_u8(a.v128[0], b.v128[0])); -} - -SIMD_INLINE c_v256 c_v256_min_s8(c_v256 a, c_v256 b) { - return c_v256_from_v128(c_v128_min_s8(a.v128[1], b.v128[1]), - c_v128_min_s8(a.v128[0], b.v128[0])); -} - -SIMD_INLINE uint32_t c_v256_movemask_8(c_v256 a) { - return ((a.s8[31] < 0) << 31) | ((a.s8[30] < 0) << 30) | - ((a.s8[29] < 0) << 29) | ((a.s8[28] < 0) << 28) | - ((a.s8[27] < 0) << 27) | ((a.s8[26] < 0) << 26) | - ((a.s8[25] < 0) << 25) | ((a.s8[24] < 0) << 24) | - ((a.s8[23] < 0) << 23) | ((a.s8[22] < 0) << 22) | - ((a.s8[21] < 0) << 21) | ((a.s8[20] < 0) << 20) | - ((a.s8[19] < 0) << 19) | ((a.s8[18] < 0) << 18) | - ((a.s8[17] < 0) << 17) | ((a.s8[16] < 0) << 16) | - ((a.s8[15] < 0) << 15) | ((a.s8[14] < 0) << 14) | - ((a.s8[13] < 0) << 13) | ((a.s8[12] < 0) << 12) | - ((a.s8[11] < 0) << 11) | ((a.s8[10] < 0) << 10) | - ((a.s8[9] < 0) << 9) | ((a.s8[8] < 0) << 8) | ((a.s8[7] < 0) << 7) | - ((a.s8[6] < 0) << 6) | ((a.s8[5] < 0) << 5) | ((a.s8[4] < 0) << 4) | - ((a.s8[3] < 0) << 3) | ((a.s8[2] < 0) << 2) | ((a.s8[1] < 0) << 1) | - ((a.s8[0] < 0) << 0); -} - -SIMD_INLINE c_v256 c_v256_blend_8(c_v256 a, c_v256 b, c_v256 c) { - c_v256 t; - for (int i = 0; i < 32; i++) t.u8[i] = c.s8[i] < 0 ? b.u8[i] : a.u8[i]; - return t; -} - -SIMD_INLINE c_v256 c_v256_max_s8(c_v256 a, c_v256 b) { - return c_v256_from_v128(c_v128_max_s8(a.v128[1], b.v128[1]), - c_v128_max_s8(a.v128[0], b.v128[0])); -} - -SIMD_INLINE c_v256 c_v256_min_s16(c_v256 a, c_v256 b) { - return c_v256_from_v128(c_v128_min_s16(a.v128[1], b.v128[1]), - c_v128_min_s16(a.v128[0], b.v128[0])); -} - -SIMD_INLINE c_v256 c_v256_max_s16(c_v256 a, c_v256 b) { - return c_v256_from_v128(c_v128_max_s16(a.v128[1], b.v128[1]), - c_v128_max_s16(a.v128[0], b.v128[0])); -} - -SIMD_INLINE c_v256 c_v256_min_s32(c_v256 a, c_v256 b) { - return c_v256_from_v128(c_v128_min_s32(a.v128[1], b.v128[1]), - c_v128_min_s32(a.v128[0], b.v128[0])); -} - -SIMD_INLINE c_v256 c_v256_max_s32(c_v256 a, c_v256 b) { - return c_v256_from_v128(c_v128_max_s32(a.v128[1], b.v128[1]), - c_v128_max_s32(a.v128[0], b.v128[0])); -} - -SIMD_INLINE c_v256 c_v256_ziplo_8(c_v256 a, c_v256 b) { - return c_v256_from_v128(c_v128_ziphi_8(a.v128[0], b.v128[0]), - c_v128_ziplo_8(a.v128[0], b.v128[0])); -} - -SIMD_INLINE c_v256 c_v256_ziphi_8(c_v256 a, c_v256 b) { - return c_v256_from_v128(c_v128_ziphi_8(a.v128[1], b.v128[1]), - c_v128_ziplo_8(a.v128[1], b.v128[1])); -} - -SIMD_INLINE c_v256 c_v256_ziplo_16(c_v256 a, c_v256 b) { - return c_v256_from_v128(c_v128_ziphi_16(a.v128[0], b.v128[0]), - c_v128_ziplo_16(a.v128[0], b.v128[0])); -} - -SIMD_INLINE c_v256 c_v256_ziphi_16(c_v256 a, c_v256 b) { - return c_v256_from_v128(c_v128_ziphi_16(a.v128[1], b.v128[1]), - c_v128_ziplo_16(a.v128[1], b.v128[1])); -} - -SIMD_INLINE c_v256 c_v256_ziplo_32(c_v256 a, c_v256 b) { - return c_v256_from_v128(c_v128_ziphi_32(a.v128[0], b.v128[0]), - c_v128_ziplo_32(a.v128[0], b.v128[0])); -} - -SIMD_INLINE c_v256 c_v256_ziphi_32(c_v256 a, c_v256 b) { - return c_v256_from_v128(c_v128_ziphi_32(a.v128[1], b.v128[1]), - c_v128_ziplo_32(a.v128[1], b.v128[1])); -} - -SIMD_INLINE c_v256 c_v256_ziplo_64(c_v256 a, c_v256 b) { - return c_v256_from_v128(c_v128_ziphi_64(a.v128[0], b.v128[0]), - c_v128_ziplo_64(a.v128[0], b.v128[0])); -} - -SIMD_INLINE c_v256 c_v256_ziphi_64(c_v256 a, c_v256 b) { - return c_v256_from_v128(c_v128_ziphi_64(a.v128[1], b.v128[1]), - c_v128_ziplo_64(a.v128[1], b.v128[1])); -} - -SIMD_INLINE c_v256 c_v256_ziplo_128(c_v256 a, c_v256 b) { - return c_v256_from_v128(a.v128[0], b.v128[0]); -} - -SIMD_INLINE c_v256 c_v256_ziphi_128(c_v256 a, c_v256 b) { - return c_v256_from_v128(a.v128[1], b.v128[1]); -} - -SIMD_INLINE c_v256 c_v256_zip_8(c_v128 a, c_v128 b) { - return c_v256_from_v128(c_v128_ziphi_8(a, b), c_v128_ziplo_8(a, b)); -} - -SIMD_INLINE c_v256 c_v256_zip_16(c_v128 a, c_v128 b) { - return c_v256_from_v128(c_v128_ziphi_16(a, b), c_v128_ziplo_16(a, b)); -} - -SIMD_INLINE c_v256 c_v256_zip_32(c_v128 a, c_v128 b) { - return c_v256_from_v128(c_v128_ziphi_32(a, b), c_v128_ziplo_32(a, b)); -} - -SIMD_INLINE c_v256 _c_v256_unzip_8(c_v256 a, c_v256 b, int mode) { - c_v256 t; - int i; - if (mode) { - for (i = 0; i < 16; i++) { - t.u8[i] = a.u8[i * 2 + 1]; - t.u8[i + 16] = b.u8[i * 2 + 1]; - } - } else { - for (i = 0; i < 16; i++) { - t.u8[i] = b.u8[i * 2]; - t.u8[i + 16] = a.u8[i * 2]; - } - } - return t; -} - -SIMD_INLINE c_v256 c_v256_unziplo_8(c_v256 a, c_v256 b) { - return CONFIG_BIG_ENDIAN ? _c_v256_unzip_8(a, b, 1) - : _c_v256_unzip_8(a, b, 0); -} - -SIMD_INLINE c_v256 c_v256_unziphi_8(c_v256 a, c_v256 b) { - return CONFIG_BIG_ENDIAN ? _c_v256_unzip_8(b, a, 0) - : _c_v256_unzip_8(b, a, 1); -} - -SIMD_INLINE c_v256 _c_v256_unzip_16(c_v256 a, c_v256 b, int mode) { - c_v256 t; - int i; - if (mode) { - for (i = 0; i < 8; i++) { - t.u16[i] = a.u16[i * 2 + 1]; - t.u16[i + 8] = b.u16[i * 2 + 1]; - } - } else { - for (i = 0; i < 8; i++) { - t.u16[i] = b.u16[i * 2]; - t.u16[i + 8] = a.u16[i * 2]; - } - } - return t; -} - -SIMD_INLINE c_v256 c_v256_unziplo_16(c_v256 a, c_v256 b) { - return CONFIG_BIG_ENDIAN ? _c_v256_unzip_16(a, b, 1) - : _c_v256_unzip_16(a, b, 0); -} - -SIMD_INLINE c_v256 c_v256_unziphi_16(c_v256 a, c_v256 b) { - return CONFIG_BIG_ENDIAN ? _c_v256_unzip_16(b, a, 0) - : _c_v256_unzip_16(b, a, 1); -} - -SIMD_INLINE c_v256 _c_v256_unzip_32(c_v256 a, c_v256 b, int mode) { - c_v256 t; - if (mode) { - t.u32[7] = b.u32[7]; - t.u32[6] = b.u32[5]; - t.u32[5] = b.u32[3]; - t.u32[4] = b.u32[1]; - t.u32[3] = a.u32[7]; - t.u32[2] = a.u32[5]; - t.u32[1] = a.u32[3]; - t.u32[0] = a.u32[1]; - } else { - t.u32[7] = a.u32[6]; - t.u32[6] = a.u32[4]; - t.u32[5] = a.u32[2]; - t.u32[4] = a.u32[0]; - t.u32[3] = b.u32[6]; - t.u32[2] = b.u32[4]; - t.u32[1] = b.u32[2]; - t.u32[0] = b.u32[0]; - } - return t; -} - -SIMD_INLINE c_v256 c_v256_unziplo_32(c_v256 a, c_v256 b) { - return CONFIG_BIG_ENDIAN ? _c_v256_unzip_32(a, b, 1) - : _c_v256_unzip_32(a, b, 0); -} - -SIMD_INLINE c_v256 c_v256_unziphi_32(c_v256 a, c_v256 b) { - return CONFIG_BIG_ENDIAN ? _c_v256_unzip_32(b, a, 0) - : _c_v256_unzip_32(b, a, 1); -} - -SIMD_INLINE c_v256 _c_v256_unzip_64(c_v256 a, c_v256 b, int mode) { - c_v256 t; - if (mode) { - t.u64[3] = b.u64[3]; - t.u64[2] = b.u64[1]; - t.u64[1] = a.u64[3]; - t.u64[0] = a.u64[1]; - } else { - t.u64[3] = a.u64[2]; - t.u64[2] = a.u64[0]; - t.u64[1] = b.u64[2]; - t.u64[0] = b.u64[0]; - } - return t; -} - -SIMD_INLINE c_v256 c_v256_unziplo_64(c_v256 a, c_v256 b) { - return CONFIG_BIG_ENDIAN ? _c_v256_unzip_64(a, b, 1) - : _c_v256_unzip_64(a, b, 0); -} - -SIMD_INLINE c_v256 c_v256_unziphi_64(c_v256 a, c_v256 b) { - return CONFIG_BIG_ENDIAN ? _c_v256_unzip_64(b, a, 0) - : _c_v256_unzip_64(b, a, 1); -} - -SIMD_INLINE c_v256 c_v256_unpack_u8_s16(c_v128 a) { - return c_v256_from_v128(c_v128_unpackhi_u8_s16(a), c_v128_unpacklo_u8_s16(a)); -} - -SIMD_INLINE c_v256 c_v256_unpacklo_u8_s16(c_v256 a) { - return c_v256_from_v128(c_v128_unpackhi_u8_s16(a.v128[0]), - c_v128_unpacklo_u8_s16(a.v128[0])); -} - -SIMD_INLINE c_v256 c_v256_unpackhi_u8_s16(c_v256 a) { - return c_v256_from_v128(c_v128_unpackhi_u8_s16(a.v128[1]), - c_v128_unpacklo_u8_s16(a.v128[1])); -} - -SIMD_INLINE c_v256 c_v256_unpack_s8_s16(c_v128 a) { - return c_v256_from_v128(c_v128_unpackhi_s8_s16(a), c_v128_unpacklo_s8_s16(a)); -} - -SIMD_INLINE c_v256 c_v256_unpacklo_s8_s16(c_v256 a) { - return c_v256_from_v128(c_v128_unpackhi_s8_s16(a.v128[0]), - c_v128_unpacklo_s8_s16(a.v128[0])); -} - -SIMD_INLINE c_v256 c_v256_unpackhi_s8_s16(c_v256 a) { - return c_v256_from_v128(c_v128_unpackhi_s8_s16(a.v128[1]), - c_v128_unpacklo_s8_s16(a.v128[1])); -} - -SIMD_INLINE c_v256 c_v256_pack_s32_s16(c_v256 a, c_v256 b) { - return c_v256_from_v128(c_v128_pack_s32_s16(a.v128[1], a.v128[0]), - c_v128_pack_s32_s16(b.v128[1], b.v128[0])); -} - -SIMD_INLINE c_v256 c_v256_pack_s32_u16(c_v256 a, c_v256 b) { - return c_v256_from_v128(c_v128_pack_s32_u16(a.v128[1], a.v128[0]), - c_v128_pack_s32_u16(b.v128[1], b.v128[0])); -} - -SIMD_INLINE c_v256 c_v256_pack_s16_u8(c_v256 a, c_v256 b) { - return c_v256_from_v128(c_v128_pack_s16_u8(a.v128[1], a.v128[0]), - c_v128_pack_s16_u8(b.v128[1], b.v128[0])); -} - -SIMD_INLINE c_v256 c_v256_pack_s16_s8(c_v256 a, c_v256 b) { - return c_v256_from_v128(c_v128_pack_s16_s8(a.v128[1], a.v128[0]), - c_v128_pack_s16_s8(b.v128[1], b.v128[0])); -} - -SIMD_INLINE c_v256 c_v256_unpack_u16_s32(c_v128 a) { - return c_v256_from_v128(c_v128_unpackhi_u16_s32(a), - c_v128_unpacklo_u16_s32(a)); -} - -SIMD_INLINE c_v256 c_v256_unpack_s16_s32(c_v128 a) { - return c_v256_from_v128(c_v128_unpackhi_s16_s32(a), - c_v128_unpacklo_s16_s32(a)); -} - -SIMD_INLINE c_v256 c_v256_unpacklo_u16_s32(c_v256 a) { - return c_v256_from_v128(c_v128_unpackhi_u16_s32(a.v128[0]), - c_v128_unpacklo_u16_s32(a.v128[0])); -} - -SIMD_INLINE c_v256 c_v256_unpacklo_s16_s32(c_v256 a) { - return c_v256_from_v128(c_v128_unpackhi_s16_s32(a.v128[0]), - c_v128_unpacklo_s16_s32(a.v128[0])); -} - -SIMD_INLINE c_v256 c_v256_unpackhi_u16_s32(c_v256 a) { - return c_v256_from_v128(c_v128_unpackhi_u16_s32(a.v128[1]), - c_v128_unpacklo_u16_s32(a.v128[1])); -} - -SIMD_INLINE c_v256 c_v256_unpackhi_s16_s32(c_v256 a) { - return c_v256_from_v128(c_v128_unpackhi_s16_s32(a.v128[1]), - c_v128_unpacklo_s16_s32(a.v128[1])); -} - -SIMD_INLINE c_v256 c_v256_shuffle_8(c_v256 a, c_v256 pattern) { - c_v256 t; - int c; - for (c = 0; c < 32; c++) - t.u8[c] = a.u8[CONFIG_BIG_ENDIAN ? 31 - (pattern.u8[c] & 31) - : pattern.u8[c] & 31]; - - return t; -} - -SIMD_INLINE c_v256 c_v256_wideshuffle_8(c_v256 a, c_v256 b, c_v256 pattern) { - c_v256 t; - int c; - for (c = 0; c < 32; c++) - t.u8[c] = (pattern.u8[c] < 32 - ? b.u8 - : a.u8)[CONFIG_BIG_ENDIAN ? 31 - (pattern.u8[c] & 31) - : pattern.u8[c] & 31]; - return t; -} - -// Pairwise / dual-lane shuffle: shuffle two 128 bit lates. -SIMD_INLINE c_v256 c_v256_pshuffle_8(c_v256 a, c_v256 pattern) { - return c_v256_from_v128( - c_v128_shuffle_8(c_v256_high_v128(a), c_v256_high_v128(pattern)), - c_v128_shuffle_8(c_v256_low_v128(a), c_v256_low_v128(pattern))); -} - -SIMD_INLINE c_v256 c_v256_cmpgt_s8(c_v256 a, c_v256 b) { - return c_v256_from_v128(c_v128_cmpgt_s8(a.v128[1], b.v128[1]), - c_v128_cmpgt_s8(a.v128[0], b.v128[0])); -} - -SIMD_INLINE c_v256 c_v256_cmplt_s8(c_v256 a, c_v256 b) { - return c_v256_from_v128(c_v128_cmplt_s8(a.v128[1], b.v128[1]), - c_v128_cmplt_s8(a.v128[0], b.v128[0])); -} - -SIMD_INLINE c_v256 c_v256_cmpeq_8(c_v256 a, c_v256 b) { - return c_v256_from_v128(c_v128_cmpeq_8(a.v128[1], b.v128[1]), - c_v128_cmpeq_8(a.v128[0], b.v128[0])); -} - -SIMD_INLINE c_v256 c_v256_cmpgt_s16(c_v256 a, c_v256 b) { - return c_v256_from_v128(c_v128_cmpgt_s16(a.v128[1], b.v128[1]), - c_v128_cmpgt_s16(a.v128[0], b.v128[0])); -} - -SIMD_INLINE c_v256 c_v256_cmplt_s16(c_v256 a, c_v256 b) { - return c_v256_from_v128(c_v128_cmplt_s16(a.v128[1], b.v128[1]), - c_v128_cmplt_s16(a.v128[0], b.v128[0])); -} - -SIMD_INLINE c_v256 c_v256_cmpeq_16(c_v256 a, c_v256 b) { - return c_v256_from_v128(c_v128_cmpeq_16(a.v128[1], b.v128[1]), - c_v128_cmpeq_16(a.v128[0], b.v128[0])); -} - -SIMD_INLINE c_v256 c_v256_cmpgt_s32(c_v256 a, c_v256 b) { - return c_v256_from_v128(c_v128_cmpgt_s32(a.v128[1], b.v128[1]), - c_v128_cmpgt_s32(a.v128[0], b.v128[0])); -} - -SIMD_INLINE c_v256 c_v256_cmplt_s32(c_v256 a, c_v256 b) { - return c_v256_from_v128(c_v128_cmplt_s32(a.v128[1], b.v128[1]), - c_v128_cmplt_s32(a.v128[0], b.v128[0])); -} - -SIMD_INLINE c_v256 c_v256_cmpeq_32(c_v256 a, c_v256 b) { - return c_v256_from_v128(c_v128_cmpeq_32(a.v128[1], b.v128[1]), - c_v128_cmpeq_32(a.v128[0], b.v128[0])); -} - -SIMD_INLINE c_v256 c_v256_shl_n_byte(c_v256 a, unsigned int n) { - if (n < 16) - return c_v256_from_v128(c_v128_or(c_v128_shl_n_byte(a.v128[1], n), - c_v128_shr_n_byte(a.v128[0], 16 - n)), - c_v128_shl_n_byte(a.v128[0], n)); - else if (n > 16) - return c_v256_from_v128(c_v128_shl_n_byte(a.v128[0], n - 16), - c_v128_zero()); - else - return c_v256_from_v128(c_v256_low_v128(a), c_v128_zero()); -} - -SIMD_INLINE c_v256 c_v256_shr_n_byte(c_v256 a, unsigned int n) { - if (n < 16) - return c_v256_from_v128(c_v128_shr_n_byte(a.v128[1], n), - c_v128_or(c_v128_shr_n_byte(a.v128[0], n), - c_v128_shl_n_byte(a.v128[1], 16 - n))); - else if (n > 16) - return c_v256_from_v128(c_v128_zero(), - c_v128_shr_n_byte(a.v128[1], n - 16)); - else - return c_v256_from_v128(c_v128_zero(), c_v256_high_v128(a)); -} - -SIMD_INLINE c_v256 c_v256_align(c_v256 a, c_v256 b, unsigned int c) { - if (SIMD_CHECK && c > 31) { - fprintf(stderr, "Error: undefined alignment %d\n", c); - abort(); - } - return c ? c_v256_or(c_v256_shr_n_byte(b, c), c_v256_shl_n_byte(a, 32 - c)) - : b; -} - -SIMD_INLINE c_v256 c_v256_shl_8(c_v256 a, unsigned int c) { - return c_v256_from_v128(c_v128_shl_8(a.v128[1], c), - c_v128_shl_8(a.v128[0], c)); -} - -SIMD_INLINE c_v256 c_v256_shr_u8(c_v256 a, unsigned int c) { - return c_v256_from_v128(c_v128_shr_u8(a.v128[1], c), - c_v128_shr_u8(a.v128[0], c)); -} - -SIMD_INLINE c_v256 c_v256_shr_s8(c_v256 a, unsigned int c) { - return c_v256_from_v128(c_v128_shr_s8(a.v128[1], c), - c_v128_shr_s8(a.v128[0], c)); -} - -SIMD_INLINE c_v256 c_v256_shl_16(c_v256 a, unsigned int c) { - return c_v256_from_v128(c_v128_shl_16(a.v128[1], c), - c_v128_shl_16(a.v128[0], c)); -} - -SIMD_INLINE c_v256 c_v256_shr_u16(c_v256 a, unsigned int c) { - return c_v256_from_v128(c_v128_shr_u16(a.v128[1], c), - c_v128_shr_u16(a.v128[0], c)); -} - -SIMD_INLINE c_v256 c_v256_shr_s16(c_v256 a, unsigned int c) { - return c_v256_from_v128(c_v128_shr_s16(a.v128[1], c), - c_v128_shr_s16(a.v128[0], c)); -} - -SIMD_INLINE c_v256 c_v256_shl_32(c_v256 a, unsigned int c) { - return c_v256_from_v128(c_v128_shl_32(a.v128[1], c), - c_v128_shl_32(a.v128[0], c)); -} - -SIMD_INLINE c_v256 c_v256_shr_u32(c_v256 a, unsigned int c) { - return c_v256_from_v128(c_v128_shr_u32(a.v128[1], c), - c_v128_shr_u32(a.v128[0], c)); -} - -SIMD_INLINE c_v256 c_v256_shr_s32(c_v256 a, unsigned int c) { - return c_v256_from_v128(c_v128_shr_s32(a.v128[1], c), - c_v128_shr_s32(a.v128[0], c)); -} - -SIMD_INLINE c_v256 c_v256_shr_s64(c_v256 a, unsigned int n) { - c_v256 t; - if (SIMD_CHECK && n > 63) { - fprintf(stderr, "Error: undefined s64 shift right %d\n", n); - abort(); - } - t.s64[3] = a.s64[3] >> n; - t.s64[2] = a.s64[2] >> n; - t.s64[1] = a.s64[1] >> n; - t.s64[0] = a.s64[0] >> n; - return t; -} - -SIMD_INLINE c_v256 c_v256_shr_u64(c_v256 a, unsigned int n) { - c_v256 t; - if (SIMD_CHECK && n > 63) { - fprintf(stderr, "Error: undefined s64 shift right %d\n", n); - abort(); - } - t.u64[3] = a.u64[3] >> n; - t.u64[2] = a.u64[2] >> n; - t.u64[1] = a.u64[1] >> n; - t.u64[0] = a.u64[0] >> n; - return t; -} - -SIMD_INLINE c_v256 c_v256_shl_64(c_v256 a, unsigned int n) { - c_v256 t; - if (SIMD_CHECK && n > 63) { - fprintf(stderr, "Error: undefined s64 shift right %d\n", n); - abort(); - } - t.u64[3] = a.u64[3] << n; - t.u64[2] = a.u64[2] << n; - t.u64[1] = a.u64[1] << n; - t.u64[0] = a.u64[0] << n; - return t; -} - -SIMD_INLINE c_v256 c_v256_shl_n_8(c_v256 a, unsigned int n) { - return c_v256_shl_8(a, n); -} - -SIMD_INLINE c_v256 c_v256_shl_n_16(c_v256 a, unsigned int n) { - return c_v256_shl_16(a, n); -} - -SIMD_INLINE c_v256 c_v256_shl_n_32(c_v256 a, unsigned int n) { - return c_v256_shl_32(a, n); -} - -SIMD_INLINE c_v256 c_v256_shl_n_64(c_v256 a, unsigned int n) { - return c_v256_shl_64(a, n); -} - -SIMD_INLINE c_v256 c_v256_shr_n_u8(c_v256 a, unsigned int n) { - return c_v256_shr_u8(a, n); -} - -SIMD_INLINE c_v256 c_v256_shr_n_u16(c_v256 a, unsigned int n) { - return c_v256_shr_u16(a, n); -} - -SIMD_INLINE c_v256 c_v256_shr_n_u32(c_v256 a, unsigned int n) { - return c_v256_shr_u32(a, n); -} - -SIMD_INLINE c_v256 c_v256_shr_n_u64(c_v256 a, unsigned int n) { - return c_v256_shr_u64(a, n); -} - -SIMD_INLINE c_v256 c_v256_shr_n_s8(c_v256 a, unsigned int n) { - return c_v256_shr_s8(a, n); -} - -SIMD_INLINE c_v256 c_v256_shr_n_s16(c_v256 a, unsigned int n) { - return c_v256_shr_s16(a, n); -} - -SIMD_INLINE c_v256 c_v256_shr_n_s32(c_v256 a, unsigned int n) { - return c_v256_shr_s32(a, n); -} - -SIMD_INLINE c_v256 c_v256_shr_n_s64(c_v256 a, unsigned int n) { - return c_v256_shr_s64(a, n); -} - -SIMD_INLINE c_v256 c_v256_shr_n_word(c_v256 a, const unsigned int n) { - return c_v256_shr_n_byte(a, 2 * n); -} -SIMD_INLINE c_v256 c_v256_shl_n_word(c_v256 a, const unsigned int n) { - return c_v256_shl_n_byte(a, 2 * n); -} - -typedef uint32_t c_sad256_internal_u16; - -SIMD_INLINE c_sad256_internal_u16 c_v256_sad_u16_init() { return 0; } - -/* Implementation dependent return value. Result must be finalised with - v256_sad_u16_sum(). */ -SIMD_INLINE c_sad256_internal_u16 c_v256_sad_u16(c_sad256_internal_u16 s, - c_v256 a, c_v256 b) { - int c; - for (c = 0; c < 16; c++) - s += a.u16[c] > b.u16[c] ? a.u16[c] - b.u16[c] : b.u16[c] - a.u16[c]; - return s; -} - -SIMD_INLINE uint32_t c_v256_sad_u16_sum(c_sad256_internal_u16 s) { return s; } - -typedef uint64_t c_ssd256_internal_s16; - -SIMD_INLINE c_ssd256_internal_s16 c_v256_ssd_s16_init() { return 0; } - -/* Implementation dependent return value. Result must be finalised with - * v256_ssd_s16_sum(). */ -SIMD_INLINE c_ssd256_internal_s16 c_v256_ssd_s16(c_ssd256_internal_s16 s, - c_v256 a, c_v256 b) { - int c; - for (c = 0; c < 16; c++) - s += (int32_t)(int16_t)(a.s16[c] - b.s16[c]) * - (int32_t)(int16_t)(a.s16[c] - b.s16[c]); - return s; -} - -SIMD_INLINE uint64_t c_v256_ssd_s16_sum(c_ssd256_internal_s16 s) { return s; } - -#endif // AOM_AOM_DSP_SIMD_V256_INTRINSICS_C_H_ diff --git a/third_party/aom/aom_dsp/simd/v256_intrinsics_v128.h b/third_party/aom/aom_dsp/simd/v256_intrinsics_v128.h deleted file mode 100644 index d5b7905ef..000000000 --- a/third_party/aom/aom_dsp/simd/v256_intrinsics_v128.h +++ /dev/null @@ -1,873 +0,0 @@ -/* - * Copyright (c) 2016, Alliance for Open Media. All rights reserved - * - * This source code is subject to the terms of the BSD 2 Clause License and - * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License - * was not distributed with this source code in the LICENSE file, you can - * obtain it at www.aomedia.org/license/software. If the Alliance for Open - * Media Patent License 1.0 was not distributed with this source code in the - * PATENTS file, you can obtain it at www.aomedia.org/license/patent. - */ - -#ifndef AOM_AOM_DSP_SIMD_V256_INTRINSICS_V128_H_ -#define AOM_AOM_DSP_SIMD_V256_INTRINSICS_V128_H_ - -#if HAVE_NEON -#include "aom_dsp/simd/v128_intrinsics_arm.h" -#elif HAVE_SSE2 -#include "aom_dsp/simd/v128_intrinsics_x86.h" -#else -#include "aom_dsp/simd/v128_intrinsics.h" -#endif - -#if HAVE_NEON -typedef int64x2x2_t v256; -#else -typedef struct { - v128 val[2]; -} v256; -#endif - -SIMD_INLINE uint32_t v256_low_u32(v256 a) { return v128_low_u32(a.val[0]); } - -SIMD_INLINE v64 v256_low_v64(v256 a) { return v128_low_v64(a.val[0]); } - -SIMD_INLINE uint64_t v256_low_u64(v256 a) { return v64_u64(v256_low_v64(a)); } - -SIMD_INLINE v128 v256_low_v128(v256 a) { return a.val[0]; } - -SIMD_INLINE v128 v256_high_v128(v256 a) { return a.val[1]; } - -SIMD_INLINE v256 v256_from_v128(v128 hi, v128 lo) { - v256 t; - t.val[1] = hi; - t.val[0] = lo; - return t; -} - -SIMD_INLINE v256 v256_from_64(uint64_t a, uint64_t b, uint64_t c, uint64_t d) { - return v256_from_v128(v128_from_64(a, b), v128_from_64(c, d)); -} - -SIMD_INLINE v256 v256_from_v64(v64 a, v64 b, v64 c, v64 d) { - return v256_from_v128(v128_from_v64(a, b), v128_from_v64(c, d)); -} - -SIMD_INLINE v256 v256_load_unaligned(const void *p) { - return v256_from_v128(v128_load_unaligned((uint8_t *)p + 16), - v128_load_unaligned(p)); -} - -SIMD_INLINE v256 v256_load_aligned(const void *p) { - return v256_from_v128(v128_load_aligned((uint8_t *)p + 16), - v128_load_aligned(p)); -} - -SIMD_INLINE void v256_store_unaligned(void *p, v256 a) { - v128_store_unaligned(p, a.val[0]); - v128_store_unaligned((uint8_t *)p + 16, a.val[1]); -} - -SIMD_INLINE void v256_store_aligned(void *p, v256 a) { - v128_store_aligned(p, a.val[0]); - v128_store_aligned((uint8_t *)p + 16, a.val[1]); -} - -SIMD_INLINE v256 v256_zero() { - return v256_from_v128(v128_zero(), v128_zero()); -} - -SIMD_INLINE v256 v256_dup_8(uint8_t x) { - v128 t = v128_dup_8(x); - return v256_from_v128(t, t); -} - -SIMD_INLINE v256 v256_dup_16(uint16_t x) { - v128 t = v128_dup_16(x); - return v256_from_v128(t, t); -} - -SIMD_INLINE v256 v256_dup_32(uint32_t x) { - v128 t = v128_dup_32(x); - return v256_from_v128(t, t); -} - -SIMD_INLINE v256 v256_dup_64(uint64_t x) { - v128 t = v128_dup_64(x); - return v256_from_v128(t, t); -} - -SIMD_INLINE int64_t v256_dotp_su8(v256 a, v256 b) { - return v128_dotp_su8(a.val[1], b.val[1]) + v128_dotp_su8(a.val[0], b.val[0]); -} - -SIMD_INLINE int64_t v256_dotp_s16(v256 a, v256 b) { - return v128_dotp_s16(a.val[1], b.val[1]) + v128_dotp_s16(a.val[0], b.val[0]); -} - -SIMD_INLINE int64_t v256_dotp_s32(v256 a, v256 b) { - return v128_dotp_s32(a.val[1], b.val[1]) + v128_dotp_s32(a.val[0], b.val[0]); -} - -SIMD_INLINE uint64_t v256_hadd_u8(v256 a) { - return v128_hadd_u8(a.val[1]) + v128_hadd_u8(a.val[0]); -} - -typedef struct { - sad128_internal val[2]; -} sad256_internal; - -SIMD_INLINE sad256_internal v256_sad_u8_init() { - sad256_internal t; - t.val[1] = v128_sad_u8_init(); - t.val[0] = v128_sad_u8_init(); - return t; -} - -/* Implementation dependent return value. Result must be finalised with - v256_sad_u8_sum(). - The result for more than 16 v256_sad_u8() calls is undefined. */ -SIMD_INLINE sad256_internal v256_sad_u8(sad256_internal s, v256 a, v256 b) { - sad256_internal t; - t.val[1] = v128_sad_u8(s.val[1], a.val[1], b.val[1]); - t.val[0] = v128_sad_u8(s.val[0], a.val[0], b.val[0]); - return t; -} - -SIMD_INLINE uint32_t v256_sad_u8_sum(sad256_internal s) { - return v128_sad_u8_sum(s.val[1]) + v128_sad_u8_sum(s.val[0]); -} - -typedef struct { - ssd128_internal val[2]; -} ssd256_internal; - -SIMD_INLINE ssd256_internal v256_ssd_u8_init() { - ssd256_internal t; - t.val[1] = v128_ssd_u8_init(); - t.val[0] = v128_ssd_u8_init(); - return t; -} - -/* Implementation dependent return value. Result must be finalised with - * v256_ssd_u8_sum(). */ -SIMD_INLINE ssd256_internal v256_ssd_u8(ssd256_internal s, v256 a, v256 b) { - ssd256_internal t; - t.val[1] = v128_ssd_u8(s.val[1], a.val[1], b.val[1]); - t.val[0] = v128_ssd_u8(s.val[0], a.val[0], b.val[0]); - return t; -} - -SIMD_INLINE uint32_t v256_ssd_u8_sum(ssd256_internal s) { - return v128_ssd_u8_sum(s.val[1]) + v128_ssd_u8_sum(s.val[0]); -} - -SIMD_INLINE v256 v256_or(v256 a, v256 b) { - return v256_from_v128(v128_or(a.val[1], b.val[1]), - v128_or(a.val[0], b.val[0])); -} - -SIMD_INLINE v256 v256_xor(v256 a, v256 b) { - return v256_from_v128(v128_xor(a.val[1], b.val[1]), - v128_xor(a.val[0], b.val[0])); -} - -SIMD_INLINE v256 v256_and(v256 a, v256 b) { - return v256_from_v128(v128_and(a.val[1], b.val[1]), - v128_and(a.val[0], b.val[0])); -} - -SIMD_INLINE v256 v256_andn(v256 a, v256 b) { - return v256_from_v128(v128_andn(a.val[1], b.val[1]), - v128_andn(a.val[0], b.val[0])); -} - -SIMD_INLINE v256 v256_add_8(v256 a, v256 b) { - return v256_from_v128(v128_add_8(a.val[1], b.val[1]), - v128_add_8(a.val[0], b.val[0])); -} - -SIMD_INLINE v256 v256_add_16(v256 a, v256 b) { - return v256_from_v128(v128_add_16(a.val[1], b.val[1]), - v128_add_16(a.val[0], b.val[0])); -} - -SIMD_INLINE v256 v256_sadd_s8(v256 a, v256 b) { - return v256_from_v128(v128_sadd_s8(a.val[1], b.val[1]), - v128_sadd_s8(a.val[0], b.val[0])); -} - -SIMD_INLINE v256 v256_sadd_u8(v256 a, v256 b) { - return v256_from_v128(v128_sadd_u8(a.val[1], b.val[1]), - v128_sadd_u8(a.val[0], b.val[0])); -} - -SIMD_INLINE v256 v256_sadd_s16(v256 a, v256 b) { - return v256_from_v128(v128_sadd_s16(a.val[1], b.val[1]), - v128_sadd_s16(a.val[0], b.val[0])); -} - -SIMD_INLINE v256 v256_add_32(v256 a, v256 b) { - return v256_from_v128(v128_add_32(a.val[1], b.val[1]), - v128_add_32(a.val[0], b.val[0])); -} - -SIMD_INLINE v256 v256_add_64(v256 a, v256 b) { - return v256_from_v128(v128_add_64(a.val[1], b.val[1]), - v128_add_64(a.val[0], b.val[0])); -} - -SIMD_INLINE v256 v256_padd_u8(v256 a) { - return v256_from_v128(v128_padd_u8(a.val[1]), v128_padd_u8(a.val[0])); -} - -SIMD_INLINE v256 v256_padd_s16(v256 a) { - return v256_from_v128(v128_padd_s16(a.val[1]), v128_padd_s16(a.val[0])); -} - -SIMD_INLINE v256 v256_sub_8(v256 a, v256 b) { - return v256_from_v128(v128_sub_8(a.val[1], b.val[1]), - v128_sub_8(a.val[0], b.val[0])); -} - -SIMD_INLINE v256 v256_ssub_u8(v256 a, v256 b) { - return v256_from_v128(v128_ssub_u8(a.val[1], b.val[1]), - v128_ssub_u8(a.val[0], b.val[0])); -} - -SIMD_INLINE v256 v256_ssub_s8(v256 a, v256 b) { - return v256_from_v128(v128_ssub_s8(a.val[1], b.val[1]), - v128_ssub_s8(a.val[0], b.val[0])); -} - -SIMD_INLINE v256 v256_sub_16(v256 a, v256 b) { - return v256_from_v128(v128_sub_16(a.val[1], b.val[1]), - v128_sub_16(a.val[0], b.val[0])); -} - -SIMD_INLINE v256 v256_ssub_s16(v256 a, v256 b) { - return v256_from_v128(v128_ssub_s16(a.val[1], b.val[1]), - v128_ssub_s16(a.val[0], b.val[0])); -} - -SIMD_INLINE v256 v256_ssub_u16(v256 a, v256 b) { - return v256_from_v128(v128_ssub_u16(a.val[1], b.val[1]), - v128_ssub_u16(a.val[0], b.val[0])); -} - -SIMD_INLINE v256 v256_sub_32(v256 a, v256 b) { - return v256_from_v128(v128_sub_32(a.val[1], b.val[1]), - v128_sub_32(a.val[0], b.val[0])); -} - -SIMD_INLINE v256 v256_sub_64(v256 a, v256 b) { - return v256_from_v128(v128_sub_64(a.val[1], b.val[1]), - v128_sub_64(a.val[0], b.val[0])); -} - -SIMD_INLINE v256 v256_abs_s16(v256 a) { - return v256_from_v128(v128_abs_s16(a.val[1]), v128_abs_s16(a.val[0])); -} - -SIMD_INLINE v256 v256_abs_s8(v256 a) { - return v256_from_v128(v128_abs_s8(a.val[1]), v128_abs_s8(a.val[0])); -} - -SIMD_INLINE v256 v256_mul_s16(v128 a, v128 b) { - v128 lo_bits = v128_mullo_s16(a, b); - v128 hi_bits = v128_mulhi_s16(a, b); - return v256_from_v128(v128_ziphi_16(hi_bits, lo_bits), - v128_ziplo_16(hi_bits, lo_bits)); -} - -SIMD_INLINE v256 v256_mullo_s16(v256 a, v256 b) { - return v256_from_v128(v128_mullo_s16(a.val[1], b.val[1]), - v128_mullo_s16(a.val[0], b.val[0])); -} - -SIMD_INLINE v256 v256_mulhi_s16(v256 a, v256 b) { - return v256_from_v128(v128_mulhi_s16(a.val[1], b.val[1]), - v128_mulhi_s16(a.val[0], b.val[0])); -} - -SIMD_INLINE v256 v256_mullo_s32(v256 a, v256 b) { - return v256_from_v128(v128_mullo_s32(a.val[1], b.val[1]), - v128_mullo_s32(a.val[0], b.val[0])); -} - -SIMD_INLINE v256 v256_madd_s16(v256 a, v256 b) { - return v256_from_v128(v128_madd_s16(a.val[1], b.val[1]), - v128_madd_s16(a.val[0], b.val[0])); -} - -SIMD_INLINE v256 v256_madd_us8(v256 a, v256 b) { - return v256_from_v128(v128_madd_us8(a.val[1], b.val[1]), - v128_madd_us8(a.val[0], b.val[0])); -} - -SIMD_INLINE v256 v256_avg_u8(v256 a, v256 b) { - return v256_from_v128(v128_avg_u8(a.val[1], b.val[1]), - v128_avg_u8(a.val[0], b.val[0])); -} - -SIMD_INLINE v256 v256_rdavg_u8(v256 a, v256 b) { - return v256_from_v128(v128_rdavg_u8(a.val[1], b.val[1]), - v128_rdavg_u8(a.val[0], b.val[0])); -} - -SIMD_INLINE v256 v256_rdavg_u16(v256 a, v256 b) { - return v256_from_v128(v128_rdavg_u16(a.val[1], b.val[1]), - v128_rdavg_u16(a.val[0], b.val[0])); -} - -SIMD_INLINE v256 v256_avg_u16(v256 a, v256 b) { - return v256_from_v128(v128_avg_u16(a.val[1], b.val[1]), - v128_avg_u16(a.val[0], b.val[0])); -} - -SIMD_INLINE v256 v256_min_u8(v256 a, v256 b) { - return v256_from_v128(v128_min_u8(a.val[1], b.val[1]), - v128_min_u8(a.val[0], b.val[0])); -} - -SIMD_INLINE v256 v256_max_u8(v256 a, v256 b) { - return v256_from_v128(v128_max_u8(a.val[1], b.val[1]), - v128_max_u8(a.val[0], b.val[0])); -} - -SIMD_INLINE v256 v256_min_s8(v256 a, v256 b) { - return v256_from_v128(v128_min_s8(a.val[1], b.val[1]), - v128_min_s8(a.val[0], b.val[0])); -} - -SIMD_INLINE uint32_t v256_movemask_8(v256 a) { - return (v128_movemask_8(v256_high_v128(a)) << 16) | - v128_movemask_8(v256_low_v128(a)); -} - -SIMD_INLINE v256 v256_blend_8(v256 a, v256 b, v256 c) { - return v256_from_v128(v128_blend_8(a.val[1], b.val[1], c.val[1]), - v128_blend_8(a.val[0], b.val[0], c.val[0])); -} - -SIMD_INLINE v256 v256_max_s8(v256 a, v256 b) { - return v256_from_v128(v128_max_s8(a.val[1], b.val[1]), - v128_max_s8(a.val[0], b.val[0])); -} - -SIMD_INLINE v256 v256_min_s16(v256 a, v256 b) { - return v256_from_v128(v128_min_s16(a.val[1], b.val[1]), - v128_min_s16(a.val[0], b.val[0])); -} - -SIMD_INLINE v256 v256_max_s16(v256 a, v256 b) { - return v256_from_v128(v128_max_s16(a.val[1], b.val[1]), - v128_max_s16(a.val[0], b.val[0])); -} - -SIMD_INLINE v256 v256_min_s32(v256 a, v256 b) { - return v256_from_v128(v128_min_s32(a.val[1], b.val[1]), - v128_min_s32(a.val[0], b.val[0])); -} - -SIMD_INLINE v256 v256_max_s32(v256 a, v256 b) { - return v256_from_v128(v128_max_s32(a.val[1], b.val[1]), - v128_max_s32(a.val[0], b.val[0])); -} - -SIMD_INLINE v256 v256_ziplo_8(v256 a, v256 b) { - return v256_from_v128(v128_ziphi_8(a.val[0], b.val[0]), - v128_ziplo_8(a.val[0], b.val[0])); -} - -SIMD_INLINE v256 v256_ziphi_8(v256 a, v256 b) { - return v256_from_v128(v128_ziphi_8(a.val[1], b.val[1]), - v128_ziplo_8(a.val[1], b.val[1])); -} - -SIMD_INLINE v256 v256_ziplo_16(v256 a, v256 b) { - return v256_from_v128(v128_ziphi_16(a.val[0], b.val[0]), - v128_ziplo_16(a.val[0], b.val[0])); -} - -SIMD_INLINE v256 v256_ziphi_16(v256 a, v256 b) { - return v256_from_v128(v128_ziphi_16(a.val[1], b.val[1]), - v128_ziplo_16(a.val[1], b.val[1])); -} - -SIMD_INLINE v256 v256_ziplo_32(v256 a, v256 b) { - return v256_from_v128(v128_ziphi_32(a.val[0], b.val[0]), - v128_ziplo_32(a.val[0], b.val[0])); -} - -SIMD_INLINE v256 v256_ziphi_32(v256 a, v256 b) { - return v256_from_v128(v128_ziphi_32(a.val[1], b.val[1]), - v128_ziplo_32(a.val[1], b.val[1])); -} - -SIMD_INLINE v256 v256_ziplo_64(v256 a, v256 b) { - return v256_from_v128(v128_ziphi_64(a.val[0], b.val[0]), - v128_ziplo_64(a.val[0], b.val[0])); -} - -SIMD_INLINE v256 v256_ziphi_64(v256 a, v256 b) { - return v256_from_v128(v128_ziphi_64(a.val[1], b.val[1]), - v128_ziplo_64(a.val[1], b.val[1])); -} - -SIMD_INLINE v256 v256_ziplo_128(v256 a, v256 b) { - return v256_from_v128(a.val[0], b.val[0]); -} - -SIMD_INLINE v256 v256_ziphi_128(v256 a, v256 b) { - return v256_from_v128(a.val[1], b.val[1]); -} - -SIMD_INLINE v256 v256_zip_8(v128 a, v128 b) { - return v256_from_v128(v128_ziphi_8(a, b), v128_ziplo_8(a, b)); -} - -SIMD_INLINE v256 v256_zip_16(v128 a, v128 b) { - return v256_from_v128(v128_ziphi_16(a, b), v128_ziplo_16(a, b)); -} - -SIMD_INLINE v256 v256_zip_32(v128 a, v128 b) { - return v256_from_v128(v128_ziphi_32(a, b), v128_ziplo_32(a, b)); -} - -SIMD_INLINE v256 v256_unziplo_8(v256 a, v256 b) { - return v256_from_v128(v128_unziplo_8(a.val[1], a.val[0]), - v128_unziplo_8(b.val[1], b.val[0])); -} - -SIMD_INLINE v256 v256_unziphi_8(v256 a, v256 b) { - return v256_from_v128(v128_unziphi_8(a.val[1], a.val[0]), - v128_unziphi_8(b.val[1], b.val[0])); -} - -SIMD_INLINE v256 v256_unziplo_16(v256 a, v256 b) { - return v256_from_v128(v128_unziplo_16(a.val[1], a.val[0]), - v128_unziplo_16(b.val[1], b.val[0])); -} - -SIMD_INLINE v256 v256_unziphi_16(v256 a, v256 b) { - return v256_from_v128(v128_unziphi_16(a.val[1], a.val[0]), - v128_unziphi_16(b.val[1], b.val[0])); -} - -SIMD_INLINE v256 v256_unziplo_32(v256 a, v256 b) { - return v256_from_v128(v128_unziplo_32(a.val[1], a.val[0]), - v128_unziplo_32(b.val[1], b.val[0])); -} - -SIMD_INLINE v256 v256_unziphi_32(v256 a, v256 b) { - return v256_from_v128(v128_unziphi_32(a.val[1], a.val[0]), - v128_unziphi_32(b.val[1], b.val[0])); -} - -SIMD_INLINE v256 v256_unziplo_64(v256 a, v256 b) { -#if HAVE_SSE2 - return v256_from_v128( - _mm_castpd_si128(_mm_shuffle_pd(_mm_castsi128_pd(a.val[0]), - _mm_castsi128_pd(a.val[1]), 0)), - _mm_castpd_si128(_mm_shuffle_pd(_mm_castsi128_pd(b.val[0]), - _mm_castsi128_pd(b.val[1]), 0))); -#else - return v256_from_v64(v128_low_v64(a.val[1]), v128_low_v64(a.val[0]), - v128_low_v64(b.val[1]), v128_low_v64(b.val[0])); -#endif -} - -SIMD_INLINE v256 v256_unziphi_64(v256 a, v256 b) { -#if HAVE_SSE2 - return v256_from_v128( - _mm_castpd_si128(_mm_shuffle_pd(_mm_castsi128_pd(a.val[0]), - _mm_castsi128_pd(a.val[1]), 3)), - _mm_castpd_si128(_mm_shuffle_pd(_mm_castsi128_pd(b.val[0]), - _mm_castsi128_pd(b.val[1]), 3))); -#else - return v256_from_v64(v128_high_v64(a.val[1]), v128_high_v64(a.val[0]), - v128_high_v64(b.val[1]), v128_high_v64(b.val[0])); -#endif -} - -SIMD_INLINE v256 v256_unpack_u8_s16(v128 a) { - return v256_from_v128(v128_unpackhi_u8_s16(a), v128_unpacklo_u8_s16(a)); -} - -SIMD_INLINE v256 v256_unpacklo_u8_s16(v256 a) { - return v256_from_v128(v128_unpackhi_u8_s16(a.val[0]), - v128_unpacklo_u8_s16(a.val[0])); -} - -SIMD_INLINE v256 v256_unpackhi_u8_s16(v256 a) { - return v256_from_v128(v128_unpackhi_u8_s16(a.val[1]), - v128_unpacklo_u8_s16(a.val[1])); -} - -SIMD_INLINE v256 v256_unpack_s8_s16(v128 a) { - return v256_from_v128(v128_unpackhi_s8_s16(a), v128_unpacklo_s8_s16(a)); -} - -SIMD_INLINE v256 v256_unpacklo_s8_s16(v256 a) { - return v256_from_v128(v128_unpackhi_s8_s16(a.val[0]), - v128_unpacklo_s8_s16(a.val[0])); -} - -SIMD_INLINE v256 v256_unpackhi_s8_s16(v256 a) { - return v256_from_v128(v128_unpackhi_s8_s16(a.val[1]), - v128_unpacklo_s8_s16(a.val[1])); -} - -SIMD_INLINE v256 v256_pack_s32_s16(v256 a, v256 b) { - return v256_from_v128(v128_pack_s32_s16(a.val[1], a.val[0]), - v128_pack_s32_s16(b.val[1], b.val[0])); -} - -SIMD_INLINE v256 v256_pack_s32_u16(v256 a, v256 b) { - return v256_from_v128(v128_pack_s32_u16(a.val[1], a.val[0]), - v128_pack_s32_u16(b.val[1], b.val[0])); -} - -SIMD_INLINE v256 v256_pack_s16_u8(v256 a, v256 b) { - return v256_from_v128(v128_pack_s16_u8(a.val[1], a.val[0]), - v128_pack_s16_u8(b.val[1], b.val[0])); -} - -SIMD_INLINE v256 v256_pack_s16_s8(v256 a, v256 b) { - return v256_from_v128(v128_pack_s16_s8(a.val[1], a.val[0]), - v128_pack_s16_s8(b.val[1], b.val[0])); -} - -SIMD_INLINE v256 v256_unpack_u16_s32(v128 a) { - return v256_from_v128(v128_unpackhi_u16_s32(a), v128_unpacklo_u16_s32(a)); -} - -SIMD_INLINE v256 v256_unpack_s16_s32(v128 a) { - return v256_from_v128(v128_unpackhi_s16_s32(a), v128_unpacklo_s16_s32(a)); -} - -SIMD_INLINE v256 v256_unpacklo_u16_s32(v256 a) { - return v256_from_v128(v128_unpackhi_u16_s32(a.val[0]), - v128_unpacklo_u16_s32(a.val[0])); -} - -SIMD_INLINE v256 v256_unpacklo_s16_s32(v256 a) { - return v256_from_v128(v128_unpackhi_s16_s32(a.val[0]), - v128_unpacklo_s16_s32(a.val[0])); -} - -SIMD_INLINE v256 v256_unpackhi_u16_s32(v256 a) { - return v256_from_v128(v128_unpackhi_u16_s32(a.val[1]), - v128_unpacklo_u16_s32(a.val[1])); -} - -SIMD_INLINE v256 v256_unpackhi_s16_s32(v256 a) { - return v256_from_v128(v128_unpackhi_s16_s32(a.val[1]), - v128_unpacklo_s16_s32(a.val[1])); -} - -SIMD_INLINE v256 v256_cmpgt_s8(v256 a, v256 b) { - return v256_from_v128(v128_cmpgt_s8(a.val[1], b.val[1]), - v128_cmpgt_s8(a.val[0], b.val[0])); -} - -SIMD_INLINE v256 v256_cmplt_s8(v256 a, v256 b) { - return v256_from_v128(v128_cmplt_s8(a.val[1], b.val[1]), - v128_cmplt_s8(a.val[0], b.val[0])); -} - -SIMD_INLINE v256 v256_cmpeq_8(v256 a, v256 b) { - return v256_from_v128(v128_cmpeq_8(a.val[1], b.val[1]), - v128_cmpeq_8(a.val[0], b.val[0])); -} - -SIMD_INLINE v256 v256_cmpgt_s16(v256 a, v256 b) { - return v256_from_v128(v128_cmpgt_s16(a.val[1], b.val[1]), - v128_cmpgt_s16(a.val[0], b.val[0])); -} - -SIMD_INLINE v256 v256_cmplt_s16(v256 a, v256 b) { - return v256_from_v128(v128_cmplt_s16(a.val[1], b.val[1]), - v128_cmplt_s16(a.val[0], b.val[0])); -} - -SIMD_INLINE v256 v256_cmpeq_16(v256 a, v256 b) { - return v256_from_v128(v128_cmpeq_16(a.val[1], b.val[1]), - v128_cmpeq_16(a.val[0], b.val[0])); -} - -SIMD_INLINE v256 v256_cmpgt_s32(v256 a, v256 b) { - return v256_from_v128(v128_cmpgt_s32(a.val[1], b.val[1]), - v128_cmpgt_s32(a.val[0], b.val[0])); -} - -SIMD_INLINE v256 v256_cmplt_s32(v256 a, v256 b) { - return v256_from_v128(v128_cmplt_s32(a.val[1], b.val[1]), - v128_cmplt_s32(a.val[0], b.val[0])); -} - -SIMD_INLINE v256 v256_cmpeq_32(v256 a, v256 b) { - return v256_from_v128(v128_cmpeq_32(a.val[1], b.val[1]), - v128_cmpeq_32(a.val[0], b.val[0])); -} - -SIMD_INLINE v256 v256_shuffle_8(v256 x, v256 pattern) { -#if HAVE_NEON -#if defined(__aarch64__) - uint8x16x2_t p = { { vreinterpretq_u8_s64(x.val[0]), - vreinterpretq_u8_s64(x.val[1]) } }; - return v256_from_v128( - vreinterpretq_s64_u8(vqtbl2q_u8(p, vreinterpretq_u8_s64(pattern.val[1]))), - vreinterpretq_s64_u8( - vqtbl2q_u8(p, vreinterpretq_u8_s64(pattern.val[0])))); -#else - uint8x8x4_t p = { { vget_low_u8(vreinterpretq_u8_s64(x.val[0])), - vget_high_u8(vreinterpretq_u8_s64(x.val[0])), - vget_low_u8(vreinterpretq_u8_s64(x.val[1])), - vget_high_u8(vreinterpretq_u8_s64(x.val[1])) } }; - return v256_from_64( - (uint64_t)vreinterpret_s64_u8( - vtbl4_u8(p, vreinterpret_u8_s64(vget_high_s64(pattern.val[1])))), - (uint64_t)vreinterpret_s64_u8( - vtbl4_u8(p, vreinterpret_u8_s64(vget_low_s64(pattern.val[1])))), - (uint64_t)vreinterpret_s64_u8( - vtbl4_u8(p, vreinterpret_u8_s64(vget_high_s64(pattern.val[0])))), - (uint64_t)vreinterpret_s64_u8( - vtbl4_u8(p, vreinterpret_u8_s64(vget_low_s64(pattern.val[0]))))); -#endif -#else - v128 c16 = v128_dup_8(16); - v128 maskhi = v128_cmplt_s8(pattern.val[1], c16); - v128 masklo = v128_cmplt_s8(pattern.val[0], c16); - return v256_from_v128( - v128_blend_8(v128_shuffle_8(x.val[1], v128_sub_8(pattern.val[1], c16)), - v128_shuffle_8(x.val[0], pattern.val[1]), maskhi), - v128_blend_8(v128_shuffle_8(x.val[1], v128_sub_8(pattern.val[0], c16)), - v128_shuffle_8(x.val[0], pattern.val[0]), masklo)); -#endif -} - -SIMD_INLINE v256 v256_wideshuffle_8(v256 x, v256 y, v256 pattern) { -#if HAVE_NEON -#if defined(__aarch64__) - uint8x16x4_t p = { { - vreinterpretq_u8_s64(y.val[0]), - vreinterpretq_u8_s64(y.val[1]), - vreinterpretq_u8_s64(x.val[0]), - vreinterpretq_u8_s64(x.val[1]), - } }; - return v256_from_v128( - vreinterpretq_s64_u8(vqtbl4q_u8(p, vreinterpretq_u8_s64(pattern.val[1]))), - vreinterpretq_s64_u8( - vqtbl4q_u8(p, vreinterpretq_u8_s64(pattern.val[0])))); -#else - v256 c32 = v256_dup_8(32); - v256 p32 = v256_sub_8(pattern, c32); - uint8x8x4_t p = { { vget_low_u8(vreinterpretq_u8_s64(x.val[0])), - vget_high_u8(vreinterpretq_u8_s64(x.val[0])), - vget_low_u8(vreinterpretq_u8_s64(x.val[1])), - vget_high_u8(vreinterpretq_u8_s64(x.val[1])) } }; - uint8x8x4_t q = { { vget_low_u8(vreinterpretq_u8_s64(y.val[0])), - vget_high_u8(vreinterpretq_u8_s64(y.val[0])), - vget_low_u8(vreinterpretq_u8_s64(y.val[1])), - vget_high_u8(vreinterpretq_u8_s64(y.val[1])) } }; - v256 r1 = - v256_from_64((uint64_t)vreinterpret_s64_u8(vtbl4_u8( - p, vreinterpret_u8_s64(vget_high_s64(p32.val[1])))), - (uint64_t)vreinterpret_s64_u8(vtbl4_u8( - p, vreinterpret_u8_s64(vget_low_s64(p32.val[1])))), - (uint64_t)vreinterpret_s64_u8(vtbl4_u8( - p, vreinterpret_u8_s64(vget_high_s64(p32.val[0])))), - (uint64_t)vreinterpret_s64_u8(vtbl4_u8( - p, vreinterpret_u8_s64(vget_low_s64(p32.val[0]))))); - v256 r2 = - v256_from_64((uint64_t)vreinterpret_s64_u8(vtbl4_u8( - q, vreinterpret_u8_s64(vget_high_s64(pattern.val[1])))), - (uint64_t)vreinterpret_s64_u8(vtbl4_u8( - q, vreinterpret_u8_s64(vget_low_s64(pattern.val[1])))), - (uint64_t)vreinterpret_s64_u8(vtbl4_u8( - q, vreinterpret_u8_s64(vget_high_s64(pattern.val[0])))), - (uint64_t)vreinterpret_s64_u8(vtbl4_u8( - q, vreinterpret_u8_s64(vget_low_s64(pattern.val[0]))))); - return v256_blend_8(r1, r2, v256_cmplt_s8(pattern, c32)); -#endif -#else - v128 c16 = v128_dup_8(16); - v128 c32 = v128_dup_8(32); - v128 c48 = v128_dup_8(48); - v128 maskhi16 = v128_cmpgt_s8(c16, pattern.val[1]); - v128 masklo16 = v128_cmpgt_s8(c16, pattern.val[0]); - v128 maskhi48 = v128_cmpgt_s8(c48, pattern.val[1]); - v128 masklo48 = v128_cmpgt_s8(c48, pattern.val[0]); - v256 r1 = v256_from_v128( - v128_blend_8(v128_shuffle_8(x.val[1], v128_sub_8(pattern.val[1], c48)), - v128_shuffle_8(x.val[0], v128_sub_8(pattern.val[1], c32)), - maskhi48), - v128_blend_8(v128_shuffle_8(x.val[1], v128_sub_8(pattern.val[0], c48)), - v128_shuffle_8(x.val[0], v128_sub_8(pattern.val[0], c32)), - masklo48)); - v256 r2 = v256_from_v128( - v128_blend_8(v128_shuffle_8(y.val[1], v128_sub_8(pattern.val[1], c16)), - v128_shuffle_8(y.val[0], pattern.val[1]), maskhi16), - v128_blend_8(v128_shuffle_8(y.val[1], v128_sub_8(pattern.val[0], c16)), - v128_shuffle_8(y.val[0], pattern.val[0]), masklo16)); - return v256_blend_8(r1, r2, v256_cmpgt_s8(v256_from_v128(c32, c32), pattern)); -#endif -} - -SIMD_INLINE v256 v256_pshuffle_8(v256 a, v256 pattern) { - return v256_from_v128( - v128_shuffle_8(v256_high_v128(a), v256_high_v128(pattern)), - v128_shuffle_8(v256_low_v128(a), v256_low_v128(pattern))); -} - -SIMD_INLINE v256 v256_shl_8(v256 a, const unsigned int c) { - return v256_from_v128(v128_shl_8(a.val[1], c), v128_shl_8(a.val[0], c)); -} - -SIMD_INLINE v256 v256_shr_u8(v256 a, const unsigned int c) { - return v256_from_v128(v128_shr_u8(a.val[1], c), v128_shr_u8(a.val[0], c)); -} - -SIMD_INLINE v256 v256_shr_s8(v256 a, const unsigned int c) { - return v256_from_v128(v128_shr_s8(a.val[1], c), v128_shr_s8(a.val[0], c)); -} - -SIMD_INLINE v256 v256_shl_16(v256 a, const unsigned int c) { - return v256_from_v128(v128_shl_16(a.val[1], c), v128_shl_16(a.val[0], c)); -} - -SIMD_INLINE v256 v256_shr_u16(v256 a, const unsigned int c) { - return v256_from_v128(v128_shr_u16(a.val[1], c), v128_shr_u16(a.val[0], c)); -} - -SIMD_INLINE v256 v256_shr_s16(v256 a, const unsigned int c) { - return v256_from_v128(v128_shr_s16(a.val[1], c), v128_shr_s16(a.val[0], c)); -} - -SIMD_INLINE v256 v256_shl_32(v256 a, const unsigned int c) { - return v256_from_v128(v128_shl_32(a.val[1], c), v128_shl_32(a.val[0], c)); -} - -SIMD_INLINE v256 v256_shr_u32(v256 a, const unsigned int c) { - return v256_from_v128(v128_shr_u32(a.val[1], c), v128_shr_u32(a.val[0], c)); -} - -SIMD_INLINE v256 v256_shr_s32(v256 a, const unsigned int c) { - return v256_from_v128(v128_shr_s32(a.val[1], c), v128_shr_s32(a.val[0], c)); -} - -SIMD_INLINE v256 v256_shl_64(v256 a, const unsigned int c) { - return v256_from_v128(v128_shl_64(a.val[1], c), v128_shl_64(a.val[0], c)); -} - -SIMD_INLINE v256 v256_shr_u64(v256 a, const unsigned int c) { - return v256_from_v128(v128_shr_u64(a.val[1], c), v128_shr_u64(a.val[0], c)); -} - -SIMD_INLINE v256 v256_shr_s64(v256 a, const unsigned int c) { - return v256_from_v128(v128_shr_s64(a.val[1], c), v128_shr_s64(a.val[0], c)); -} - -/* These intrinsics require immediate values, so we must use #defines - to enforce that. */ -#define v256_shl_n_byte(a, n) \ - ((n) < 16 ? v256_from_v128(v128_or(v128_shl_n_byte(a.val[1], n), \ - v128_shr_n_byte(a.val[0], 16 - (n))), \ - v128_shl_n_byte(a.val[0], (n))) \ - : v256_from_v128( \ - (n) > 16 ? v128_shl_n_byte(a.val[0], (n)-16) : a.val[0], \ - v128_zero())) - -#define v256_shr_n_byte(a, n) \ - ((n) < 16 ? v256_from_v128(v128_shr_n_byte(a.val[1], n), \ - v128_or(v128_shr_n_byte(a.val[0], n), \ - v128_shl_n_byte(a.val[1], 16 - (n)))) \ - : v256_from_v128( \ - v128_zero(), \ - (n) > 16 ? v128_shr_n_byte(a.val[1], (n)-16) : a.val[1])) - -#define v256_align(a, b, c) \ - ((c) ? v256_or(v256_shr_n_byte(b, c), v256_shl_n_byte(a, 32 - (c))) : b) - -#define v256_shl_n_8(a, n) \ - v256_from_v128(v128_shl_n_8(a.val[1], n), v128_shl_n_8(a.val[0], n)) -#define v256_shl_n_16(a, n) \ - v256_from_v128(v128_shl_n_16(a.val[1], n), v128_shl_n_16(a.val[0], n)) -#define v256_shl_n_32(a, n) \ - v256_from_v128(v128_shl_n_32(a.val[1], n), v128_shl_n_32(a.val[0], n)) -#define v256_shl_n_64(a, n) \ - v256_from_v128(v128_shl_n_64(a.val[1], n), v128_shl_n_64(a.val[0], n)) -#define v256_shr_n_u8(a, n) \ - v256_from_v128(v128_shr_n_u8(a.val[1], n), v128_shr_n_u8(a.val[0], n)) -#define v256_shr_n_u16(a, n) \ - v256_from_v128(v128_shr_n_u16(a.val[1], n), v128_shr_n_u16(a.val[0], n)) -#define v256_shr_n_u32(a, n) \ - v256_from_v128(v128_shr_n_u32(a.val[1], n), v128_shr_n_u32(a.val[0], n)) -#define v256_shr_n_u64(a, n) \ - v256_from_v128(v128_shr_n_u64(a.val[1], n), v128_shr_n_u64(a.val[0], n)) -#define v256_shr_n_s8(a, n) \ - v256_from_v128(v128_shr_n_s8(a.val[1], n), v128_shr_n_s8(a.val[0], n)) -#define v256_shr_n_s16(a, n) \ - v256_from_v128(v128_shr_n_s16(a.val[1], n), v128_shr_n_s16(a.val[0], n)) -#define v256_shr_n_s32(a, n) \ - v256_from_v128(v128_shr_n_s32(a.val[1], n), v128_shr_n_s32(a.val[0], n)) -#define v256_shr_n_s64(a, n) \ - v256_from_v128(v128_shr_n_s64(a.val[1], n), v128_shr_n_s64(a.val[0], n)) - -#define v256_shr_n_word(a, n) v256_shr_n_byte(a, 2 * (n)) -#define v256_shl_n_word(a, n) v256_shl_n_byte(a, 2 * (n)) - -typedef struct { - sad128_internal_u16 val[2]; -} sad256_internal_u16; - -SIMD_INLINE sad256_internal_u16 v256_sad_u16_init() { - sad256_internal_u16 t; - t.val[1] = v128_sad_u16_init(); - t.val[0] = v128_sad_u16_init(); - return t; -} - -/* Implementation dependent return value. Result must be finalised with - v256_sad_u16_sum(). - The result for more than 16 v256_sad_u16() calls is undefined. */ -SIMD_INLINE sad256_internal_u16 v256_sad_u16(sad256_internal_u16 s, v256 a, - v256 b) { - sad256_internal_u16 t; - t.val[1] = v128_sad_u16(s.val[1], a.val[1], b.val[1]); - t.val[0] = v128_sad_u16(s.val[0], a.val[0], b.val[0]); - return t; -} - -SIMD_INLINE uint32_t v256_sad_u16_sum(sad256_internal_u16 s) { - return v128_sad_u16_sum(s.val[1]) + v128_sad_u16_sum(s.val[0]); -} - -typedef struct { - ssd128_internal_s16 val[2]; -} ssd256_internal_s16; - -SIMD_INLINE ssd256_internal_s16 v256_ssd_s16_init() { - ssd256_internal_s16 t; - t.val[1] = v128_ssd_s16_init(); - t.val[0] = v128_ssd_s16_init(); - return t; -} - -/* Implementation dependent return value. Result must be finalised with - * v256_ssd_s16_sum(). */ -SIMD_INLINE ssd256_internal_s16 v256_ssd_s16(ssd256_internal_s16 s, v256 a, - v256 b) { - ssd256_internal_s16 t; - t.val[1] = v128_ssd_s16(s.val[1], a.val[1], b.val[1]); - t.val[0] = v128_ssd_s16(s.val[0], a.val[0], b.val[0]); - return t; -} - -SIMD_INLINE uint64_t v256_ssd_s16_sum(ssd256_internal_s16 s) { - return v128_ssd_s16_sum(s.val[1]) + v128_ssd_s16_sum(s.val[0]); -} - -#endif // AOM_AOM_DSP_SIMD_V256_INTRINSICS_V128_H_ diff --git a/third_party/aom/aom_dsp/simd/v256_intrinsics_x86.h b/third_party/aom/aom_dsp/simd/v256_intrinsics_x86.h deleted file mode 100644 index 44594bc41..000000000 --- a/third_party/aom/aom_dsp/simd/v256_intrinsics_x86.h +++ /dev/null @@ -1,750 +0,0 @@ -/* - * Copyright (c) 2016, Alliance for Open Media. All rights reserved - * - * This source code is subject to the terms of the BSD 2 Clause License and - * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License - * was not distributed with this source code in the LICENSE file, you can - * obtain it at www.aomedia.org/license/software. If the Alliance for Open - * Media Patent License 1.0 was not distributed with this source code in the - * PATENTS file, you can obtain it at www.aomedia.org/license/patent. - */ - -#ifndef AOM_AOM_DSP_SIMD_V256_INTRINSICS_X86_H_ -#define AOM_AOM_DSP_SIMD_V256_INTRINSICS_X86_H_ - -#if !defined(__AVX2__) - -#include "aom_dsp/simd/v256_intrinsics_v128.h" - -#else - -// The _m256i type seems to cause problems for g++'s mangling prior to -// version 5, but adding -fabi-version=0 fixes this. -#if !defined(__clang__) && defined(__GNUC__) && __GNUC__ < 5 && \ - defined(__AVX2__) && defined(__cplusplus) -#pragma GCC optimize "-fabi-version=0" -#endif - -#include <immintrin.h> - -#include "aom_dsp/simd/v128_intrinsics_x86.h" - -typedef __m256i v256; - -SIMD_INLINE uint32_t v256_low_u32(v256 a) { - return (uint32_t)_mm_cvtsi128_si32(_mm256_extracti128_si256(a, 0)); -} - -SIMD_INLINE v64 v256_low_v64(v256 a) { - return _mm_unpacklo_epi64(_mm256_extracti128_si256(a, 0), v64_zero()); -} - -SIMD_INLINE uint64_t v256_low_u64(v256 a) { return v64_u64(v256_low_v64(a)); } - -SIMD_INLINE v128 v256_low_v128(v256 a) { return _mm256_castsi256_si128(a); } - -SIMD_INLINE v128 v256_high_v128(v256 a) { - return _mm256_extracti128_si256(a, 1); -} - -SIMD_INLINE v256 v256_from_v128(v128 a, v128 b) { - // gcc seems to be missing _mm256_set_m128i() - return _mm256_inserti128_si256(_mm256_castsi128_si256(b), a, 1); -} - -SIMD_INLINE v256 v256_from_v64(v64 a, v64 b, v64 c, v64 d) { - return v256_from_v128(v128_from_v64(a, b), v128_from_v64(c, d)); -} - -SIMD_INLINE v256 v256_from_64(uint64_t a, uint64_t b, uint64_t c, uint64_t d) { - return v256_from_v128(v128_from_64(a, b), v128_from_64(c, d)); -} - -SIMD_INLINE v256 v256_load_aligned(const void *p) { - return _mm256_load_si256((const __m256i *)p); -} - -SIMD_INLINE v256 v256_load_unaligned(const void *p) { - return _mm256_loadu_si256((const __m256i *)p); -} - -SIMD_INLINE void v256_store_aligned(void *p, v256 a) { - _mm256_store_si256((__m256i *)p, a); -} - -SIMD_INLINE void v256_store_unaligned(void *p, v256 a) { - _mm256_storeu_si256((__m256i *)p, a); -} - -SIMD_INLINE v256 v256_zero() { return _mm256_setzero_si256(); } - -SIMD_INLINE v256 v256_dup_8(uint8_t x) { return _mm256_set1_epi8(x); } - -SIMD_INLINE v256 v256_dup_16(uint16_t x) { return _mm256_set1_epi16(x); } - -SIMD_INLINE v256 v256_dup_32(uint32_t x) { return _mm256_set1_epi32(x); } - -SIMD_INLINE v256 v256_dup_64(uint64_t x) { return _mm256_set1_epi64x(x); } - -SIMD_INLINE v256 v256_add_8(v256 a, v256 b) { return _mm256_add_epi8(a, b); } - -SIMD_INLINE v256 v256_add_16(v256 a, v256 b) { return _mm256_add_epi16(a, b); } - -SIMD_INLINE v256 v256_sadd_u8(v256 a, v256 b) { return _mm256_adds_epu8(a, b); } - -SIMD_INLINE v256 v256_sadd_s8(v256 a, v256 b) { return _mm256_adds_epi8(a, b); } - -SIMD_INLINE v256 v256_sadd_s16(v256 a, v256 b) { - return _mm256_adds_epi16(a, b); -} - -SIMD_INLINE v256 v256_add_32(v256 a, v256 b) { return _mm256_add_epi32(a, b); } - -SIMD_INLINE v256 v256_add_64(v256 a, v256 b) { return _mm256_add_epi64(a, b); } - -SIMD_INLINE v256 v256_padd_u8(v256 a) { - return _mm256_maddubs_epi16(a, _mm256_set1_epi8(1)); -} - -SIMD_INLINE v256 v256_padd_s16(v256 a) { - return _mm256_madd_epi16(a, _mm256_set1_epi16(1)); -} - -SIMD_INLINE v256 v256_sub_8(v256 a, v256 b) { return _mm256_sub_epi8(a, b); } - -SIMD_INLINE v256 v256_ssub_u8(v256 a, v256 b) { return _mm256_subs_epu8(a, b); } - -SIMD_INLINE v256 v256_ssub_s8(v256 a, v256 b) { return _mm256_subs_epi8(a, b); } - -SIMD_INLINE v256 v256_sub_16(v256 a, v256 b) { return _mm256_sub_epi16(a, b); } - -SIMD_INLINE v256 v256_ssub_s16(v256 a, v256 b) { - return _mm256_subs_epi16(a, b); -} - -SIMD_INLINE v256 v256_ssub_u16(v256 a, v256 b) { - return _mm256_subs_epu16(a, b); -} - -SIMD_INLINE v256 v256_sub_32(v256 a, v256 b) { return _mm256_sub_epi32(a, b); } - -SIMD_INLINE v256 v256_sub_64(v256 a, v256 b) { return _mm256_sub_epi64(a, b); } - -SIMD_INLINE v256 v256_abs_s16(v256 a) { return _mm256_abs_epi16(a); } - -SIMD_INLINE v256 v256_abs_s8(v256 a) { return _mm256_abs_epi8(a); } - -// AVX doesn't have the direct intrinsics to zip/unzip 8, 16, 32 bit -// lanes of lower or upper halves of a 256bit vector because the -// unpack/pack intrinsics operate on the 256 bit input vector as 2 -// independent 128 bit vectors. -SIMD_INLINE v256 v256_ziplo_8(v256 a, v256 b) { - return _mm256_unpacklo_epi8( - _mm256_permute4x64_epi64(b, _MM_SHUFFLE(3, 1, 2, 0)), - _mm256_permute4x64_epi64(a, _MM_SHUFFLE(3, 1, 2, 0))); -} - -SIMD_INLINE v256 v256_ziphi_8(v256 a, v256 b) { - return _mm256_unpackhi_epi8( - _mm256_permute4x64_epi64(b, _MM_SHUFFLE(3, 1, 2, 0)), - _mm256_permute4x64_epi64(a, _MM_SHUFFLE(3, 1, 2, 0))); -} - -SIMD_INLINE v256 v256_ziplo_16(v256 a, v256 b) { - return _mm256_unpacklo_epi16( - _mm256_permute4x64_epi64(b, _MM_SHUFFLE(3, 1, 2, 0)), - _mm256_permute4x64_epi64(a, _MM_SHUFFLE(3, 1, 2, 0))); -} - -SIMD_INLINE v256 v256_ziphi_16(v256 a, v256 b) { - return _mm256_unpackhi_epi16( - _mm256_permute4x64_epi64(b, _MM_SHUFFLE(3, 1, 2, 0)), - _mm256_permute4x64_epi64(a, _MM_SHUFFLE(3, 1, 2, 0))); -} - -SIMD_INLINE v256 v256_ziplo_32(v256 a, v256 b) { - return _mm256_unpacklo_epi32( - _mm256_permute4x64_epi64(b, _MM_SHUFFLE(3, 1, 2, 0)), - _mm256_permute4x64_epi64(a, _MM_SHUFFLE(3, 1, 2, 0))); -} - -SIMD_INLINE v256 v256_ziphi_32(v256 a, v256 b) { - return _mm256_unpackhi_epi32( - _mm256_permute4x64_epi64(b, _MM_SHUFFLE(3, 1, 2, 0)), - _mm256_permute4x64_epi64(a, _MM_SHUFFLE(3, 1, 2, 0))); -} - -SIMD_INLINE v256 v256_ziplo_64(v256 a, v256 b) { - return _mm256_unpacklo_epi64( - _mm256_permute4x64_epi64(b, _MM_SHUFFLE(3, 1, 2, 0)), - _mm256_permute4x64_epi64(a, _MM_SHUFFLE(3, 1, 2, 0))); -} - -SIMD_INLINE v256 v256_ziphi_64(v256 a, v256 b) { - return _mm256_unpackhi_epi64( - _mm256_permute4x64_epi64(b, _MM_SHUFFLE(3, 1, 2, 0)), - _mm256_permute4x64_epi64(a, _MM_SHUFFLE(3, 1, 2, 0))); -} - -SIMD_INLINE v256 v256_ziplo_128(v256 a, v256 b) { - return v256_from_v128(v256_low_v128(a), v256_low_v128(b)); -} - -SIMD_INLINE v256 v256_ziphi_128(v256 a, v256 b) { - return v256_from_v128(v256_high_v128(a), v256_high_v128(b)); -} - -SIMD_INLINE v256 v256_zip_8(v128 a, v128 b) { - return v256_from_v128(v128_ziphi_8(a, b), v128_ziplo_8(a, b)); -} - -SIMD_INLINE v256 v256_zip_16(v128 a, v128 b) { - return v256_from_v128(v128_ziphi_16(a, b), v128_ziplo_16(a, b)); -} - -SIMD_INLINE v256 v256_zip_32(v128 a, v128 b) { - return v256_from_v128(v128_ziphi_32(a, b), v128_ziplo_32(a, b)); -} - -SIMD_INLINE v256 v256_unziphi_8(v256 a, v256 b) { - return _mm256_permute4x64_epi64( - _mm256_packs_epi16(_mm256_srai_epi16(b, 8), _mm256_srai_epi16(a, 8)), - _MM_SHUFFLE(3, 1, 2, 0)); -} - -SIMD_INLINE v256 v256_unziplo_8(v256 a, v256 b) { - return v256_unziphi_8(_mm256_slli_si256(a, 1), _mm256_slli_si256(b, 1)); -} - -SIMD_INLINE v256 v256_unziphi_16(v256 a, v256 b) { - return _mm256_permute4x64_epi64( - _mm256_packs_epi32(_mm256_srai_epi32(b, 16), _mm256_srai_epi32(a, 16)), - _MM_SHUFFLE(3, 1, 2, 0)); -} - -SIMD_INLINE v256 v256_unziplo_16(v256 a, v256 b) { - return v256_unziphi_16(_mm256_slli_si256(a, 2), _mm256_slli_si256(b, 2)); -} - -SIMD_INLINE v256 v256_unziphi_32(v256 a, v256 b) { - return _mm256_permute4x64_epi64( - _mm256_castps_si256(_mm256_shuffle_ps(_mm256_castsi256_ps(b), - _mm256_castsi256_ps(a), - _MM_SHUFFLE(3, 1, 3, 1))), - _MM_SHUFFLE(3, 1, 2, 0)); -} - -SIMD_INLINE v256 v256_unziplo_32(v256 a, v256 b) { - return _mm256_permute4x64_epi64( - _mm256_castps_si256(_mm256_shuffle_ps(_mm256_castsi256_ps(b), - _mm256_castsi256_ps(a), - _MM_SHUFFLE(2, 0, 2, 0))), - _MM_SHUFFLE(3, 1, 2, 0)); -} - -SIMD_INLINE v256 v256_unziphi_64(v256 a, v256 b) { - return _mm256_permute4x64_epi64( - _mm256_castpd_si256(_mm256_shuffle_pd(_mm256_castsi256_pd(b), - _mm256_castsi256_pd(a), 15)), - _MM_SHUFFLE(3, 1, 2, 0)); -} - -SIMD_INLINE v256 v256_unziplo_64(v256 a, v256 b) { - return _mm256_permute4x64_epi64( - _mm256_castpd_si256( - _mm256_shuffle_pd(_mm256_castsi256_pd(b), _mm256_castsi256_pd(a), 0)), - _MM_SHUFFLE(3, 1, 2, 0)); -} - -SIMD_INLINE v256 v256_unpack_u8_s16(v128 a) { - return v256_from_v128(v128_unpackhi_u8_s16(a), v128_unpacklo_u8_s16(a)); -} - -SIMD_INLINE v256 v256_unpacklo_u8_s16(v256 a) { - return _mm256_unpacklo_epi8( - _mm256_permute4x64_epi64(a, _MM_SHUFFLE(3, 1, 2, 0)), - _mm256_setzero_si256()); -} - -SIMD_INLINE v256 v256_unpackhi_u8_s16(v256 a) { - return _mm256_unpackhi_epi8( - _mm256_permute4x64_epi64(a, _MM_SHUFFLE(3, 1, 2, 0)), - _mm256_setzero_si256()); -} - -SIMD_INLINE v256 v256_unpack_s8_s16(v128 a) { - return v256_from_v128(v128_unpackhi_s8_s16(a), v128_unpacklo_s8_s16(a)); -} - -SIMD_INLINE v256 v256_unpacklo_s8_s16(v256 a) { - return _mm256_srai_epi16( - _mm256_unpacklo_epi8( - a, _mm256_permute4x64_epi64(a, _MM_SHUFFLE(3, 1, 2, 0))), - 8); -} - -SIMD_INLINE v256 v256_unpackhi_s8_s16(v256 a) { - return _mm256_srai_epi16( - _mm256_unpackhi_epi8( - a, _mm256_permute4x64_epi64(a, _MM_SHUFFLE(3, 1, 2, 0))), - 8); -} - -SIMD_INLINE v256 v256_pack_s32_s16(v256 a, v256 b) { - return _mm256_permute4x64_epi64(_mm256_packs_epi32(b, a), - _MM_SHUFFLE(3, 1, 2, 0)); -} - -SIMD_INLINE v256 v256_pack_s32_u16(v256 a, v256 b) { - return _mm256_permute4x64_epi64(_mm256_packus_epi32(b, a), - _MM_SHUFFLE(3, 1, 2, 0)); -} - -SIMD_INLINE v256 v256_pack_s16_u8(v256 a, v256 b) { - return _mm256_permute4x64_epi64(_mm256_packus_epi16(b, a), - _MM_SHUFFLE(3, 1, 2, 0)); -} - -SIMD_INLINE v256 v256_pack_s16_s8(v256 a, v256 b) { - return _mm256_permute4x64_epi64(_mm256_packs_epi16(b, a), - _MM_SHUFFLE(3, 1, 2, 0)); -} - -SIMD_INLINE v256 v256_unpack_u16_s32(v128 a) { - return v256_from_v128(v128_unpackhi_u16_s32(a), v128_unpacklo_u16_s32(a)); -} - -SIMD_INLINE v256 v256_unpack_s16_s32(v128 a) { - return v256_from_v128(v128_unpackhi_s16_s32(a), v128_unpacklo_s16_s32(a)); -} - -SIMD_INLINE v256 v256_unpacklo_u16_s32(v256 a) { - return _mm256_unpacklo_epi16( - _mm256_permute4x64_epi64(a, _MM_SHUFFLE(3, 1, 2, 0)), - _mm256_setzero_si256()); -} - -SIMD_INLINE v256 v256_unpacklo_s16_s32(v256 a) { - return _mm256_srai_epi32( - _mm256_unpacklo_epi16( - a, _mm256_permute4x64_epi64(a, _MM_SHUFFLE(3, 1, 2, 0))), - 16); -} - -SIMD_INLINE v256 v256_unpackhi_u16_s32(v256 a) { - return _mm256_unpackhi_epi16( - _mm256_permute4x64_epi64(a, _MM_SHUFFLE(3, 1, 2, 0)), - _mm256_setzero_si256()); -} - -SIMD_INLINE v256 v256_unpackhi_s16_s32(v256 a) { - return _mm256_srai_epi32( - _mm256_unpackhi_epi16( - a, _mm256_permute4x64_epi64(a, _MM_SHUFFLE(3, 1, 2, 0))), - 16); -} - -SIMD_INLINE v256 v256_shuffle_8(v256 a, v256 pattern) { - return _mm256_blendv_epi8( - _mm256_shuffle_epi8( - _mm256_permute2x128_si256(a, a, _MM_SHUFFLE(0, 1, 0, 1)), pattern), - _mm256_shuffle_epi8( - _mm256_permute2x128_si256(a, a, _MM_SHUFFLE(0, 0, 0, 0)), pattern), - _mm256_cmpgt_epi8(v256_dup_8(16), pattern)); -} - -SIMD_INLINE v256 v256_wideshuffle_8(v256 a, v256 b, v256 pattern) { - v256 c32 = v256_dup_8(32); - v256 p32 = v256_sub_8(pattern, c32); - v256 r1 = _mm256_blendv_epi8( - _mm256_shuffle_epi8( - _mm256_permute2x128_si256(a, b, _MM_SHUFFLE(0, 1, 0, 1)), p32), - _mm256_shuffle_epi8( - _mm256_permute2x128_si256(a, b, _MM_SHUFFLE(0, 0, 0, 0)), p32), - _mm256_cmpgt_epi8(v256_dup_8(48), pattern)); - v256 r2 = _mm256_blendv_epi8( - _mm256_shuffle_epi8( - _mm256_permute2x128_si256(a, b, _MM_SHUFFLE(0, 3, 0, 3)), pattern), - _mm256_shuffle_epi8( - _mm256_permute2x128_si256(a, b, _MM_SHUFFLE(0, 2, 0, 2)), pattern), - _mm256_cmpgt_epi8(v256_dup_8(16), pattern)); - return _mm256_blendv_epi8(r1, r2, _mm256_cmpgt_epi8(c32, pattern)); -} - -SIMD_INLINE v256 v256_pshuffle_8(v256 a, v256 pattern) { - return _mm256_shuffle_epi8(a, pattern); -} - -SIMD_INLINE int64_t v256_dotp_su8(v256 a, v256 b) { - v256 t1 = _mm256_madd_epi16(v256_unpackhi_s8_s16(a), v256_unpackhi_u8_s16(b)); - v256 t2 = _mm256_madd_epi16(v256_unpacklo_s8_s16(a), v256_unpacklo_u8_s16(b)); - t1 = _mm256_add_epi32(t1, t2); - v128 t = _mm_add_epi32(_mm256_extracti128_si256(t1, 0), - _mm256_extracti128_si256(t1, 1)); - t = _mm_add_epi32(t, _mm_srli_si128(t, 8)); - t = _mm_add_epi32(t, _mm_srli_si128(t, 4)); - return (int32_t)v128_low_u32(t); -} - -SIMD_INLINE int64_t v256_dotp_s16(v256 a, v256 b) { - v256 r = _mm256_madd_epi16(a, b); -#if defined(__x86_64__) - v128 t; - r = _mm256_add_epi64(_mm256_cvtepi32_epi64(v256_high_v128(r)), - _mm256_cvtepi32_epi64(v256_low_v128(r))); - t = v256_low_v128(_mm256_add_epi64( - r, _mm256_permute2x128_si256(r, r, _MM_SHUFFLE(2, 0, 0, 1)))); - return _mm_cvtsi128_si64(_mm_add_epi64(t, _mm_srli_si128(t, 8))); -#else - v128 l = v256_low_v128(r); - v128 h = v256_high_v128(r); - return (int64_t)_mm_cvtsi128_si32(l) + - (int64_t)_mm_cvtsi128_si32(_mm_srli_si128(l, 4)) + - (int64_t)_mm_cvtsi128_si32(_mm_srli_si128(l, 8)) + - (int64_t)_mm_cvtsi128_si32(_mm_srli_si128(l, 12)) + - (int64_t)_mm_cvtsi128_si32(h) + - (int64_t)_mm_cvtsi128_si32(_mm_srli_si128(h, 4)) + - (int64_t)_mm_cvtsi128_si32(_mm_srli_si128(h, 8)) + - (int64_t)_mm_cvtsi128_si32(_mm_srli_si128(h, 12)); -#endif -} - -SIMD_INLINE int64_t v256_dotp_s32(v256 a, v256 b) { - v256 r = _mm256_mullo_epi32(a, b); -#if defined(__x86_64__) - v128 t; - r = _mm256_add_epi64(_mm256_cvtepi32_epi64(v256_high_v128(r)), - _mm256_cvtepi32_epi64(v256_low_v128(r))); - t = v256_low_v128(_mm256_add_epi64( - r, _mm256_permute2x128_si256(r, r, _MM_SHUFFLE(2, 0, 0, 1)))); - return _mm_cvtsi128_si64(_mm_add_epi64(t, _mm_srli_si128(t, 8))); -#else - v128 l = v256_low_v128(r); - v128 h = v256_high_v128(r); - return (int64_t)_mm_cvtsi128_si32(l) + - (int64_t)_mm_cvtsi128_si32(_mm_srli_si128(l, 4)) + - (int64_t)_mm_cvtsi128_si32(_mm_srli_si128(l, 8)) + - (int64_t)_mm_cvtsi128_si32(_mm_srli_si128(l, 12)) + - (int64_t)_mm_cvtsi128_si32(h) + - (int64_t)_mm_cvtsi128_si32(_mm_srli_si128(h, 4)) + - (int64_t)_mm_cvtsi128_si32(_mm_srli_si128(h, 8)) + - (int64_t)_mm_cvtsi128_si32(_mm_srli_si128(h, 12)); -#endif -} - -SIMD_INLINE uint64_t v256_hadd_u8(v256 a) { - v256 t = _mm256_sad_epu8(a, _mm256_setzero_si256()); - v128 lo = v256_low_v128(t); - v128 hi = v256_high_v128(t); - lo = v128_add_32(lo, hi); - return v64_low_u32(v128_low_v64(lo)) + v128_low_u32(v128_high_v64(lo)); -} - -typedef v256 sad256_internal; - -SIMD_INLINE sad256_internal v256_sad_u8_init() { - return _mm256_setzero_si256(); -} - -/* Implementation dependent return value. Result must be finalised with - v256_sad_u8_sum(). - The result for more than 32 v256_sad_u8() calls is undefined. */ -SIMD_INLINE sad256_internal v256_sad_u8(sad256_internal s, v256 a, v256 b) { - return _mm256_add_epi64(s, _mm256_sad_epu8(a, b)); -} - -SIMD_INLINE uint32_t v256_sad_u8_sum(sad256_internal s) { - v256 t = _mm256_add_epi32(s, _mm256_unpackhi_epi64(s, s)); - return v128_low_u32(_mm_add_epi32(v256_high_v128(t), v256_low_v128(t))); -} - -typedef v256 ssd256_internal; - -SIMD_INLINE ssd256_internal v256_ssd_u8_init() { - return _mm256_setzero_si256(); -} - -/* Implementation dependent return value. Result must be finalised with - * v256_ssd_u8_sum(). */ -SIMD_INLINE ssd256_internal v256_ssd_u8(ssd256_internal s, v256 a, v256 b) { - v256 l = _mm256_sub_epi16(_mm256_unpacklo_epi8(a, _mm256_setzero_si256()), - _mm256_unpacklo_epi8(b, _mm256_setzero_si256())); - v256 h = _mm256_sub_epi16(_mm256_unpackhi_epi8(a, _mm256_setzero_si256()), - _mm256_unpackhi_epi8(b, _mm256_setzero_si256())); - v256 rl = _mm256_madd_epi16(l, l); - v256 rh = _mm256_madd_epi16(h, h); - v128 c = _mm_cvtsi32_si128(32); - rl = _mm256_add_epi32(rl, _mm256_srli_si256(rl, 8)); - rl = _mm256_add_epi32(rl, _mm256_srli_si256(rl, 4)); - rh = _mm256_add_epi32(rh, _mm256_srli_si256(rh, 8)); - rh = _mm256_add_epi32(rh, _mm256_srli_si256(rh, 4)); - return _mm256_add_epi64( - s, - _mm256_srl_epi64(_mm256_sll_epi64(_mm256_unpacklo_epi64(rl, rh), c), c)); -} - -SIMD_INLINE uint32_t v256_ssd_u8_sum(ssd256_internal s) { - v256 t = _mm256_add_epi32(s, _mm256_unpackhi_epi64(s, s)); - return v128_low_u32(_mm_add_epi32(v256_high_v128(t), v256_low_v128(t))); -} - -SIMD_INLINE v256 v256_or(v256 a, v256 b) { return _mm256_or_si256(a, b); } - -SIMD_INLINE v256 v256_xor(v256 a, v256 b) { return _mm256_xor_si256(a, b); } - -SIMD_INLINE v256 v256_and(v256 a, v256 b) { return _mm256_and_si256(a, b); } - -SIMD_INLINE v256 v256_andn(v256 a, v256 b) { return _mm256_andnot_si256(b, a); } - -SIMD_INLINE v256 v256_mul_s16(v64 a, v64 b) { - v128 lo_bits = v128_mullo_s16(a, b); - v128 hi_bits = v128_mulhi_s16(a, b); - return v256_from_v128(v128_ziphi_16(hi_bits, lo_bits), - v128_ziplo_16(hi_bits, lo_bits)); -} - -SIMD_INLINE v256 v256_mullo_s16(v256 a, v256 b) { - return _mm256_mullo_epi16(a, b); -} - -SIMD_INLINE v256 v256_mulhi_s16(v256 a, v256 b) { - return _mm256_mulhi_epi16(a, b); -} - -SIMD_INLINE v256 v256_mullo_s32(v256 a, v256 b) { - return _mm256_mullo_epi32(a, b); -} - -SIMD_INLINE v256 v256_madd_s16(v256 a, v256 b) { - return _mm256_madd_epi16(a, b); -} - -SIMD_INLINE v256 v256_madd_us8(v256 a, v256 b) { - return _mm256_maddubs_epi16(a, b); -} - -SIMD_INLINE v256 v256_avg_u8(v256 a, v256 b) { return _mm256_avg_epu8(a, b); } - -SIMD_INLINE v256 v256_rdavg_u8(v256 a, v256 b) { - return _mm256_sub_epi8( - _mm256_avg_epu8(a, b), - _mm256_and_si256(_mm256_xor_si256(a, b), v256_dup_8(1))); -} - -SIMD_INLINE v256 v256_rdavg_u16(v256 a, v256 b) { - return _mm256_sub_epi16( - _mm256_avg_epu16(a, b), - _mm256_and_si256(_mm256_xor_si256(a, b), v256_dup_16(1))); -} - -SIMD_INLINE v256 v256_avg_u16(v256 a, v256 b) { return _mm256_avg_epu16(a, b); } - -SIMD_INLINE v256 v256_min_u8(v256 a, v256 b) { return _mm256_min_epu8(a, b); } - -SIMD_INLINE v256 v256_max_u8(v256 a, v256 b) { return _mm256_max_epu8(a, b); } - -SIMD_INLINE v256 v256_min_s8(v256 a, v256 b) { return _mm256_min_epi8(a, b); } - -SIMD_INLINE uint32_t v256_movemask_8(v256 a) { return _mm256_movemask_epi8(a); } - -SIMD_INLINE v256 v256_blend_8(v256 a, v256 b, v256 c) { - return _mm256_blendv_epi8(a, b, c); -} - -SIMD_INLINE v256 v256_max_s8(v256 a, v256 b) { return _mm256_max_epi8(a, b); } - -SIMD_INLINE v256 v256_min_s16(v256 a, v256 b) { return _mm256_min_epi16(a, b); } - -SIMD_INLINE v256 v256_max_s16(v256 a, v256 b) { return _mm256_max_epi16(a, b); } - -SIMD_INLINE v256 v256_min_s32(v256 a, v256 b) { return _mm256_min_epi32(a, b); } - -SIMD_INLINE v256 v256_max_s32(v256 a, v256 b) { return _mm256_max_epi32(a, b); } - -SIMD_INLINE v256 v256_cmpgt_s8(v256 a, v256 b) { - return _mm256_cmpgt_epi8(a, b); -} - -SIMD_INLINE v256 v256_cmplt_s8(v256 a, v256 b) { - return _mm256_cmpgt_epi8(b, a); -} - -SIMD_INLINE v256 v256_cmpeq_8(v256 a, v256 b) { - return _mm256_cmpeq_epi8(a, b); -} - -SIMD_INLINE v256 v256_cmpgt_s16(v256 a, v256 b) { - return _mm256_cmpgt_epi16(a, b); -} - -SIMD_INLINE v256 v256_cmplt_s16(v256 a, v256 b) { - return _mm256_cmpgt_epi16(b, a); -} - -SIMD_INLINE v256 v256_cmpeq_16(v256 a, v256 b) { - return _mm256_cmpeq_epi16(a, b); -} - -SIMD_INLINE v256 v256_cmpgt_s32(v256 a, v256 b) { - return _mm256_cmpgt_epi32(a, b); -} - -SIMD_INLINE v256 v256_cmplt_s32(v256 a, v256 b) { - return _mm256_cmpgt_epi32(b, a); -} - -SIMD_INLINE v256 v256_cmpeq_32(v256 a, v256 b) { - return _mm256_cmpeq_epi32(a, b); -} - -SIMD_INLINE v256 v256_shl_8(v256 a, unsigned int c) { - return _mm256_and_si256(_mm256_set1_epi8((uint8_t)(0xff << c)), - _mm256_sll_epi16(a, _mm_cvtsi32_si128(c))); -} - -SIMD_INLINE v256 v256_shr_u8(v256 a, unsigned int c) { - return _mm256_and_si256(_mm256_set1_epi8(0xff >> c), - _mm256_srl_epi16(a, _mm_cvtsi32_si128(c))); -} - -SIMD_INLINE v256 v256_shr_s8(v256 a, unsigned int c) { - __m128i x = _mm_cvtsi32_si128(c + 8); - return _mm256_packs_epi16(_mm256_sra_epi16(_mm256_unpacklo_epi8(a, a), x), - _mm256_sra_epi16(_mm256_unpackhi_epi8(a, a), x)); -} - -SIMD_INLINE v256 v256_shl_16(v256 a, unsigned int c) { - return _mm256_sll_epi16(a, _mm_cvtsi32_si128(c)); -} - -SIMD_INLINE v256 v256_shr_u16(v256 a, unsigned int c) { - return _mm256_srl_epi16(a, _mm_cvtsi32_si128(c)); -} - -SIMD_INLINE v256 v256_shr_s16(v256 a, unsigned int c) { - return _mm256_sra_epi16(a, _mm_cvtsi32_si128(c)); -} - -SIMD_INLINE v256 v256_shl_32(v256 a, unsigned int c) { - return _mm256_sll_epi32(a, _mm_cvtsi32_si128(c)); -} - -SIMD_INLINE v256 v256_shr_u32(v256 a, unsigned int c) { - return _mm256_srl_epi32(a, _mm_cvtsi32_si128(c)); -} - -SIMD_INLINE v256 v256_shr_s32(v256 a, unsigned int c) { - return _mm256_sra_epi32(a, _mm_cvtsi32_si128(c)); -} - -SIMD_INLINE v256 v256_shl_64(v256 a, unsigned int c) { - return _mm256_sll_epi64(a, _mm_cvtsi32_si128(c)); -} - -SIMD_INLINE v256 v256_shr_u64(v256 a, unsigned int c) { - return _mm256_srl_epi64(a, _mm_cvtsi32_si128(c)); -} - -SIMD_INLINE v256 v256_shr_s64(v256 a, unsigned int c) { -#if defined(__AVX512F__) - return _mm256_sra_epi64(a, _mm_cvtsi32_si128(c)); -#else - return v256_from_v128(v128_shr_s64(v256_high_v128(a), c), - v128_shr_s64(v256_low_v128(a), c)); -#endif -} - -/* These intrinsics require immediate values, so we must use #defines - to enforce that. */ -// _mm256_slli_si256 works on 128 bit lanes and can't be used -#define v256_shl_n_byte(a, n) \ - ((n) < 16 ? v256_from_v128( \ - v128_align(v256_high_v128(a), v256_low_v128(a), 16 - (n)), \ - v128_shl_n_byte(v256_low_v128(a), n)) \ - : _mm256_inserti128_si256( \ - _mm256_setzero_si256(), \ - v128_shl_n_byte(v256_low_v128(a), (n)-16), 1)) - -// _mm256_srli_si256 works on 128 bit lanes and can't be used -#define v256_shr_n_byte(a, n) \ - ((n) < 16 \ - ? _mm256_alignr_epi8( \ - _mm256_permute2x128_si256(a, a, _MM_SHUFFLE(2, 0, 0, 1)), a, n) \ - : _mm256_inserti128_si256( \ - _mm256_setzero_si256(), \ - v128_align(v256_high_v128(a), v256_high_v128(a), n), 0)) - -// _mm256_alignr_epi8 works on two 128 bit lanes and can't be used -#define v256_align(a, b, c) \ - ((c) ? v256_or(v256_shr_n_byte(b, c), v256_shl_n_byte(a, 32 - c)) : b) - -#define v256_shl_n_8(a, c) \ - _mm256_and_si256(_mm256_set1_epi8((uint8_t)(0xff << (c))), \ - _mm256_slli_epi16(a, c)) -#define v256_shr_n_u8(a, c) \ - _mm256_and_si256(_mm256_set1_epi8(0xff >> (c)), _mm256_srli_epi16(a, c)) -#define v256_shr_n_s8(a, c) \ - _mm256_packs_epi16(_mm256_srai_epi16(_mm256_unpacklo_epi8(a, a), (c) + 8), \ - _mm256_srai_epi16(_mm256_unpackhi_epi8(a, a), (c) + 8)) -#define v256_shl_n_16(a, c) _mm256_slli_epi16(a, c) -#define v256_shr_n_u16(a, c) _mm256_srli_epi16(a, c) -#define v256_shr_n_s16(a, c) _mm256_srai_epi16(a, c) -#define v256_shl_n_32(a, c) _mm256_slli_epi32(a, c) -#define v256_shr_n_u32(a, c) _mm256_srli_epi32(a, c) -#define v256_shr_n_s32(a, c) _mm256_srai_epi32(a, c) -#define v256_shl_n_64(a, c) _mm256_slli_epi64(a, c) -#define v256_shr_n_u64(a, c) _mm256_srli_epi64(a, c) -#define v256_shr_n_s64(a, c) \ - v256_shr_s64((a), (c)) // _mm256_srai_epi64 broken in gcc? -#define v256_shr_n_word(a, n) v256_shr_n_byte(a, 2 * (n)) -#define v256_shl_n_word(a, n) v256_shl_n_byte(a, 2 * (n)) - -typedef v256 sad256_internal_u16; - -SIMD_INLINE sad256_internal_u16 v256_sad_u16_init() { return v256_zero(); } - -/* Implementation dependent return value. Result must be finalised with - * v256_sad_u16_sum(). */ -SIMD_INLINE sad256_internal_u16 v256_sad_u16(sad256_internal_u16 s, v256 a, - v256 b) { -#if defined(__SSE4_1__) - v256 t = v256_sub_16(_mm256_max_epu16(a, b), _mm256_min_epu16(a, b)); -#else - v256 t = v256_cmplt_s16(v256_xor(a, v256_dup_16(32768)), - v256_xor(b, v256_dup_16(32768))); - t = v256_sub_16(v256_or(v256_and(b, t), v256_andn(a, t)), - v256_or(v256_and(a, t), v256_andn(b, t))); -#endif - return v256_add_32( - s, v256_add_32(v256_unpackhi_u16_s32(t), v256_unpacklo_u16_s32(t))); -} - -SIMD_INLINE uint32_t v256_sad_u16_sum(sad256_internal_u16 s) { - v128 t = v128_add_32(v256_high_v128(s), v256_low_v128(s)); - return v128_low_u32(t) + v128_low_u32(v128_shr_n_byte(t, 4)) + - v128_low_u32(v128_shr_n_byte(t, 8)) + - v128_low_u32(v128_shr_n_byte(t, 12)); -} - -typedef v256 ssd256_internal_s16; - -SIMD_INLINE ssd256_internal_s16 v256_ssd_s16_init() { return v256_zero(); } - -/* Implementation dependent return value. Result must be finalised with - * v256_ssd_s16_sum(). */ -SIMD_INLINE ssd256_internal_s16 v256_ssd_s16(ssd256_internal_s16 s, v256 a, - v256 b) { - v256 d = v256_sub_16(a, b); - d = v256_madd_s16(d, d); - return v256_add_64(s, v256_add_64(_mm256_unpackhi_epi32(d, v256_zero()), - _mm256_unpacklo_epi32(d, v256_zero()))); -} - -SIMD_INLINE uint64_t v256_ssd_s16_sum(ssd256_internal_s16 s) { - v128 t = v128_add_64(v256_high_v128(s), v256_low_v128(s)); - return v64_u64(v128_low_v64(t)) + v64_u64(v128_high_v64(t)); -} - -#endif - -#endif // AOM_AOM_DSP_SIMD_V256_INTRINSICS_X86_H_ diff --git a/third_party/aom/aom_dsp/simd/v64_intrinsics.h b/third_party/aom/aom_dsp/simd/v64_intrinsics.h deleted file mode 100644 index afc55428d..000000000 --- a/third_party/aom/aom_dsp/simd/v64_intrinsics.h +++ /dev/null @@ -1,232 +0,0 @@ -/* - * Copyright (c) 2016, Alliance for Open Media. All rights reserved - * - * This source code is subject to the terms of the BSD 2 Clause License and - * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License - * was not distributed with this source code in the LICENSE file, you can - * obtain it at www.aomedia.org/license/software. If the Alliance for Open - * Media Patent License 1.0 was not distributed with this source code in the - * PATENTS file, you can obtain it at www.aomedia.org/license/patent. - */ - -#ifndef AOM_AOM_DSP_SIMD_V64_INTRINSICS_H_ -#define AOM_AOM_DSP_SIMD_V64_INTRINSICS_H_ - -#include <stdio.h> -#include <stdlib.h> - -#include "aom_dsp/simd/v64_intrinsics_c.h" - -/* Fallback to plain, unoptimised C. */ - -typedef c_v64 v64; - -SIMD_INLINE uint32_t v64_low_u32(v64 a) { return c_v64_low_u32(a); } -SIMD_INLINE uint32_t v64_high_u32(v64 a) { return c_v64_high_u32(a); } -SIMD_INLINE int32_t v64_low_s32(v64 a) { return c_v64_low_s32(a); } -SIMD_INLINE int32_t v64_high_s32(v64 a) { return c_v64_high_s32(a); } -SIMD_INLINE v64 v64_from_32(uint32_t x, uint32_t y) { - return c_v64_from_32(x, y); -} -SIMD_INLINE v64 v64_from_64(uint64_t x) { return c_v64_from_64(x); } -SIMD_INLINE uint64_t v64_u64(v64 x) { return c_v64_u64(x); } -SIMD_INLINE v64 v64_from_16(uint16_t a, uint16_t b, uint16_t c, uint16_t d) { - return c_v64_from_16(a, b, c, d); -} - -SIMD_INLINE uint32_t u32_load_unaligned(const void *p) { - return c_u32_load_unaligned(p); -} -SIMD_INLINE uint32_t u32_load_aligned(const void *p) { - return c_u32_load_aligned(p); -} -SIMD_INLINE void u32_store_unaligned(void *p, uint32_t a) { - c_u32_store_unaligned(p, a); -} -SIMD_INLINE void u32_store_aligned(void *p, uint32_t a) { - c_u32_store_aligned(p, a); -} - -SIMD_INLINE v64 v64_load_unaligned(const void *p) { - return c_v64_load_unaligned(p); -} -SIMD_INLINE v64 v64_load_aligned(const void *p) { - return c_v64_load_aligned(p); -} - -SIMD_INLINE void v64_store_unaligned(void *p, v64 a) { - c_v64_store_unaligned(p, a); -} -SIMD_INLINE void v64_store_aligned(void *p, v64 a) { - c_v64_store_aligned(p, a); -} - -SIMD_INLINE v64 v64_align(v64 a, v64 b, unsigned int c) { - return c_v64_align(a, b, c); -} - -SIMD_INLINE v64 v64_zero() { return c_v64_zero(); } -SIMD_INLINE v64 v64_dup_8(uint8_t x) { return c_v64_dup_8(x); } -SIMD_INLINE v64 v64_dup_16(uint16_t x) { return c_v64_dup_16(x); } -SIMD_INLINE v64 v64_dup_32(uint32_t x) { return c_v64_dup_32(x); } - -SIMD_INLINE v64 v64_add_8(v64 a, v64 b) { return c_v64_add_8(a, b); } -SIMD_INLINE v64 v64_add_16(v64 a, v64 b) { return c_v64_add_16(a, b); } -SIMD_INLINE v64 v64_sadd_u8(v64 a, v64 b) { return c_v64_sadd_u8(a, b); } -SIMD_INLINE v64 v64_sadd_s8(v64 a, v64 b) { return c_v64_sadd_s8(a, b); } -SIMD_INLINE v64 v64_sadd_s16(v64 a, v64 b) { return c_v64_sadd_s16(a, b); } -SIMD_INLINE v64 v64_add_32(v64 a, v64 b) { return c_v64_add_32(a, b); } -SIMD_INLINE v64 v64_sub_8(v64 a, v64 b) { return c_v64_sub_8(a, b); } -SIMD_INLINE v64 v64_ssub_u8(v64 a, v64 b) { return c_v64_ssub_u8(a, b); } -SIMD_INLINE v64 v64_ssub_s8(v64 a, v64 b) { return c_v64_ssub_s8(a, b); } -SIMD_INLINE v64 v64_sub_16(v64 a, v64 b) { return c_v64_sub_16(a, b); } -SIMD_INLINE v64 v64_ssub_s16(v64 a, v64 b) { return c_v64_ssub_s16(a, b); } -SIMD_INLINE v64 v64_ssub_u16(v64 a, v64 b) { return c_v64_ssub_u16(a, b); } -SIMD_INLINE v64 v64_sub_32(v64 a, v64 b) { return c_v64_sub_32(a, b); } -SIMD_INLINE v64 v64_abs_s16(v64 a) { return c_v64_abs_s16(a); } -SIMD_INLINE v64 v64_abs_s8(v64 a) { return c_v64_abs_s8(a); } - -SIMD_INLINE v64 v64_ziplo_8(v64 a, v64 b) { return c_v64_ziplo_8(a, b); } -SIMD_INLINE v64 v64_ziphi_8(v64 a, v64 b) { return c_v64_ziphi_8(a, b); } -SIMD_INLINE v64 v64_ziplo_16(v64 a, v64 b) { return c_v64_ziplo_16(a, b); } -SIMD_INLINE v64 v64_ziphi_16(v64 a, v64 b) { return c_v64_ziphi_16(a, b); } -SIMD_INLINE v64 v64_ziplo_32(v64 a, v64 b) { return c_v64_ziplo_32(a, b); } -SIMD_INLINE v64 v64_ziphi_32(v64 a, v64 b) { return c_v64_ziphi_32(a, b); } -SIMD_INLINE v64 v64_unziplo_8(v64 a, v64 b) { return c_v64_unziplo_8(a, b); } -SIMD_INLINE v64 v64_unziphi_8(v64 a, v64 b) { return c_v64_unziphi_8(a, b); } -SIMD_INLINE v64 v64_unziplo_16(v64 a, v64 b) { return c_v64_unziplo_16(a, b); } -SIMD_INLINE v64 v64_unziphi_16(v64 a, v64 b) { return c_v64_unziphi_16(a, b); } -SIMD_INLINE v64 v64_unpacklo_u8_s16(v64 a) { return c_v64_unpacklo_u8_s16(a); } -SIMD_INLINE v64 v64_unpackhi_u8_s16(v64 a) { return c_v64_unpackhi_u8_s16(a); } -SIMD_INLINE v64 v64_unpacklo_s8_s16(v64 a) { return c_v64_unpacklo_s8_s16(a); } -SIMD_INLINE v64 v64_unpackhi_s8_s16(v64 a) { return c_v64_unpackhi_s8_s16(a); } -SIMD_INLINE v64 v64_pack_s32_s16(v64 a, v64 b) { - return c_v64_pack_s32_s16(a, b); -} -SIMD_INLINE v64 v64_pack_s32_u16(v64 a, v64 b) { - return c_v64_pack_s32_u16(a, b); -} -SIMD_INLINE v64 v64_pack_s16_u8(v64 a, v64 b) { - return c_v64_pack_s16_u8(a, b); -} -SIMD_INLINE v64 v64_pack_s16_s8(v64 a, v64 b) { - return c_v64_pack_s16_s8(a, b); -} -SIMD_INLINE v64 v64_unpacklo_u16_s32(v64 a) { - return c_v64_unpacklo_u16_s32(a); -} -SIMD_INLINE v64 v64_unpacklo_s16_s32(v64 a) { - return c_v64_unpacklo_s16_s32(a); -} -SIMD_INLINE v64 v64_unpackhi_u16_s32(v64 a) { - return c_v64_unpackhi_u16_s32(a); -} -SIMD_INLINE v64 v64_unpackhi_s16_s32(v64 a) { - return c_v64_unpackhi_s16_s32(a); -} -SIMD_INLINE v64 v64_shuffle_8(v64 a, v64 pattern) { - return c_v64_shuffle_8(a, pattern); -} - -typedef uint32_t sad64_internal; -SIMD_INLINE sad64_internal v64_sad_u8_init() { return c_v64_sad_u8_init(); } -SIMD_INLINE sad64_internal v64_sad_u8(sad64_internal s, v64 a, v64 b) { - return c_v64_sad_u8(s, a, b); -} -SIMD_INLINE uint32_t v64_sad_u8_sum(sad64_internal s) { - return c_v64_sad_u8_sum(s); -} -typedef uint32_t ssd64_internal; -SIMD_INLINE ssd64_internal v64_ssd_u8_init() { return c_v64_ssd_u8_init(); } -SIMD_INLINE ssd64_internal v64_ssd_u8(ssd64_internal s, v64 a, v64 b) { - return c_v64_ssd_u8(s, a, b); -} -SIMD_INLINE uint32_t v64_ssd_u8_sum(ssd64_internal s) { - return c_v64_ssd_u8_sum(s); -} -SIMD_INLINE int64_t v64_dotp_su8(v64 a, v64 b) { return c_v64_dotp_su8(a, b); } -SIMD_INLINE int64_t v64_dotp_s16(v64 a, v64 b) { return c_v64_dotp_s16(a, b); } -SIMD_INLINE uint64_t v64_hadd_u8(v64 a) { return c_v64_hadd_u8(a); } -SIMD_INLINE int64_t v64_hadd_s16(v64 a) { return c_v64_hadd_s16(a); } - -SIMD_INLINE v64 v64_or(v64 a, v64 b) { return c_v64_or(a, b); } -SIMD_INLINE v64 v64_xor(v64 a, v64 b) { return c_v64_xor(a, b); } -SIMD_INLINE v64 v64_and(v64 a, v64 b) { return c_v64_and(a, b); } -SIMD_INLINE v64 v64_andn(v64 a, v64 b) { return c_v64_andn(a, b); } - -SIMD_INLINE v64 v64_mullo_s16(v64 a, v64 b) { return c_v64_mullo_s16(a, b); } -SIMD_INLINE v64 v64_mulhi_s16(v64 a, v64 b) { return c_v64_mulhi_s16(a, b); } -SIMD_INLINE v64 v64_mullo_s32(v64 a, v64 b) { return c_v64_mullo_s32(a, b); } -SIMD_INLINE v64 v64_madd_s16(v64 a, v64 b) { return c_v64_madd_s16(a, b); } -SIMD_INLINE v64 v64_madd_us8(v64 a, v64 b) { return c_v64_madd_us8(a, b); } - -SIMD_INLINE v64 v64_avg_u8(v64 a, v64 b) { return c_v64_avg_u8(a, b); } -SIMD_INLINE v64 v64_rdavg_u8(v64 a, v64 b) { return c_v64_rdavg_u8(a, b); } -SIMD_INLINE v64 v64_rdavg_u16(v64 a, v64 b) { return c_v64_rdavg_u16(a, b); } -SIMD_INLINE v64 v64_avg_u16(v64 a, v64 b) { return c_v64_avg_u16(a, b); } -SIMD_INLINE v64 v64_min_u8(v64 a, v64 b) { return c_v64_min_u8(a, b); } -SIMD_INLINE v64 v64_max_u8(v64 a, v64 b) { return c_v64_max_u8(a, b); } -SIMD_INLINE v64 v64_min_s8(v64 a, v64 b) { return c_v64_min_s8(a, b); } -SIMD_INLINE v64 v64_max_s8(v64 a, v64 b) { return c_v64_max_s8(a, b); } -SIMD_INLINE v64 v64_min_s16(v64 a, v64 b) { return c_v64_min_s16(a, b); } -SIMD_INLINE v64 v64_max_s16(v64 a, v64 b) { return c_v64_max_s16(a, b); } - -SIMD_INLINE v64 v64_cmpgt_s8(v64 a, v64 b) { return c_v64_cmpgt_s8(a, b); } -SIMD_INLINE v64 v64_cmplt_s8(v64 a, v64 b) { return c_v64_cmplt_s8(a, b); } -SIMD_INLINE v64 v64_cmpeq_8(v64 a, v64 b) { return c_v64_cmpeq_8(a, b); } -SIMD_INLINE v64 v64_cmpgt_s16(v64 a, v64 b) { return c_v64_cmpgt_s16(a, b); } -SIMD_INLINE v64 v64_cmplt_s16(v64 a, v64 b) { return c_v64_cmplt_s16(a, b); } -SIMD_INLINE v64 v64_cmpeq_16(v64 a, v64 b) { return c_v64_cmpeq_16(a, b); } - -SIMD_INLINE v64 v64_shl_8(v64 a, unsigned int n) { return c_v64_shl_8(a, n); } -SIMD_INLINE v64 v64_shr_u8(v64 a, unsigned int n) { return c_v64_shr_u8(a, n); } -SIMD_INLINE v64 v64_shr_s8(v64 a, unsigned int n) { return c_v64_shr_s8(a, n); } -SIMD_INLINE v64 v64_shl_16(v64 a, unsigned int n) { return c_v64_shl_16(a, n); } -SIMD_INLINE v64 v64_shr_u16(v64 a, unsigned int n) { - return c_v64_shr_u16(a, n); -} -SIMD_INLINE v64 v64_shr_s16(v64 a, unsigned int n) { - return c_v64_shr_s16(a, n); -} -SIMD_INLINE v64 v64_shl_32(v64 a, unsigned int n) { return c_v64_shl_32(a, n); } -SIMD_INLINE v64 v64_shr_u32(v64 a, unsigned int n) { - return c_v64_shr_u32(a, n); -} -SIMD_INLINE v64 v64_shr_s32(v64 a, unsigned int n) { - return c_v64_shr_s32(a, n); -} -SIMD_INLINE v64 v64_shr_n_byte(v64 a, unsigned int n) { - return c_v64_shr_n_byte(a, n); -} -SIMD_INLINE v64 v64_shl_n_byte(v64 a, unsigned int n) { - return c_v64_shl_n_byte(a, n); -} -SIMD_INLINE v64 v64_shl_n_8(v64 a, unsigned int c) { - return c_v64_shl_n_8(a, c); -} -SIMD_INLINE v64 v64_shr_n_u8(v64 a, unsigned int c) { - return c_v64_shr_n_u8(a, c); -} -SIMD_INLINE v64 v64_shr_n_s8(v64 a, unsigned int c) { - return c_v64_shr_n_s8(a, c); -} -SIMD_INLINE v64 v64_shl_n_16(v64 a, unsigned int c) { - return c_v64_shl_n_16(a, c); -} -SIMD_INLINE v64 v64_shr_n_u16(v64 a, unsigned int c) { - return c_v64_shr_n_u16(a, c); -} -SIMD_INLINE v64 v64_shr_n_s16(v64 a, unsigned int c) { - return c_v64_shr_n_s16(a, c); -} -SIMD_INLINE v64 v64_shl_n_32(v64 a, unsigned int c) { - return c_v64_shl_n_32(a, c); -} -SIMD_INLINE v64 v64_shr_n_u32(v64 a, unsigned int c) { - return c_v64_shr_n_u32(a, c); -} -SIMD_INLINE v64 v64_shr_n_s32(v64 a, unsigned int c) { - return c_v64_shr_n_s32(a, c); -} - -#endif // AOM_AOM_DSP_SIMD_V64_INTRINSICS_H_ diff --git a/third_party/aom/aom_dsp/simd/v64_intrinsics_arm.h b/third_party/aom/aom_dsp/simd/v64_intrinsics_arm.h deleted file mode 100644 index 8f39ad6e8..000000000 --- a/third_party/aom/aom_dsp/simd/v64_intrinsics_arm.h +++ /dev/null @@ -1,680 +0,0 @@ -/* - * Copyright (c) 2016, Alliance for Open Media. All rights reserved - * - * This source code is subject to the terms of the BSD 2 Clause License and - * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License - * was not distributed with this source code in the LICENSE file, you can - * obtain it at www.aomedia.org/license/software. If the Alliance for Open - * Media Patent License 1.0 was not distributed with this source code in the - * PATENTS file, you can obtain it at www.aomedia.org/license/patent. - */ - -#ifndef AOM_AOM_DSP_SIMD_V64_INTRINSICS_ARM_H_ -#define AOM_AOM_DSP_SIMD_V64_INTRINSICS_ARM_H_ - -#include <arm_neon.h> - -#include "aom_dsp/simd/v64_intrinsics_arm.h" -#include "aom_ports/arm.h" - -#ifdef AOM_INCOMPATIBLE_GCC -#error Incompatible gcc -#endif - -typedef int64x1_t v64; - -SIMD_INLINE uint32_t v64_low_u32(v64 a) { - return vget_lane_u32(vreinterpret_u32_s64(a), 0); -} - -SIMD_INLINE uint32_t v64_high_u32(v64 a) { - return vget_lane_u32(vreinterpret_u32_s64(a), 1); -} - -SIMD_INLINE int32_t v64_low_s32(v64 a) { - return vget_lane_s32(vreinterpret_s32_s64(a), 0); -} - -SIMD_INLINE int32_t v64_high_s32(v64 a) { - return vget_lane_s32(vreinterpret_s32_s64(a), 1); -} - -SIMD_INLINE v64 v64_from_16(uint16_t a, uint16_t b, uint16_t c, uint16_t d) { - return vcreate_s64((uint64_t)a << 48 | (uint64_t)b << 32 | (uint64_t)c << 16 | - d); -} - -SIMD_INLINE v64 v64_from_32(uint32_t x, uint32_t y) { - return vcreate_s64((uint64_t)x << 32 | y); -} - -SIMD_INLINE v64 v64_from_64(uint64_t x) { return vcreate_s64(x); } - -SIMD_INLINE uint64_t v64_u64(v64 x) { return (uint64_t)x; } - -SIMD_INLINE uint32_t u32_load_aligned(const void *p) { - return *((uint32_t *)p); -} - -SIMD_INLINE uint32_t u32_load_unaligned(const void *p) { - return vget_lane_u32(vreinterpret_u32_u8(vld1_u8((const uint8_t *)p)), 0); -} - -SIMD_INLINE void u32_store_aligned(void *p, uint32_t a) { - *((uint32_t *)p) = a; -} - -SIMD_INLINE void u32_store_unaligned(void *p, uint32_t a) { -#if defined(__clang__) - vst1_lane_u32((uint32_t *)p, vreinterpret_u32_s64((uint64x1_t)(uint64_t)a), - 0); -#elif defined(__CC_ARM) - *(__packed uint32_t *)p) = a; -#elif defined(__GNUC__) - *((__attribute((packed)) uint32_t *)p) = a; -#else - vst1_lane_u32((uint32_t *)p, vreinterpret_u32_s64((uint64x1_t)(uint64_t)a), - 0); -#endif -} - -SIMD_INLINE v64 v64_load_aligned(const void *p) { - return vreinterpret_s64_u8(vld1_u8((const uint8_t *)p)); -} - -SIMD_INLINE v64 v64_load_unaligned(const void *p) { - return v64_load_aligned(p); -} - -SIMD_INLINE void v64_store_aligned(void *p, v64 r) { - vst1_u8((uint8_t *)p, vreinterpret_u8_s64(r)); -} - -SIMD_INLINE void v64_store_unaligned(void *p, v64 r) { - vst1_u8((uint8_t *)p, vreinterpret_u8_s64(r)); -} - -// The following function requires an immediate. -// Some compilers will check this if it's optimising, others wont. -SIMD_INLINE v64 v64_align(v64 a, v64 b, unsigned int c) { -#if defined(__OPTIMIZE__) && __OPTIMIZE__ && !defined(__clang__) - return c ? vreinterpret_s64_s8( - vext_s8(vreinterpret_s8_s64(b), vreinterpret_s8_s64(a), c)) - : b; -#else - return c ? v64_from_64(((uint64_t)b >> c * 8) | ((uint64_t)a << (8 - c) * 8)) - : b; -#endif -} - -SIMD_INLINE v64 v64_zero() { return vreinterpret_s64_u8(vdup_n_u8(0)); } - -SIMD_INLINE v64 v64_dup_8(uint8_t x) { - return vreinterpret_s64_u8(vdup_n_u8(x)); -} - -SIMD_INLINE v64 v64_dup_16(uint16_t x) { - return vreinterpret_s64_u16(vdup_n_u16(x)); -} - -SIMD_INLINE v64 v64_dup_32(uint32_t x) { - return vreinterpret_s64_u32(vdup_n_u32(x)); -} - -SIMD_INLINE int64_t v64_dotp_su8(v64 x, v64 y) { - int16x8_t t = - vmulq_s16(vmovl_s8(vreinterpret_s8_s64(x)), - vreinterpretq_s16_u16(vmovl_u8(vreinterpret_u8_s64(y)))); -#if defined(__aarch64__) - return vaddlvq_s16(t); -#else - int64x2_t r = vpaddlq_s32(vpaddlq_s16(t)); - return (int64_t)vadd_s64(vget_high_s64(r), vget_low_s64(r)); -#endif -} - -SIMD_INLINE int64_t v64_dotp_s16(v64 x, v64 y) { -#if defined(__aarch64__) - return vaddlvq_s32( - vmull_s16(vreinterpret_s16_s64(x), vreinterpret_s16_s64(y))); -#else - int64x2_t r = - vpaddlq_s32(vmull_s16(vreinterpret_s16_s64(x), vreinterpret_s16_s64(y))); - return (int64_t)(vget_high_s64(r) + vget_low_s64(r)); -#endif -} - -SIMD_INLINE uint64_t v64_hadd_u8(v64 x) { -#if defined(__aarch64__) - return vaddlv_u8(vreinterpret_u8_s64(x)); -#else - return (uint64_t)vpaddl_u32(vpaddl_u16(vpaddl_u8(vreinterpret_u8_s64(x)))); -#endif -} - -SIMD_INLINE int64_t v64_hadd_s16(v64 a) { - return (int64_t)vpaddl_s32(vpaddl_s16(vreinterpret_s16_s64(a))); -} - -typedef uint16x8_t sad64_internal; - -SIMD_INLINE sad64_internal v64_sad_u8_init() { return vdupq_n_u16(0); } - -// Implementation dependent return value. Result must be finalised with -// v64_sad_u8_sum(). -SIMD_INLINE sad64_internal v64_sad_u8(sad64_internal s, v64 a, v64 b) { - return vabal_u8(s, vreinterpret_u8_s64(a), vreinterpret_u8_s64(b)); -} - -SIMD_INLINE uint32_t v64_sad_u8_sum(sad64_internal s) { -#if defined(__aarch64__) - return vaddlvq_u16(s); -#else - uint64x2_t r = vpaddlq_u32(vpaddlq_u16(s)); - return (uint32_t)(uint64_t)(vget_high_u64(r) + vget_low_u64(r)); -#endif -} - -typedef uint32x4_t ssd64_internal; - -SIMD_INLINE ssd64_internal v64_ssd_u8_init() { return vdupq_n_u32(0); } - -// Implementation dependent return value. Result must be finalised with -// v64_ssd_u8_sum(). -SIMD_INLINE ssd64_internal v64_ssd_u8(ssd64_internal s, v64 a, v64 b) { - uint8x8_t t = vabd_u8(vreinterpret_u8_s64(a), vreinterpret_u8_s64(b)); - return vaddq_u32(s, vpaddlq_u16(vmull_u8(t, t))); -} - -SIMD_INLINE uint32_t v64_ssd_u8_sum(ssd64_internal s) { -#if defined(__aarch64__) - return vaddvq_u32(s); -#else - uint64x2_t t = vpaddlq_u32(s); - return vget_lane_u32( - vreinterpret_u32_u64(vadd_u64(vget_high_u64(t), vget_low_u64(t))), 0); -#endif -} - -SIMD_INLINE v64 v64_or(v64 x, v64 y) { return vorr_s64(x, y); } - -SIMD_INLINE v64 v64_xor(v64 x, v64 y) { return veor_s64(x, y); } - -SIMD_INLINE v64 v64_and(v64 x, v64 y) { return vand_s64(x, y); } - -SIMD_INLINE v64 v64_andn(v64 x, v64 y) { return vbic_s64(x, y); } - -SIMD_INLINE v64 v64_add_8(v64 x, v64 y) { - return vreinterpret_s64_u8( - vadd_u8(vreinterpret_u8_s64(x), vreinterpret_u8_s64(y))); -} - -SIMD_INLINE v64 v64_sadd_u8(v64 x, v64 y) { - return vreinterpret_s64_u8( - vqadd_u8(vreinterpret_u8_s64(x), vreinterpret_u8_s64(y))); -} - -SIMD_INLINE v64 v64_sadd_s8(v64 x, v64 y) { - return vreinterpret_s64_s8( - vqadd_s8(vreinterpret_s8_s64(x), vreinterpret_s8_s64(y))); -} - -SIMD_INLINE v64 v64_add_16(v64 x, v64 y) { - return vreinterpret_s64_s16( - vadd_s16(vreinterpret_s16_s64(x), vreinterpret_s16_s64(y))); -} - -SIMD_INLINE v64 v64_sadd_s16(v64 x, v64 y) { - return vreinterpret_s64_s16( - vqadd_s16(vreinterpret_s16_s64(x), vreinterpret_s16_s64(y))); -} - -SIMD_INLINE v64 v64_add_32(v64 x, v64 y) { - return vreinterpret_s64_u32( - vadd_u32(vreinterpret_u32_s64(x), vreinterpret_u32_s64(y))); -} - -SIMD_INLINE v64 v64_sub_8(v64 x, v64 y) { - return vreinterpret_s64_u8( - vsub_u8(vreinterpret_u8_s64(x), vreinterpret_u8_s64(y))); -} - -SIMD_INLINE v64 v64_sub_16(v64 x, v64 y) { - return vreinterpret_s64_s16( - vsub_s16(vreinterpret_s16_s64(x), vreinterpret_s16_s64(y))); -} - -SIMD_INLINE v64 v64_ssub_s16(v64 x, v64 y) { - return vreinterpret_s64_s16( - vqsub_s16(vreinterpret_s16_s64(x), vreinterpret_s16_s64(y))); -} - -SIMD_INLINE v64 v64_ssub_u16(v64 x, v64 y) { - return vreinterpret_s64_u16( - vqsub_u16(vreinterpret_u16_s64(x), vreinterpret_u16_s64(y))); -} - -SIMD_INLINE v64 v64_ssub_u8(v64 x, v64 y) { - return vreinterpret_s64_u8( - vqsub_u8(vreinterpret_u8_s64(x), vreinterpret_u8_s64(y))); -} - -SIMD_INLINE v64 v64_ssub_s8(v64 x, v64 y) { - return vreinterpret_s64_s8( - vqsub_s8(vreinterpret_s8_s64(x), vreinterpret_s8_s64(y))); -} - -SIMD_INLINE v64 v64_sub_32(v64 x, v64 y) { - return vreinterpret_s64_s32( - vsub_s32(vreinterpret_s32_s64(x), vreinterpret_s32_s64(y))); -} - -SIMD_INLINE v64 v64_abs_s16(v64 x) { - return vreinterpret_s64_s16(vabs_s16(vreinterpret_s16_s64(x))); -} - -SIMD_INLINE v64 v64_abs_s8(v64 x) { - return vreinterpret_s64_s8(vabs_s8(vreinterpret_s8_s64(x))); -} - -SIMD_INLINE v64 v64_mullo_s16(v64 x, v64 y) { - return vreinterpret_s64_s16( - vmul_s16(vreinterpret_s16_s64(x), vreinterpret_s16_s64(y))); -} - -SIMD_INLINE v64 v64_mulhi_s16(v64 x, v64 y) { -#if defined(__aarch64__) - int16x8_t t = vreinterpretq_s16_s32( - vmull_s16(vreinterpret_s16_s64(x), vreinterpret_s16_s64(y))); - return vget_low_s64(vreinterpretq_s64_s16(vuzp2q_s16(t, t))); -#else - return vreinterpret_s64_s16(vmovn_s32(vshrq_n_s32( - vmull_s16(vreinterpret_s16_s64(x), vreinterpret_s16_s64(y)), 16))); -#endif -} - -SIMD_INLINE v64 v64_mullo_s32(v64 x, v64 y) { - return vreinterpret_s64_s32( - vmul_s32(vreinterpret_s32_s64(x), vreinterpret_s32_s64(y))); -} - -SIMD_INLINE v64 v64_madd_s16(v64 x, v64 y) { - int32x4_t t = vmull_s16(vreinterpret_s16_s64(x), vreinterpret_s16_s64(y)); - return vreinterpret_s64_s32( - vpadd_s32(vreinterpret_s32_s64(vget_low_s64(vreinterpretq_s64_s32(t))), - vreinterpret_s32_s64(vget_high_s64(vreinterpretq_s64_s32(t))))); -} - -SIMD_INLINE v64 v64_madd_us8(v64 x, v64 y) { - int16x8_t t = - vmulq_s16(vreinterpretq_s16_u16(vmovl_u8(vreinterpret_u8_s64(x))), - vmovl_s8(vreinterpret_s8_s64(y))); - return vreinterpret_s64_s16(vqmovn_s32(vpaddlq_s16(t))); -} - -SIMD_INLINE v64 v64_avg_u8(v64 x, v64 y) { - return vreinterpret_s64_u8( - vrhadd_u8(vreinterpret_u8_s64(x), vreinterpret_u8_s64(y))); -} - -SIMD_INLINE v64 v64_rdavg_u8(v64 x, v64 y) { - return vreinterpret_s64_u8( - vhadd_u8(vreinterpret_u8_s64(x), vreinterpret_u8_s64(y))); -} - -SIMD_INLINE v64 v64_rdavg_u16(v64 x, v64 y) { - return vreinterpret_s64_u16( - vhadd_u16(vreinterpret_u16_s64(x), vreinterpret_u16_s64(y))); -} - -SIMD_INLINE v64 v64_avg_u16(v64 x, v64 y) { - return vreinterpret_s64_u16( - vrhadd_u16(vreinterpret_u16_s64(x), vreinterpret_u16_s64(y))); -} - -SIMD_INLINE v64 v64_max_u8(v64 x, v64 y) { - return vreinterpret_s64_u8( - vmax_u8(vreinterpret_u8_s64(x), vreinterpret_u8_s64(y))); -} - -SIMD_INLINE v64 v64_min_u8(v64 x, v64 y) { - return vreinterpret_s64_u8( - vmin_u8(vreinterpret_u8_s64(x), vreinterpret_u8_s64(y))); -} - -SIMD_INLINE v64 v64_max_s8(v64 x, v64 y) { - return vreinterpret_s64_s8( - vmax_s8(vreinterpret_s8_s64(x), vreinterpret_s8_s64(y))); -} - -SIMD_INLINE v64 v64_min_s8(v64 x, v64 y) { - return vreinterpret_s64_s8( - vmin_s8(vreinterpret_s8_s64(x), vreinterpret_s8_s64(y))); -} - -SIMD_INLINE v64 v64_max_s16(v64 x, v64 y) { - return vreinterpret_s64_s16( - vmax_s16(vreinterpret_s16_s64(x), vreinterpret_s16_s64(y))); -} - -SIMD_INLINE v64 v64_min_s16(v64 x, v64 y) { - return vreinterpret_s64_s16( - vmin_s16(vreinterpret_s16_s64(x), vreinterpret_s16_s64(y))); -} - -SIMD_INLINE v64 v64_ziplo_8(v64 x, v64 y) { -#if defined(__aarch64__) - return vreinterpret_s64_u8( - vzip1_u8(vreinterpret_u8_s64(y), vreinterpret_u8_s64(x))); -#else - uint8x8x2_t r = vzip_u8(vreinterpret_u8_s64(y), vreinterpret_u8_s64(x)); - return vreinterpret_s64_u8(r.val[0]); -#endif -} - -SIMD_INLINE v64 v64_ziphi_8(v64 x, v64 y) { -#if defined(__aarch64__) - return vreinterpret_s64_u8( - vzip2_u8(vreinterpret_u8_s64(y), vreinterpret_u8_s64(x))); -#else - uint8x8x2_t r = vzip_u8(vreinterpret_u8_s64(y), vreinterpret_u8_s64(x)); - return vreinterpret_s64_u8(r.val[1]); -#endif -} - -SIMD_INLINE v64 v64_ziplo_16(v64 x, v64 y) { -#if defined(__aarch64__) - return vreinterpret_s64_u16( - vzip1_u16(vreinterpret_u16_s64(y), vreinterpret_u16_s64(x))); -#else - int16x4x2_t r = vzip_s16(vreinterpret_s16_s64(y), vreinterpret_s16_s64(x)); - return vreinterpret_s64_s16(r.val[0]); -#endif -} - -SIMD_INLINE v64 v64_ziphi_16(v64 x, v64 y) { -#if defined(__aarch64__) - return vreinterpret_s64_u16( - vzip2_u16(vreinterpret_u16_s64(y), vreinterpret_u16_s64(x))); -#else - int16x4x2_t r = vzip_s16(vreinterpret_s16_s64(y), vreinterpret_s16_s64(x)); - return vreinterpret_s64_s16(r.val[1]); -#endif -} - -SIMD_INLINE v64 v64_ziplo_32(v64 x, v64 y) { -#if defined(__aarch64__) - return vreinterpret_s64_u32( - vzip1_u32(vreinterpret_u32_s64(y), vreinterpret_u32_s64(x))); -#else - int32x2x2_t r = vzip_s32(vreinterpret_s32_s64(y), vreinterpret_s32_s64(x)); - return vreinterpret_s64_s32(r.val[0]); -#endif -} - -SIMD_INLINE v64 v64_ziphi_32(v64 x, v64 y) { -#if defined(__aarch64__) - return vreinterpret_s64_u32( - vzip2_u32(vreinterpret_u32_s64(y), vreinterpret_u32_s64(x))); -#else - int32x2x2_t r = vzip_s32(vreinterpret_s32_s64(y), vreinterpret_s32_s64(x)); - return vreinterpret_s64_s32(r.val[1]); -#endif -} - -SIMD_INLINE v64 v64_unpacklo_u8_s16(v64 a) { - return vreinterpret_s64_u16(vget_low_u16(vmovl_u8(vreinterpret_u8_s64(a)))); -} - -SIMD_INLINE v64 v64_unpackhi_u8_s16(v64 a) { - return vreinterpret_s64_u16(vget_high_u16(vmovl_u8(vreinterpret_u8_s64(a)))); -} - -SIMD_INLINE v64 v64_unpacklo_s8_s16(v64 a) { - return vreinterpret_s64_s16(vget_low_s16(vmovl_s8(vreinterpret_s8_s64(a)))); -} - -SIMD_INLINE v64 v64_unpackhi_s8_s16(v64 a) { - return vreinterpret_s64_s16(vget_high_s16(vmovl_s8(vreinterpret_s8_s64(a)))); -} - -SIMD_INLINE v64 v64_pack_s32_s16(v64 x, v64 y) { - return vreinterpret_s64_s16(vqmovn_s32( - vcombine_s32(vreinterpret_s32_s64(y), vreinterpret_s32_s64(x)))); -} - -SIMD_INLINE v64 v64_pack_s32_u16(v64 x, v64 y) { - return vreinterpret_s64_u16(vqmovun_s32( - vcombine_s32(vreinterpret_s32_s64(y), vreinterpret_s32_s64(x)))); -} - -SIMD_INLINE v64 v64_pack_s16_u8(v64 x, v64 y) { - return vreinterpret_s64_u8(vqmovun_s16(vreinterpretq_s16_s32( - vcombine_s32(vreinterpret_s32_s64(y), vreinterpret_s32_s64(x))))); -} - -SIMD_INLINE v64 v64_pack_s16_s8(v64 x, v64 y) { - return vreinterpret_s64_s8(vqmovn_s16(vreinterpretq_s16_s32( - vcombine_s32(vreinterpret_s32_s64(y), vreinterpret_s32_s64(x))))); -} - -SIMD_INLINE v64 v64_unziplo_8(v64 x, v64 y) { -#if defined(__aarch64__) - return vreinterpret_s64_u8( - vuzp1_u8(vreinterpret_u8_s64(y), vreinterpret_u8_s64(x))); -#else - uint8x8x2_t r = vuzp_u8(vreinterpret_u8_s64(y), vreinterpret_u8_s64(x)); - return vreinterpret_s64_u8(r.val[0]); -#endif -} - -SIMD_INLINE v64 v64_unziphi_8(v64 x, v64 y) { -#if defined(__aarch64__) - return vreinterpret_s64_u8( - vuzp2_u8(vreinterpret_u8_s64(y), vreinterpret_u8_s64(x))); -#else - uint8x8x2_t r = vuzp_u8(vreinterpret_u8_s64(y), vreinterpret_u8_s64(x)); - return vreinterpret_s64_u8(r.val[1]); -#endif -} - -SIMD_INLINE v64 v64_unziplo_16(v64 x, v64 y) { -#if defined(__aarch64__) - return vreinterpret_s64_u16( - vuzp1_u16(vreinterpret_u16_s64(y), vreinterpret_u16_s64(x))); -#else - uint16x4x2_t r = vuzp_u16(vreinterpret_u16_s64(y), vreinterpret_u16_s64(x)); - return vreinterpret_s64_u16(r.val[0]); -#endif -} - -SIMD_INLINE v64 v64_unziphi_16(v64 x, v64 y) { -#if defined(__aarch64__) - return vreinterpret_s64_u16( - vuzp2_u16(vreinterpret_u16_s64(y), vreinterpret_u16_s64(x))); -#else - uint16x4x2_t r = vuzp_u16(vreinterpret_u16_s64(y), vreinterpret_u16_s64(x)); - return vreinterpret_s64_u16(r.val[1]); -#endif -} - -SIMD_INLINE v64 v64_unpacklo_s16_s32(v64 x) { - return vreinterpret_s64_s32(vget_low_s32(vmovl_s16(vreinterpret_s16_s64(x)))); -} - -SIMD_INLINE v64 v64_unpacklo_u16_s32(v64 x) { - return vreinterpret_s64_u32(vget_low_u32(vmovl_u16(vreinterpret_u16_s64(x)))); -} - -SIMD_INLINE v64 v64_unpackhi_s16_s32(v64 x) { - return vreinterpret_s64_s32( - vget_high_s32(vmovl_s16(vreinterpret_s16_s64(x)))); -} - -SIMD_INLINE v64 v64_unpackhi_u16_s32(v64 x) { - return vreinterpret_s64_u32( - vget_high_u32(vmovl_u16(vreinterpret_u16_s64(x)))); -} - -SIMD_INLINE v64 v64_shuffle_8(v64 x, v64 pattern) { - return vreinterpret_s64_u8( - vtbl1_u8(vreinterpret_u8_s64(x), vreinterpret_u8_s64(pattern))); -} - -SIMD_INLINE v64 v64_cmpgt_s8(v64 x, v64 y) { - return vreinterpret_s64_u8( - vcgt_s8(vreinterpret_s8_s64(x), vreinterpret_s8_s64(y))); -} - -SIMD_INLINE v64 v64_cmplt_s8(v64 x, v64 y) { - return vreinterpret_s64_u8( - vclt_s8(vreinterpret_s8_s64(x), vreinterpret_s8_s64(y))); -} - -SIMD_INLINE v64 v64_cmpeq_8(v64 x, v64 y) { - return vreinterpret_s64_u8( - vceq_u8(vreinterpret_u8_s64(x), vreinterpret_u8_s64(y))); -} - -SIMD_INLINE v64 v64_cmpgt_s16(v64 x, v64 y) { - return vreinterpret_s64_u16( - vcgt_s16(vreinterpret_s16_s64(x), vreinterpret_s16_s64(y))); -} - -SIMD_INLINE v64 v64_cmplt_s16(v64 x, v64 y) { - return vreinterpret_s64_u16( - vclt_s16(vreinterpret_s16_s64(x), vreinterpret_s16_s64(y))); -} - -SIMD_INLINE v64 v64_cmpeq_16(v64 x, v64 y) { - return vreinterpret_s64_u16( - vceq_s16(vreinterpret_s16_s64(x), vreinterpret_s16_s64(y))); -} - -SIMD_INLINE v64 v64_shl_8(v64 a, unsigned int c) { - return vreinterpret_s64_u8(vshl_u8(vreinterpret_u8_s64(a), vdup_n_s8(c))); -} - -SIMD_INLINE v64 v64_shr_u8(v64 a, unsigned int c) { - return vreinterpret_s64_u8(vshl_u8(vreinterpret_u8_s64(a), vdup_n_s8(-c))); -} - -SIMD_INLINE v64 v64_shr_s8(v64 a, unsigned int c) { - return vreinterpret_s64_s8(vshl_s8(vreinterpret_s8_s64(a), vdup_n_s8(-c))); -} - -SIMD_INLINE v64 v64_shl_16(v64 a, unsigned int c) { - return vreinterpret_s64_u16(vshl_u16(vreinterpret_u16_s64(a), vdup_n_s16(c))); -} - -SIMD_INLINE v64 v64_shr_u16(v64 a, unsigned int c) { - return vreinterpret_s64_u16( - vshl_u16(vreinterpret_u16_s64(a), vdup_n_s16(-(int)c))); -} - -SIMD_INLINE v64 v64_shr_s16(v64 a, unsigned int c) { - return vreinterpret_s64_s16( - vshl_s16(vreinterpret_s16_s64(a), vdup_n_s16(-(int)c))); -} - -SIMD_INLINE v64 v64_shl_32(v64 a, unsigned int c) { - return vreinterpret_s64_u32(vshl_u32(vreinterpret_u32_s64(a), vdup_n_s32(c))); -} - -SIMD_INLINE v64 v64_shr_u32(v64 a, unsigned int c) { - return vreinterpret_s64_u32( - vshl_u32(vreinterpret_u32_s64(a), vdup_n_s32(-(int)c))); -} - -SIMD_INLINE v64 v64_shr_s32(v64 a, unsigned int c) { - return vreinterpret_s64_s32( - vshl_s32(vreinterpret_s32_s64(a), vdup_n_s32(-(int)c))); -} - -// The following functions require an immediate. -// Some compilers will check this during optimisation, others wont. -#if defined(__OPTIMIZE__) && __OPTIMIZE__ && !defined(__clang__) - -SIMD_INLINE v64 v64_shl_n_byte(v64 a, unsigned int c) { - return vshl_n_s64(a, c * 8); -} - -SIMD_INLINE v64 v64_shr_n_byte(v64 a, unsigned int c) { - return c ? (v64)vshr_n_u64(vreinterpret_u64_s64(a), c * 8) : a; -} - -SIMD_INLINE v64 v64_shl_n_8(v64 a, unsigned int c) { - return vreinterpret_s64_u8(vshl_n_u8(vreinterpret_u8_s64(a), c)); -} - -SIMD_INLINE v64 v64_shr_n_u8(v64 a, unsigned int c) { - return vreinterpret_s64_u8(vshr_n_u8(vreinterpret_u8_s64(a), c)); -} - -SIMD_INLINE v64 v64_shr_n_s8(v64 a, unsigned int c) { - return vreinterpret_s64_s8(vshr_n_s8(vreinterpret_s8_s64(a), c)); -} - -SIMD_INLINE v64 v64_shl_n_16(v64 a, unsigned int c) { - return vreinterpret_s64_u16(vshl_n_u16(vreinterpret_u16_s64(a), c)); -} - -SIMD_INLINE v64 v64_shr_n_u16(v64 a, unsigned int c) { - return vreinterpret_s64_u16(vshr_n_u16(vreinterpret_u16_s64(a), c)); -} - -SIMD_INLINE v64 v64_shr_n_s16(v64 a, unsigned int c) { - return vreinterpret_s64_s16(vshr_n_s16(vreinterpret_s16_s64(a), c)); -} - -SIMD_INLINE v64 v64_shl_n_32(v64 a, unsigned int c) { - return vreinterpret_s64_u32(vshl_n_u32(vreinterpret_u32_s64(a), c)); -} - -SIMD_INLINE v64 v64_shr_n_u32(v64 a, unsigned int c) { - return vreinterpret_s64_u32(vshr_n_u32(vreinterpret_u32_s64(a), c)); -} - -SIMD_INLINE v64 v64_shr_n_s32(v64 a, unsigned int c) { - return vreinterpret_s64_s32(vshr_n_s32(vreinterpret_s32_s64(a), c)); -} - -#else - -SIMD_INLINE v64 v64_shl_n_byte(v64 a, unsigned int c) { - return v64_from_64(v64_u64(a) << c * 8); -} - -SIMD_INLINE v64 v64_shr_n_byte(v64 a, unsigned int c) { - return v64_from_64(v64_u64(a) >> c * 8); -} - -SIMD_INLINE v64 v64_shl_n_8(v64 a, unsigned int c) { return v64_shl_8(a, c); } - -SIMD_INLINE v64 v64_shr_n_u8(v64 a, unsigned int c) { return v64_shr_u8(a, c); } - -SIMD_INLINE v64 v64_shr_n_s8(v64 a, unsigned int c) { return v64_shr_s8(a, c); } - -SIMD_INLINE v64 v64_shl_n_16(v64 a, unsigned int c) { return v64_shl_16(a, c); } - -SIMD_INLINE v64 v64_shr_n_u16(v64 a, unsigned int c) { - return v64_shr_u16(a, c); -} - -SIMD_INLINE v64 v64_shr_n_s16(v64 a, unsigned int c) { - return v64_shr_s16(a, c); -} - -SIMD_INLINE v64 v64_shl_n_32(v64 a, unsigned int c) { return v64_shl_32(a, c); } - -SIMD_INLINE v64 v64_shr_n_u32(v64 a, unsigned int c) { - return v64_shr_u32(a, c); -} - -SIMD_INLINE v64 v64_shr_n_s32(v64 a, unsigned int c) { - return v64_shr_s32(a, c); -} - -#endif - -#endif // AOM_AOM_DSP_SIMD_V64_INTRINSICS_ARM_H_ diff --git a/third_party/aom/aom_dsp/simd/v64_intrinsics_c.h b/third_party/aom/aom_dsp/simd/v64_intrinsics_c.h deleted file mode 100644 index 028d68c4f..000000000 --- a/third_party/aom/aom_dsp/simd/v64_intrinsics_c.h +++ /dev/null @@ -1,968 +0,0 @@ -/* - * Copyright (c) 2016, Alliance for Open Media. All rights reserved - * - * This source code is subject to the terms of the BSD 2 Clause License and - * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License - * was not distributed with this source code in the LICENSE file, you can - * obtain it at www.aomedia.org/license/software. If the Alliance for Open - * Media Patent License 1.0 was not distributed with this source code in the - * PATENTS file, you can obtain it at www.aomedia.org/license/patent. - */ - -#ifndef AOM_AOM_DSP_SIMD_V64_INTRINSICS_C_H_ -#define AOM_AOM_DSP_SIMD_V64_INTRINSICS_C_H_ - -/* Note: This implements the intrinsics in plain, unoptimised C. - Intended for reference, porting or debugging. */ - -#include <stdio.h> -#include <stdlib.h> - -#include "config/aom_config.h" - -typedef union { - uint8_t u8[8]; - uint16_t u16[4]; - uint32_t u32[2]; - uint64_t u64; - int8_t s8[8]; - int16_t s16[4]; - int32_t s32[2]; - int64_t s64; -} c_v64; - -SIMD_INLINE uint32_t c_v64_low_u32(c_v64 a) { - return a.u32[!!CONFIG_BIG_ENDIAN]; -} - -SIMD_INLINE uint32_t c_v64_high_u32(c_v64 a) { - return a.u32[!CONFIG_BIG_ENDIAN]; -} - -SIMD_INLINE int32_t c_v64_low_s32(c_v64 a) { - return a.s32[!!CONFIG_BIG_ENDIAN]; -} - -SIMD_INLINE int32_t c_v64_high_s32(c_v64 a) { - return a.s32[!CONFIG_BIG_ENDIAN]; -} - -SIMD_INLINE c_v64 c_v64_from_32(uint32_t x, uint32_t y) { - c_v64 t; - t.u32[!CONFIG_BIG_ENDIAN] = x; - t.u32[!!CONFIG_BIG_ENDIAN] = y; - return t; -} - -SIMD_INLINE c_v64 c_v64_from_64(uint64_t x) { - c_v64 t; - t.u64 = x; - return t; -} - -SIMD_INLINE uint64_t c_v64_u64(c_v64 x) { return x.u64; } - -SIMD_INLINE c_v64 c_v64_from_16(uint16_t a, uint16_t b, uint16_t c, - uint16_t d) { - c_v64 t; - if (CONFIG_BIG_ENDIAN) { - t.u16[0] = a; - t.u16[1] = b; - t.u16[2] = c; - t.u16[3] = d; - } else { - t.u16[3] = a; - t.u16[2] = b; - t.u16[1] = c; - t.u16[0] = d; - } - return t; -} - -SIMD_INLINE uint32_t c_u32_load_unaligned(const void *p) { - uint32_t t; - uint8_t *pp = (uint8_t *)p; - uint8_t *q = (uint8_t *)&t; - int c; - for (c = 0; c < 4; c++) q[c] = pp[c]; - return t; -} - -SIMD_INLINE void c_u32_store_unaligned(void *p, uint32_t a) { - uint8_t *pp = (uint8_t *)p; - uint8_t *q = (uint8_t *)&a; - int c; - for (c = 0; c < 4; c++) pp[c] = q[c]; -} - -SIMD_INLINE uint32_t c_u32_load_aligned(const void *p) { - if (SIMD_CHECK && (uintptr_t)p & 3) { - fprintf(stderr, "Error: Unaligned u32 load at %p\n", p); - abort(); - } - return c_u32_load_unaligned(p); -} - -SIMD_INLINE void c_u32_store_aligned(void *p, uint32_t a) { - if (SIMD_CHECK && (uintptr_t)p & 3) { - fprintf(stderr, "Error: Unaligned u32 store at %p\n", p); - abort(); - } - c_u32_store_unaligned(p, a); -} - -SIMD_INLINE c_v64 c_v64_load_unaligned(const void *p) { - c_v64 t; - uint8_t *pp = (uint8_t *)p; - uint8_t *q = (uint8_t *)&t; - int c; - for (c = 0; c < 8; c++) q[c] = pp[c]; - return t; -} - -SIMD_INLINE c_v64 c_v64_load_aligned(const void *p) { - if (SIMD_CHECK && (uintptr_t)p & 7) { - fprintf(stderr, "Error: Unaligned c_v64 load at %p\n", p); - abort(); - } - return c_v64_load_unaligned(p); -} - -SIMD_INLINE void c_v64_store_unaligned(void *p, c_v64 a) { - uint8_t *q = (uint8_t *)p; - uint8_t *r = (uint8_t *)&a; - int c; - for (c = 0; c < 8; c++) q[c] = r[c]; -} - -SIMD_INLINE void c_v64_store_aligned(void *p, c_v64 a) { - if (SIMD_CHECK && (uintptr_t)p & 7) { - fprintf(stderr, "Error: Unaligned c_v64 store at %p\n", p); - abort(); - } - c_v64_store_unaligned(p, a); -} - -SIMD_INLINE c_v64 c_v64_zero() { - c_v64 t; - t.u64 = 0; - return t; -} - -SIMD_INLINE c_v64 c_v64_dup_8(uint8_t x) { - c_v64 t; - t.u8[0] = t.u8[1] = t.u8[2] = t.u8[3] = t.u8[4] = t.u8[5] = t.u8[6] = - t.u8[7] = x; - return t; -} - -SIMD_INLINE c_v64 c_v64_dup_16(uint16_t x) { - c_v64 t; - t.u16[0] = t.u16[1] = t.u16[2] = t.u16[3] = x; - return t; -} - -SIMD_INLINE c_v64 c_v64_dup_32(uint32_t x) { - c_v64 t; - t.u32[0] = t.u32[1] = x; - return t; -} - -SIMD_INLINE c_v64 c_v64_add_8(c_v64 a, c_v64 b) { - c_v64 t; - int c; - for (c = 0; c < 8; c++) t.u8[c] = a.u8[c] + b.u8[c]; - return t; -} - -SIMD_INLINE c_v64 c_v64_add_16(c_v64 a, c_v64 b) { - c_v64 t; - int c; - for (c = 0; c < 4; c++) t.u16[c] = a.u16[c] + b.u16[c]; - return t; -} - -SIMD_INLINE c_v64 c_v64_sadd_u8(c_v64 a, c_v64 b) { - c_v64 t; - int c; - for (c = 0; c < 8; c++) - t.u8[c] = (int16_t)a.u8[c] + (int16_t)b.u8[c] > 255 - ? 255 - : (int16_t)a.u8[c] + (int16_t)b.u8[c] < 0 - ? 0 - : (int16_t)a.u8[c] + (int16_t)b.u8[c]; - return t; -} - -SIMD_INLINE c_v64 c_v64_sadd_s8(c_v64 a, c_v64 b) { - c_v64 t; - int c; - for (c = 0; c < 8; c++) - t.s8[c] = (int16_t)a.s8[c] + (int16_t)b.s8[c] > 127 - ? 127 - : (int16_t)a.s8[c] + (int16_t)b.s8[c] < -128 - ? -128 - : (int16_t)a.s8[c] + (int16_t)b.s8[c]; - return t; -} - -SIMD_INLINE c_v64 c_v64_sadd_s16(c_v64 a, c_v64 b) { - c_v64 t; - int c; - for (c = 0; c < 4; c++) - t.s16[c] = (int32_t)a.s16[c] + (int32_t)b.s16[c] > 32767 - ? 32767 - : (int32_t)a.s16[c] + (int32_t)b.s16[c] < -32768 - ? -32768 - : (int32_t)a.s16[c] + (int32_t)b.s16[c]; - return t; -} - -SIMD_INLINE c_v64 c_v64_add_32(c_v64 a, c_v64 b) { - c_v64 t; - t.u32[0] = (uint32_t)((uint64_t)a.u32[0] + b.u32[0]); - t.u32[1] = (uint32_t)((uint64_t)a.u32[1] + b.u32[1]); - return t; -} - -SIMD_INLINE c_v64 c_v64_sub_8(c_v64 a, c_v64 b) { - c_v64 t; - int c; - for (c = 0; c < 8; c++) t.u8[c] = a.u8[c] - b.u8[c]; - return t; -} - -SIMD_INLINE c_v64 c_v64_ssub_u8(c_v64 a, c_v64 b) { - c_v64 t; - int c; - for (c = 0; c < 8; c++) t.u8[c] = a.u8[c] < b.u8[c] ? 0 : a.u8[c] - b.u8[c]; - return t; -} - -SIMD_INLINE c_v64 c_v64_ssub_s8(c_v64 a, c_v64 b) { - c_v64 t; - int c; - for (c = 0; c < 8; c++) { - int16_t d = (int16_t)a.s8[c] - (int16_t)b.s8[c]; - t.s8[c] = d > 127 ? 127 : (d < -128 ? -128 : d); - } - return t; -} - -SIMD_INLINE c_v64 c_v64_sub_16(c_v64 a, c_v64 b) { - c_v64 t; - int c; - for (c = 0; c < 4; c++) t.u16[c] = a.u16[c] - b.u16[c]; - return t; -} - -SIMD_INLINE c_v64 c_v64_ssub_s16(c_v64 a, c_v64 b) { - c_v64 t; - int c; - for (c = 0; c < 4; c++) - t.s16[c] = (int32_t)a.s16[c] - (int32_t)b.s16[c] < -32768 - ? -32768 - : (int32_t)a.s16[c] - (int32_t)b.s16[c] > 32767 - ? 32767 - : (int32_t)a.s16[c] - (int32_t)b.s16[c]; - return t; -} - -SIMD_INLINE c_v64 c_v64_ssub_u16(c_v64 a, c_v64 b) { - c_v64 t; - int c; - for (c = 0; c < 4; c++) - t.u16[c] = - (int32_t)a.u16[c] - (int32_t)b.u16[c] < 0 ? 0 : a.u16[c] - b.u16[c]; - return t; -} - -SIMD_INLINE c_v64 c_v64_sub_32(c_v64 a, c_v64 b) { - c_v64 t; - t.u32[0] = (uint32_t)((int64_t)a.u32[0] - b.u32[0]); - t.u32[1] = (uint32_t)((int64_t)a.u32[1] - b.u32[1]); - return t; -} - -SIMD_INLINE c_v64 c_v64_abs_s16(c_v64 a) { - c_v64 t; - int c; - for (c = 0; c < 4; c++) - t.u16[c] = (int16_t)a.u16[c] > 0 ? a.u16[c] : -a.u16[c]; - return t; -} - -SIMD_INLINE c_v64 c_v64_abs_s8(c_v64 a) { - c_v64 t; - int c; - for (c = 0; c < 8; c++) t.u8[c] = (int8_t)a.u8[c] > 0 ? a.u8[c] : -a.u8[c]; - return t; -} - -SIMD_INLINE c_v64 _c_v64_zip_8(c_v64 a, c_v64 b, int mode) { - c_v64 t; - if (mode) { - t.u8[7] = a.u8[7]; - t.u8[6] = b.u8[7]; - t.u8[5] = a.u8[6]; - t.u8[4] = b.u8[6]; - t.u8[3] = a.u8[5]; - t.u8[2] = b.u8[5]; - t.u8[1] = a.u8[4]; - t.u8[0] = b.u8[4]; - } else { - t.u8[7] = a.u8[3]; - t.u8[6] = b.u8[3]; - t.u8[5] = a.u8[2]; - t.u8[4] = b.u8[2]; - t.u8[3] = a.u8[1]; - t.u8[2] = b.u8[1]; - t.u8[1] = a.u8[0]; - t.u8[0] = b.u8[0]; - } - return t; -} - -SIMD_INLINE c_v64 c_v64_ziplo_8(c_v64 a, c_v64 b) { - return CONFIG_BIG_ENDIAN ? _c_v64_zip_8(b, a, 1) : _c_v64_zip_8(a, b, 0); -} - -SIMD_INLINE c_v64 c_v64_ziphi_8(c_v64 a, c_v64 b) { - return CONFIG_BIG_ENDIAN ? _c_v64_zip_8(b, a, 0) : _c_v64_zip_8(a, b, 1); -} - -SIMD_INLINE c_v64 _c_v64_zip_16(c_v64 a, c_v64 b, int mode) { - c_v64 t; - if (mode) { - t.u16[3] = a.u16[3]; - t.u16[2] = b.u16[3]; - t.u16[1] = a.u16[2]; - t.u16[0] = b.u16[2]; - } else { - t.u16[3] = a.u16[1]; - t.u16[2] = b.u16[1]; - t.u16[1] = a.u16[0]; - t.u16[0] = b.u16[0]; - } - return t; -} - -SIMD_INLINE c_v64 c_v64_ziplo_16(c_v64 a, c_v64 b) { - return CONFIG_BIG_ENDIAN ? _c_v64_zip_16(b, a, 1) : _c_v64_zip_16(a, b, 0); -} - -SIMD_INLINE c_v64 c_v64_ziphi_16(c_v64 a, c_v64 b) { - return CONFIG_BIG_ENDIAN ? _c_v64_zip_16(b, a, 0) : _c_v64_zip_16(a, b, 1); -} - -SIMD_INLINE c_v64 _c_v64_zip_32(c_v64 a, c_v64 b, int mode) { - c_v64 t; - if (mode) { - t.u32[1] = a.u32[1]; - t.u32[0] = b.u32[1]; - } else { - t.u32[1] = a.u32[0]; - t.u32[0] = b.u32[0]; - } - return t; -} - -SIMD_INLINE c_v64 c_v64_ziplo_32(c_v64 a, c_v64 b) { - return CONFIG_BIG_ENDIAN ? _c_v64_zip_32(b, a, 1) : _c_v64_zip_32(a, b, 0); -} - -SIMD_INLINE c_v64 c_v64_ziphi_32(c_v64 a, c_v64 b) { - return CONFIG_BIG_ENDIAN ? _c_v64_zip_32(b, a, 0) : _c_v64_zip_32(a, b, 1); -} - -SIMD_INLINE c_v64 _c_v64_unzip_8(c_v64 a, c_v64 b, int mode) { - c_v64 t; - if (mode) { - t.u8[7] = b.u8[7]; - t.u8[6] = b.u8[5]; - t.u8[5] = b.u8[3]; - t.u8[4] = b.u8[1]; - t.u8[3] = a.u8[7]; - t.u8[2] = a.u8[5]; - t.u8[1] = a.u8[3]; - t.u8[0] = a.u8[1]; - } else { - t.u8[7] = a.u8[6]; - t.u8[6] = a.u8[4]; - t.u8[5] = a.u8[2]; - t.u8[4] = a.u8[0]; - t.u8[3] = b.u8[6]; - t.u8[2] = b.u8[4]; - t.u8[1] = b.u8[2]; - t.u8[0] = b.u8[0]; - } - return t; -} - -SIMD_INLINE c_v64 c_v64_unziplo_8(c_v64 a, c_v64 b) { - return CONFIG_BIG_ENDIAN ? _c_v64_unzip_8(a, b, 1) : _c_v64_unzip_8(a, b, 0); -} - -SIMD_INLINE c_v64 c_v64_unziphi_8(c_v64 a, c_v64 b) { - return CONFIG_BIG_ENDIAN ? _c_v64_unzip_8(b, a, 0) : _c_v64_unzip_8(b, a, 1); -} - -SIMD_INLINE c_v64 _c_v64_unzip_16(c_v64 a, c_v64 b, int mode) { - c_v64 t; - if (mode) { - t.u16[3] = b.u16[3]; - t.u16[2] = b.u16[1]; - t.u16[1] = a.u16[3]; - t.u16[0] = a.u16[1]; - } else { - t.u16[3] = a.u16[2]; - t.u16[2] = a.u16[0]; - t.u16[1] = b.u16[2]; - t.u16[0] = b.u16[0]; - } - return t; -} - -SIMD_INLINE c_v64 c_v64_unziplo_16(c_v64 a, c_v64 b) { - return CONFIG_BIG_ENDIAN ? _c_v64_unzip_16(a, b, 1) - : _c_v64_unzip_16(a, b, 0); -} - -SIMD_INLINE c_v64 c_v64_unziphi_16(c_v64 a, c_v64 b) { - return CONFIG_BIG_ENDIAN ? _c_v64_unzip_16(b, a, 0) - : _c_v64_unzip_16(b, a, 1); -} - -SIMD_INLINE c_v64 c_v64_unpacklo_u8_s16(c_v64 a) { - c_v64 t; - int endian = !!CONFIG_BIG_ENDIAN * 4; - t.s16[3] = (int16_t)a.u8[3 + endian]; - t.s16[2] = (int16_t)a.u8[2 + endian]; - t.s16[1] = (int16_t)a.u8[1 + endian]; - t.s16[0] = (int16_t)a.u8[0 + endian]; - return t; -} - -SIMD_INLINE c_v64 c_v64_unpackhi_u8_s16(c_v64 a) { - c_v64 t; - int endian = !!CONFIG_BIG_ENDIAN * 4; - t.s16[3] = (int16_t)a.u8[7 - endian]; - t.s16[2] = (int16_t)a.u8[6 - endian]; - t.s16[1] = (int16_t)a.u8[5 - endian]; - t.s16[0] = (int16_t)a.u8[4 - endian]; - return t; -} - -SIMD_INLINE c_v64 c_v64_unpacklo_s8_s16(c_v64 a) { - c_v64 t; - int endian = !!CONFIG_BIG_ENDIAN * 4; - t.s16[3] = (int16_t)a.s8[3 + endian]; - t.s16[2] = (int16_t)a.s8[2 + endian]; - t.s16[1] = (int16_t)a.s8[1 + endian]; - t.s16[0] = (int16_t)a.s8[0 + endian]; - return t; -} - -SIMD_INLINE c_v64 c_v64_unpackhi_s8_s16(c_v64 a) { - c_v64 t; - int endian = !!CONFIG_BIG_ENDIAN * 4; - t.s16[3] = (int16_t)a.s8[7 - endian]; - t.s16[2] = (int16_t)a.s8[6 - endian]; - t.s16[1] = (int16_t)a.s8[5 - endian]; - t.s16[0] = (int16_t)a.s8[4 - endian]; - return t; -} - -SIMD_INLINE c_v64 c_v64_pack_s32_s16(c_v64 a, c_v64 b) { - c_v64 t; - if (CONFIG_BIG_ENDIAN) { - c_v64 u = a; - a = b; - b = u; - } - t.s16[3] = a.s32[1] > 32767 ? 32767 : a.s32[1] < -32768 ? -32768 : a.s32[1]; - t.s16[2] = a.s32[0] > 32767 ? 32767 : a.s32[0] < -32768 ? -32768 : a.s32[0]; - t.s16[1] = b.s32[1] > 32767 ? 32767 : b.s32[1] < -32768 ? -32768 : b.s32[1]; - t.s16[0] = b.s32[0] > 32767 ? 32767 : b.s32[0] < -32768 ? -32768 : b.s32[0]; - return t; -} - -SIMD_INLINE c_v64 c_v64_pack_s32_u16(c_v64 a, c_v64 b) { - c_v64 t; - if (CONFIG_BIG_ENDIAN) { - c_v64 u = a; - a = b; - b = u; - } - t.u16[3] = a.s32[1] > 65535 ? 65535 : a.s32[1] < 0 ? 0 : a.s32[1]; - t.u16[2] = a.s32[0] > 65535 ? 65535 : a.s32[0] < 0 ? 0 : a.s32[0]; - t.u16[1] = b.s32[1] > 65535 ? 65535 : b.s32[1] < 0 ? 0 : b.s32[1]; - t.u16[0] = b.s32[0] > 65535 ? 65535 : b.s32[0] < 0 ? 0 : b.s32[0]; - return t; -} - -SIMD_INLINE c_v64 c_v64_pack_s16_u8(c_v64 a, c_v64 b) { - c_v64 t; - if (CONFIG_BIG_ENDIAN) { - c_v64 u = a; - a = b; - b = u; - } - t.u8[7] = a.s16[3] > 255 ? 255 : a.s16[3] < 0 ? 0 : a.s16[3]; - t.u8[6] = a.s16[2] > 255 ? 255 : a.s16[2] < 0 ? 0 : a.s16[2]; - t.u8[5] = a.s16[1] > 255 ? 255 : a.s16[1] < 0 ? 0 : a.s16[1]; - t.u8[4] = a.s16[0] > 255 ? 255 : a.s16[0] < 0 ? 0 : a.s16[0]; - t.u8[3] = b.s16[3] > 255 ? 255 : b.s16[3] < 0 ? 0 : b.s16[3]; - t.u8[2] = b.s16[2] > 255 ? 255 : b.s16[2] < 0 ? 0 : b.s16[2]; - t.u8[1] = b.s16[1] > 255 ? 255 : b.s16[1] < 0 ? 0 : b.s16[1]; - t.u8[0] = b.s16[0] > 255 ? 255 : b.s16[0] < 0 ? 0 : b.s16[0]; - return t; -} - -SIMD_INLINE c_v64 c_v64_pack_s16_s8(c_v64 a, c_v64 b) { - c_v64 t; - if (CONFIG_BIG_ENDIAN) { - c_v64 u = a; - a = b; - b = u; - } - t.u8[7] = a.s16[3] > 127 ? 127 : a.s16[3] < -128 ? 128 : a.s16[3]; - t.u8[6] = a.s16[2] > 127 ? 127 : a.s16[2] < -128 ? 128 : a.s16[2]; - t.u8[5] = a.s16[1] > 127 ? 127 : a.s16[1] < -128 ? 128 : a.s16[1]; - t.u8[4] = a.s16[0] > 127 ? 127 : a.s16[0] < -128 ? 128 : a.s16[0]; - t.u8[3] = b.s16[3] > 127 ? 127 : b.s16[3] < -128 ? 128 : b.s16[3]; - t.u8[2] = b.s16[2] > 127 ? 127 : b.s16[2] < -128 ? 128 : b.s16[2]; - t.u8[1] = b.s16[1] > 127 ? 127 : b.s16[1] < -128 ? 128 : b.s16[1]; - t.u8[0] = b.s16[0] > 127 ? 127 : b.s16[0] < -128 ? 128 : b.s16[0]; - return t; -} - -SIMD_INLINE c_v64 c_v64_unpacklo_u16_s32(c_v64 a) { - c_v64 t; - t.s32[1] = a.u16[1 + !!CONFIG_BIG_ENDIAN * 2]; - t.s32[0] = a.u16[0 + !!CONFIG_BIG_ENDIAN * 2]; - return t; -} - -SIMD_INLINE c_v64 c_v64_unpacklo_s16_s32(c_v64 a) { - c_v64 t; - t.s32[1] = a.s16[1 + !!CONFIG_BIG_ENDIAN * 2]; - t.s32[0] = a.s16[0 + !!CONFIG_BIG_ENDIAN * 2]; - return t; -} - -SIMD_INLINE c_v64 c_v64_unpackhi_u16_s32(c_v64 a) { - c_v64 t; - t.s32[1] = a.u16[3 - !!CONFIG_BIG_ENDIAN * 2]; - t.s32[0] = a.u16[2 - !!CONFIG_BIG_ENDIAN * 2]; - return t; -} - -SIMD_INLINE c_v64 c_v64_unpackhi_s16_s32(c_v64 a) { - c_v64 t; - t.s32[1] = a.s16[3 - !!CONFIG_BIG_ENDIAN * 2]; - t.s32[0] = a.s16[2 - !!CONFIG_BIG_ENDIAN * 2]; - return t; -} - -SIMD_INLINE c_v64 c_v64_shuffle_8(c_v64 a, c_v64 pattern) { - c_v64 t; - int c; - for (c = 0; c < 8; c++) { - if (SIMD_CHECK && (pattern.u8[c] & ~7)) { - fprintf(stderr, "Error: Undefined v64_shuffle_8 index %d/%d\n", - pattern.u8[c], c); - abort(); - } - t.u8[c] = - a.u8[CONFIG_BIG_ENDIAN ? 7 - (pattern.u8[c] & 7) : pattern.u8[c] & 7]; - } - return t; -} - -SIMD_INLINE int64_t c_v64_dotp_su8(c_v64 a, c_v64 b) { - return a.s8[7] * b.u8[7] + a.s8[6] * b.u8[6] + a.s8[5] * b.u8[5] + - a.s8[4] * b.u8[4] + a.s8[3] * b.u8[3] + a.s8[2] * b.u8[2] + - a.s8[1] * b.u8[1] + a.s8[0] * b.u8[0]; -} - -SIMD_INLINE int64_t c_v64_dotp_s16(c_v64 a, c_v64 b) { - return (int64_t)(a.s16[3] * b.s16[3] + a.s16[2] * b.s16[2]) + - (int64_t)(a.s16[1] * b.s16[1] + a.s16[0] * b.s16[0]); -} - -SIMD_INLINE uint64_t c_v64_hadd_u8(c_v64 a) { - return a.u8[7] + a.u8[6] + a.u8[5] + a.u8[4] + a.u8[3] + a.u8[2] + a.u8[1] + - a.u8[0]; -} - -SIMD_INLINE int64_t c_v64_hadd_s16(c_v64 a) { - return a.s16[3] + a.s16[2] + a.s16[1] + a.s16[0]; -} - -typedef uint32_t c_sad64_internal; - -/* Implementation dependent return value. Result must be finalised with - v64_sad_u8_sum(). - The result for more than 32 v64_sad_u8() calls is undefined. */ -SIMD_INLINE c_sad64_internal c_v64_sad_u8_init() { return 0; } - -SIMD_INLINE c_sad64_internal c_v64_sad_u8(c_sad64_internal s, c_v64 a, - c_v64 b) { - int c; - for (c = 0; c < 8; c++) - s += a.u8[c] > b.u8[c] ? a.u8[c] - b.u8[c] : b.u8[c] - a.u8[c]; - return s; -} - -SIMD_INLINE uint32_t c_v64_sad_u8_sum(c_sad64_internal s) { return s; } - -typedef uint32_t c_ssd64_internal; - -/* Implementation dependent return value. Result must be finalised with - * v64_ssd_u8_sum(). */ -SIMD_INLINE c_ssd64_internal c_v64_ssd_u8_init() { return 0; } - -SIMD_INLINE c_ssd64_internal c_v64_ssd_u8(c_ssd64_internal s, c_v64 a, - c_v64 b) { - int c; - for (c = 0; c < 8; c++) s += (a.u8[c] - b.u8[c]) * (a.u8[c] - b.u8[c]); - return s; -} - -SIMD_INLINE uint32_t c_v64_ssd_u8_sum(c_ssd64_internal s) { return s; } - -SIMD_INLINE c_v64 c_v64_or(c_v64 a, c_v64 b) { - c_v64 t; - t.u64 = a.u64 | b.u64; - return t; -} - -SIMD_INLINE c_v64 c_v64_xor(c_v64 a, c_v64 b) { - c_v64 t; - t.u64 = a.u64 ^ b.u64; - return t; -} - -SIMD_INLINE c_v64 c_v64_and(c_v64 a, c_v64 b) { - c_v64 t; - t.u64 = a.u64 & b.u64; - return t; -} - -SIMD_INLINE c_v64 c_v64_andn(c_v64 a, c_v64 b) { - c_v64 t; - t.u64 = a.u64 & ~b.u64; - return t; -} - -SIMD_INLINE c_v64 c_v64_mullo_s16(c_v64 a, c_v64 b) { - c_v64 t; - int c; - for (c = 0; c < 4; c++) t.s16[c] = (int16_t)(a.s16[c] * b.s16[c]); - return t; -} - -SIMD_INLINE c_v64 c_v64_mulhi_s16(c_v64 a, c_v64 b) { - c_v64 t; - int c; - for (c = 0; c < 4; c++) t.s16[c] = (a.s16[c] * b.s16[c]) >> 16; - return t; -} - -SIMD_INLINE c_v64 c_v64_mullo_s32(c_v64 a, c_v64 b) { - c_v64 t; - t.s32[0] = (int32_t)((int64_t)a.s32[0] * b.s32[0]); - t.s32[1] = (int32_t)((int64_t)a.s32[1] * b.s32[1]); - return t; -} - -SIMD_INLINE c_v64 c_v64_madd_s16(c_v64 a, c_v64 b) { - c_v64 t; - t.s32[0] = a.s16[0] * b.s16[0] + a.s16[1] * b.s16[1]; - t.s32[1] = a.s16[2] * b.s16[2] + a.s16[3] * b.s16[3]; - return t; -} - -SIMD_INLINE c_v64 c_v64_madd_us8(c_v64 a, c_v64 b) { - c_v64 t; - int32_t u; - u = a.u8[0] * b.s8[0] + a.u8[1] * b.s8[1]; - t.s16[0] = u > 32767 ? 32767 : u < -32768 ? -32768 : u; - u = a.u8[2] * b.s8[2] + a.u8[3] * b.s8[3]; - t.s16[1] = u > 32767 ? 32767 : u < -32768 ? -32768 : u; - u = a.u8[4] * b.s8[4] + a.u8[5] * b.s8[5]; - t.s16[2] = u > 32767 ? 32767 : u < -32768 ? -32768 : u; - u = a.u8[6] * b.s8[6] + a.u8[7] * b.s8[7]; - t.s16[3] = u > 32767 ? 32767 : u < -32768 ? -32768 : u; - return t; -} - -SIMD_INLINE c_v64 c_v64_avg_u8(c_v64 a, c_v64 b) { - c_v64 t; - int c; - for (c = 0; c < 8; c++) t.u8[c] = (a.u8[c] + b.u8[c] + 1) >> 1; - return t; -} - -SIMD_INLINE c_v64 c_v64_rdavg_u8(c_v64 a, c_v64 b) { - c_v64 t; - int c; - for (c = 0; c < 8; c++) t.u8[c] = (a.u8[c] + b.u8[c]) >> 1; - return t; -} - -SIMD_INLINE c_v64 c_v64_rdavg_u16(c_v64 a, c_v64 b) { - c_v64 t; - int c; - for (c = 0; c < 4; c++) t.u16[c] = (a.u16[c] + b.u16[c]) >> 1; - return t; -} - -SIMD_INLINE c_v64 c_v64_avg_u16(c_v64 a, c_v64 b) { - c_v64 t; - int c; - for (c = 0; c < 4; c++) t.u16[c] = (a.u16[c] + b.u16[c] + 1) >> 1; - return t; -} - -SIMD_INLINE c_v64 c_v64_min_u8(c_v64 a, c_v64 b) { - c_v64 t; - int c; - for (c = 0; c < 8; c++) t.u8[c] = a.u8[c] > b.u8[c] ? b.u8[c] : a.u8[c]; - return t; -} - -SIMD_INLINE c_v64 c_v64_max_u8(c_v64 a, c_v64 b) { - c_v64 t; - int c; - for (c = 0; c < 8; c++) t.u8[c] = a.u8[c] > b.u8[c] ? a.u8[c] : b.u8[c]; - return t; -} - -SIMD_INLINE c_v64 c_v64_min_s8(c_v64 a, c_v64 b) { - c_v64 t; - int c; - for (c = 0; c < 8; c++) t.s8[c] = a.s8[c] > b.s8[c] ? b.s8[c] : a.s8[c]; - return t; -} - -SIMD_INLINE c_v64 c_v64_max_s8(c_v64 a, c_v64 b) { - c_v64 t; - int c; - for (c = 0; c < 8; c++) t.s8[c] = a.s8[c] > b.s8[c] ? a.s8[c] : b.s8[c]; - return t; -} - -SIMD_INLINE c_v64 c_v64_min_s16(c_v64 a, c_v64 b) { - c_v64 t; - int c; - for (c = 0; c < 4; c++) t.s16[c] = a.s16[c] > b.s16[c] ? b.s16[c] : a.s16[c]; - return t; -} - -SIMD_INLINE c_v64 c_v64_max_s16(c_v64 a, c_v64 b) { - c_v64 t; - int c; - for (c = 0; c < 4; c++) t.s16[c] = a.s16[c] > b.s16[c] ? a.s16[c] : b.s16[c]; - return t; -} - -SIMD_INLINE c_v64 c_v64_cmpgt_s8(c_v64 a, c_v64 b) { - c_v64 t; - int c; - for (c = 0; c < 8; c++) t.s8[c] = -(a.s8[c] > b.s8[c]); - return t; -} - -SIMD_INLINE c_v64 c_v64_cmplt_s8(c_v64 a, c_v64 b) { - c_v64 t; - int c; - for (c = 0; c < 8; c++) t.s8[c] = -(a.s8[c] < b.s8[c]); - return t; -} - -SIMD_INLINE c_v64 c_v64_cmpeq_8(c_v64 a, c_v64 b) { - c_v64 t; - int c; - for (c = 0; c < 8; c++) t.s8[c] = -(a.u8[c] == b.u8[c]); - return t; -} - -SIMD_INLINE c_v64 c_v64_cmpgt_s16(c_v64 a, c_v64 b) { - c_v64 t; - int c; - for (c = 0; c < 4; c++) t.s16[c] = -(a.s16[c] > b.s16[c]); - return t; -} - -SIMD_INLINE c_v64 c_v64_cmplt_s16(c_v64 a, c_v64 b) { - c_v64 t; - int c; - for (c = 0; c < 4; c++) t.s16[c] = -(a.s16[c] < b.s16[c]); - return t; -} - -SIMD_INLINE c_v64 c_v64_cmpeq_16(c_v64 a, c_v64 b) { - c_v64 t; - int c; - for (c = 0; c < 4; c++) t.s16[c] = -(a.u16[c] == b.u16[c]); - return t; -} - -SIMD_INLINE c_v64 c_v64_shl_8(c_v64 a, unsigned int n) { - c_v64 t; - int c; - if (SIMD_CHECK && n > 7) { - fprintf(stderr, "Error: Undefined u8 shift left %d\n", n); - abort(); - } - for (c = 0; c < 8; c++) t.s8[c] = a.u8[c] << n; - return t; -} - -SIMD_INLINE c_v64 c_v64_shr_u8(c_v64 a, unsigned int n) { - c_v64 t; - int c; - if (SIMD_CHECK && n > 7) { - fprintf(stderr, "Error: Undefined u8 shift right %d\n", n); - abort(); - } - for (c = 0; c < 8; c++) t.u8[c] = a.u8[c] >> n; - return t; -} - -SIMD_INLINE c_v64 c_v64_shr_s8(c_v64 a, unsigned int n) { - c_v64 t; - int c; - if (SIMD_CHECK && n > 7) { - fprintf(stderr, "Error: Undefined s8 shift right %d\n", n); - abort(); - } - for (c = 0; c < 8; c++) t.s8[c] = a.s8[c] >> n; - return t; -} - -SIMD_INLINE c_v64 c_v64_shl_16(c_v64 a, unsigned int n) { - c_v64 t; - int c; - if (SIMD_CHECK && n > 15) { - fprintf(stderr, "Error: Undefined u16 shift left %d\n", n); - abort(); - } - for (c = 0; c < 4; c++) t.u16[c] = a.u16[c] << n; - return t; -} - -SIMD_INLINE c_v64 c_v64_shr_u16(c_v64 a, unsigned int n) { - c_v64 t; - int c; - if (SIMD_CHECK && n > 15) { - fprintf(stderr, "Error: Undefined u16 shift right %d\n", n); - abort(); - } - for (c = 0; c < 4; c++) t.u16[c] = a.u16[c] >> n; - return t; -} - -SIMD_INLINE c_v64 c_v64_shr_s16(c_v64 a, unsigned int n) { - c_v64 t; - int c; - if (SIMD_CHECK && n > 15) { - fprintf(stderr, "Error: undefined s16 shift right %d\n", n); - abort(); - } - for (c = 0; c < 4; c++) t.s16[c] = a.s16[c] >> n; - return t; -} - -SIMD_INLINE c_v64 c_v64_shl_32(c_v64 a, unsigned int n) { - c_v64 t; - if (SIMD_CHECK && n > 31) { - fprintf(stderr, "Error: undefined u32 shift left %d\n", n); - abort(); - } - t.u32[1] = a.u32[1] << n; - t.u32[0] = a.u32[0] << n; - return t; -} - -SIMD_INLINE c_v64 c_v64_shr_u32(c_v64 a, unsigned int n) { - c_v64 t; - if (SIMD_CHECK && n > 31) { - fprintf(stderr, "Error: undefined u32 shift right %d\n", n); - abort(); - } - t.u32[1] = a.u32[1] >> n; - t.u32[0] = a.u32[0] >> n; - return t; -} - -SIMD_INLINE c_v64 c_v64_shr_s32(c_v64 a, unsigned int n) { - c_v64 t; - if (SIMD_CHECK && n > 31) { - fprintf(stderr, "Error: undefined s32 shift right %d\n", n); - abort(); - } - t.s32[1] = a.s32[1] >> n; - t.s32[0] = a.s32[0] >> n; - return t; -} - -SIMD_INLINE c_v64 c_v64_shr_n_byte(c_v64 x, unsigned int i) { - c_v64 t; - t.u64 = x.u64 >> i * 8; - return t; -} - -SIMD_INLINE c_v64 c_v64_shl_n_byte(c_v64 x, unsigned int i) { - c_v64 t; - t.u64 = x.u64 << i * 8; - return t; -} - -SIMD_INLINE c_v64 c_v64_align(c_v64 a, c_v64 b, unsigned int c) { - if (SIMD_CHECK && c > 7) { - fprintf(stderr, "Error: undefined alignment %d\n", c); - abort(); - } - return c ? c_v64_or(c_v64_shr_n_byte(b, c), c_v64_shl_n_byte(a, 8 - c)) : b; -} - -SIMD_INLINE c_v64 c_v64_shl_n_8(c_v64 a, unsigned int c) { - return c_v64_shl_8(a, c); -} - -SIMD_INLINE c_v64 c_v64_shr_n_u8(c_v64 a, unsigned int c) { - return c_v64_shr_u8(a, c); -} - -SIMD_INLINE c_v64 c_v64_shr_n_s8(c_v64 a, unsigned int c) { - return c_v64_shr_s8(a, c); -} - -SIMD_INLINE c_v64 c_v64_shl_n_16(c_v64 a, unsigned int c) { - return c_v64_shl_16(a, c); -} - -SIMD_INLINE c_v64 c_v64_shr_n_u16(c_v64 a, unsigned int c) { - return c_v64_shr_u16(a, c); -} - -SIMD_INLINE c_v64 c_v64_shr_n_s16(c_v64 a, unsigned int c) { - return c_v64_shr_s16(a, c); -} - -SIMD_INLINE c_v64 c_v64_shl_n_32(c_v64 a, unsigned int c) { - return c_v64_shl_32(a, c); -} - -SIMD_INLINE c_v64 c_v64_shr_n_u32(c_v64 a, unsigned int c) { - return c_v64_shr_u32(a, c); -} - -SIMD_INLINE c_v64 c_v64_shr_n_s32(c_v64 a, unsigned int c) { - return c_v64_shr_s32(a, c); -} - -#endif // AOM_AOM_DSP_SIMD_V64_INTRINSICS_C_H_ diff --git a/third_party/aom/aom_dsp/simd/v64_intrinsics_x86.h b/third_party/aom/aom_dsp/simd/v64_intrinsics_x86.h deleted file mode 100644 index 5f9a57b37..000000000 --- a/third_party/aom/aom_dsp/simd/v64_intrinsics_x86.h +++ /dev/null @@ -1,491 +0,0 @@ -/* - * Copyright (c) 2016, Alliance for Open Media. All rights reserved - * - * This source code is subject to the terms of the BSD 2 Clause License and - * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License - * was not distributed with this source code in the LICENSE file, you can - * obtain it at www.aomedia.org/license/software. If the Alliance for Open - * Media Patent License 1.0 was not distributed with this source code in the - * PATENTS file, you can obtain it at www.aomedia.org/license/patent. - */ - -#ifndef AOM_AOM_DSP_SIMD_V64_INTRINSICS_X86_H_ -#define AOM_AOM_DSP_SIMD_V64_INTRINSICS_X86_H_ - -#include <emmintrin.h> -#if defined(__SSSE3__) -#include <tmmintrin.h> -#endif -#if defined(__SSE4_1__) -#include <smmintrin.h> -#endif - -typedef __m128i v64; - -SIMD_INLINE uint32_t v64_low_u32(v64 a) { - return (uint32_t)_mm_cvtsi128_si32(a); -} - -SIMD_INLINE uint32_t v64_high_u32(v64 a) { - return (uint32_t)_mm_cvtsi128_si32(_mm_srli_si128(a, 4)); -} - -SIMD_INLINE int32_t v64_low_s32(v64 a) { return (int32_t)_mm_cvtsi128_si32(a); } - -SIMD_INLINE int32_t v64_high_s32(v64 a) { - return (int32_t)_mm_cvtsi128_si32(_mm_srli_si128(a, 4)); -} - -SIMD_INLINE v64 v64_from_16(uint16_t a, uint16_t b, uint16_t c, uint16_t d) { - return _mm_packs_epi32( - _mm_set_epi32((int16_t)a, (int16_t)b, (int16_t)c, (int16_t)d), - _mm_setzero_si128()); -} - -SIMD_INLINE v64 v64_from_32(uint32_t x, uint32_t y) { - return _mm_set_epi32(0, 0, x, y); -} - -SIMD_INLINE v64 v64_from_64(uint64_t x) { -#ifdef __x86_64__ - return _mm_cvtsi64_si128(x); -#else - return _mm_set_epi32(0, 0, x >> 32, (uint32_t)x); -#endif -} - -SIMD_INLINE uint64_t v64_u64(v64 x) { - return (uint64_t)v64_low_u32(x) | ((uint64_t)v64_high_u32(x) << 32); -} - -SIMD_INLINE uint32_t u32_load_aligned(const void *p) { - return *((uint32_t *)p); -} - -SIMD_INLINE uint32_t u32_load_unaligned(const void *p) { - return *((uint32_t *)p); -} - -SIMD_INLINE void u32_store_aligned(void *p, uint32_t a) { - *((uint32_t *)p) = a; -} - -SIMD_INLINE void u32_store_unaligned(void *p, uint32_t a) { - *((uint32_t *)p) = a; -} - -SIMD_INLINE v64 v64_load_aligned(const void *p) { - return _mm_loadl_epi64((__m128i *)p); -} - -SIMD_INLINE v64 v64_load_unaligned(const void *p) { - return _mm_loadl_epi64((__m128i *)p); -} - -SIMD_INLINE void v64_store_aligned(void *p, v64 a) { - _mm_storel_epi64((__m128i *)p, a); -} - -SIMD_INLINE void v64_store_unaligned(void *p, v64 a) { - _mm_storel_epi64((__m128i *)p, a); -} - -#if defined(__OPTIMIZE__) && __OPTIMIZE__ && !defined(__clang__) -#define v64_align(a, b, c) \ - ((c) ? _mm_srli_si128(_mm_unpacklo_epi64(b, a), (c)) : b) -#else -#define v64_align(a, b, c) \ - ((c) ? v64_from_64((v64_u64(b) >> (c)*8) | (v64_u64(a) << (8 - (c)) * 8)) \ - : (b)) -#endif - -SIMD_INLINE v64 v64_zero() { return _mm_setzero_si128(); } - -SIMD_INLINE v64 v64_dup_8(uint8_t x) { return _mm_set1_epi8(x); } - -SIMD_INLINE v64 v64_dup_16(uint16_t x) { return _mm_set1_epi16(x); } - -SIMD_INLINE v64 v64_dup_32(uint32_t x) { return _mm_set1_epi32(x); } - -SIMD_INLINE v64 v64_add_8(v64 a, v64 b) { return _mm_add_epi8(a, b); } - -SIMD_INLINE v64 v64_add_16(v64 a, v64 b) { return _mm_add_epi16(a, b); } - -SIMD_INLINE v64 v64_sadd_u8(v64 a, v64 b) { return _mm_adds_epu8(a, b); } - -SIMD_INLINE v64 v64_sadd_s8(v64 a, v64 b) { return _mm_adds_epi8(a, b); } - -SIMD_INLINE v64 v64_sadd_s16(v64 a, v64 b) { return _mm_adds_epi16(a, b); } - -SIMD_INLINE v64 v64_add_32(v64 a, v64 b) { return _mm_add_epi32(a, b); } - -SIMD_INLINE v64 v64_sub_8(v64 a, v64 b) { return _mm_sub_epi8(a, b); } - -SIMD_INLINE v64 v64_ssub_u8(v64 a, v64 b) { return _mm_subs_epu8(a, b); } - -SIMD_INLINE v64 v64_ssub_s8(v64 a, v64 b) { return _mm_subs_epi8(a, b); } - -SIMD_INLINE v64 v64_sub_16(v64 a, v64 b) { return _mm_sub_epi16(a, b); } - -SIMD_INLINE v64 v64_ssub_s16(v64 a, v64 b) { return _mm_subs_epi16(a, b); } - -SIMD_INLINE v64 v64_ssub_u16(v64 a, v64 b) { return _mm_subs_epu16(a, b); } - -SIMD_INLINE v64 v64_sub_32(v64 a, v64 b) { return _mm_sub_epi32(a, b); } - -SIMD_INLINE v64 v64_abs_s16(v64 a) { -#if defined(__SSSE3__) - return _mm_abs_epi16(a); -#else - return _mm_max_epi16(a, _mm_sub_epi16(_mm_setzero_si128(), a)); -#endif -} - -SIMD_INLINE v64 v64_abs_s8(v64 a) { -#if defined(__SSSE3__) - return _mm_abs_epi8(a); -#else - v64 sign = _mm_cmplt_epi8(a, _mm_setzero_si128()); - return _mm_xor_si128(sign, _mm_add_epi8(a, sign)); -#endif -} - -SIMD_INLINE v64 v64_ziplo_8(v64 a, v64 b) { return _mm_unpacklo_epi8(b, a); } - -SIMD_INLINE v64 v64_ziphi_8(v64 a, v64 b) { - return _mm_srli_si128(_mm_unpacklo_epi8(b, a), 8); -} - -SIMD_INLINE v64 v64_ziplo_16(v64 a, v64 b) { return _mm_unpacklo_epi16(b, a); } - -SIMD_INLINE v64 v64_ziphi_16(v64 a, v64 b) { - return _mm_srli_si128(_mm_unpacklo_epi16(b, a), 8); -} - -SIMD_INLINE v64 v64_ziplo_32(v64 a, v64 b) { return _mm_unpacklo_epi32(b, a); } - -SIMD_INLINE v64 v64_ziphi_32(v64 a, v64 b) { - return _mm_srli_si128(_mm_unpacklo_epi32(b, a), 8); -} - -SIMD_INLINE v64 v64_pack_s32_s16(v64 a, v64 b) { - __m128i t = _mm_unpacklo_epi64(b, a); - return _mm_packs_epi32(t, t); -} - -SIMD_INLINE v64 v64_pack_s32_u16(v64 a, v64 b) { -#if defined(__SSE4_1__) - __m128i t = _mm_unpacklo_epi64(b, a); - return _mm_packus_epi32(t, t); -#else - int32_t ah = v64_high_u32(a); - int32_t al = v64_low_u32(a); - int32_t bh = v64_high_u32(b); - int32_t bl = v64_low_u32(b); - return v64_from_16(ah > 65535 ? 65535 : ah < 0 ? 0 : ah, - al > 65535 ? 65535 : al < 0 ? 0 : al, - bh > 65535 ? 65535 : bh < 0 ? 0 : bh, - bl > 65535 ? 65535 : bl < 0 ? 0 : bl); -#endif -} - -SIMD_INLINE v64 v64_pack_s16_u8(v64 a, v64 b) { - __m128i t = _mm_unpacklo_epi64(b, a); - return _mm_packus_epi16(t, t); -} - -SIMD_INLINE v64 v64_pack_s16_s8(v64 a, v64 b) { - __m128i t = _mm_unpacklo_epi64(b, a); - return _mm_packs_epi16(t, t); -} - -SIMD_INLINE v64 v64_unziphi_8(v64 a, v64 b) { -#if defined(__SSSE3__) - return _mm_shuffle_epi8(_mm_unpacklo_epi64(b, a), - v64_from_64(0x0f0d0b0907050301LL)); -#else - return _mm_packus_epi16( - _mm_unpacklo_epi64(_mm_srli_epi16(b, 8), _mm_srli_epi16(a, 8)), - _mm_setzero_si128()); -#endif -} - -SIMD_INLINE v64 v64_unziplo_8(v64 a, v64 b) { -#if defined(__SSSE3__) - return _mm_shuffle_epi8(_mm_unpacklo_epi64(b, a), - v64_from_64(0x0e0c0a0806040200LL)); -#else - return v64_unziphi_8(_mm_slli_si128(a, 1), _mm_slli_si128(b, 1)); -#endif -} - -SIMD_INLINE v64 v64_unziphi_16(v64 a, v64 b) { -#if defined(__SSSE3__) - return _mm_shuffle_epi8(_mm_unpacklo_epi64(b, a), - v64_from_64(0x0f0e0b0a07060302LL)); -#else - return _mm_packs_epi32( - _mm_unpacklo_epi64(_mm_srai_epi32(b, 16), _mm_srai_epi32(a, 16)), - _mm_setzero_si128()); -#endif -} - -SIMD_INLINE v64 v64_unziplo_16(v64 a, v64 b) { -#if defined(__SSSE3__) - return _mm_shuffle_epi8(_mm_unpacklo_epi64(b, a), - v64_from_64(0x0d0c090805040100LL)); -#else - return v64_unziphi_16(_mm_slli_si128(a, 2), _mm_slli_si128(b, 2)); -#endif -} - -SIMD_INLINE v64 v64_unpacklo_u8_s16(v64 a) { - return _mm_unpacklo_epi8(a, _mm_setzero_si128()); -} - -SIMD_INLINE v64 v64_unpackhi_u8_s16(v64 a) { - return _mm_srli_si128(_mm_unpacklo_epi8(a, _mm_setzero_si128()), 8); -} - -SIMD_INLINE v64 v64_unpacklo_s8_s16(v64 a) { - return _mm_srai_epi16(_mm_unpacklo_epi8(a, a), 8); -} - -SIMD_INLINE v64 v64_unpackhi_s8_s16(v64 a) { - return _mm_srli_si128(_mm_srai_epi16(_mm_unpacklo_epi8(a, a), 8), 8); -} - -SIMD_INLINE v64 v64_unpacklo_u16_s32(v64 a) { - return _mm_unpacklo_epi16(a, _mm_setzero_si128()); -} - -SIMD_INLINE v64 v64_unpacklo_s16_s32(v64 a) { - return _mm_srai_epi32(_mm_unpacklo_epi16(_mm_setzero_si128(), a), 16); -} - -SIMD_INLINE v64 v64_unpackhi_u16_s32(v64 a) { - return _mm_srli_si128(_mm_unpacklo_epi16(a, _mm_setzero_si128()), 8); -} - -SIMD_INLINE v64 v64_unpackhi_s16_s32(v64 a) { - return _mm_srli_si128( - _mm_srai_epi32(_mm_unpacklo_epi16(_mm_setzero_si128(), a), 16), 8); -} - -SIMD_INLINE v64 v64_shuffle_8(v64 x, v64 pattern) { -#if defined(__SSSE3__) - return _mm_shuffle_epi8(x, pattern); -#else - v64 output; - unsigned char *input = (unsigned char *)&x; - unsigned char *index = (unsigned char *)&pattern; - char *selected = (char *)&output; - int counter; - - for (counter = 0; counter < 8; counter++) { - selected[counter] = input[index[counter]]; - } - - return output; -#endif -} - -SIMD_INLINE int64_t v64_dotp_su8(v64 a, v64 b) { - __m128i t = _mm_madd_epi16(_mm_srai_epi16(_mm_unpacklo_epi8(a, a), 8), - _mm_unpacklo_epi8(b, _mm_setzero_si128())); - t = _mm_add_epi32(t, _mm_srli_si128(t, 8)); - t = _mm_add_epi32(t, _mm_srli_si128(t, 4)); - return (int32_t)v64_low_u32(t); -} - -SIMD_INLINE int64_t v64_dotp_s16(v64 a, v64 b) { - __m128i r = _mm_madd_epi16(a, b); -#if defined(__SSE4_1__) && defined(__x86_64__) - __m128i x = _mm_cvtepi32_epi64(r); - return _mm_cvtsi128_si64(_mm_add_epi64(x, _mm_srli_si128(x, 8))); -#else - return (int64_t)_mm_cvtsi128_si32(_mm_srli_si128(r, 4)) + - (int64_t)_mm_cvtsi128_si32(r); -#endif -} - -SIMD_INLINE uint64_t v64_hadd_u8(v64 a) { - return v64_low_u32(_mm_sad_epu8(a, _mm_setzero_si128())); -} - -SIMD_INLINE int64_t v64_hadd_s16(v64 a) { - return v64_dotp_s16(a, v64_dup_16(1)); -} - -typedef v64 sad64_internal; - -SIMD_INLINE sad64_internal v64_sad_u8_init() { return _mm_setzero_si128(); } - -/* Implementation dependent return value. Result must be finalised with - v64_sad_u8_sum(). - The result for more than 32 v64_sad_u8() calls is undefined. */ -SIMD_INLINE sad64_internal v64_sad_u8(sad64_internal s, v64 a, v64 b) { - return _mm_add_epi64(s, _mm_sad_epu8(a, b)); -} - -SIMD_INLINE uint32_t v64_sad_u8_sum(sad64_internal s) { return v64_low_u32(s); } - -typedef v64 ssd64_internal; - -SIMD_INLINE ssd64_internal v64_ssd_u8_init() { return _mm_setzero_si128(); } - -/* Implementation dependent return value. Result must be finalised with - * v64_ssd_u8_sum(). */ -SIMD_INLINE ssd64_internal v64_ssd_u8(ssd64_internal s, v64 a, v64 b) { - v64 l = v64_sub_16(v64_ziplo_8(v64_zero(), a), v64_ziplo_8(v64_zero(), b)); - v64 h = v64_sub_16(v64_ziphi_8(v64_zero(), a), v64_ziphi_8(v64_zero(), b)); - v64 r = v64_add_32(_mm_madd_epi16(l, l), _mm_madd_epi16(h, h)); - return _mm_add_epi64( - s, v64_ziplo_32(v64_zero(), _mm_add_epi32(r, _mm_srli_si128(r, 4)))); -} - -SIMD_INLINE uint32_t v64_ssd_u8_sum(sad64_internal s) { return v64_low_u32(s); } - -SIMD_INLINE v64 v64_or(v64 a, v64 b) { return _mm_or_si128(a, b); } - -SIMD_INLINE v64 v64_xor(v64 a, v64 b) { return _mm_xor_si128(a, b); } - -SIMD_INLINE v64 v64_and(v64 a, v64 b) { return _mm_and_si128(a, b); } - -SIMD_INLINE v64 v64_andn(v64 a, v64 b) { return _mm_andnot_si128(b, a); } - -SIMD_INLINE v64 v64_mullo_s16(v64 a, v64 b) { return _mm_mullo_epi16(a, b); } - -SIMD_INLINE v64 v64_mulhi_s16(v64 a, v64 b) { return _mm_mulhi_epi16(a, b); } - -SIMD_INLINE v64 v64_mullo_s32(v64 a, v64 b) { -#if defined(__SSE4_1__) - return _mm_mullo_epi32(a, b); -#else - return _mm_unpacklo_epi32( - _mm_mul_epu32(a, b), - _mm_mul_epu32(_mm_srli_si128(a, 4), _mm_srli_si128(b, 4))); -#endif -} - -SIMD_INLINE v64 v64_madd_s16(v64 a, v64 b) { return _mm_madd_epi16(a, b); } - -SIMD_INLINE v64 v64_madd_us8(v64 a, v64 b) { -#if defined(__SSSE3__) - return _mm_maddubs_epi16(a, b); -#else - __m128i t = _mm_madd_epi16(_mm_unpacklo_epi8(a, _mm_setzero_si128()), - _mm_srai_epi16(_mm_unpacklo_epi8(b, b), 8)); - return _mm_packs_epi32(t, t); -#endif -} - -SIMD_INLINE v64 v64_avg_u8(v64 a, v64 b) { return _mm_avg_epu8(a, b); } - -SIMD_INLINE v64 v64_rdavg_u8(v64 a, v64 b) { - return _mm_sub_epi8(_mm_avg_epu8(a, b), - _mm_and_si128(_mm_xor_si128(a, b), v64_dup_8(1))); -} - -SIMD_INLINE v64 v64_rdavg_u16(v64 a, v64 b) { - return _mm_sub_epi16(_mm_avg_epu16(a, b), - _mm_and_si128(_mm_xor_si128(a, b), v64_dup_16(1))); -} - -SIMD_INLINE v64 v64_avg_u16(v64 a, v64 b) { return _mm_avg_epu16(a, b); } - -SIMD_INLINE v64 v64_min_u8(v64 a, v64 b) { return _mm_min_epu8(a, b); } - -SIMD_INLINE v64 v64_max_u8(v64 a, v64 b) { return _mm_max_epu8(a, b); } - -SIMD_INLINE v64 v64_min_s8(v64 a, v64 b) { -#if defined(__SSE4_1__) - return _mm_min_epi8(a, b); -#else - v64 mask = _mm_cmplt_epi8(a, b); - return _mm_or_si128(_mm_andnot_si128(mask, b), _mm_and_si128(mask, a)); -#endif -} - -SIMD_INLINE v64 v64_max_s8(v64 a, v64 b) { -#if defined(__SSE4_1__) - return _mm_max_epi8(a, b); -#else - v64 mask = _mm_cmplt_epi8(b, a); - return _mm_or_si128(_mm_andnot_si128(mask, b), _mm_and_si128(mask, a)); -#endif -} - -SIMD_INLINE v64 v64_min_s16(v64 a, v64 b) { return _mm_min_epi16(a, b); } - -SIMD_INLINE v64 v64_max_s16(v64 a, v64 b) { return _mm_max_epi16(a, b); } - -SIMD_INLINE v64 v64_cmpgt_s8(v64 a, v64 b) { return _mm_cmpgt_epi8(a, b); } - -SIMD_INLINE v64 v64_cmplt_s8(v64 a, v64 b) { return _mm_cmplt_epi8(a, b); } - -SIMD_INLINE v64 v64_cmpeq_8(v64 a, v64 b) { return _mm_cmpeq_epi8(a, b); } - -SIMD_INLINE v64 v64_cmpgt_s16(v64 a, v64 b) { return _mm_cmpgt_epi16(a, b); } - -SIMD_INLINE v64 v64_cmplt_s16(v64 a, v64 b) { return _mm_cmplt_epi16(a, b); } - -SIMD_INLINE v64 v64_cmpeq_16(v64 a, v64 b) { return _mm_cmpeq_epi16(a, b); } - -SIMD_INLINE v64 v64_shl_8(v64 a, unsigned int c) { - return _mm_and_si128(_mm_set1_epi8((uint8_t)(0xff << c)), - _mm_sll_epi16(a, _mm_cvtsi32_si128(c))); -} - -SIMD_INLINE v64 v64_shr_u8(v64 a, unsigned int c) { - return _mm_and_si128(_mm_set1_epi8(0xff >> c), - _mm_srl_epi16(a, _mm_cvtsi32_si128(c))); -} - -SIMD_INLINE v64 v64_shr_s8(v64 a, unsigned int c) { - return _mm_packs_epi16( - _mm_sra_epi16(_mm_unpacklo_epi8(a, a), _mm_cvtsi32_si128(c + 8)), a); -} - -SIMD_INLINE v64 v64_shl_16(v64 a, unsigned int c) { - return _mm_sll_epi16(a, _mm_cvtsi32_si128(c)); -} - -SIMD_INLINE v64 v64_shr_u16(v64 a, unsigned int c) { - return _mm_srl_epi16(a, _mm_cvtsi32_si128(c)); -} - -SIMD_INLINE v64 v64_shr_s16(v64 a, unsigned int c) { - return _mm_sra_epi16(a, _mm_cvtsi32_si128(c)); -} - -SIMD_INLINE v64 v64_shl_32(v64 a, unsigned int c) { - return _mm_sll_epi32(a, _mm_cvtsi32_si128(c)); -} - -SIMD_INLINE v64 v64_shr_u32(v64 a, unsigned int c) { - return _mm_srl_epi32(a, _mm_cvtsi32_si128(c)); -} - -SIMD_INLINE v64 v64_shr_s32(v64 a, unsigned int c) { - return _mm_sra_epi32(a, _mm_cvtsi32_si128(c)); -} - -/* These intrinsics require immediate values, so we must use #defines - to enforce that. */ -#define v64_shl_n_byte(a, c) _mm_slli_si128(a, c) -#define v64_shr_n_byte(a, c) _mm_srli_si128(_mm_unpacklo_epi64(a, a), c + 8) -#define v64_shl_n_8(a, c) \ - _mm_and_si128(_mm_set1_epi8((uint8_t)(0xff << (c))), _mm_slli_epi16(a, c)) -#define v64_shr_n_u8(a, c) \ - _mm_and_si128(_mm_set1_epi8(0xff >> (c)), _mm_srli_epi16(a, c)) -#define v64_shr_n_s8(a, c) \ - _mm_packs_epi16(_mm_srai_epi16(_mm_unpacklo_epi8(a, a), (c) + 8), a) -#define v64_shl_n_16(a, c) _mm_slli_epi16(a, c) -#define v64_shr_n_u16(a, c) _mm_srli_epi16(a, c) -#define v64_shr_n_s16(a, c) _mm_srai_epi16(a, c) -#define v64_shl_n_32(a, c) _mm_slli_epi32(a, c) -#define v64_shr_n_u32(a, c) _mm_srli_epi32(a, c) -#define v64_shr_n_s32(a, c) _mm_srai_epi32(a, c) - -#endif // AOM_AOM_DSP_SIMD_V64_INTRINSICS_X86_H_ |