1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
|
/*
* Copyright (c) 2018, Alliance for Open Media. All rights reserved
*
* This source code is subject to the terms of the BSD 2 Clause License and
* the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
* was not distributed with this source code in the LICENSE file, you can
* obtain it at www.aomedia.org/license/software. If the Alliance for Open
* Media Patent License 1.0 was not distributed with this source code in the
* PATENTS file, you can obtain it at www.aomedia.org/license/patent.
*/
#include <assert.h>
#include <immintrin.h>
#include <smmintrin.h>
#include "aom_dsp/x86/synonyms.h"
#include "aom_dsp/x86/synonyms_avx2.h"
#include "aom/aom_integer.h"
#include "av1/common/reconinter.h"
#define MAX_MASK_VALUE (1 << WEDGE_WEIGHT_BITS)
/**
* See av1_wedge_sse_from_residuals_c
*/
uint64_t av1_wedge_sse_from_residuals_avx2(const int16_t *r1, const int16_t *d,
const uint8_t *m, int N) {
int n = -N;
uint64_t csse;
const __m256i v_mask_max_w = _mm256_set1_epi16(MAX_MASK_VALUE);
const __m256i v_zext_q = yy_set1_64_from_32i(0xffffffff);
__m256i v_acc0_q = _mm256_setzero_si256();
assert(N % 64 == 0);
r1 += N;
d += N;
m += N;
do {
const __m256i v_r0_w = _mm256_lddqu_si256((__m256i *)(r1 + n));
const __m256i v_d0_w = _mm256_lddqu_si256((__m256i *)(d + n));
const __m128i v_m01_b = _mm_lddqu_si128((__m128i *)(m + n));
const __m256i v_rd0l_w = _mm256_unpacklo_epi16(v_d0_w, v_r0_w);
const __m256i v_rd0h_w = _mm256_unpackhi_epi16(v_d0_w, v_r0_w);
const __m256i v_m0_w = _mm256_cvtepu8_epi16(v_m01_b);
const __m256i v_m0l_w = _mm256_unpacklo_epi16(v_m0_w, v_mask_max_w);
const __m256i v_m0h_w = _mm256_unpackhi_epi16(v_m0_w, v_mask_max_w);
const __m256i v_t0l_d = _mm256_madd_epi16(v_rd0l_w, v_m0l_w);
const __m256i v_t0h_d = _mm256_madd_epi16(v_rd0h_w, v_m0h_w);
const __m256i v_t0_w = _mm256_packs_epi32(v_t0l_d, v_t0h_d);
const __m256i v_sq0_d = _mm256_madd_epi16(v_t0_w, v_t0_w);
const __m256i v_sum0_q = _mm256_add_epi64(
_mm256_and_si256(v_sq0_d, v_zext_q), _mm256_srli_epi64(v_sq0_d, 32));
v_acc0_q = _mm256_add_epi64(v_acc0_q, v_sum0_q);
n += 16;
} while (n);
v_acc0_q = _mm256_add_epi64(v_acc0_q, _mm256_srli_si256(v_acc0_q, 8));
__m128i v_acc_q_0 = _mm256_castsi256_si128(v_acc0_q);
__m128i v_acc_q_1 = _mm256_extracti128_si256(v_acc0_q, 1);
v_acc_q_0 = _mm_add_epi64(v_acc_q_0, v_acc_q_1);
#if ARCH_X86_64
csse = (uint64_t)_mm_extract_epi64(v_acc_q_0, 0);
#else
xx_storel_64(&csse, v_acc_q_0);
#endif
return ROUND_POWER_OF_TWO(csse, 2 * WEDGE_WEIGHT_BITS);
}
/**
* See av1_wedge_sign_from_residuals_c
*/
int av1_wedge_sign_from_residuals_avx2(const int16_t *ds, const uint8_t *m,
int N, int64_t limit) {
int64_t acc;
__m256i v_acc0_d = _mm256_setzero_si256();
// Input size limited to 8192 by the use of 32 bit accumulators and m
// being between [0, 64]. Overflow might happen at larger sizes,
// though it is practically impossible on real video input.
assert(N < 8192);
assert(N % 64 == 0);
do {
const __m256i v_m01_b = _mm256_lddqu_si256((__m256i *)(m));
const __m256i v_m23_b = _mm256_lddqu_si256((__m256i *)(m + 32));
const __m256i v_d0_w = _mm256_lddqu_si256((__m256i *)(ds));
const __m256i v_d1_w = _mm256_lddqu_si256((__m256i *)(ds + 16));
const __m256i v_d2_w = _mm256_lddqu_si256((__m256i *)(ds + 32));
const __m256i v_d3_w = _mm256_lddqu_si256((__m256i *)(ds + 48));
const __m256i v_m0_w =
_mm256_cvtepu8_epi16(_mm256_castsi256_si128(v_m01_b));
const __m256i v_m1_w =
_mm256_cvtepu8_epi16(_mm256_extracti128_si256(v_m01_b, 1));
const __m256i v_m2_w =
_mm256_cvtepu8_epi16(_mm256_castsi256_si128(v_m23_b));
const __m256i v_m3_w =
_mm256_cvtepu8_epi16(_mm256_extracti128_si256(v_m23_b, 1));
const __m256i v_p0_d = _mm256_madd_epi16(v_d0_w, v_m0_w);
const __m256i v_p1_d = _mm256_madd_epi16(v_d1_w, v_m1_w);
const __m256i v_p2_d = _mm256_madd_epi16(v_d2_w, v_m2_w);
const __m256i v_p3_d = _mm256_madd_epi16(v_d3_w, v_m3_w);
const __m256i v_p01_d = _mm256_add_epi32(v_p0_d, v_p1_d);
const __m256i v_p23_d = _mm256_add_epi32(v_p2_d, v_p3_d);
const __m256i v_p0123_d = _mm256_add_epi32(v_p01_d, v_p23_d);
v_acc0_d = _mm256_add_epi32(v_acc0_d, v_p0123_d);
ds += 64;
m += 64;
N -= 64;
} while (N);
__m256i v_sign_d = _mm256_srai_epi32(v_acc0_d, 31);
v_acc0_d = _mm256_add_epi64(_mm256_unpacklo_epi32(v_acc0_d, v_sign_d),
_mm256_unpackhi_epi32(v_acc0_d, v_sign_d));
__m256i v_acc_q = _mm256_add_epi64(v_acc0_d, _mm256_srli_si256(v_acc0_d, 8));
__m128i v_acc_q_0 = _mm256_castsi256_si128(v_acc_q);
__m128i v_acc_q_1 = _mm256_extracti128_si256(v_acc_q, 1);
v_acc_q_0 = _mm_add_epi64(v_acc_q_0, v_acc_q_1);
#if ARCH_X86_64
acc = (uint64_t)_mm_extract_epi64(v_acc_q_0, 0);
#else
xx_storel_64(&acc, v_acc_q_0);
#endif
return acc > limit;
}
/**
* av1_wedge_compute_delta_squares_c
*/
void av1_wedge_compute_delta_squares_avx2(int16_t *d, const int16_t *a,
const int16_t *b, int N) {
const __m256i v_neg_w = _mm256_set1_epi32(0xffff0001);
assert(N % 64 == 0);
do {
const __m256i v_a0_w = _mm256_lddqu_si256((__m256i *)(a));
const __m256i v_b0_w = _mm256_lddqu_si256((__m256i *)(b));
const __m256i v_a1_w = _mm256_lddqu_si256((__m256i *)(a + 16));
const __m256i v_b1_w = _mm256_lddqu_si256((__m256i *)(b + 16));
const __m256i v_a2_w = _mm256_lddqu_si256((__m256i *)(a + 32));
const __m256i v_b2_w = _mm256_lddqu_si256((__m256i *)(b + 32));
const __m256i v_a3_w = _mm256_lddqu_si256((__m256i *)(a + 48));
const __m256i v_b3_w = _mm256_lddqu_si256((__m256i *)(b + 48));
const __m256i v_ab0l_w = _mm256_unpacklo_epi16(v_a0_w, v_b0_w);
const __m256i v_ab0h_w = _mm256_unpackhi_epi16(v_a0_w, v_b0_w);
const __m256i v_ab1l_w = _mm256_unpacklo_epi16(v_a1_w, v_b1_w);
const __m256i v_ab1h_w = _mm256_unpackhi_epi16(v_a1_w, v_b1_w);
const __m256i v_ab2l_w = _mm256_unpacklo_epi16(v_a2_w, v_b2_w);
const __m256i v_ab2h_w = _mm256_unpackhi_epi16(v_a2_w, v_b2_w);
const __m256i v_ab3l_w = _mm256_unpacklo_epi16(v_a3_w, v_b3_w);
const __m256i v_ab3h_w = _mm256_unpackhi_epi16(v_a3_w, v_b3_w);
// Negate top word of pairs
const __m256i v_abl0n_w = _mm256_sign_epi16(v_ab0l_w, v_neg_w);
const __m256i v_abh0n_w = _mm256_sign_epi16(v_ab0h_w, v_neg_w);
const __m256i v_abl1n_w = _mm256_sign_epi16(v_ab1l_w, v_neg_w);
const __m256i v_abh1n_w = _mm256_sign_epi16(v_ab1h_w, v_neg_w);
const __m256i v_abl2n_w = _mm256_sign_epi16(v_ab2l_w, v_neg_w);
const __m256i v_abh2n_w = _mm256_sign_epi16(v_ab2h_w, v_neg_w);
const __m256i v_abl3n_w = _mm256_sign_epi16(v_ab3l_w, v_neg_w);
const __m256i v_abh3n_w = _mm256_sign_epi16(v_ab3h_w, v_neg_w);
const __m256i v_r0l_w = _mm256_madd_epi16(v_ab0l_w, v_abl0n_w);
const __m256i v_r0h_w = _mm256_madd_epi16(v_ab0h_w, v_abh0n_w);
const __m256i v_r1l_w = _mm256_madd_epi16(v_ab1l_w, v_abl1n_w);
const __m256i v_r1h_w = _mm256_madd_epi16(v_ab1h_w, v_abh1n_w);
const __m256i v_r2l_w = _mm256_madd_epi16(v_ab2l_w, v_abl2n_w);
const __m256i v_r2h_w = _mm256_madd_epi16(v_ab2h_w, v_abh2n_w);
const __m256i v_r3l_w = _mm256_madd_epi16(v_ab3l_w, v_abl3n_w);
const __m256i v_r3h_w = _mm256_madd_epi16(v_ab3h_w, v_abh3n_w);
const __m256i v_r0_w = _mm256_packs_epi32(v_r0l_w, v_r0h_w);
const __m256i v_r1_w = _mm256_packs_epi32(v_r1l_w, v_r1h_w);
const __m256i v_r2_w = _mm256_packs_epi32(v_r2l_w, v_r2h_w);
const __m256i v_r3_w = _mm256_packs_epi32(v_r3l_w, v_r3h_w);
_mm256_store_si256((__m256i *)(d), v_r0_w);
_mm256_store_si256((__m256i *)(d + 16), v_r1_w);
_mm256_store_si256((__m256i *)(d + 32), v_r2_w);
_mm256_store_si256((__m256i *)(d + 48), v_r3_w);
a += 64;
b += 64;
d += 64;
N -= 64;
} while (N);
}
|
