1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
|
/*
* Copyright (c) 2016, Alliance for Open Media. All rights reserved
*
* This source code is subject to the terms of the BSD 2 Clause License and
* the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
* was not distributed with this source code in the LICENSE file, you can
* obtain it at www.aomedia.org/license/software. If the Alliance for Open
* Media Patent License 1.0 was not distributed with this source code in the
* PATENTS file, you can obtain it at www.aomedia.org/license/patent.
*/
#include <assert.h>
#include <emmintrin.h> // SSE2
#include <tmmintrin.h>
#include "config/aom_config.h"
#include "config/aom_dsp_rtcd.h"
#include "config/av1_rtcd.h"
#include "aom_dsp/x86/synonyms.h"
void aom_var_filter_block2d_bil_first_pass_ssse3(
const uint8_t *a, uint16_t *b, unsigned int src_pixels_per_line,
unsigned int pixel_step, unsigned int output_height,
unsigned int output_width, const uint8_t *filter);
void aom_var_filter_block2d_bil_second_pass_ssse3(
const uint16_t *a, uint8_t *b, unsigned int src_pixels_per_line,
unsigned int pixel_step, unsigned int output_height,
unsigned int output_width, const uint8_t *filter);
static INLINE void compute_jnt_comp_avg(__m128i *p0, __m128i *p1,
const __m128i *w, const __m128i *r,
void *const result) {
__m128i p_lo = _mm_unpacklo_epi8(*p0, *p1);
__m128i mult_lo = _mm_maddubs_epi16(p_lo, *w);
__m128i round_lo = _mm_add_epi16(mult_lo, *r);
__m128i shift_lo = _mm_srai_epi16(round_lo, DIST_PRECISION_BITS);
__m128i p_hi = _mm_unpackhi_epi8(*p0, *p1);
__m128i mult_hi = _mm_maddubs_epi16(p_hi, *w);
__m128i round_hi = _mm_add_epi16(mult_hi, *r);
__m128i shift_hi = _mm_srai_epi16(round_hi, DIST_PRECISION_BITS);
xx_storeu_128(result, _mm_packus_epi16(shift_lo, shift_hi));
}
void aom_jnt_comp_avg_pred_ssse3(uint8_t *comp_pred, const uint8_t *pred,
int width, int height, const uint8_t *ref,
int ref_stride,
const JNT_COMP_PARAMS *jcp_param) {
int i;
const uint8_t w0 = (uint8_t)jcp_param->fwd_offset;
const uint8_t w1 = (uint8_t)jcp_param->bck_offset;
const __m128i w = _mm_set_epi8(w1, w0, w1, w0, w1, w0, w1, w0, w1, w0, w1, w0,
w1, w0, w1, w0);
const uint16_t round = ((1 << DIST_PRECISION_BITS) >> 1);
const __m128i r =
_mm_set_epi16(round, round, round, round, round, round, round, round);
if (width >= 16) {
// Read 16 pixels one row at a time
assert(!(width & 15));
for (i = 0; i < height; ++i) {
int j;
for (j = 0; j < width; j += 16) {
__m128i p0 = xx_loadu_128(ref);
__m128i p1 = xx_loadu_128(pred);
compute_jnt_comp_avg(&p0, &p1, &w, &r, comp_pred);
comp_pred += 16;
pred += 16;
ref += 16;
}
ref += ref_stride - width;
}
} else if (width >= 8) {
// Read 8 pixels two row at a time
assert(!(width & 7));
assert(!(width & 1));
for (i = 0; i < height; i += 2) {
__m128i p0_0 = xx_loadl_64(ref + 0 * ref_stride);
__m128i p0_1 = xx_loadl_64(ref + 1 * ref_stride);
__m128i p0 = _mm_unpacklo_epi64(p0_0, p0_1);
__m128i p1 = xx_loadu_128(pred);
compute_jnt_comp_avg(&p0, &p1, &w, &r, comp_pred);
comp_pred += 16;
pred += 16;
ref += 2 * ref_stride;
}
} else {
// Read 4 pixels four row at a time
assert(!(width & 3));
assert(!(height & 3));
for (i = 0; i < height; i += 4) {
const uint8_t *row0 = ref + 0 * ref_stride;
const uint8_t *row1 = ref + 1 * ref_stride;
const uint8_t *row2 = ref + 2 * ref_stride;
const uint8_t *row3 = ref + 3 * ref_stride;
__m128i p0 =
_mm_setr_epi8(row0[0], row0[1], row0[2], row0[3], row1[0], row1[1],
row1[2], row1[3], row2[0], row2[1], row2[2], row2[3],
row3[0], row3[1], row3[2], row3[3]);
__m128i p1 = xx_loadu_128(pred);
compute_jnt_comp_avg(&p0, &p1, &w, &r, comp_pred);
comp_pred += 16;
pred += 16;
ref += 4 * ref_stride;
}
}
}
void aom_jnt_comp_avg_upsampled_pred_ssse3(
MACROBLOCKD *xd, const struct AV1Common *const cm, int mi_row, int mi_col,
const MV *const mv, uint8_t *comp_pred, const uint8_t *pred, int width,
int height, int subpel_x_q3, int subpel_y_q3, const uint8_t *ref,
int ref_stride, const JNT_COMP_PARAMS *jcp_param, int subpel_search) {
int n;
int i;
aom_upsampled_pred(xd, cm, mi_row, mi_col, mv, comp_pred, width, height,
subpel_x_q3, subpel_y_q3, ref, ref_stride, subpel_search);
/*The total number of pixels must be a multiple of 16 (e.g., 4x4).*/
assert(!(width * height & 15));
n = width * height >> 4;
const uint8_t w0 = (uint8_t)jcp_param->fwd_offset;
const uint8_t w1 = (uint8_t)jcp_param->bck_offset;
const __m128i w = _mm_set_epi8(w1, w0, w1, w0, w1, w0, w1, w0, w1, w0, w1, w0,
w1, w0, w1, w0);
const uint16_t round = ((1 << DIST_PRECISION_BITS) >> 1);
const __m128i r =
_mm_set_epi16(round, round, round, round, round, round, round, round);
for (i = 0; i < n; i++) {
__m128i p0 = xx_loadu_128(comp_pred);
__m128i p1 = xx_loadu_128(pred);
compute_jnt_comp_avg(&p0, &p1, &w, &r, comp_pred);
comp_pred += 16;
pred += 16;
}
}
#define JNT_SUBPIX_AVG_VAR(W, H) \
uint32_t aom_jnt_sub_pixel_avg_variance##W##x##H##_ssse3( \
const uint8_t *a, int a_stride, int xoffset, int yoffset, \
const uint8_t *b, int b_stride, uint32_t *sse, \
const uint8_t *second_pred, const JNT_COMP_PARAMS *jcp_param) { \
uint16_t fdata3[(H + 1) * W]; \
uint8_t temp2[H * W]; \
DECLARE_ALIGNED(16, uint8_t, temp3[H * W]); \
\
aom_var_filter_block2d_bil_first_pass_ssse3( \
a, fdata3, a_stride, 1, H + 1, W, bilinear_filters_2t[xoffset]); \
aom_var_filter_block2d_bil_second_pass_ssse3( \
fdata3, temp2, W, W, H, W, bilinear_filters_2t[yoffset]); \
\
aom_jnt_comp_avg_pred_ssse3(temp3, second_pred, W, H, temp2, W, \
jcp_param); \
\
return aom_variance##W##x##H(temp3, W, b, b_stride, sse); \
}
JNT_SUBPIX_AVG_VAR(128, 128)
JNT_SUBPIX_AVG_VAR(128, 64)
JNT_SUBPIX_AVG_VAR(64, 128)
JNT_SUBPIX_AVG_VAR(64, 64)
JNT_SUBPIX_AVG_VAR(64, 32)
JNT_SUBPIX_AVG_VAR(32, 64)
JNT_SUBPIX_AVG_VAR(32, 32)
JNT_SUBPIX_AVG_VAR(32, 16)
JNT_SUBPIX_AVG_VAR(16, 32)
JNT_SUBPIX_AVG_VAR(16, 16)
JNT_SUBPIX_AVG_VAR(16, 8)
JNT_SUBPIX_AVG_VAR(8, 16)
JNT_SUBPIX_AVG_VAR(8, 8)
JNT_SUBPIX_AVG_VAR(8, 4)
JNT_SUBPIX_AVG_VAR(4, 8)
JNT_SUBPIX_AVG_VAR(4, 4)
JNT_SUBPIX_AVG_VAR(4, 16)
JNT_SUBPIX_AVG_VAR(16, 4)
JNT_SUBPIX_AVG_VAR(8, 32)
JNT_SUBPIX_AVG_VAR(32, 8)
JNT_SUBPIX_AVG_VAR(16, 64)
JNT_SUBPIX_AVG_VAR(64, 16)
|
