1 // Copyright 2015 Google Inc. All Rights Reserved. 2 // 3 // Use of this source code is governed by a BSD-style license 4 // that can be found in the COPYING file in the root of the source 5 // tree. An additional intellectual property rights grant can be found 6 // in the file PATENTS. All contributing project authors may 7 // be found in the AUTHORS file in the root of the source tree. 8 // ----------------------------------------------------------------------------- 9 // 10 // SSE2 variant of alpha filters 11 // 12 // Author: Skal (pascal.massimino (at) gmail.com) 13 14 #include "src/dsp/dsp.h" 15 16 #if defined(WEBP_USE_SSE2) 17 18 #include <assert.h> 19 #include <emmintrin.h> 20 #include <stdlib.h> 21 #include <string.h> 22 23 //------------------------------------------------------------------------------ 24 // Helpful macro. 25 26 # define SANITY_CHECK(in, out) \ 27 assert((in) != NULL); \ 28 assert((out) != NULL); \ 29 assert(width > 0); \ 30 assert(height > 0); \ 31 assert(stride >= width); \ 32 assert(row >= 0 && num_rows > 0 && row + num_rows <= height); \ 33 (void)height; // Silence unused warning. 34 35 static void PredictLineTop_SSE2(const uint8_t* src, const uint8_t* pred, 36 uint8_t* dst, int length) { 37 int i; 38 const int max_pos = length & ~31; 39 assert(length >= 0); 40 for (i = 0; i < max_pos; i += 32) { 41 const __m128i A0 = _mm_loadu_si128((const __m128i*)&src[i + 0]); 42 const __m128i A1 = _mm_loadu_si128((const __m128i*)&src[i + 16]); 43 const __m128i B0 = _mm_loadu_si128((const __m128i*)&pred[i + 0]); 44 const __m128i B1 = _mm_loadu_si128((const __m128i*)&pred[i + 16]); 45 const __m128i C0 = _mm_sub_epi8(A0, B0); 46 const __m128i C1 = _mm_sub_epi8(A1, B1); 47 _mm_storeu_si128((__m128i*)&dst[i + 0], C0); 48 _mm_storeu_si128((__m128i*)&dst[i + 16], C1); 49 } 50 for (; i < length; ++i) dst[i] = src[i] - pred[i]; 51 } 52 53 // Special case for left-based prediction (when preds==dst-1 or preds==src-1). 54 static void PredictLineLeft_SSE2(const uint8_t* src, uint8_t* dst, int length) { 55 int i; 56 const int max_pos = length & ~31; 57 assert(length >= 0); 58 for (i = 0; i < max_pos; i += 32) { 59 const __m128i A0 = _mm_loadu_si128((const __m128i*)(src + i + 0 )); 60 const __m128i B0 = _mm_loadu_si128((const __m128i*)(src + i + 0 - 1)); 61 const __m128i A1 = _mm_loadu_si128((const __m128i*)(src + i + 16 )); 62 const __m128i B1 = _mm_loadu_si128((const __m128i*)(src + i + 16 - 1)); 63 const __m128i C0 = _mm_sub_epi8(A0, B0); 64 const __m128i C1 = _mm_sub_epi8(A1, B1); 65 _mm_storeu_si128((__m128i*)(dst + i + 0), C0); 66 _mm_storeu_si128((__m128i*)(dst + i + 16), C1); 67 } 68 for (; i < length; ++i) dst[i] = src[i] - src[i - 1]; 69 } 70 71 //------------------------------------------------------------------------------ 72 // Horizontal filter. 73 74 static WEBP_INLINE void DoHorizontalFilter_SSE2(const uint8_t* in, 75 int width, int height, 76 int stride, 77 int row, int num_rows, 78 uint8_t* out) { 79 const size_t start_offset = row * stride; 80 const int last_row = row + num_rows; 81 SANITY_CHECK(in, out); 82 in += start_offset; 83 out += start_offset; 84 85 if (row == 0) { 86 // Leftmost pixel is the same as input for topmost scanline. 87 out[0] = in[0]; 88 PredictLineLeft_SSE2(in + 1, out + 1, width - 1); 89 row = 1; 90 in += stride; 91 out += stride; 92 } 93 94 // Filter line-by-line. 95 while (row < last_row) { 96 // Leftmost pixel is predicted from above. 97 out[0] = in[0] - in[-stride]; 98 PredictLineLeft_SSE2(in + 1, out + 1, width - 1); 99 ++row; 100 in += stride; 101 out += stride; 102 } 103 } 104 105 //------------------------------------------------------------------------------ 106 // Vertical filter. 107 108 static WEBP_INLINE void DoVerticalFilter_SSE2(const uint8_t* in, 109 int width, int height, int stride, 110 int row, int num_rows, 111 uint8_t* out) { 112 const size_t start_offset = row * stride; 113 const int last_row = row + num_rows; 114 SANITY_CHECK(in, out); 115 in += start_offset; 116 out += start_offset; 117 118 if (row == 0) { 119 // Very first top-left pixel is copied. 120 out[0] = in[0]; 121 // Rest of top scan-line is left-predicted. 122 PredictLineLeft_SSE2(in + 1, out + 1, width - 1); 123 row = 1; 124 in += stride; 125 out += stride; 126 } 127 128 // Filter line-by-line. 129 while (row < last_row) { 130 PredictLineTop_SSE2(in, in - stride, out, width); 131 ++row; 132 in += stride; 133 out += stride; 134 } 135 } 136 137 //------------------------------------------------------------------------------ 138 // Gradient filter. 139 140 static WEBP_INLINE int GradientPredictor_SSE2(uint8_t a, uint8_t b, uint8_t c) { 141 const int g = a + b - c; 142 return ((g & ~0xff) == 0) ? g : (g < 0) ? 0 : 255; // clip to 8bit 143 } 144 145 static void GradientPredictDirect_SSE2(const uint8_t* const row, 146 const uint8_t* const top, 147 uint8_t* const out, int length) { 148 const int max_pos = length & ~7; 149 int i; 150 const __m128i zero = _mm_setzero_si128(); 151 for (i = 0; i < max_pos; i += 8) { 152 const __m128i A0 = _mm_loadl_epi64((const __m128i*)&row[i - 1]); 153 const __m128i B0 = _mm_loadl_epi64((const __m128i*)&top[i]); 154 const __m128i C0 = _mm_loadl_epi64((const __m128i*)&top[i - 1]); 155 const __m128i D = _mm_loadl_epi64((const __m128i*)&row[i]); 156 const __m128i A1 = _mm_unpacklo_epi8(A0, zero); 157 const __m128i B1 = _mm_unpacklo_epi8(B0, zero); 158 const __m128i C1 = _mm_unpacklo_epi8(C0, zero); 159 const __m128i E = _mm_add_epi16(A1, B1); 160 const __m128i F = _mm_sub_epi16(E, C1); 161 const __m128i G = _mm_packus_epi16(F, zero); 162 const __m128i H = _mm_sub_epi8(D, G); 163 _mm_storel_epi64((__m128i*)(out + i), H); 164 } 165 for (; i < length; ++i) { 166 out[i] = row[i] - GradientPredictor_SSE2(row[i - 1], top[i], top[i - 1]); 167 } 168 } 169 170 static WEBP_INLINE void DoGradientFilter_SSE2(const uint8_t* in, 171 int width, int height, int stride, 172 int row, int num_rows, 173 uint8_t* out) { 174 const size_t start_offset = row * stride; 175 const int last_row = row + num_rows; 176 SANITY_CHECK(in, out); 177 in += start_offset; 178 out += start_offset; 179 180 // left prediction for top scan-line 181 if (row == 0) { 182 out[0] = in[0]; 183 PredictLineLeft_SSE2(in + 1, out + 1, width - 1); 184 row = 1; 185 in += stride; 186 out += stride; 187 } 188 189 // Filter line-by-line. 190 while (row < last_row) { 191 out[0] = in[0] - in[-stride]; 192 GradientPredictDirect_SSE2(in + 1, in + 1 - stride, out + 1, width - 1); 193 ++row; 194 in += stride; 195 out += stride; 196 } 197 } 198 199 #undef SANITY_CHECK 200 201 //------------------------------------------------------------------------------ 202 203 static void HorizontalFilter_SSE2(const uint8_t* data, int width, int height, 204 int stride, uint8_t* filtered_data) { 205 DoHorizontalFilter_SSE2(data, width, height, stride, 0, height, 206 filtered_data); 207 } 208 209 static void VerticalFilter_SSE2(const uint8_t* data, int width, int height, 210 int stride, uint8_t* filtered_data) { 211 DoVerticalFilter_SSE2(data, width, height, stride, 0, height, filtered_data); 212 } 213 214 static void GradientFilter_SSE2(const uint8_t* data, int width, int height, 215 int stride, uint8_t* filtered_data) { 216 DoGradientFilter_SSE2(data, width, height, stride, 0, height, filtered_data); 217 } 218 219 //------------------------------------------------------------------------------ 220 // Inverse transforms 221 222 static void HorizontalUnfilter_SSE2(const uint8_t* prev, const uint8_t* in, 223 uint8_t* out, int width) { 224 int i; 225 __m128i last; 226 out[0] = in[0] + (prev == NULL ? 0 : prev[0]); 227 if (width <= 1) return; 228 last = _mm_set_epi32(0, 0, 0, out[0]); 229 for (i = 1; i + 8 <= width; i += 8) { 230 const __m128i A0 = _mm_loadl_epi64((const __m128i*)(in + i)); 231 const __m128i A1 = _mm_add_epi8(A0, last); 232 const __m128i A2 = _mm_slli_si128(A1, 1); 233 const __m128i A3 = _mm_add_epi8(A1, A2); 234 const __m128i A4 = _mm_slli_si128(A3, 2); 235 const __m128i A5 = _mm_add_epi8(A3, A4); 236 const __m128i A6 = _mm_slli_si128(A5, 4); 237 const __m128i A7 = _mm_add_epi8(A5, A6); 238 _mm_storel_epi64((__m128i*)(out + i), A7); 239 last = _mm_srli_epi64(A7, 56); 240 } 241 for (; i < width; ++i) out[i] = in[i] + out[i - 1]; 242 } 243 244 static void VerticalUnfilter_SSE2(const uint8_t* prev, const uint8_t* in, 245 uint8_t* out, int width) { 246 if (prev == NULL) { 247 HorizontalUnfilter_SSE2(NULL, in, out, width); 248 } else { 249 int i; 250 const int max_pos = width & ~31; 251 assert(width >= 0); 252 for (i = 0; i < max_pos; i += 32) { 253 const __m128i A0 = _mm_loadu_si128((const __m128i*)&in[i + 0]); 254 const __m128i A1 = _mm_loadu_si128((const __m128i*)&in[i + 16]); 255 const __m128i B0 = _mm_loadu_si128((const __m128i*)&prev[i + 0]); 256 const __m128i B1 = _mm_loadu_si128((const __m128i*)&prev[i + 16]); 257 const __m128i C0 = _mm_add_epi8(A0, B0); 258 const __m128i C1 = _mm_add_epi8(A1, B1); 259 _mm_storeu_si128((__m128i*)&out[i + 0], C0); 260 _mm_storeu_si128((__m128i*)&out[i + 16], C1); 261 } 262 for (; i < width; ++i) out[i] = in[i] + prev[i]; 263 } 264 } 265 266 static void GradientPredictInverse_SSE2(const uint8_t* const in, 267 const uint8_t* const top, 268 uint8_t* const row, int length) { 269 if (length > 0) { 270 int i; 271 const int max_pos = length & ~7; 272 const __m128i zero = _mm_setzero_si128(); 273 __m128i A = _mm_set_epi32(0, 0, 0, row[-1]); // left sample 274 for (i = 0; i < max_pos; i += 8) { 275 const __m128i tmp0 = _mm_loadl_epi64((const __m128i*)&top[i]); 276 const __m128i tmp1 = _mm_loadl_epi64((const __m128i*)&top[i - 1]); 277 const __m128i B = _mm_unpacklo_epi8(tmp0, zero); 278 const __m128i C = _mm_unpacklo_epi8(tmp1, zero); 279 const __m128i D = _mm_loadl_epi64((const __m128i*)&in[i]); // base input 280 const __m128i E = _mm_sub_epi16(B, C); // unclipped gradient basis B - C 281 __m128i out = zero; // accumulator for output 282 __m128i mask_hi = _mm_set_epi32(0, 0, 0, 0xff); 283 int k = 8; 284 while (1) { 285 const __m128i tmp3 = _mm_add_epi16(A, E); // delta = A + B - C 286 const __m128i tmp4 = _mm_packus_epi16(tmp3, zero); // saturate delta 287 const __m128i tmp5 = _mm_add_epi8(tmp4, D); // add to in[] 288 A = _mm_and_si128(tmp5, mask_hi); // 1-complement clip 289 out = _mm_or_si128(out, A); // accumulate output 290 if (--k == 0) break; 291 A = _mm_slli_si128(A, 1); // rotate left sample 292 mask_hi = _mm_slli_si128(mask_hi, 1); // rotate mask 293 A = _mm_unpacklo_epi8(A, zero); // convert 8b->16b 294 } 295 A = _mm_srli_si128(A, 7); // prepare left sample for next iteration 296 _mm_storel_epi64((__m128i*)&row[i], out); 297 } 298 for (; i < length; ++i) { 299 row[i] = in[i] + GradientPredictor_SSE2(row[i - 1], top[i], top[i - 1]); 300 } 301 } 302 } 303 304 static void GradientUnfilter_SSE2(const uint8_t* prev, const uint8_t* in, 305 uint8_t* out, int width) { 306 if (prev == NULL) { 307 HorizontalUnfilter_SSE2(NULL, in, out, width); 308 } else { 309 out[0] = in[0] + prev[0]; // predict from above 310 GradientPredictInverse_SSE2(in + 1, prev + 1, out + 1, width - 1); 311 } 312 } 313 314 //------------------------------------------------------------------------------ 315 // Entry point 316 317 extern void VP8FiltersInitSSE2(void); 318 319 WEBP_TSAN_IGNORE_FUNCTION void VP8FiltersInitSSE2(void) { 320 WebPUnfilters[WEBP_FILTER_HORIZONTAL] = HorizontalUnfilter_SSE2; 321 WebPUnfilters[WEBP_FILTER_VERTICAL] = VerticalUnfilter_SSE2; 322 WebPUnfilters[WEBP_FILTER_GRADIENT] = GradientUnfilter_SSE2; 323 324 WebPFilters[WEBP_FILTER_HORIZONTAL] = HorizontalFilter_SSE2; 325 WebPFilters[WEBP_FILTER_VERTICAL] = VerticalFilter_SSE2; 326 WebPFilters[WEBP_FILTER_GRADIENT] = GradientFilter_SSE2; 327 } 328 329 #else // !WEBP_USE_SSE2 330 331 WEBP_DSP_INIT_STUB(VP8FiltersInitSSE2) 332 333 #endif // WEBP_USE_SSE2 334