1 // Copyright 2015 Google Inc. All Rights Reserved. 2 // 3 // Use of this source code is governed by a BSD-style license 4 // that can be found in the COPYING file in the root of the source 5 // tree. An additional intellectual property rights grant can be found 6 // in the file PATENTS. All contributing project authors may 7 // be found in the AUTHORS file in the root of the source tree. 8 // ----------------------------------------------------------------------------- 9 // 10 // SSE2 variant of alpha filters 11 // 12 // Author: Skal (pascal.massimino (at) gmail.com) 13 14 #include "./dsp.h" 15 16 #if defined(WEBP_USE_SSE2) 17 18 #include <assert.h> 19 #include <emmintrin.h> 20 #include <stdlib.h> 21 #include <string.h> 22 23 //------------------------------------------------------------------------------ 24 // Helpful macro. 25 26 # define SANITY_CHECK(in, out) \ 27 assert(in != NULL); \ 28 assert(out != NULL); \ 29 assert(width > 0); \ 30 assert(height > 0); \ 31 assert(stride >= width); \ 32 assert(row >= 0 && num_rows > 0 && row + num_rows <= height); \ 33 (void)height; // Silence unused warning. 34 35 static void PredictLineTop(const uint8_t* src, const uint8_t* pred, 36 uint8_t* dst, int length) { 37 int i; 38 const int max_pos = length & ~31; 39 assert(length >= 0); 40 for (i = 0; i < max_pos; i += 32) { 41 const __m128i A0 = _mm_loadu_si128((const __m128i*)&src[i + 0]); 42 const __m128i A1 = _mm_loadu_si128((const __m128i*)&src[i + 16]); 43 const __m128i B0 = _mm_loadu_si128((const __m128i*)&pred[i + 0]); 44 const __m128i B1 = _mm_loadu_si128((const __m128i*)&pred[i + 16]); 45 const __m128i C0 = _mm_sub_epi8(A0, B0); 46 const __m128i C1 = _mm_sub_epi8(A1, B1); 47 _mm_storeu_si128((__m128i*)&dst[i + 0], C0); 48 _mm_storeu_si128((__m128i*)&dst[i + 16], C1); 49 } 50 for (; i < length; ++i) dst[i] = src[i] - pred[i]; 51 } 52 53 // Special case for left-based prediction (when preds==dst-1 or preds==src-1). 54 static void PredictLineLeft(const uint8_t* src, uint8_t* dst, int length) { 55 int i; 56 const int max_pos = length & ~31; 57 assert(length >= 0); 58 for (i = 0; i < max_pos; i += 32) { 59 const __m128i A0 = _mm_loadu_si128((const __m128i*)(src + i + 0 )); 60 const __m128i B0 = _mm_loadu_si128((const __m128i*)(src + i + 0 - 1)); 61 const __m128i A1 = _mm_loadu_si128((const __m128i*)(src + i + 16 )); 62 const __m128i B1 = _mm_loadu_si128((const __m128i*)(src + i + 16 - 1)); 63 const __m128i C0 = _mm_sub_epi8(A0, B0); 64 const __m128i C1 = _mm_sub_epi8(A1, B1); 65 _mm_storeu_si128((__m128i*)(dst + i + 0), C0); 66 _mm_storeu_si128((__m128i*)(dst + i + 16), C1); 67 } 68 for (; i < length; ++i) dst[i] = src[i] - src[i - 1]; 69 } 70 71 //------------------------------------------------------------------------------ 72 // Horizontal filter. 73 74 static WEBP_INLINE void DoHorizontalFilter(const uint8_t* in, 75 int width, int height, int stride, 76 int row, int num_rows, 77 uint8_t* out) { 78 const size_t start_offset = row * stride; 79 const int last_row = row + num_rows; 80 SANITY_CHECK(in, out); 81 in += start_offset; 82 out += start_offset; 83 84 if (row == 0) { 85 // Leftmost pixel is the same as input for topmost scanline. 86 out[0] = in[0]; 87 PredictLineLeft(in + 1, out + 1, width - 1); 88 row = 1; 89 in += stride; 90 out += stride; 91 } 92 93 // Filter line-by-line. 94 while (row < last_row) { 95 // Leftmost pixel is predicted from above. 96 out[0] = in[0] - in[-stride]; 97 PredictLineLeft(in + 1, out + 1, width - 1); 98 ++row; 99 in += stride; 100 out += stride; 101 } 102 } 103 104 //------------------------------------------------------------------------------ 105 // Vertical filter. 106 107 static WEBP_INLINE void DoVerticalFilter(const uint8_t* in, 108 int width, int height, int stride, 109 int row, int num_rows, uint8_t* out) { 110 const size_t start_offset = row * stride; 111 const int last_row = row + num_rows; 112 SANITY_CHECK(in, out); 113 in += start_offset; 114 out += start_offset; 115 116 if (row == 0) { 117 // Very first top-left pixel is copied. 118 out[0] = in[0]; 119 // Rest of top scan-line is left-predicted. 120 PredictLineLeft(in + 1, out + 1, width - 1); 121 row = 1; 122 in += stride; 123 out += stride; 124 } 125 126 // Filter line-by-line. 127 while (row < last_row) { 128 PredictLineTop(in, in - stride, out, width); 129 ++row; 130 in += stride; 131 out += stride; 132 } 133 } 134 135 //------------------------------------------------------------------------------ 136 // Gradient filter. 137 138 static WEBP_INLINE int GradientPredictorC(uint8_t a, uint8_t b, uint8_t c) { 139 const int g = a + b - c; 140 return ((g & ~0xff) == 0) ? g : (g < 0) ? 0 : 255; // clip to 8bit 141 } 142 143 static void GradientPredictDirect(const uint8_t* const row, 144 const uint8_t* const top, 145 uint8_t* const out, int length) { 146 const int max_pos = length & ~7; 147 int i; 148 const __m128i zero = _mm_setzero_si128(); 149 for (i = 0; i < max_pos; i += 8) { 150 const __m128i A0 = _mm_loadl_epi64((const __m128i*)&row[i - 1]); 151 const __m128i B0 = _mm_loadl_epi64((const __m128i*)&top[i]); 152 const __m128i C0 = _mm_loadl_epi64((const __m128i*)&top[i - 1]); 153 const __m128i D = _mm_loadl_epi64((const __m128i*)&row[i]); 154 const __m128i A1 = _mm_unpacklo_epi8(A0, zero); 155 const __m128i B1 = _mm_unpacklo_epi8(B0, zero); 156 const __m128i C1 = _mm_unpacklo_epi8(C0, zero); 157 const __m128i E = _mm_add_epi16(A1, B1); 158 const __m128i F = _mm_sub_epi16(E, C1); 159 const __m128i G = _mm_packus_epi16(F, zero); 160 const __m128i H = _mm_sub_epi8(D, G); 161 _mm_storel_epi64((__m128i*)(out + i), H); 162 } 163 for (; i < length; ++i) { 164 out[i] = row[i] - GradientPredictorC(row[i - 1], top[i], top[i - 1]); 165 } 166 } 167 168 static WEBP_INLINE void DoGradientFilter(const uint8_t* in, 169 int width, int height, int stride, 170 int row, int num_rows, 171 uint8_t* out) { 172 const size_t start_offset = row * stride; 173 const int last_row = row + num_rows; 174 SANITY_CHECK(in, out); 175 in += start_offset; 176 out += start_offset; 177 178 // left prediction for top scan-line 179 if (row == 0) { 180 out[0] = in[0]; 181 PredictLineLeft(in + 1, out + 1, width - 1); 182 row = 1; 183 in += stride; 184 out += stride; 185 } 186 187 // Filter line-by-line. 188 while (row < last_row) { 189 out[0] = in[0] - in[-stride]; 190 GradientPredictDirect(in + 1, in + 1 - stride, out + 1, width - 1); 191 ++row; 192 in += stride; 193 out += stride; 194 } 195 } 196 197 #undef SANITY_CHECK 198 199 //------------------------------------------------------------------------------ 200 201 static void HorizontalFilter(const uint8_t* data, int width, int height, 202 int stride, uint8_t* filtered_data) { 203 DoHorizontalFilter(data, width, height, stride, 0, height, filtered_data); 204 } 205 206 static void VerticalFilter(const uint8_t* data, int width, int height, 207 int stride, uint8_t* filtered_data) { 208 DoVerticalFilter(data, width, height, stride, 0, height, filtered_data); 209 } 210 211 static void GradientFilter(const uint8_t* data, int width, int height, 212 int stride, uint8_t* filtered_data) { 213 DoGradientFilter(data, width, height, stride, 0, height, filtered_data); 214 } 215 216 //------------------------------------------------------------------------------ 217 // Inverse transforms 218 219 static void HorizontalUnfilter(const uint8_t* prev, const uint8_t* in, 220 uint8_t* out, int width) { 221 int i; 222 __m128i last; 223 out[0] = in[0] + (prev == NULL ? 0 : prev[0]); 224 if (width <= 1) return; 225 last = _mm_set_epi32(0, 0, 0, out[0]); 226 for (i = 1; i + 8 <= width; i += 8) { 227 const __m128i A0 = _mm_loadl_epi64((const __m128i*)(in + i)); 228 const __m128i A1 = _mm_add_epi8(A0, last); 229 const __m128i A2 = _mm_slli_si128(A1, 1); 230 const __m128i A3 = _mm_add_epi8(A1, A2); 231 const __m128i A4 = _mm_slli_si128(A3, 2); 232 const __m128i A5 = _mm_add_epi8(A3, A4); 233 const __m128i A6 = _mm_slli_si128(A5, 4); 234 const __m128i A7 = _mm_add_epi8(A5, A6); 235 _mm_storel_epi64((__m128i*)(out + i), A7); 236 last = _mm_srli_epi64(A7, 56); 237 } 238 for (; i < width; ++i) out[i] = in[i] + out[i - 1]; 239 } 240 241 static void VerticalUnfilter(const uint8_t* prev, const uint8_t* in, 242 uint8_t* out, int width) { 243 if (prev == NULL) { 244 HorizontalUnfilter(NULL, in, out, width); 245 } else { 246 int i; 247 const int max_pos = width & ~31; 248 assert(width >= 0); 249 for (i = 0; i < max_pos; i += 32) { 250 const __m128i A0 = _mm_loadu_si128((const __m128i*)&in[i + 0]); 251 const __m128i A1 = _mm_loadu_si128((const __m128i*)&in[i + 16]); 252 const __m128i B0 = _mm_loadu_si128((const __m128i*)&prev[i + 0]); 253 const __m128i B1 = _mm_loadu_si128((const __m128i*)&prev[i + 16]); 254 const __m128i C0 = _mm_add_epi8(A0, B0); 255 const __m128i C1 = _mm_add_epi8(A1, B1); 256 _mm_storeu_si128((__m128i*)&out[i + 0], C0); 257 _mm_storeu_si128((__m128i*)&out[i + 16], C1); 258 } 259 for (; i < width; ++i) out[i] = in[i] + prev[i]; 260 } 261 } 262 263 static void GradientPredictInverse(const uint8_t* const in, 264 const uint8_t* const top, 265 uint8_t* const row, int length) { 266 if (length > 0) { 267 int i; 268 const int max_pos = length & ~7; 269 const __m128i zero = _mm_setzero_si128(); 270 __m128i A = _mm_set_epi32(0, 0, 0, row[-1]); // left sample 271 for (i = 0; i < max_pos; i += 8) { 272 const __m128i tmp0 = _mm_loadl_epi64((const __m128i*)&top[i]); 273 const __m128i tmp1 = _mm_loadl_epi64((const __m128i*)&top[i - 1]); 274 const __m128i B = _mm_unpacklo_epi8(tmp0, zero); 275 const __m128i C = _mm_unpacklo_epi8(tmp1, zero); 276 const __m128i D = _mm_loadl_epi64((const __m128i*)&in[i]); // base input 277 const __m128i E = _mm_sub_epi16(B, C); // unclipped gradient basis B - C 278 __m128i out = zero; // accumulator for output 279 __m128i mask_hi = _mm_set_epi32(0, 0, 0, 0xff); 280 int k = 8; 281 while (1) { 282 const __m128i tmp3 = _mm_add_epi16(A, E); // delta = A + B - C 283 const __m128i tmp4 = _mm_packus_epi16(tmp3, zero); // saturate delta 284 const __m128i tmp5 = _mm_add_epi8(tmp4, D); // add to in[] 285 A = _mm_and_si128(tmp5, mask_hi); // 1-complement clip 286 out = _mm_or_si128(out, A); // accumulate output 287 if (--k == 0) break; 288 A = _mm_slli_si128(A, 1); // rotate left sample 289 mask_hi = _mm_slli_si128(mask_hi, 1); // rotate mask 290 A = _mm_unpacklo_epi8(A, zero); // convert 8b->16b 291 } 292 A = _mm_srli_si128(A, 7); // prepare left sample for next iteration 293 _mm_storel_epi64((__m128i*)&row[i], out); 294 } 295 for (; i < length; ++i) { 296 row[i] = in[i] + GradientPredictorC(row[i - 1], top[i], top[i - 1]); 297 } 298 } 299 } 300 301 static void GradientUnfilter(const uint8_t* prev, const uint8_t* in, 302 uint8_t* out, int width) { 303 if (prev == NULL) { 304 HorizontalUnfilter(NULL, in, out, width); 305 } else { 306 out[0] = in[0] + prev[0]; // predict from above 307 GradientPredictInverse(in + 1, prev + 1, out + 1, width - 1); 308 } 309 } 310 311 //------------------------------------------------------------------------------ 312 // Entry point 313 314 extern void VP8FiltersInitSSE2(void); 315 316 WEBP_TSAN_IGNORE_FUNCTION void VP8FiltersInitSSE2(void) { 317 WebPUnfilters[WEBP_FILTER_HORIZONTAL] = HorizontalUnfilter; 318 WebPUnfilters[WEBP_FILTER_VERTICAL] = VerticalUnfilter; 319 WebPUnfilters[WEBP_FILTER_GRADIENT] = GradientUnfilter; 320 321 WebPFilters[WEBP_FILTER_HORIZONTAL] = HorizontalFilter; 322 WebPFilters[WEBP_FILTER_VERTICAL] = VerticalFilter; 323 WebPFilters[WEBP_FILTER_GRADIENT] = GradientFilter; 324 } 325 326 #else // !WEBP_USE_SSE2 327 328 WEBP_DSP_INIT_STUB(VP8FiltersInitSSE2) 329 330 #endif // WEBP_USE_SSE2 331