1 // Copyright 2012 Google Inc. All Rights Reserved. 2 // 3 // Use of this source code is governed by a BSD-style license 4 // that can be found in the COPYING file in the root of the source 5 // tree. An additional intellectual property rights grant can be found 6 // in the file PATENTS. All contributing project authors may 7 // be found in the AUTHORS file in the root of the source tree. 8 // ----------------------------------------------------------------------------- 9 // 10 // ARM NEON version of dsp functions and loop filtering. 11 // 12 // Authors: Somnath Banerjee (somnath (at) google.com) 13 // Johann Koenig (johannkoenig (at) google.com) 14 15 #include "./dsp.h" 16 17 #if defined(WEBP_USE_NEON) 18 19 #include "./neon.h" 20 #include "../dec/vp8i_dec.h" 21 22 //------------------------------------------------------------------------------ 23 // NxM Loading functions 24 25 // Load/Store vertical edge 26 #define LOAD8x4(c1, c2, c3, c4, b1, b2, stride) \ 27 "vld4.8 {" #c1 "[0]," #c2 "[0]," #c3 "[0]," #c4 "[0]}," #b1 "," #stride "\n" \ 28 "vld4.8 {" #c1 "[1]," #c2 "[1]," #c3 "[1]," #c4 "[1]}," #b2 "," #stride "\n" \ 29 "vld4.8 {" #c1 "[2]," #c2 "[2]," #c3 "[2]," #c4 "[2]}," #b1 "," #stride "\n" \ 30 "vld4.8 {" #c1 "[3]," #c2 "[3]," #c3 "[3]," #c4 "[3]}," #b2 "," #stride "\n" \ 31 "vld4.8 {" #c1 "[4]," #c2 "[4]," #c3 "[4]," #c4 "[4]}," #b1 "," #stride "\n" \ 32 "vld4.8 {" #c1 "[5]," #c2 "[5]," #c3 "[5]," #c4 "[5]}," #b2 "," #stride "\n" \ 33 "vld4.8 {" #c1 "[6]," #c2 "[6]," #c3 "[6]," #c4 "[6]}," #b1 "," #stride "\n" \ 34 "vld4.8 {" #c1 "[7]," #c2 "[7]," #c3 "[7]," #c4 "[7]}," #b2 "," #stride "\n" 35 36 #define STORE8x2(c1, c2, p, stride) \ 37 "vst2.8 {" #c1 "[0], " #c2 "[0]}," #p "," #stride " \n" \ 38 "vst2.8 {" #c1 "[1], " #c2 "[1]}," #p "," #stride " \n" \ 39 "vst2.8 {" #c1 "[2], " #c2 "[2]}," #p "," #stride " \n" \ 40 "vst2.8 {" #c1 "[3], " #c2 "[3]}," #p "," #stride " \n" \ 41 "vst2.8 {" #c1 "[4], " #c2 "[4]}," #p "," #stride " \n" \ 42 "vst2.8 {" #c1 "[5], " #c2 "[5]}," #p "," #stride " \n" \ 43 "vst2.8 {" #c1 "[6], " #c2 "[6]}," #p "," #stride " \n" \ 44 "vst2.8 {" #c1 "[7], " #c2 "[7]}," #p "," #stride " \n" 45 46 #if !defined(WORK_AROUND_GCC) 47 48 // This intrinsics version makes gcc-4.6.3 crash during Load4x??() compilation 49 // (register alloc, probably). The variants somewhat mitigate the problem, but 50 // not quite. HFilter16i() remains problematic. 51 static WEBP_INLINE uint8x8x4_t Load4x8(const uint8_t* const src, int stride) { 52 const uint8x8_t zero = vdup_n_u8(0); 53 uint8x8x4_t out; 54 INIT_VECTOR4(out, zero, zero, zero, zero); 55 out = vld4_lane_u8(src + 0 * stride, out, 0); 56 out = vld4_lane_u8(src + 1 * stride, out, 1); 57 out = vld4_lane_u8(src + 2 * stride, out, 2); 58 out = vld4_lane_u8(src + 3 * stride, out, 3); 59 out = vld4_lane_u8(src + 4 * stride, out, 4); 60 out = vld4_lane_u8(src + 5 * stride, out, 5); 61 out = vld4_lane_u8(src + 6 * stride, out, 6); 62 out = vld4_lane_u8(src + 7 * stride, out, 7); 63 return out; 64 } 65 66 static WEBP_INLINE void Load4x16(const uint8_t* const src, int stride, 67 uint8x16_t* const p1, uint8x16_t* const p0, 68 uint8x16_t* const q0, uint8x16_t* const q1) { 69 // row0 = p1[0..7]|p0[0..7]|q0[0..7]|q1[0..7] 70 // row8 = p1[8..15]|p0[8..15]|q0[8..15]|q1[8..15] 71 const uint8x8x4_t row0 = Load4x8(src - 2 + 0 * stride, stride); 72 const uint8x8x4_t row8 = Load4x8(src - 2 + 8 * stride, stride); 73 *p1 = vcombine_u8(row0.val[0], row8.val[0]); 74 *p0 = vcombine_u8(row0.val[1], row8.val[1]); 75 *q0 = vcombine_u8(row0.val[2], row8.val[2]); 76 *q1 = vcombine_u8(row0.val[3], row8.val[3]); 77 } 78 79 #else // WORK_AROUND_GCC 80 81 #define LOADQ_LANE_32b(VALUE, LANE) do { \ 82 (VALUE) = vld1q_lane_u32((const uint32_t*)src, (VALUE), (LANE)); \ 83 src += stride; \ 84 } while (0) 85 86 static WEBP_INLINE void Load4x16(const uint8_t* src, int stride, 87 uint8x16_t* const p1, uint8x16_t* const p0, 88 uint8x16_t* const q0, uint8x16_t* const q1) { 89 const uint32x4_t zero = vdupq_n_u32(0); 90 uint32x4x4_t in; 91 INIT_VECTOR4(in, zero, zero, zero, zero); 92 src -= 2; 93 LOADQ_LANE_32b(in.val[0], 0); 94 LOADQ_LANE_32b(in.val[1], 0); 95 LOADQ_LANE_32b(in.val[2], 0); 96 LOADQ_LANE_32b(in.val[3], 0); 97 LOADQ_LANE_32b(in.val[0], 1); 98 LOADQ_LANE_32b(in.val[1], 1); 99 LOADQ_LANE_32b(in.val[2], 1); 100 LOADQ_LANE_32b(in.val[3], 1); 101 LOADQ_LANE_32b(in.val[0], 2); 102 LOADQ_LANE_32b(in.val[1], 2); 103 LOADQ_LANE_32b(in.val[2], 2); 104 LOADQ_LANE_32b(in.val[3], 2); 105 LOADQ_LANE_32b(in.val[0], 3); 106 LOADQ_LANE_32b(in.val[1], 3); 107 LOADQ_LANE_32b(in.val[2], 3); 108 LOADQ_LANE_32b(in.val[3], 3); 109 // Transpose four 4x4 parts: 110 { 111 const uint8x16x2_t row01 = vtrnq_u8(vreinterpretq_u8_u32(in.val[0]), 112 vreinterpretq_u8_u32(in.val[1])); 113 const uint8x16x2_t row23 = vtrnq_u8(vreinterpretq_u8_u32(in.val[2]), 114 vreinterpretq_u8_u32(in.val[3])); 115 const uint16x8x2_t row02 = vtrnq_u16(vreinterpretq_u16_u8(row01.val[0]), 116 vreinterpretq_u16_u8(row23.val[0])); 117 const uint16x8x2_t row13 = vtrnq_u16(vreinterpretq_u16_u8(row01.val[1]), 118 vreinterpretq_u16_u8(row23.val[1])); 119 *p1 = vreinterpretq_u8_u16(row02.val[0]); 120 *p0 = vreinterpretq_u8_u16(row13.val[0]); 121 *q0 = vreinterpretq_u8_u16(row02.val[1]); 122 *q1 = vreinterpretq_u8_u16(row13.val[1]); 123 } 124 } 125 #undef LOADQ_LANE_32b 126 127 #endif // !WORK_AROUND_GCC 128 129 static WEBP_INLINE void Load8x16(const uint8_t* const src, int stride, 130 uint8x16_t* const p3, uint8x16_t* const p2, 131 uint8x16_t* const p1, uint8x16_t* const p0, 132 uint8x16_t* const q0, uint8x16_t* const q1, 133 uint8x16_t* const q2, uint8x16_t* const q3) { 134 Load4x16(src - 2, stride, p3, p2, p1, p0); 135 Load4x16(src + 2, stride, q0, q1, q2, q3); 136 } 137 138 static WEBP_INLINE void Load16x4(const uint8_t* const src, int stride, 139 uint8x16_t* const p1, uint8x16_t* const p0, 140 uint8x16_t* const q0, uint8x16_t* const q1) { 141 *p1 = vld1q_u8(src - 2 * stride); 142 *p0 = vld1q_u8(src - 1 * stride); 143 *q0 = vld1q_u8(src + 0 * stride); 144 *q1 = vld1q_u8(src + 1 * stride); 145 } 146 147 static WEBP_INLINE void Load16x8(const uint8_t* const src, int stride, 148 uint8x16_t* const p3, uint8x16_t* const p2, 149 uint8x16_t* const p1, uint8x16_t* const p0, 150 uint8x16_t* const q0, uint8x16_t* const q1, 151 uint8x16_t* const q2, uint8x16_t* const q3) { 152 Load16x4(src - 2 * stride, stride, p3, p2, p1, p0); 153 Load16x4(src + 2 * stride, stride, q0, q1, q2, q3); 154 } 155 156 static WEBP_INLINE void Load8x8x2(const uint8_t* const u, 157 const uint8_t* const v, 158 int stride, 159 uint8x16_t* const p3, uint8x16_t* const p2, 160 uint8x16_t* const p1, uint8x16_t* const p0, 161 uint8x16_t* const q0, uint8x16_t* const q1, 162 uint8x16_t* const q2, uint8x16_t* const q3) { 163 // We pack the 8x8 u-samples in the lower half of the uint8x16_t destination 164 // and the v-samples on the higher half. 165 *p3 = vcombine_u8(vld1_u8(u - 4 * stride), vld1_u8(v - 4 * stride)); 166 *p2 = vcombine_u8(vld1_u8(u - 3 * stride), vld1_u8(v - 3 * stride)); 167 *p1 = vcombine_u8(vld1_u8(u - 2 * stride), vld1_u8(v - 2 * stride)); 168 *p0 = vcombine_u8(vld1_u8(u - 1 * stride), vld1_u8(v - 1 * stride)); 169 *q0 = vcombine_u8(vld1_u8(u + 0 * stride), vld1_u8(v + 0 * stride)); 170 *q1 = vcombine_u8(vld1_u8(u + 1 * stride), vld1_u8(v + 1 * stride)); 171 *q2 = vcombine_u8(vld1_u8(u + 2 * stride), vld1_u8(v + 2 * stride)); 172 *q3 = vcombine_u8(vld1_u8(u + 3 * stride), vld1_u8(v + 3 * stride)); 173 } 174 175 #if !defined(WORK_AROUND_GCC) 176 177 #define LOAD_UV_8(ROW) \ 178 vcombine_u8(vld1_u8(u - 4 + (ROW) * stride), vld1_u8(v - 4 + (ROW) * stride)) 179 180 static WEBP_INLINE void Load8x8x2T(const uint8_t* const u, 181 const uint8_t* const v, 182 int stride, 183 uint8x16_t* const p3, uint8x16_t* const p2, 184 uint8x16_t* const p1, uint8x16_t* const p0, 185 uint8x16_t* const q0, uint8x16_t* const q1, 186 uint8x16_t* const q2, uint8x16_t* const q3) { 187 // We pack the 8x8 u-samples in the lower half of the uint8x16_t destination 188 // and the v-samples on the higher half. 189 const uint8x16_t row0 = LOAD_UV_8(0); 190 const uint8x16_t row1 = LOAD_UV_8(1); 191 const uint8x16_t row2 = LOAD_UV_8(2); 192 const uint8x16_t row3 = LOAD_UV_8(3); 193 const uint8x16_t row4 = LOAD_UV_8(4); 194 const uint8x16_t row5 = LOAD_UV_8(5); 195 const uint8x16_t row6 = LOAD_UV_8(6); 196 const uint8x16_t row7 = LOAD_UV_8(7); 197 // Perform two side-by-side 8x8 transposes 198 // u00 u01 u02 u03 u04 u05 u06 u07 | v00 v01 v02 v03 v04 v05 v06 v07 199 // u10 u11 u12 u13 u14 u15 u16 u17 | v10 v11 v12 ... 200 // u20 u21 u22 u23 u24 u25 u26 u27 | v20 v21 ... 201 // u30 u31 u32 u33 u34 u35 u36 u37 | ... 202 // u40 u41 u42 u43 u44 u45 u46 u47 | ... 203 // u50 u51 u52 u53 u54 u55 u56 u57 | ... 204 // u60 u61 u62 u63 u64 u65 u66 u67 | v60 ... 205 // u70 u71 u72 u73 u74 u75 u76 u77 | v70 v71 v72 ... 206 const uint8x16x2_t row01 = vtrnq_u8(row0, row1); // u00 u10 u02 u12 ... 207 // u01 u11 u03 u13 ... 208 const uint8x16x2_t row23 = vtrnq_u8(row2, row3); // u20 u30 u22 u32 ... 209 // u21 u31 u23 u33 ... 210 const uint8x16x2_t row45 = vtrnq_u8(row4, row5); // ... 211 const uint8x16x2_t row67 = vtrnq_u8(row6, row7); // ... 212 const uint16x8x2_t row02 = vtrnq_u16(vreinterpretq_u16_u8(row01.val[0]), 213 vreinterpretq_u16_u8(row23.val[0])); 214 const uint16x8x2_t row13 = vtrnq_u16(vreinterpretq_u16_u8(row01.val[1]), 215 vreinterpretq_u16_u8(row23.val[1])); 216 const uint16x8x2_t row46 = vtrnq_u16(vreinterpretq_u16_u8(row45.val[0]), 217 vreinterpretq_u16_u8(row67.val[0])); 218 const uint16x8x2_t row57 = vtrnq_u16(vreinterpretq_u16_u8(row45.val[1]), 219 vreinterpretq_u16_u8(row67.val[1])); 220 const uint32x4x2_t row04 = vtrnq_u32(vreinterpretq_u32_u16(row02.val[0]), 221 vreinterpretq_u32_u16(row46.val[0])); 222 const uint32x4x2_t row26 = vtrnq_u32(vreinterpretq_u32_u16(row02.val[1]), 223 vreinterpretq_u32_u16(row46.val[1])); 224 const uint32x4x2_t row15 = vtrnq_u32(vreinterpretq_u32_u16(row13.val[0]), 225 vreinterpretq_u32_u16(row57.val[0])); 226 const uint32x4x2_t row37 = vtrnq_u32(vreinterpretq_u32_u16(row13.val[1]), 227 vreinterpretq_u32_u16(row57.val[1])); 228 *p3 = vreinterpretq_u8_u32(row04.val[0]); 229 *p2 = vreinterpretq_u8_u32(row15.val[0]); 230 *p1 = vreinterpretq_u8_u32(row26.val[0]); 231 *p0 = vreinterpretq_u8_u32(row37.val[0]); 232 *q0 = vreinterpretq_u8_u32(row04.val[1]); 233 *q1 = vreinterpretq_u8_u32(row15.val[1]); 234 *q2 = vreinterpretq_u8_u32(row26.val[1]); 235 *q3 = vreinterpretq_u8_u32(row37.val[1]); 236 } 237 #undef LOAD_UV_8 238 239 #endif // !WORK_AROUND_GCC 240 241 static WEBP_INLINE void Store2x8(const uint8x8x2_t v, 242 uint8_t* const dst, int stride) { 243 vst2_lane_u8(dst + 0 * stride, v, 0); 244 vst2_lane_u8(dst + 1 * stride, v, 1); 245 vst2_lane_u8(dst + 2 * stride, v, 2); 246 vst2_lane_u8(dst + 3 * stride, v, 3); 247 vst2_lane_u8(dst + 4 * stride, v, 4); 248 vst2_lane_u8(dst + 5 * stride, v, 5); 249 vst2_lane_u8(dst + 6 * stride, v, 6); 250 vst2_lane_u8(dst + 7 * stride, v, 7); 251 } 252 253 static WEBP_INLINE void Store2x16(const uint8x16_t p0, const uint8x16_t q0, 254 uint8_t* const dst, int stride) { 255 uint8x8x2_t lo, hi; 256 lo.val[0] = vget_low_u8(p0); 257 lo.val[1] = vget_low_u8(q0); 258 hi.val[0] = vget_high_u8(p0); 259 hi.val[1] = vget_high_u8(q0); 260 Store2x8(lo, dst - 1 + 0 * stride, stride); 261 Store2x8(hi, dst - 1 + 8 * stride, stride); 262 } 263 264 #if !defined(WORK_AROUND_GCC) 265 static WEBP_INLINE void Store4x8(const uint8x8x4_t v, 266 uint8_t* const dst, int stride) { 267 vst4_lane_u8(dst + 0 * stride, v, 0); 268 vst4_lane_u8(dst + 1 * stride, v, 1); 269 vst4_lane_u8(dst + 2 * stride, v, 2); 270 vst4_lane_u8(dst + 3 * stride, v, 3); 271 vst4_lane_u8(dst + 4 * stride, v, 4); 272 vst4_lane_u8(dst + 5 * stride, v, 5); 273 vst4_lane_u8(dst + 6 * stride, v, 6); 274 vst4_lane_u8(dst + 7 * stride, v, 7); 275 } 276 277 static WEBP_INLINE void Store4x16(const uint8x16_t p1, const uint8x16_t p0, 278 const uint8x16_t q0, const uint8x16_t q1, 279 uint8_t* const dst, int stride) { 280 uint8x8x4_t lo, hi; 281 INIT_VECTOR4(lo, 282 vget_low_u8(p1), vget_low_u8(p0), 283 vget_low_u8(q0), vget_low_u8(q1)); 284 INIT_VECTOR4(hi, 285 vget_high_u8(p1), vget_high_u8(p0), 286 vget_high_u8(q0), vget_high_u8(q1)); 287 Store4x8(lo, dst - 2 + 0 * stride, stride); 288 Store4x8(hi, dst - 2 + 8 * stride, stride); 289 } 290 #endif // !WORK_AROUND_GCC 291 292 static WEBP_INLINE void Store16x2(const uint8x16_t p0, const uint8x16_t q0, 293 uint8_t* const dst, int stride) { 294 vst1q_u8(dst - stride, p0); 295 vst1q_u8(dst, q0); 296 } 297 298 static WEBP_INLINE void Store16x4(const uint8x16_t p1, const uint8x16_t p0, 299 const uint8x16_t q0, const uint8x16_t q1, 300 uint8_t* const dst, int stride) { 301 Store16x2(p1, p0, dst - stride, stride); 302 Store16x2(q0, q1, dst + stride, stride); 303 } 304 305 static WEBP_INLINE void Store8x2x2(const uint8x16_t p0, const uint8x16_t q0, 306 uint8_t* const u, uint8_t* const v, 307 int stride) { 308 // p0 and q0 contain the u+v samples packed in low/high halves. 309 vst1_u8(u - stride, vget_low_u8(p0)); 310 vst1_u8(u, vget_low_u8(q0)); 311 vst1_u8(v - stride, vget_high_u8(p0)); 312 vst1_u8(v, vget_high_u8(q0)); 313 } 314 315 static WEBP_INLINE void Store8x4x2(const uint8x16_t p1, const uint8x16_t p0, 316 const uint8x16_t q0, const uint8x16_t q1, 317 uint8_t* const u, uint8_t* const v, 318 int stride) { 319 // The p1...q1 registers contain the u+v samples packed in low/high halves. 320 Store8x2x2(p1, p0, u - stride, v - stride, stride); 321 Store8x2x2(q0, q1, u + stride, v + stride, stride); 322 } 323 324 #if !defined(WORK_AROUND_GCC) 325 326 #define STORE6_LANE(DST, VAL0, VAL1, LANE) do { \ 327 vst3_lane_u8((DST) - 3, (VAL0), (LANE)); \ 328 vst3_lane_u8((DST) + 0, (VAL1), (LANE)); \ 329 (DST) += stride; \ 330 } while (0) 331 332 static WEBP_INLINE void Store6x8x2(const uint8x16_t p2, const uint8x16_t p1, 333 const uint8x16_t p0, const uint8x16_t q0, 334 const uint8x16_t q1, const uint8x16_t q2, 335 uint8_t* u, uint8_t* v, 336 int stride) { 337 uint8x8x3_t u0, u1, v0, v1; 338 INIT_VECTOR3(u0, vget_low_u8(p2), vget_low_u8(p1), vget_low_u8(p0)); 339 INIT_VECTOR3(u1, vget_low_u8(q0), vget_low_u8(q1), vget_low_u8(q2)); 340 INIT_VECTOR3(v0, vget_high_u8(p2), vget_high_u8(p1), vget_high_u8(p0)); 341 INIT_VECTOR3(v1, vget_high_u8(q0), vget_high_u8(q1), vget_high_u8(q2)); 342 STORE6_LANE(u, u0, u1, 0); 343 STORE6_LANE(u, u0, u1, 1); 344 STORE6_LANE(u, u0, u1, 2); 345 STORE6_LANE(u, u0, u1, 3); 346 STORE6_LANE(u, u0, u1, 4); 347 STORE6_LANE(u, u0, u1, 5); 348 STORE6_LANE(u, u0, u1, 6); 349 STORE6_LANE(u, u0, u1, 7); 350 STORE6_LANE(v, v0, v1, 0); 351 STORE6_LANE(v, v0, v1, 1); 352 STORE6_LANE(v, v0, v1, 2); 353 STORE6_LANE(v, v0, v1, 3); 354 STORE6_LANE(v, v0, v1, 4); 355 STORE6_LANE(v, v0, v1, 5); 356 STORE6_LANE(v, v0, v1, 6); 357 STORE6_LANE(v, v0, v1, 7); 358 } 359 #undef STORE6_LANE 360 361 static WEBP_INLINE void Store4x8x2(const uint8x16_t p1, const uint8x16_t p0, 362 const uint8x16_t q0, const uint8x16_t q1, 363 uint8_t* const u, uint8_t* const v, 364 int stride) { 365 uint8x8x4_t u0, v0; 366 INIT_VECTOR4(u0, 367 vget_low_u8(p1), vget_low_u8(p0), 368 vget_low_u8(q0), vget_low_u8(q1)); 369 INIT_VECTOR4(v0, 370 vget_high_u8(p1), vget_high_u8(p0), 371 vget_high_u8(q0), vget_high_u8(q1)); 372 vst4_lane_u8(u - 2 + 0 * stride, u0, 0); 373 vst4_lane_u8(u - 2 + 1 * stride, u0, 1); 374 vst4_lane_u8(u - 2 + 2 * stride, u0, 2); 375 vst4_lane_u8(u - 2 + 3 * stride, u0, 3); 376 vst4_lane_u8(u - 2 + 4 * stride, u0, 4); 377 vst4_lane_u8(u - 2 + 5 * stride, u0, 5); 378 vst4_lane_u8(u - 2 + 6 * stride, u0, 6); 379 vst4_lane_u8(u - 2 + 7 * stride, u0, 7); 380 vst4_lane_u8(v - 2 + 0 * stride, v0, 0); 381 vst4_lane_u8(v - 2 + 1 * stride, v0, 1); 382 vst4_lane_u8(v - 2 + 2 * stride, v0, 2); 383 vst4_lane_u8(v - 2 + 3 * stride, v0, 3); 384 vst4_lane_u8(v - 2 + 4 * stride, v0, 4); 385 vst4_lane_u8(v - 2 + 5 * stride, v0, 5); 386 vst4_lane_u8(v - 2 + 6 * stride, v0, 6); 387 vst4_lane_u8(v - 2 + 7 * stride, v0, 7); 388 } 389 390 #endif // !WORK_AROUND_GCC 391 392 // Zero extend 'v' to an int16x8_t. 393 static WEBP_INLINE int16x8_t ConvertU8ToS16(uint8x8_t v) { 394 return vreinterpretq_s16_u16(vmovl_u8(v)); 395 } 396 397 // Performs unsigned 8b saturation on 'dst01' and 'dst23' storing the result 398 // to the corresponding rows of 'dst'. 399 static WEBP_INLINE void SaturateAndStore4x4(uint8_t* const dst, 400 const int16x8_t dst01, 401 const int16x8_t dst23) { 402 // Unsigned saturate to 8b. 403 const uint8x8_t dst01_u8 = vqmovun_s16(dst01); 404 const uint8x8_t dst23_u8 = vqmovun_s16(dst23); 405 406 // Store the results. 407 vst1_lane_u32((uint32_t*)(dst + 0 * BPS), vreinterpret_u32_u8(dst01_u8), 0); 408 vst1_lane_u32((uint32_t*)(dst + 1 * BPS), vreinterpret_u32_u8(dst01_u8), 1); 409 vst1_lane_u32((uint32_t*)(dst + 2 * BPS), vreinterpret_u32_u8(dst23_u8), 0); 410 vst1_lane_u32((uint32_t*)(dst + 3 * BPS), vreinterpret_u32_u8(dst23_u8), 1); 411 } 412 413 static WEBP_INLINE void Add4x4(const int16x8_t row01, const int16x8_t row23, 414 uint8_t* const dst) { 415 uint32x2_t dst01 = vdup_n_u32(0); 416 uint32x2_t dst23 = vdup_n_u32(0); 417 418 // Load the source pixels. 419 dst01 = vld1_lane_u32((uint32_t*)(dst + 0 * BPS), dst01, 0); 420 dst23 = vld1_lane_u32((uint32_t*)(dst + 2 * BPS), dst23, 0); 421 dst01 = vld1_lane_u32((uint32_t*)(dst + 1 * BPS), dst01, 1); 422 dst23 = vld1_lane_u32((uint32_t*)(dst + 3 * BPS), dst23, 1); 423 424 { 425 // Convert to 16b. 426 const int16x8_t dst01_s16 = ConvertU8ToS16(vreinterpret_u8_u32(dst01)); 427 const int16x8_t dst23_s16 = ConvertU8ToS16(vreinterpret_u8_u32(dst23)); 428 429 // Descale with rounding. 430 const int16x8_t out01 = vrsraq_n_s16(dst01_s16, row01, 3); 431 const int16x8_t out23 = vrsraq_n_s16(dst23_s16, row23, 3); 432 // Add the inverse transform. 433 SaturateAndStore4x4(dst, out01, out23); 434 } 435 } 436 437 //----------------------------------------------------------------------------- 438 // Simple In-loop filtering (Paragraph 15.2) 439 440 static uint8x16_t NeedsFilter(const uint8x16_t p1, const uint8x16_t p0, 441 const uint8x16_t q0, const uint8x16_t q1, 442 int thresh) { 443 const uint8x16_t thresh_v = vdupq_n_u8((uint8_t)thresh); 444 const uint8x16_t a_p0_q0 = vabdq_u8(p0, q0); // abs(p0-q0) 445 const uint8x16_t a_p1_q1 = vabdq_u8(p1, q1); // abs(p1-q1) 446 const uint8x16_t a_p0_q0_2 = vqaddq_u8(a_p0_q0, a_p0_q0); // 2 * abs(p0-q0) 447 const uint8x16_t a_p1_q1_2 = vshrq_n_u8(a_p1_q1, 1); // abs(p1-q1) / 2 448 const uint8x16_t sum = vqaddq_u8(a_p0_q0_2, a_p1_q1_2); 449 const uint8x16_t mask = vcgeq_u8(thresh_v, sum); 450 return mask; 451 } 452 453 static int8x16_t FlipSign(const uint8x16_t v) { 454 const uint8x16_t sign_bit = vdupq_n_u8(0x80); 455 return vreinterpretq_s8_u8(veorq_u8(v, sign_bit)); 456 } 457 458 static uint8x16_t FlipSignBack(const int8x16_t v) { 459 const int8x16_t sign_bit = vdupq_n_s8(0x80); 460 return vreinterpretq_u8_s8(veorq_s8(v, sign_bit)); 461 } 462 463 static int8x16_t GetBaseDelta(const int8x16_t p1, const int8x16_t p0, 464 const int8x16_t q0, const int8x16_t q1) { 465 const int8x16_t q0_p0 = vqsubq_s8(q0, p0); // (q0-p0) 466 const int8x16_t p1_q1 = vqsubq_s8(p1, q1); // (p1-q1) 467 const int8x16_t s1 = vqaddq_s8(p1_q1, q0_p0); // (p1-q1) + 1 * (q0 - p0) 468 const int8x16_t s2 = vqaddq_s8(q0_p0, s1); // (p1-q1) + 2 * (q0 - p0) 469 const int8x16_t s3 = vqaddq_s8(q0_p0, s2); // (p1-q1) + 3 * (q0 - p0) 470 return s3; 471 } 472 473 static int8x16_t GetBaseDelta0(const int8x16_t p0, const int8x16_t q0) { 474 const int8x16_t q0_p0 = vqsubq_s8(q0, p0); // (q0-p0) 475 const int8x16_t s1 = vqaddq_s8(q0_p0, q0_p0); // 2 * (q0 - p0) 476 const int8x16_t s2 = vqaddq_s8(q0_p0, s1); // 3 * (q0 - p0) 477 return s2; 478 } 479 480 //------------------------------------------------------------------------------ 481 482 static void ApplyFilter2NoFlip(const int8x16_t p0s, const int8x16_t q0s, 483 const int8x16_t delta, 484 int8x16_t* const op0, int8x16_t* const oq0) { 485 const int8x16_t kCst3 = vdupq_n_s8(0x03); 486 const int8x16_t kCst4 = vdupq_n_s8(0x04); 487 const int8x16_t delta_p3 = vqaddq_s8(delta, kCst3); 488 const int8x16_t delta_p4 = vqaddq_s8(delta, kCst4); 489 const int8x16_t delta3 = vshrq_n_s8(delta_p3, 3); 490 const int8x16_t delta4 = vshrq_n_s8(delta_p4, 3); 491 *op0 = vqaddq_s8(p0s, delta3); 492 *oq0 = vqsubq_s8(q0s, delta4); 493 } 494 495 #if defined(WEBP_USE_INTRINSICS) 496 497 static void ApplyFilter2(const int8x16_t p0s, const int8x16_t q0s, 498 const int8x16_t delta, 499 uint8x16_t* const op0, uint8x16_t* const oq0) { 500 const int8x16_t kCst3 = vdupq_n_s8(0x03); 501 const int8x16_t kCst4 = vdupq_n_s8(0x04); 502 const int8x16_t delta_p3 = vqaddq_s8(delta, kCst3); 503 const int8x16_t delta_p4 = vqaddq_s8(delta, kCst4); 504 const int8x16_t delta3 = vshrq_n_s8(delta_p3, 3); 505 const int8x16_t delta4 = vshrq_n_s8(delta_p4, 3); 506 const int8x16_t sp0 = vqaddq_s8(p0s, delta3); 507 const int8x16_t sq0 = vqsubq_s8(q0s, delta4); 508 *op0 = FlipSignBack(sp0); 509 *oq0 = FlipSignBack(sq0); 510 } 511 512 static void DoFilter2(const uint8x16_t p1, const uint8x16_t p0, 513 const uint8x16_t q0, const uint8x16_t q1, 514 const uint8x16_t mask, 515 uint8x16_t* const op0, uint8x16_t* const oq0) { 516 const int8x16_t p1s = FlipSign(p1); 517 const int8x16_t p0s = FlipSign(p0); 518 const int8x16_t q0s = FlipSign(q0); 519 const int8x16_t q1s = FlipSign(q1); 520 const int8x16_t delta0 = GetBaseDelta(p1s, p0s, q0s, q1s); 521 const int8x16_t delta1 = vandq_s8(delta0, vreinterpretq_s8_u8(mask)); 522 ApplyFilter2(p0s, q0s, delta1, op0, oq0); 523 } 524 525 static void SimpleVFilter16(uint8_t* p, int stride, int thresh) { 526 uint8x16_t p1, p0, q0, q1, op0, oq0; 527 Load16x4(p, stride, &p1, &p0, &q0, &q1); 528 { 529 const uint8x16_t mask = NeedsFilter(p1, p0, q0, q1, thresh); 530 DoFilter2(p1, p0, q0, q1, mask, &op0, &oq0); 531 } 532 Store16x2(op0, oq0, p, stride); 533 } 534 535 static void SimpleHFilter16(uint8_t* p, int stride, int thresh) { 536 uint8x16_t p1, p0, q0, q1, oq0, op0; 537 Load4x16(p, stride, &p1, &p0, &q0, &q1); 538 { 539 const uint8x16_t mask = NeedsFilter(p1, p0, q0, q1, thresh); 540 DoFilter2(p1, p0, q0, q1, mask, &op0, &oq0); 541 } 542 Store2x16(op0, oq0, p, stride); 543 } 544 545 #else 546 547 #define QRegs "q0", "q1", "q2", "q3", \ 548 "q8", "q9", "q10", "q11", "q12", "q13", "q14", "q15" 549 550 #define FLIP_SIGN_BIT2(a, b, s) \ 551 "veor " #a "," #a "," #s " \n" \ 552 "veor " #b "," #b "," #s " \n" \ 553 554 #define FLIP_SIGN_BIT4(a, b, c, d, s) \ 555 FLIP_SIGN_BIT2(a, b, s) \ 556 FLIP_SIGN_BIT2(c, d, s) \ 557 558 #define NEEDS_FILTER(p1, p0, q0, q1, thresh, mask) \ 559 "vabd.u8 q15," #p0 "," #q0 " \n" /* abs(p0 - q0) */ \ 560 "vabd.u8 q14," #p1 "," #q1 " \n" /* abs(p1 - q1) */ \ 561 "vqadd.u8 q15, q15, q15 \n" /* abs(p0 - q0) * 2 */ \ 562 "vshr.u8 q14, q14, #1 \n" /* abs(p1 - q1) / 2 */ \ 563 "vqadd.u8 q15, q15, q14 \n" /* abs(p0 - q0) * 2 + abs(p1 - q1) / 2 */ \ 564 "vdup.8 q14, " #thresh " \n" \ 565 "vcge.u8 " #mask ", q14, q15 \n" /* mask <= thresh */ 566 567 #define GET_BASE_DELTA(p1, p0, q0, q1, o) \ 568 "vqsub.s8 q15," #q0 "," #p0 " \n" /* (q0 - p0) */ \ 569 "vqsub.s8 " #o "," #p1 "," #q1 " \n" /* (p1 - q1) */ \ 570 "vqadd.s8 " #o "," #o ", q15 \n" /* (p1 - q1) + 1 * (p0 - q0) */ \ 571 "vqadd.s8 " #o "," #o ", q15 \n" /* (p1 - q1) + 2 * (p0 - q0) */ \ 572 "vqadd.s8 " #o "," #o ", q15 \n" /* (p1 - q1) + 3 * (p0 - q0) */ 573 574 #define DO_SIMPLE_FILTER(p0, q0, fl) \ 575 "vmov.i8 q15, #0x03 \n" \ 576 "vqadd.s8 q15, q15, " #fl " \n" /* filter1 = filter + 3 */ \ 577 "vshr.s8 q15, q15, #3 \n" /* filter1 >> 3 */ \ 578 "vqadd.s8 " #p0 "," #p0 ", q15 \n" /* p0 += filter1 */ \ 579 \ 580 "vmov.i8 q15, #0x04 \n" \ 581 "vqadd.s8 q15, q15, " #fl " \n" /* filter1 = filter + 4 */ \ 582 "vshr.s8 q15, q15, #3 \n" /* filter2 >> 3 */ \ 583 "vqsub.s8 " #q0 "," #q0 ", q15 \n" /* q0 -= filter2 */ 584 585 // Applies filter on 2 pixels (p0 and q0) 586 #define DO_FILTER2(p1, p0, q0, q1, thresh) \ 587 NEEDS_FILTER(p1, p0, q0, q1, thresh, q9) /* filter mask in q9 */ \ 588 "vmov.i8 q10, #0x80 \n" /* sign bit */ \ 589 FLIP_SIGN_BIT4(p1, p0, q0, q1, q10) /* convert to signed value */ \ 590 GET_BASE_DELTA(p1, p0, q0, q1, q11) /* get filter level */ \ 591 "vand q9, q9, q11 \n" /* apply filter mask */ \ 592 DO_SIMPLE_FILTER(p0, q0, q9) /* apply filter */ \ 593 FLIP_SIGN_BIT2(p0, q0, q10) 594 595 static void SimpleVFilter16(uint8_t* p, int stride, int thresh) { 596 __asm__ volatile ( 597 "sub %[p], %[p], %[stride], lsl #1 \n" // p -= 2 * stride 598 599 "vld1.u8 {q1}, [%[p]], %[stride] \n" // p1 600 "vld1.u8 {q2}, [%[p]], %[stride] \n" // p0 601 "vld1.u8 {q3}, [%[p]], %[stride] \n" // q0 602 "vld1.u8 {q12}, [%[p]] \n" // q1 603 604 DO_FILTER2(q1, q2, q3, q12, %[thresh]) 605 606 "sub %[p], %[p], %[stride], lsl #1 \n" // p -= 2 * stride 607 608 "vst1.u8 {q2}, [%[p]], %[stride] \n" // store op0 609 "vst1.u8 {q3}, [%[p]] \n" // store oq0 610 : [p] "+r"(p) 611 : [stride] "r"(stride), [thresh] "r"(thresh) 612 : "memory", QRegs 613 ); 614 } 615 616 static void SimpleHFilter16(uint8_t* p, int stride, int thresh) { 617 __asm__ volatile ( 618 "sub r4, %[p], #2 \n" // base1 = p - 2 619 "lsl r6, %[stride], #1 \n" // r6 = 2 * stride 620 "add r5, r4, %[stride] \n" // base2 = base1 + stride 621 622 LOAD8x4(d2, d3, d4, d5, [r4], [r5], r6) 623 LOAD8x4(d24, d25, d26, d27, [r4], [r5], r6) 624 "vswp d3, d24 \n" // p1:q1 p0:q3 625 "vswp d5, d26 \n" // q0:q2 q1:q4 626 "vswp q2, q12 \n" // p1:q1 p0:q2 q0:q3 q1:q4 627 628 DO_FILTER2(q1, q2, q12, q13, %[thresh]) 629 630 "sub %[p], %[p], #1 \n" // p - 1 631 632 "vswp d5, d24 \n" 633 STORE8x2(d4, d5, [%[p]], %[stride]) 634 STORE8x2(d24, d25, [%[p]], %[stride]) 635 636 : [p] "+r"(p) 637 : [stride] "r"(stride), [thresh] "r"(thresh) 638 : "memory", "r4", "r5", "r6", QRegs 639 ); 640 } 641 642 #endif // WEBP_USE_INTRINSICS 643 644 static void SimpleVFilter16i(uint8_t* p, int stride, int thresh) { 645 uint32_t k; 646 for (k = 3; k != 0; --k) { 647 p += 4 * stride; 648 SimpleVFilter16(p, stride, thresh); 649 } 650 } 651 652 static void SimpleHFilter16i(uint8_t* p, int stride, int thresh) { 653 uint32_t k; 654 for (k = 3; k != 0; --k) { 655 p += 4; 656 SimpleHFilter16(p, stride, thresh); 657 } 658 } 659 660 //------------------------------------------------------------------------------ 661 // Complex In-loop filtering (Paragraph 15.3) 662 663 static uint8x16_t NeedsHev(const uint8x16_t p1, const uint8x16_t p0, 664 const uint8x16_t q0, const uint8x16_t q1, 665 int hev_thresh) { 666 const uint8x16_t hev_thresh_v = vdupq_n_u8((uint8_t)hev_thresh); 667 const uint8x16_t a_p1_p0 = vabdq_u8(p1, p0); // abs(p1 - p0) 668 const uint8x16_t a_q1_q0 = vabdq_u8(q1, q0); // abs(q1 - q0) 669 const uint8x16_t a_max = vmaxq_u8(a_p1_p0, a_q1_q0); 670 const uint8x16_t mask = vcgtq_u8(a_max, hev_thresh_v); 671 return mask; 672 } 673 674 static uint8x16_t NeedsFilter2(const uint8x16_t p3, const uint8x16_t p2, 675 const uint8x16_t p1, const uint8x16_t p0, 676 const uint8x16_t q0, const uint8x16_t q1, 677 const uint8x16_t q2, const uint8x16_t q3, 678 int ithresh, int thresh) { 679 const uint8x16_t ithresh_v = vdupq_n_u8((uint8_t)ithresh); 680 const uint8x16_t a_p3_p2 = vabdq_u8(p3, p2); // abs(p3 - p2) 681 const uint8x16_t a_p2_p1 = vabdq_u8(p2, p1); // abs(p2 - p1) 682 const uint8x16_t a_p1_p0 = vabdq_u8(p1, p0); // abs(p1 - p0) 683 const uint8x16_t a_q3_q2 = vabdq_u8(q3, q2); // abs(q3 - q2) 684 const uint8x16_t a_q2_q1 = vabdq_u8(q2, q1); // abs(q2 - q1) 685 const uint8x16_t a_q1_q0 = vabdq_u8(q1, q0); // abs(q1 - q0) 686 const uint8x16_t max1 = vmaxq_u8(a_p3_p2, a_p2_p1); 687 const uint8x16_t max2 = vmaxq_u8(a_p1_p0, a_q3_q2); 688 const uint8x16_t max3 = vmaxq_u8(a_q2_q1, a_q1_q0); 689 const uint8x16_t max12 = vmaxq_u8(max1, max2); 690 const uint8x16_t max123 = vmaxq_u8(max12, max3); 691 const uint8x16_t mask2 = vcgeq_u8(ithresh_v, max123); 692 const uint8x16_t mask1 = NeedsFilter(p1, p0, q0, q1, thresh); 693 const uint8x16_t mask = vandq_u8(mask1, mask2); 694 return mask; 695 } 696 697 // 4-points filter 698 699 static void ApplyFilter4( 700 const int8x16_t p1, const int8x16_t p0, 701 const int8x16_t q0, const int8x16_t q1, 702 const int8x16_t delta0, 703 uint8x16_t* const op1, uint8x16_t* const op0, 704 uint8x16_t* const oq0, uint8x16_t* const oq1) { 705 const int8x16_t kCst3 = vdupq_n_s8(0x03); 706 const int8x16_t kCst4 = vdupq_n_s8(0x04); 707 const int8x16_t delta1 = vqaddq_s8(delta0, kCst4); 708 const int8x16_t delta2 = vqaddq_s8(delta0, kCst3); 709 const int8x16_t a1 = vshrq_n_s8(delta1, 3); 710 const int8x16_t a2 = vshrq_n_s8(delta2, 3); 711 const int8x16_t a3 = vrshrq_n_s8(a1, 1); // a3 = (a1 + 1) >> 1 712 *op0 = FlipSignBack(vqaddq_s8(p0, a2)); // clip(p0 + a2) 713 *oq0 = FlipSignBack(vqsubq_s8(q0, a1)); // clip(q0 - a1) 714 *op1 = FlipSignBack(vqaddq_s8(p1, a3)); // clip(p1 + a3) 715 *oq1 = FlipSignBack(vqsubq_s8(q1, a3)); // clip(q1 - a3) 716 } 717 718 static void DoFilter4( 719 const uint8x16_t p1, const uint8x16_t p0, 720 const uint8x16_t q0, const uint8x16_t q1, 721 const uint8x16_t mask, const uint8x16_t hev_mask, 722 uint8x16_t* const op1, uint8x16_t* const op0, 723 uint8x16_t* const oq0, uint8x16_t* const oq1) { 724 // This is a fused version of DoFilter2() calling ApplyFilter2 directly 725 const int8x16_t p1s = FlipSign(p1); 726 int8x16_t p0s = FlipSign(p0); 727 int8x16_t q0s = FlipSign(q0); 728 const int8x16_t q1s = FlipSign(q1); 729 const uint8x16_t simple_lf_mask = vandq_u8(mask, hev_mask); 730 731 // do_filter2 part (simple loopfilter on pixels with hev) 732 { 733 const int8x16_t delta = GetBaseDelta(p1s, p0s, q0s, q1s); 734 const int8x16_t simple_lf_delta = 735 vandq_s8(delta, vreinterpretq_s8_u8(simple_lf_mask)); 736 ApplyFilter2NoFlip(p0s, q0s, simple_lf_delta, &p0s, &q0s); 737 } 738 739 // do_filter4 part (complex loopfilter on pixels without hev) 740 { 741 const int8x16_t delta0 = GetBaseDelta0(p0s, q0s); 742 // we use: (mask & hev_mask) ^ mask = mask & !hev_mask 743 const uint8x16_t complex_lf_mask = veorq_u8(simple_lf_mask, mask); 744 const int8x16_t complex_lf_delta = 745 vandq_s8(delta0, vreinterpretq_s8_u8(complex_lf_mask)); 746 ApplyFilter4(p1s, p0s, q0s, q1s, complex_lf_delta, op1, op0, oq0, oq1); 747 } 748 } 749 750 // 6-points filter 751 752 static void ApplyFilter6( 753 const int8x16_t p2, const int8x16_t p1, const int8x16_t p0, 754 const int8x16_t q0, const int8x16_t q1, const int8x16_t q2, 755 const int8x16_t delta, 756 uint8x16_t* const op2, uint8x16_t* const op1, uint8x16_t* const op0, 757 uint8x16_t* const oq0, uint8x16_t* const oq1, uint8x16_t* const oq2) { 758 // We have to compute: X = (9*a+63) >> 7, Y = (18*a+63)>>7, Z = (27*a+63) >> 7 759 // Turns out, there's a common sub-expression S=9 * a - 1 that can be used 760 // with the special vqrshrn_n_s16 rounding-shift-and-narrow instruction: 761 // X = (S + 64) >> 7, Y = (S + 32) >> 6, Z = (18 * a + S + 64) >> 7 762 const int8x8_t delta_lo = vget_low_s8(delta); 763 const int8x8_t delta_hi = vget_high_s8(delta); 764 const int8x8_t kCst9 = vdup_n_s8(9); 765 const int16x8_t kCstm1 = vdupq_n_s16(-1); 766 const int8x8_t kCst18 = vdup_n_s8(18); 767 const int16x8_t S_lo = vmlal_s8(kCstm1, kCst9, delta_lo); // S = 9 * a - 1 768 const int16x8_t S_hi = vmlal_s8(kCstm1, kCst9, delta_hi); 769 const int16x8_t Z_lo = vmlal_s8(S_lo, kCst18, delta_lo); // S + 18 * a 770 const int16x8_t Z_hi = vmlal_s8(S_hi, kCst18, delta_hi); 771 const int8x8_t a3_lo = vqrshrn_n_s16(S_lo, 7); // (9 * a + 63) >> 7 772 const int8x8_t a3_hi = vqrshrn_n_s16(S_hi, 7); 773 const int8x8_t a2_lo = vqrshrn_n_s16(S_lo, 6); // (9 * a + 31) >> 6 774 const int8x8_t a2_hi = vqrshrn_n_s16(S_hi, 6); 775 const int8x8_t a1_lo = vqrshrn_n_s16(Z_lo, 7); // (27 * a + 63) >> 7 776 const int8x8_t a1_hi = vqrshrn_n_s16(Z_hi, 7); 777 const int8x16_t a1 = vcombine_s8(a1_lo, a1_hi); 778 const int8x16_t a2 = vcombine_s8(a2_lo, a2_hi); 779 const int8x16_t a3 = vcombine_s8(a3_lo, a3_hi); 780 781 *op0 = FlipSignBack(vqaddq_s8(p0, a1)); // clip(p0 + a1) 782 *oq0 = FlipSignBack(vqsubq_s8(q0, a1)); // clip(q0 - q1) 783 *oq1 = FlipSignBack(vqsubq_s8(q1, a2)); // clip(q1 - a2) 784 *op1 = FlipSignBack(vqaddq_s8(p1, a2)); // clip(p1 + a2) 785 *oq2 = FlipSignBack(vqsubq_s8(q2, a3)); // clip(q2 - a3) 786 *op2 = FlipSignBack(vqaddq_s8(p2, a3)); // clip(p2 + a3) 787 } 788 789 static void DoFilter6( 790 const uint8x16_t p2, const uint8x16_t p1, const uint8x16_t p0, 791 const uint8x16_t q0, const uint8x16_t q1, const uint8x16_t q2, 792 const uint8x16_t mask, const uint8x16_t hev_mask, 793 uint8x16_t* const op2, uint8x16_t* const op1, uint8x16_t* const op0, 794 uint8x16_t* const oq0, uint8x16_t* const oq1, uint8x16_t* const oq2) { 795 // This is a fused version of DoFilter2() calling ApplyFilter2 directly 796 const int8x16_t p2s = FlipSign(p2); 797 const int8x16_t p1s = FlipSign(p1); 798 int8x16_t p0s = FlipSign(p0); 799 int8x16_t q0s = FlipSign(q0); 800 const int8x16_t q1s = FlipSign(q1); 801 const int8x16_t q2s = FlipSign(q2); 802 const uint8x16_t simple_lf_mask = vandq_u8(mask, hev_mask); 803 const int8x16_t delta0 = GetBaseDelta(p1s, p0s, q0s, q1s); 804 805 // do_filter2 part (simple loopfilter on pixels with hev) 806 { 807 const int8x16_t simple_lf_delta = 808 vandq_s8(delta0, vreinterpretq_s8_u8(simple_lf_mask)); 809 ApplyFilter2NoFlip(p0s, q0s, simple_lf_delta, &p0s, &q0s); 810 } 811 812 // do_filter6 part (complex loopfilter on pixels without hev) 813 { 814 // we use: (mask & hev_mask) ^ mask = mask & !hev_mask 815 const uint8x16_t complex_lf_mask = veorq_u8(simple_lf_mask, mask); 816 const int8x16_t complex_lf_delta = 817 vandq_s8(delta0, vreinterpretq_s8_u8(complex_lf_mask)); 818 ApplyFilter6(p2s, p1s, p0s, q0s, q1s, q2s, complex_lf_delta, 819 op2, op1, op0, oq0, oq1, oq2); 820 } 821 } 822 823 // on macroblock edges 824 825 static void VFilter16(uint8_t* p, int stride, 826 int thresh, int ithresh, int hev_thresh) { 827 uint8x16_t p3, p2, p1, p0, q0, q1, q2, q3; 828 Load16x8(p, stride, &p3, &p2, &p1, &p0, &q0, &q1, &q2, &q3); 829 { 830 const uint8x16_t mask = NeedsFilter2(p3, p2, p1, p0, q0, q1, q2, q3, 831 ithresh, thresh); 832 const uint8x16_t hev_mask = NeedsHev(p1, p0, q0, q1, hev_thresh); 833 uint8x16_t op2, op1, op0, oq0, oq1, oq2; 834 DoFilter6(p2, p1, p0, q0, q1, q2, mask, hev_mask, 835 &op2, &op1, &op0, &oq0, &oq1, &oq2); 836 Store16x2(op2, op1, p - 2 * stride, stride); 837 Store16x2(op0, oq0, p + 0 * stride, stride); 838 Store16x2(oq1, oq2, p + 2 * stride, stride); 839 } 840 } 841 842 static void HFilter16(uint8_t* p, int stride, 843 int thresh, int ithresh, int hev_thresh) { 844 uint8x16_t p3, p2, p1, p0, q0, q1, q2, q3; 845 Load8x16(p, stride, &p3, &p2, &p1, &p0, &q0, &q1, &q2, &q3); 846 { 847 const uint8x16_t mask = NeedsFilter2(p3, p2, p1, p0, q0, q1, q2, q3, 848 ithresh, thresh); 849 const uint8x16_t hev_mask = NeedsHev(p1, p0, q0, q1, hev_thresh); 850 uint8x16_t op2, op1, op0, oq0, oq1, oq2; 851 DoFilter6(p2, p1, p0, q0, q1, q2, mask, hev_mask, 852 &op2, &op1, &op0, &oq0, &oq1, &oq2); 853 Store2x16(op2, op1, p - 2, stride); 854 Store2x16(op0, oq0, p + 0, stride); 855 Store2x16(oq1, oq2, p + 2, stride); 856 } 857 } 858 859 // on three inner edges 860 static void VFilter16i(uint8_t* p, int stride, 861 int thresh, int ithresh, int hev_thresh) { 862 uint32_t k; 863 uint8x16_t p3, p2, p1, p0; 864 Load16x4(p + 2 * stride, stride, &p3, &p2, &p1, &p0); 865 for (k = 3; k != 0; --k) { 866 uint8x16_t q0, q1, q2, q3; 867 p += 4 * stride; 868 Load16x4(p + 2 * stride, stride, &q0, &q1, &q2, &q3); 869 { 870 const uint8x16_t mask = 871 NeedsFilter2(p3, p2, p1, p0, q0, q1, q2, q3, ithresh, thresh); 872 const uint8x16_t hev_mask = NeedsHev(p1, p0, q0, q1, hev_thresh); 873 // p3 and p2 are not just temporary variables here: they will be 874 // re-used for next span. And q2/q3 will become p1/p0 accordingly. 875 DoFilter4(p1, p0, q0, q1, mask, hev_mask, &p1, &p0, &p3, &p2); 876 Store16x4(p1, p0, p3, p2, p, stride); 877 p1 = q2; 878 p0 = q3; 879 } 880 } 881 } 882 883 #if !defined(WORK_AROUND_GCC) 884 static void HFilter16i(uint8_t* p, int stride, 885 int thresh, int ithresh, int hev_thresh) { 886 uint32_t k; 887 uint8x16_t p3, p2, p1, p0; 888 Load4x16(p + 2, stride, &p3, &p2, &p1, &p0); 889 for (k = 3; k != 0; --k) { 890 uint8x16_t q0, q1, q2, q3; 891 p += 4; 892 Load4x16(p + 2, stride, &q0, &q1, &q2, &q3); 893 { 894 const uint8x16_t mask = 895 NeedsFilter2(p3, p2, p1, p0, q0, q1, q2, q3, ithresh, thresh); 896 const uint8x16_t hev_mask = NeedsHev(p1, p0, q0, q1, hev_thresh); 897 DoFilter4(p1, p0, q0, q1, mask, hev_mask, &p1, &p0, &p3, &p2); 898 Store4x16(p1, p0, p3, p2, p, stride); 899 p1 = q2; 900 p0 = q3; 901 } 902 } 903 } 904 #endif // !WORK_AROUND_GCC 905 906 // 8-pixels wide variant, for chroma filtering 907 static void VFilter8(uint8_t* u, uint8_t* v, int stride, 908 int thresh, int ithresh, int hev_thresh) { 909 uint8x16_t p3, p2, p1, p0, q0, q1, q2, q3; 910 Load8x8x2(u, v, stride, &p3, &p2, &p1, &p0, &q0, &q1, &q2, &q3); 911 { 912 const uint8x16_t mask = NeedsFilter2(p3, p2, p1, p0, q0, q1, q2, q3, 913 ithresh, thresh); 914 const uint8x16_t hev_mask = NeedsHev(p1, p0, q0, q1, hev_thresh); 915 uint8x16_t op2, op1, op0, oq0, oq1, oq2; 916 DoFilter6(p2, p1, p0, q0, q1, q2, mask, hev_mask, 917 &op2, &op1, &op0, &oq0, &oq1, &oq2); 918 Store8x2x2(op2, op1, u - 2 * stride, v - 2 * stride, stride); 919 Store8x2x2(op0, oq0, u + 0 * stride, v + 0 * stride, stride); 920 Store8x2x2(oq1, oq2, u + 2 * stride, v + 2 * stride, stride); 921 } 922 } 923 static void VFilter8i(uint8_t* u, uint8_t* v, int stride, 924 int thresh, int ithresh, int hev_thresh) { 925 uint8x16_t p3, p2, p1, p0, q0, q1, q2, q3; 926 u += 4 * stride; 927 v += 4 * stride; 928 Load8x8x2(u, v, stride, &p3, &p2, &p1, &p0, &q0, &q1, &q2, &q3); 929 { 930 const uint8x16_t mask = NeedsFilter2(p3, p2, p1, p0, q0, q1, q2, q3, 931 ithresh, thresh); 932 const uint8x16_t hev_mask = NeedsHev(p1, p0, q0, q1, hev_thresh); 933 uint8x16_t op1, op0, oq0, oq1; 934 DoFilter4(p1, p0, q0, q1, mask, hev_mask, &op1, &op0, &oq0, &oq1); 935 Store8x4x2(op1, op0, oq0, oq1, u, v, stride); 936 } 937 } 938 939 #if !defined(WORK_AROUND_GCC) 940 static void HFilter8(uint8_t* u, uint8_t* v, int stride, 941 int thresh, int ithresh, int hev_thresh) { 942 uint8x16_t p3, p2, p1, p0, q0, q1, q2, q3; 943 Load8x8x2T(u, v, stride, &p3, &p2, &p1, &p0, &q0, &q1, &q2, &q3); 944 { 945 const uint8x16_t mask = NeedsFilter2(p3, p2, p1, p0, q0, q1, q2, q3, 946 ithresh, thresh); 947 const uint8x16_t hev_mask = NeedsHev(p1, p0, q0, q1, hev_thresh); 948 uint8x16_t op2, op1, op0, oq0, oq1, oq2; 949 DoFilter6(p2, p1, p0, q0, q1, q2, mask, hev_mask, 950 &op2, &op1, &op0, &oq0, &oq1, &oq2); 951 Store6x8x2(op2, op1, op0, oq0, oq1, oq2, u, v, stride); 952 } 953 } 954 955 static void HFilter8i(uint8_t* u, uint8_t* v, int stride, 956 int thresh, int ithresh, int hev_thresh) { 957 uint8x16_t p3, p2, p1, p0, q0, q1, q2, q3; 958 u += 4; 959 v += 4; 960 Load8x8x2T(u, v, stride, &p3, &p2, &p1, &p0, &q0, &q1, &q2, &q3); 961 { 962 const uint8x16_t mask = NeedsFilter2(p3, p2, p1, p0, q0, q1, q2, q3, 963 ithresh, thresh); 964 const uint8x16_t hev_mask = NeedsHev(p1, p0, q0, q1, hev_thresh); 965 uint8x16_t op1, op0, oq0, oq1; 966 DoFilter4(p1, p0, q0, q1, mask, hev_mask, &op1, &op0, &oq0, &oq1); 967 Store4x8x2(op1, op0, oq0, oq1, u, v, stride); 968 } 969 } 970 #endif // !WORK_AROUND_GCC 971 972 //----------------------------------------------------------------------------- 973 // Inverse transforms (Paragraph 14.4) 974 975 // Technically these are unsigned but vqdmulh is only available in signed. 976 // vqdmulh returns high half (effectively >> 16) but also doubles the value, 977 // changing the >> 16 to >> 15 and requiring an additional >> 1. 978 // We use this to our advantage with kC2. The canonical value is 35468. 979 // However, the high bit is set so treating it as signed will give incorrect 980 // results. We avoid this by down shifting by 1 here to clear the highest bit. 981 // Combined with the doubling effect of vqdmulh we get >> 16. 982 // This can not be applied to kC1 because the lowest bit is set. Down shifting 983 // the constant would reduce precision. 984 985 // libwebp uses a trick to avoid some extra addition that libvpx does. 986 // Instead of: 987 // temp2 = ip[12] + ((ip[12] * cospi8sqrt2minus1) >> 16); 988 // libwebp adds 1 << 16 to cospi8sqrt2minus1 (kC1). However, this causes the 989 // same issue with kC1 and vqdmulh that we work around by down shifting kC2 990 991 static const int16_t kC1 = 20091; 992 static const int16_t kC2 = 17734; // half of kC2, actually. See comment above. 993 994 #if defined(WEBP_USE_INTRINSICS) 995 static WEBP_INLINE void Transpose8x2(const int16x8_t in0, const int16x8_t in1, 996 int16x8x2_t* const out) { 997 // a0 a1 a2 a3 | b0 b1 b2 b3 => a0 b0 c0 d0 | a1 b1 c1 d1 998 // c0 c1 c2 c3 | d0 d1 d2 d3 a2 b2 c2 d2 | a3 b3 c3 d3 999 const int16x8x2_t tmp0 = vzipq_s16(in0, in1); // a0 c0 a1 c1 a2 c2 ... 1000 // b0 d0 b1 d1 b2 d2 ... 1001 *out = vzipq_s16(tmp0.val[0], tmp0.val[1]); 1002 } 1003 1004 static WEBP_INLINE void TransformPass(int16x8x2_t* const rows) { 1005 // {rows} = in0 | in4 1006 // in8 | in12 1007 // B1 = in4 | in12 1008 const int16x8_t B1 = 1009 vcombine_s16(vget_high_s16(rows->val[0]), vget_high_s16(rows->val[1])); 1010 // C0 = kC1 * in4 | kC1 * in12 1011 // C1 = kC2 * in4 | kC2 * in12 1012 const int16x8_t C0 = vsraq_n_s16(B1, vqdmulhq_n_s16(B1, kC1), 1); 1013 const int16x8_t C1 = vqdmulhq_n_s16(B1, kC2); 1014 const int16x4_t a = vqadd_s16(vget_low_s16(rows->val[0]), 1015 vget_low_s16(rows->val[1])); // in0 + in8 1016 const int16x4_t b = vqsub_s16(vget_low_s16(rows->val[0]), 1017 vget_low_s16(rows->val[1])); // in0 - in8 1018 // c = kC2 * in4 - kC1 * in12 1019 // d = kC1 * in4 + kC2 * in12 1020 const int16x4_t c = vqsub_s16(vget_low_s16(C1), vget_high_s16(C0)); 1021 const int16x4_t d = vqadd_s16(vget_low_s16(C0), vget_high_s16(C1)); 1022 const int16x8_t D0 = vcombine_s16(a, b); // D0 = a | b 1023 const int16x8_t D1 = vcombine_s16(d, c); // D1 = d | c 1024 const int16x8_t E0 = vqaddq_s16(D0, D1); // a+d | b+c 1025 const int16x8_t E_tmp = vqsubq_s16(D0, D1); // a-d | b-c 1026 const int16x8_t E1 = vcombine_s16(vget_high_s16(E_tmp), vget_low_s16(E_tmp)); 1027 Transpose8x2(E0, E1, rows); 1028 } 1029 1030 static void TransformOne(const int16_t* in, uint8_t* dst) { 1031 int16x8x2_t rows; 1032 INIT_VECTOR2(rows, vld1q_s16(in + 0), vld1q_s16(in + 8)); 1033 TransformPass(&rows); 1034 TransformPass(&rows); 1035 Add4x4(rows.val[0], rows.val[1], dst); 1036 } 1037 1038 #else 1039 1040 static void TransformOne(const int16_t* in, uint8_t* dst) { 1041 const int kBPS = BPS; 1042 // kC1, kC2. Padded because vld1.16 loads 8 bytes 1043 const int16_t constants[4] = { kC1, kC2, 0, 0 }; 1044 /* Adapted from libvpx: vp8/common/arm/neon/shortidct4x4llm_neon.asm */ 1045 __asm__ volatile ( 1046 "vld1.16 {q1, q2}, [%[in]] \n" 1047 "vld1.16 {d0}, [%[constants]] \n" 1048 1049 /* d2: in[0] 1050 * d3: in[8] 1051 * d4: in[4] 1052 * d5: in[12] 1053 */ 1054 "vswp d3, d4 \n" 1055 1056 /* q8 = {in[4], in[12]} * kC1 * 2 >> 16 1057 * q9 = {in[4], in[12]} * kC2 >> 16 1058 */ 1059 "vqdmulh.s16 q8, q2, d0[0] \n" 1060 "vqdmulh.s16 q9, q2, d0[1] \n" 1061 1062 /* d22 = a = in[0] + in[8] 1063 * d23 = b = in[0] - in[8] 1064 */ 1065 "vqadd.s16 d22, d2, d3 \n" 1066 "vqsub.s16 d23, d2, d3 \n" 1067 1068 /* The multiplication should be x * kC1 >> 16 1069 * However, with vqdmulh we get x * kC1 * 2 >> 16 1070 * (multiply, double, return high half) 1071 * We avoided this in kC2 by pre-shifting the constant. 1072 * q8 = in[4]/[12] * kC1 >> 16 1073 */ 1074 "vshr.s16 q8, q8, #1 \n" 1075 1076 /* Add {in[4], in[12]} back after the multiplication. This is handled by 1077 * adding 1 << 16 to kC1 in the libwebp C code. 1078 */ 1079 "vqadd.s16 q8, q2, q8 \n" 1080 1081 /* d20 = c = in[4]*kC2 - in[12]*kC1 1082 * d21 = d = in[4]*kC1 + in[12]*kC2 1083 */ 1084 "vqsub.s16 d20, d18, d17 \n" 1085 "vqadd.s16 d21, d19, d16 \n" 1086 1087 /* d2 = tmp[0] = a + d 1088 * d3 = tmp[1] = b + c 1089 * d4 = tmp[2] = b - c 1090 * d5 = tmp[3] = a - d 1091 */ 1092 "vqadd.s16 d2, d22, d21 \n" 1093 "vqadd.s16 d3, d23, d20 \n" 1094 "vqsub.s16 d4, d23, d20 \n" 1095 "vqsub.s16 d5, d22, d21 \n" 1096 1097 "vzip.16 q1, q2 \n" 1098 "vzip.16 q1, q2 \n" 1099 1100 "vswp d3, d4 \n" 1101 1102 /* q8 = {tmp[4], tmp[12]} * kC1 * 2 >> 16 1103 * q9 = {tmp[4], tmp[12]} * kC2 >> 16 1104 */ 1105 "vqdmulh.s16 q8, q2, d0[0] \n" 1106 "vqdmulh.s16 q9, q2, d0[1] \n" 1107 1108 /* d22 = a = tmp[0] + tmp[8] 1109 * d23 = b = tmp[0] - tmp[8] 1110 */ 1111 "vqadd.s16 d22, d2, d3 \n" 1112 "vqsub.s16 d23, d2, d3 \n" 1113 1114 /* See long winded explanations prior */ 1115 "vshr.s16 q8, q8, #1 \n" 1116 "vqadd.s16 q8, q2, q8 \n" 1117 1118 /* d20 = c = in[4]*kC2 - in[12]*kC1 1119 * d21 = d = in[4]*kC1 + in[12]*kC2 1120 */ 1121 "vqsub.s16 d20, d18, d17 \n" 1122 "vqadd.s16 d21, d19, d16 \n" 1123 1124 /* d2 = tmp[0] = a + d 1125 * d3 = tmp[1] = b + c 1126 * d4 = tmp[2] = b - c 1127 * d5 = tmp[3] = a - d 1128 */ 1129 "vqadd.s16 d2, d22, d21 \n" 1130 "vqadd.s16 d3, d23, d20 \n" 1131 "vqsub.s16 d4, d23, d20 \n" 1132 "vqsub.s16 d5, d22, d21 \n" 1133 1134 "vld1.32 d6[0], [%[dst]], %[kBPS] \n" 1135 "vld1.32 d6[1], [%[dst]], %[kBPS] \n" 1136 "vld1.32 d7[0], [%[dst]], %[kBPS] \n" 1137 "vld1.32 d7[1], [%[dst]], %[kBPS] \n" 1138 1139 "sub %[dst], %[dst], %[kBPS], lsl #2 \n" 1140 1141 /* (val) + 4 >> 3 */ 1142 "vrshr.s16 d2, d2, #3 \n" 1143 "vrshr.s16 d3, d3, #3 \n" 1144 "vrshr.s16 d4, d4, #3 \n" 1145 "vrshr.s16 d5, d5, #3 \n" 1146 1147 "vzip.16 q1, q2 \n" 1148 "vzip.16 q1, q2 \n" 1149 1150 /* Must accumulate before saturating */ 1151 "vmovl.u8 q8, d6 \n" 1152 "vmovl.u8 q9, d7 \n" 1153 1154 "vqadd.s16 q1, q1, q8 \n" 1155 "vqadd.s16 q2, q2, q9 \n" 1156 1157 "vqmovun.s16 d0, q1 \n" 1158 "vqmovun.s16 d1, q2 \n" 1159 1160 "vst1.32 d0[0], [%[dst]], %[kBPS] \n" 1161 "vst1.32 d0[1], [%[dst]], %[kBPS] \n" 1162 "vst1.32 d1[0], [%[dst]], %[kBPS] \n" 1163 "vst1.32 d1[1], [%[dst]] \n" 1164 1165 : [in] "+r"(in), [dst] "+r"(dst) /* modified registers */ 1166 : [kBPS] "r"(kBPS), [constants] "r"(constants) /* constants */ 1167 : "memory", "q0", "q1", "q2", "q8", "q9", "q10", "q11" /* clobbered */ 1168 ); 1169 } 1170 1171 #endif // WEBP_USE_INTRINSICS 1172 1173 static void TransformTwo(const int16_t* in, uint8_t* dst, int do_two) { 1174 TransformOne(in, dst); 1175 if (do_two) { 1176 TransformOne(in + 16, dst + 4); 1177 } 1178 } 1179 1180 static void TransformDC(const int16_t* in, uint8_t* dst) { 1181 const int16x8_t DC = vdupq_n_s16(in[0]); 1182 Add4x4(DC, DC, dst); 1183 } 1184 1185 //------------------------------------------------------------------------------ 1186 1187 #define STORE_WHT(dst, col, rows) do { \ 1188 *dst = vgetq_lane_s32(rows.val[0], col); (dst) += 16; \ 1189 *dst = vgetq_lane_s32(rows.val[1], col); (dst) += 16; \ 1190 *dst = vgetq_lane_s32(rows.val[2], col); (dst) += 16; \ 1191 *dst = vgetq_lane_s32(rows.val[3], col); (dst) += 16; \ 1192 } while (0) 1193 1194 static void TransformWHT(const int16_t* in, int16_t* out) { 1195 int32x4x4_t tmp; 1196 1197 { 1198 // Load the source. 1199 const int16x4_t in00_03 = vld1_s16(in + 0); 1200 const int16x4_t in04_07 = vld1_s16(in + 4); 1201 const int16x4_t in08_11 = vld1_s16(in + 8); 1202 const int16x4_t in12_15 = vld1_s16(in + 12); 1203 const int32x4_t a0 = vaddl_s16(in00_03, in12_15); // in[0..3] + in[12..15] 1204 const int32x4_t a1 = vaddl_s16(in04_07, in08_11); // in[4..7] + in[8..11] 1205 const int32x4_t a2 = vsubl_s16(in04_07, in08_11); // in[4..7] - in[8..11] 1206 const int32x4_t a3 = vsubl_s16(in00_03, in12_15); // in[0..3] - in[12..15] 1207 tmp.val[0] = vaddq_s32(a0, a1); 1208 tmp.val[1] = vaddq_s32(a3, a2); 1209 tmp.val[2] = vsubq_s32(a0, a1); 1210 tmp.val[3] = vsubq_s32(a3, a2); 1211 // Arrange the temporary results column-wise. 1212 tmp = Transpose4x4(tmp); 1213 } 1214 1215 { 1216 const int32x4_t kCst3 = vdupq_n_s32(3); 1217 const int32x4_t dc = vaddq_s32(tmp.val[0], kCst3); // add rounder 1218 const int32x4_t a0 = vaddq_s32(dc, tmp.val[3]); 1219 const int32x4_t a1 = vaddq_s32(tmp.val[1], tmp.val[2]); 1220 const int32x4_t a2 = vsubq_s32(tmp.val[1], tmp.val[2]); 1221 const int32x4_t a3 = vsubq_s32(dc, tmp.val[3]); 1222 1223 tmp.val[0] = vaddq_s32(a0, a1); 1224 tmp.val[1] = vaddq_s32(a3, a2); 1225 tmp.val[2] = vsubq_s32(a0, a1); 1226 tmp.val[3] = vsubq_s32(a3, a2); 1227 1228 // right shift the results by 3. 1229 tmp.val[0] = vshrq_n_s32(tmp.val[0], 3); 1230 tmp.val[1] = vshrq_n_s32(tmp.val[1], 3); 1231 tmp.val[2] = vshrq_n_s32(tmp.val[2], 3); 1232 tmp.val[3] = vshrq_n_s32(tmp.val[3], 3); 1233 1234 STORE_WHT(out, 0, tmp); 1235 STORE_WHT(out, 1, tmp); 1236 STORE_WHT(out, 2, tmp); 1237 STORE_WHT(out, 3, tmp); 1238 } 1239 } 1240 1241 #undef STORE_WHT 1242 1243 //------------------------------------------------------------------------------ 1244 1245 #define MUL(a, b) (((a) * (b)) >> 16) 1246 static void TransformAC3(const int16_t* in, uint8_t* dst) { 1247 static const int kC1_full = 20091 + (1 << 16); 1248 static const int kC2_full = 35468; 1249 const int16x4_t A = vld1_dup_s16(in); 1250 const int16x4_t c4 = vdup_n_s16(MUL(in[4], kC2_full)); 1251 const int16x4_t d4 = vdup_n_s16(MUL(in[4], kC1_full)); 1252 const int c1 = MUL(in[1], kC2_full); 1253 const int d1 = MUL(in[1], kC1_full); 1254 const uint64_t cd = (uint64_t)( d1 & 0xffff) << 0 | 1255 (uint64_t)( c1 & 0xffff) << 16 | 1256 (uint64_t)(-c1 & 0xffff) << 32 | 1257 (uint64_t)(-d1 & 0xffff) << 48; 1258 const int16x4_t CD = vcreate_s16(cd); 1259 const int16x4_t B = vqadd_s16(A, CD); 1260 const int16x8_t m0_m1 = vcombine_s16(vqadd_s16(B, d4), vqadd_s16(B, c4)); 1261 const int16x8_t m2_m3 = vcombine_s16(vqsub_s16(B, c4), vqsub_s16(B, d4)); 1262 Add4x4(m0_m1, m2_m3, dst); 1263 } 1264 #undef MUL 1265 1266 //------------------------------------------------------------------------------ 1267 // 4x4 1268 1269 static void DC4(uint8_t* dst) { // DC 1270 const uint8x8_t A = vld1_u8(dst - BPS); // top row 1271 const uint16x4_t p0 = vpaddl_u8(A); // cascading summation of the top 1272 const uint16x4_t p1 = vpadd_u16(p0, p0); 1273 const uint16x8_t L0 = vmovl_u8(vld1_u8(dst + 0 * BPS - 1)); 1274 const uint16x8_t L1 = vmovl_u8(vld1_u8(dst + 1 * BPS - 1)); 1275 const uint16x8_t L2 = vmovl_u8(vld1_u8(dst + 2 * BPS - 1)); 1276 const uint16x8_t L3 = vmovl_u8(vld1_u8(dst + 3 * BPS - 1)); 1277 const uint16x8_t s0 = vaddq_u16(L0, L1); 1278 const uint16x8_t s1 = vaddq_u16(L2, L3); 1279 const uint16x8_t s01 = vaddq_u16(s0, s1); 1280 const uint16x8_t sum = vaddq_u16(s01, vcombine_u16(p1, p1)); 1281 const uint8x8_t dc0 = vrshrn_n_u16(sum, 3); // (sum + 4) >> 3 1282 const uint8x8_t dc = vdup_lane_u8(dc0, 0); 1283 int i; 1284 for (i = 0; i < 4; ++i) { 1285 vst1_lane_u32((uint32_t*)(dst + i * BPS), vreinterpret_u32_u8(dc), 0); 1286 } 1287 } 1288 1289 // TrueMotion (4x4 + 8x8) 1290 static WEBP_INLINE void TrueMotion(uint8_t* dst, int size) { 1291 const uint8x8_t TL = vld1_dup_u8(dst - BPS - 1); // top-left pixel 'A[-1]' 1292 const uint8x8_t T = vld1_u8(dst - BPS); // top row 'A[0..3]' 1293 const int16x8_t d = vreinterpretq_s16_u16(vsubl_u8(T, TL)); // A[c] - A[-1] 1294 int y; 1295 for (y = 0; y < size; y += 4) { 1296 // left edge 1297 const int16x8_t L0 = ConvertU8ToS16(vld1_dup_u8(dst + 0 * BPS - 1)); 1298 const int16x8_t L1 = ConvertU8ToS16(vld1_dup_u8(dst + 1 * BPS - 1)); 1299 const int16x8_t L2 = ConvertU8ToS16(vld1_dup_u8(dst + 2 * BPS - 1)); 1300 const int16x8_t L3 = ConvertU8ToS16(vld1_dup_u8(dst + 3 * BPS - 1)); 1301 const int16x8_t r0 = vaddq_s16(L0, d); // L[r] + A[c] - A[-1] 1302 const int16x8_t r1 = vaddq_s16(L1, d); 1303 const int16x8_t r2 = vaddq_s16(L2, d); 1304 const int16x8_t r3 = vaddq_s16(L3, d); 1305 // Saturate and store the result. 1306 const uint32x2_t r0_u32 = vreinterpret_u32_u8(vqmovun_s16(r0)); 1307 const uint32x2_t r1_u32 = vreinterpret_u32_u8(vqmovun_s16(r1)); 1308 const uint32x2_t r2_u32 = vreinterpret_u32_u8(vqmovun_s16(r2)); 1309 const uint32x2_t r3_u32 = vreinterpret_u32_u8(vqmovun_s16(r3)); 1310 if (size == 4) { 1311 vst1_lane_u32((uint32_t*)(dst + 0 * BPS), r0_u32, 0); 1312 vst1_lane_u32((uint32_t*)(dst + 1 * BPS), r1_u32, 0); 1313 vst1_lane_u32((uint32_t*)(dst + 2 * BPS), r2_u32, 0); 1314 vst1_lane_u32((uint32_t*)(dst + 3 * BPS), r3_u32, 0); 1315 } else { 1316 vst1_u32((uint32_t*)(dst + 0 * BPS), r0_u32); 1317 vst1_u32((uint32_t*)(dst + 1 * BPS), r1_u32); 1318 vst1_u32((uint32_t*)(dst + 2 * BPS), r2_u32); 1319 vst1_u32((uint32_t*)(dst + 3 * BPS), r3_u32); 1320 } 1321 dst += 4 * BPS; 1322 } 1323 } 1324 1325 static void TM4(uint8_t* dst) { TrueMotion(dst, 4); } 1326 1327 static void VE4(uint8_t* dst) { // vertical 1328 // NB: avoid vld1_u64 here as an alignment hint may be added -> SIGBUS. 1329 const uint64x1_t A0 = vreinterpret_u64_u8(vld1_u8(dst - BPS - 1)); // top row 1330 const uint64x1_t A1 = vshr_n_u64(A0, 8); 1331 const uint64x1_t A2 = vshr_n_u64(A0, 16); 1332 const uint8x8_t ABCDEFGH = vreinterpret_u8_u64(A0); 1333 const uint8x8_t BCDEFGH0 = vreinterpret_u8_u64(A1); 1334 const uint8x8_t CDEFGH00 = vreinterpret_u8_u64(A2); 1335 const uint8x8_t b = vhadd_u8(ABCDEFGH, CDEFGH00); 1336 const uint8x8_t avg = vrhadd_u8(b, BCDEFGH0); 1337 int i; 1338 for (i = 0; i < 4; ++i) { 1339 vst1_lane_u32((uint32_t*)(dst + i * BPS), vreinterpret_u32_u8(avg), 0); 1340 } 1341 } 1342 1343 static void RD4(uint8_t* dst) { // Down-right 1344 const uint8x8_t XABCD_u8 = vld1_u8(dst - BPS - 1); 1345 const uint64x1_t XABCD = vreinterpret_u64_u8(XABCD_u8); 1346 const uint64x1_t ____XABC = vshl_n_u64(XABCD, 32); 1347 const uint32_t I = dst[-1 + 0 * BPS]; 1348 const uint32_t J = dst[-1 + 1 * BPS]; 1349 const uint32_t K = dst[-1 + 2 * BPS]; 1350 const uint32_t L = dst[-1 + 3 * BPS]; 1351 const uint64x1_t LKJI____ = vcreate_u64(L | (K << 8) | (J << 16) | (I << 24)); 1352 const uint64x1_t LKJIXABC = vorr_u64(LKJI____, ____XABC); 1353 const uint8x8_t KJIXABC_ = vreinterpret_u8_u64(vshr_n_u64(LKJIXABC, 8)); 1354 const uint8x8_t JIXABC__ = vreinterpret_u8_u64(vshr_n_u64(LKJIXABC, 16)); 1355 const uint8_t D = vget_lane_u8(XABCD_u8, 4); 1356 const uint8x8_t JIXABCD_ = vset_lane_u8(D, JIXABC__, 6); 1357 const uint8x8_t LKJIXABC_u8 = vreinterpret_u8_u64(LKJIXABC); 1358 const uint8x8_t avg1 = vhadd_u8(JIXABCD_, LKJIXABC_u8); 1359 const uint8x8_t avg2 = vrhadd_u8(avg1, KJIXABC_); 1360 const uint64x1_t avg2_u64 = vreinterpret_u64_u8(avg2); 1361 const uint32x2_t r3 = vreinterpret_u32_u8(avg2); 1362 const uint32x2_t r2 = vreinterpret_u32_u64(vshr_n_u64(avg2_u64, 8)); 1363 const uint32x2_t r1 = vreinterpret_u32_u64(vshr_n_u64(avg2_u64, 16)); 1364 const uint32x2_t r0 = vreinterpret_u32_u64(vshr_n_u64(avg2_u64, 24)); 1365 vst1_lane_u32((uint32_t*)(dst + 0 * BPS), r0, 0); 1366 vst1_lane_u32((uint32_t*)(dst + 1 * BPS), r1, 0); 1367 vst1_lane_u32((uint32_t*)(dst + 2 * BPS), r2, 0); 1368 vst1_lane_u32((uint32_t*)(dst + 3 * BPS), r3, 0); 1369 } 1370 1371 static void LD4(uint8_t* dst) { // Down-left 1372 // Note using the same shift trick as VE4() is slower here. 1373 const uint8x8_t ABCDEFGH = vld1_u8(dst - BPS + 0); 1374 const uint8x8_t BCDEFGH0 = vld1_u8(dst - BPS + 1); 1375 const uint8x8_t CDEFGH00 = vld1_u8(dst - BPS + 2); 1376 const uint8x8_t CDEFGHH0 = vset_lane_u8(dst[-BPS + 7], CDEFGH00, 6); 1377 const uint8x8_t avg1 = vhadd_u8(ABCDEFGH, CDEFGHH0); 1378 const uint8x8_t avg2 = vrhadd_u8(avg1, BCDEFGH0); 1379 const uint64x1_t avg2_u64 = vreinterpret_u64_u8(avg2); 1380 const uint32x2_t r0 = vreinterpret_u32_u8(avg2); 1381 const uint32x2_t r1 = vreinterpret_u32_u64(vshr_n_u64(avg2_u64, 8)); 1382 const uint32x2_t r2 = vreinterpret_u32_u64(vshr_n_u64(avg2_u64, 16)); 1383 const uint32x2_t r3 = vreinterpret_u32_u64(vshr_n_u64(avg2_u64, 24)); 1384 vst1_lane_u32((uint32_t*)(dst + 0 * BPS), r0, 0); 1385 vst1_lane_u32((uint32_t*)(dst + 1 * BPS), r1, 0); 1386 vst1_lane_u32((uint32_t*)(dst + 2 * BPS), r2, 0); 1387 vst1_lane_u32((uint32_t*)(dst + 3 * BPS), r3, 0); 1388 } 1389 1390 //------------------------------------------------------------------------------ 1391 // Chroma 1392 1393 static void VE8uv(uint8_t* dst) { // vertical 1394 const uint8x8_t top = vld1_u8(dst - BPS); 1395 int j; 1396 for (j = 0; j < 8; ++j) { 1397 vst1_u8(dst + j * BPS, top); 1398 } 1399 } 1400 1401 static void HE8uv(uint8_t* dst) { // horizontal 1402 int j; 1403 for (j = 0; j < 8; ++j) { 1404 const uint8x8_t left = vld1_dup_u8(dst - 1); 1405 vst1_u8(dst, left); 1406 dst += BPS; 1407 } 1408 } 1409 1410 static WEBP_INLINE void DC8(uint8_t* dst, int do_top, int do_left) { 1411 uint16x8_t sum_top; 1412 uint16x8_t sum_left; 1413 uint8x8_t dc0; 1414 1415 if (do_top) { 1416 const uint8x8_t A = vld1_u8(dst - BPS); // top row 1417 const uint16x4_t p0 = vpaddl_u8(A); // cascading summation of the top 1418 const uint16x4_t p1 = vpadd_u16(p0, p0); 1419 const uint16x4_t p2 = vpadd_u16(p1, p1); 1420 sum_top = vcombine_u16(p2, p2); 1421 } 1422 1423 if (do_left) { 1424 const uint16x8_t L0 = vmovl_u8(vld1_u8(dst + 0 * BPS - 1)); 1425 const uint16x8_t L1 = vmovl_u8(vld1_u8(dst + 1 * BPS - 1)); 1426 const uint16x8_t L2 = vmovl_u8(vld1_u8(dst + 2 * BPS - 1)); 1427 const uint16x8_t L3 = vmovl_u8(vld1_u8(dst + 3 * BPS - 1)); 1428 const uint16x8_t L4 = vmovl_u8(vld1_u8(dst + 4 * BPS - 1)); 1429 const uint16x8_t L5 = vmovl_u8(vld1_u8(dst + 5 * BPS - 1)); 1430 const uint16x8_t L6 = vmovl_u8(vld1_u8(dst + 6 * BPS - 1)); 1431 const uint16x8_t L7 = vmovl_u8(vld1_u8(dst + 7 * BPS - 1)); 1432 const uint16x8_t s0 = vaddq_u16(L0, L1); 1433 const uint16x8_t s1 = vaddq_u16(L2, L3); 1434 const uint16x8_t s2 = vaddq_u16(L4, L5); 1435 const uint16x8_t s3 = vaddq_u16(L6, L7); 1436 const uint16x8_t s01 = vaddq_u16(s0, s1); 1437 const uint16x8_t s23 = vaddq_u16(s2, s3); 1438 sum_left = vaddq_u16(s01, s23); 1439 } 1440 1441 if (do_top && do_left) { 1442 const uint16x8_t sum = vaddq_u16(sum_left, sum_top); 1443 dc0 = vrshrn_n_u16(sum, 4); 1444 } else if (do_top) { 1445 dc0 = vrshrn_n_u16(sum_top, 3); 1446 } else if (do_left) { 1447 dc0 = vrshrn_n_u16(sum_left, 3); 1448 } else { 1449 dc0 = vdup_n_u8(0x80); 1450 } 1451 1452 { 1453 const uint8x8_t dc = vdup_lane_u8(dc0, 0); 1454 int i; 1455 for (i = 0; i < 8; ++i) { 1456 vst1_u32((uint32_t*)(dst + i * BPS), vreinterpret_u32_u8(dc)); 1457 } 1458 } 1459 } 1460 1461 static void DC8uv(uint8_t* dst) { DC8(dst, 1, 1); } 1462 static void DC8uvNoTop(uint8_t* dst) { DC8(dst, 0, 1); } 1463 static void DC8uvNoLeft(uint8_t* dst) { DC8(dst, 1, 0); } 1464 static void DC8uvNoTopLeft(uint8_t* dst) { DC8(dst, 0, 0); } 1465 1466 static void TM8uv(uint8_t* dst) { TrueMotion(dst, 8); } 1467 1468 //------------------------------------------------------------------------------ 1469 // 16x16 1470 1471 static void VE16(uint8_t* dst) { // vertical 1472 const uint8x16_t top = vld1q_u8(dst - BPS); 1473 int j; 1474 for (j = 0; j < 16; ++j) { 1475 vst1q_u8(dst + j * BPS, top); 1476 } 1477 } 1478 1479 static void HE16(uint8_t* dst) { // horizontal 1480 int j; 1481 for (j = 0; j < 16; ++j) { 1482 const uint8x16_t left = vld1q_dup_u8(dst - 1); 1483 vst1q_u8(dst, left); 1484 dst += BPS; 1485 } 1486 } 1487 1488 static WEBP_INLINE void DC16(uint8_t* dst, int do_top, int do_left) { 1489 uint16x8_t sum_top; 1490 uint16x8_t sum_left; 1491 uint8x8_t dc0; 1492 1493 if (do_top) { 1494 const uint8x16_t A = vld1q_u8(dst - BPS); // top row 1495 const uint16x8_t p0 = vpaddlq_u8(A); // cascading summation of the top 1496 const uint16x4_t p1 = vadd_u16(vget_low_u16(p0), vget_high_u16(p0)); 1497 const uint16x4_t p2 = vpadd_u16(p1, p1); 1498 const uint16x4_t p3 = vpadd_u16(p2, p2); 1499 sum_top = vcombine_u16(p3, p3); 1500 } 1501 1502 if (do_left) { 1503 int i; 1504 sum_left = vdupq_n_u16(0); 1505 for (i = 0; i < 16; i += 8) { 1506 const uint16x8_t L0 = vmovl_u8(vld1_u8(dst + (i + 0) * BPS - 1)); 1507 const uint16x8_t L1 = vmovl_u8(vld1_u8(dst + (i + 1) * BPS - 1)); 1508 const uint16x8_t L2 = vmovl_u8(vld1_u8(dst + (i + 2) * BPS - 1)); 1509 const uint16x8_t L3 = vmovl_u8(vld1_u8(dst + (i + 3) * BPS - 1)); 1510 const uint16x8_t L4 = vmovl_u8(vld1_u8(dst + (i + 4) * BPS - 1)); 1511 const uint16x8_t L5 = vmovl_u8(vld1_u8(dst + (i + 5) * BPS - 1)); 1512 const uint16x8_t L6 = vmovl_u8(vld1_u8(dst + (i + 6) * BPS - 1)); 1513 const uint16x8_t L7 = vmovl_u8(vld1_u8(dst + (i + 7) * BPS - 1)); 1514 const uint16x8_t s0 = vaddq_u16(L0, L1); 1515 const uint16x8_t s1 = vaddq_u16(L2, L3); 1516 const uint16x8_t s2 = vaddq_u16(L4, L5); 1517 const uint16x8_t s3 = vaddq_u16(L6, L7); 1518 const uint16x8_t s01 = vaddq_u16(s0, s1); 1519 const uint16x8_t s23 = vaddq_u16(s2, s3); 1520 const uint16x8_t sum = vaddq_u16(s01, s23); 1521 sum_left = vaddq_u16(sum_left, sum); 1522 } 1523 } 1524 1525 if (do_top && do_left) { 1526 const uint16x8_t sum = vaddq_u16(sum_left, sum_top); 1527 dc0 = vrshrn_n_u16(sum, 5); 1528 } else if (do_top) { 1529 dc0 = vrshrn_n_u16(sum_top, 4); 1530 } else if (do_left) { 1531 dc0 = vrshrn_n_u16(sum_left, 4); 1532 } else { 1533 dc0 = vdup_n_u8(0x80); 1534 } 1535 1536 { 1537 const uint8x16_t dc = vdupq_lane_u8(dc0, 0); 1538 int i; 1539 for (i = 0; i < 16; ++i) { 1540 vst1q_u8(dst + i * BPS, dc); 1541 } 1542 } 1543 } 1544 1545 static void DC16TopLeft(uint8_t* dst) { DC16(dst, 1, 1); } 1546 static void DC16NoTop(uint8_t* dst) { DC16(dst, 0, 1); } 1547 static void DC16NoLeft(uint8_t* dst) { DC16(dst, 1, 0); } 1548 static void DC16NoTopLeft(uint8_t* dst) { DC16(dst, 0, 0); } 1549 1550 static void TM16(uint8_t* dst) { 1551 const uint8x8_t TL = vld1_dup_u8(dst - BPS - 1); // top-left pixel 'A[-1]' 1552 const uint8x16_t T = vld1q_u8(dst - BPS); // top row 'A[0..15]' 1553 // A[c] - A[-1] 1554 const int16x8_t d_lo = vreinterpretq_s16_u16(vsubl_u8(vget_low_u8(T), TL)); 1555 const int16x8_t d_hi = vreinterpretq_s16_u16(vsubl_u8(vget_high_u8(T), TL)); 1556 int y; 1557 for (y = 0; y < 16; y += 4) { 1558 // left edge 1559 const int16x8_t L0 = ConvertU8ToS16(vld1_dup_u8(dst + 0 * BPS - 1)); 1560 const int16x8_t L1 = ConvertU8ToS16(vld1_dup_u8(dst + 1 * BPS - 1)); 1561 const int16x8_t L2 = ConvertU8ToS16(vld1_dup_u8(dst + 2 * BPS - 1)); 1562 const int16x8_t L3 = ConvertU8ToS16(vld1_dup_u8(dst + 3 * BPS - 1)); 1563 const int16x8_t r0_lo = vaddq_s16(L0, d_lo); // L[r] + A[c] - A[-1] 1564 const int16x8_t r1_lo = vaddq_s16(L1, d_lo); 1565 const int16x8_t r2_lo = vaddq_s16(L2, d_lo); 1566 const int16x8_t r3_lo = vaddq_s16(L3, d_lo); 1567 const int16x8_t r0_hi = vaddq_s16(L0, d_hi); 1568 const int16x8_t r1_hi = vaddq_s16(L1, d_hi); 1569 const int16x8_t r2_hi = vaddq_s16(L2, d_hi); 1570 const int16x8_t r3_hi = vaddq_s16(L3, d_hi); 1571 // Saturate and store the result. 1572 const uint8x16_t row0 = vcombine_u8(vqmovun_s16(r0_lo), vqmovun_s16(r0_hi)); 1573 const uint8x16_t row1 = vcombine_u8(vqmovun_s16(r1_lo), vqmovun_s16(r1_hi)); 1574 const uint8x16_t row2 = vcombine_u8(vqmovun_s16(r2_lo), vqmovun_s16(r2_hi)); 1575 const uint8x16_t row3 = vcombine_u8(vqmovun_s16(r3_lo), vqmovun_s16(r3_hi)); 1576 vst1q_u8(dst + 0 * BPS, row0); 1577 vst1q_u8(dst + 1 * BPS, row1); 1578 vst1q_u8(dst + 2 * BPS, row2); 1579 vst1q_u8(dst + 3 * BPS, row3); 1580 dst += 4 * BPS; 1581 } 1582 } 1583 1584 //------------------------------------------------------------------------------ 1585 // Entry point 1586 1587 extern void VP8DspInitNEON(void); 1588 1589 WEBP_TSAN_IGNORE_FUNCTION void VP8DspInitNEON(void) { 1590 VP8Transform = TransformTwo; 1591 VP8TransformAC3 = TransformAC3; 1592 VP8TransformDC = TransformDC; 1593 VP8TransformWHT = TransformWHT; 1594 1595 VP8VFilter16 = VFilter16; 1596 VP8VFilter16i = VFilter16i; 1597 VP8HFilter16 = HFilter16; 1598 #if !defined(WORK_AROUND_GCC) 1599 VP8HFilter16i = HFilter16i; 1600 #endif 1601 VP8VFilter8 = VFilter8; 1602 VP8VFilter8i = VFilter8i; 1603 #if !defined(WORK_AROUND_GCC) 1604 VP8HFilter8 = HFilter8; 1605 VP8HFilter8i = HFilter8i; 1606 #endif 1607 VP8SimpleVFilter16 = SimpleVFilter16; 1608 VP8SimpleHFilter16 = SimpleHFilter16; 1609 VP8SimpleVFilter16i = SimpleVFilter16i; 1610 VP8SimpleHFilter16i = SimpleHFilter16i; 1611 1612 VP8PredLuma4[0] = DC4; 1613 VP8PredLuma4[1] = TM4; 1614 VP8PredLuma4[2] = VE4; 1615 VP8PredLuma4[4] = RD4; 1616 VP8PredLuma4[6] = LD4; 1617 1618 VP8PredLuma16[0] = DC16TopLeft; 1619 VP8PredLuma16[1] = TM16; 1620 VP8PredLuma16[2] = VE16; 1621 VP8PredLuma16[3] = HE16; 1622 VP8PredLuma16[4] = DC16NoTop; 1623 VP8PredLuma16[5] = DC16NoLeft; 1624 VP8PredLuma16[6] = DC16NoTopLeft; 1625 1626 VP8PredChroma8[0] = DC8uv; 1627 VP8PredChroma8[1] = TM8uv; 1628 VP8PredChroma8[2] = VE8uv; 1629 VP8PredChroma8[3] = HE8uv; 1630 VP8PredChroma8[4] = DC8uvNoTop; 1631 VP8PredChroma8[5] = DC8uvNoLeft; 1632 VP8PredChroma8[6] = DC8uvNoTopLeft; 1633 } 1634 1635 #else // !WEBP_USE_NEON 1636 1637 WEBP_DSP_INIT_STUB(VP8DspInitNEON) 1638 1639 #endif // WEBP_USE_NEON 1640
