1 /* 2 * Copyright (c) 2015 The WebM project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #include <stdlib.h> 12 13 #include "./vpx_config.h" 14 #include "./vpx_dsp_rtcd.h" 15 #include "vpx_dsp/vpx_dsp_common.h" 16 #include "vpx_ports/mem.h" 17 18 static INLINE int8_t signed_char_clamp(int t) { 19 return (int8_t)clamp(t, -128, 127); 20 } 21 22 #if CONFIG_VP9_HIGHBITDEPTH 23 static INLINE int16_t signed_char_clamp_high(int t, int bd) { 24 switch (bd) { 25 case 10: return (int16_t)clamp(t, -128 * 4, 128 * 4 - 1); 26 case 12: return (int16_t)clamp(t, -128 * 16, 128 * 16 - 1); 27 case 8: 28 default: return (int16_t)clamp(t, -128, 128 - 1); 29 } 30 } 31 #endif 32 33 // Should we apply any filter at all: 11111111 yes, 00000000 no 34 static INLINE int8_t filter_mask(uint8_t limit, uint8_t blimit, uint8_t p3, 35 uint8_t p2, uint8_t p1, uint8_t p0, uint8_t q0, 36 uint8_t q1, uint8_t q2, uint8_t q3) { 37 int8_t mask = 0; 38 mask |= (abs(p3 - p2) > limit) * -1; 39 mask |= (abs(p2 - p1) > limit) * -1; 40 mask |= (abs(p1 - p0) > limit) * -1; 41 mask |= (abs(q1 - q0) > limit) * -1; 42 mask |= (abs(q2 - q1) > limit) * -1; 43 mask |= (abs(q3 - q2) > limit) * -1; 44 mask |= (abs(p0 - q0) * 2 + abs(p1 - q1) / 2 > blimit) * -1; 45 return ~mask; 46 } 47 48 static INLINE int8_t flat_mask4(uint8_t thresh, uint8_t p3, uint8_t p2, 49 uint8_t p1, uint8_t p0, uint8_t q0, uint8_t q1, 50 uint8_t q2, uint8_t q3) { 51 int8_t mask = 0; 52 mask |= (abs(p1 - p0) > thresh) * -1; 53 mask |= (abs(q1 - q0) > thresh) * -1; 54 mask |= (abs(p2 - p0) > thresh) * -1; 55 mask |= (abs(q2 - q0) > thresh) * -1; 56 mask |= (abs(p3 - p0) > thresh) * -1; 57 mask |= (abs(q3 - q0) > thresh) * -1; 58 return ~mask; 59 } 60 61 static INLINE int8_t flat_mask5(uint8_t thresh, uint8_t p4, uint8_t p3, 62 uint8_t p2, uint8_t p1, uint8_t p0, uint8_t q0, 63 uint8_t q1, uint8_t q2, uint8_t q3, 64 uint8_t q4) { 65 int8_t mask = ~flat_mask4(thresh, p3, p2, p1, p0, q0, q1, q2, q3); 66 mask |= (abs(p4 - p0) > thresh) * -1; 67 mask |= (abs(q4 - q0) > thresh) * -1; 68 return ~mask; 69 } 70 71 // Is there high edge variance internal edge: 11111111 yes, 00000000 no 72 static INLINE int8_t hev_mask(uint8_t thresh, uint8_t p1, uint8_t p0, 73 uint8_t q0, uint8_t q1) { 74 int8_t hev = 0; 75 hev |= (abs(p1 - p0) > thresh) * -1; 76 hev |= (abs(q1 - q0) > thresh) * -1; 77 return hev; 78 } 79 80 static INLINE void filter4(int8_t mask, uint8_t thresh, uint8_t *op1, 81 uint8_t *op0, uint8_t *oq0, uint8_t *oq1) { 82 int8_t filter1, filter2; 83 84 const int8_t ps1 = (int8_t)*op1 ^ 0x80; 85 const int8_t ps0 = (int8_t)*op0 ^ 0x80; 86 const int8_t qs0 = (int8_t)*oq0 ^ 0x80; 87 const int8_t qs1 = (int8_t)*oq1 ^ 0x80; 88 const uint8_t hev = hev_mask(thresh, *op1, *op0, *oq0, *oq1); 89 90 // add outer taps if we have high edge variance 91 int8_t filter = signed_char_clamp(ps1 - qs1) & hev; 92 93 // inner taps 94 filter = signed_char_clamp(filter + 3 * (qs0 - ps0)) & mask; 95 96 // save bottom 3 bits so that we round one side +4 and the other +3 97 // if it equals 4 we'll set it to adjust by -1 to account for the fact 98 // we'd round it by 3 the other way 99 filter1 = signed_char_clamp(filter + 4) >> 3; 100 filter2 = signed_char_clamp(filter + 3) >> 3; 101 102 *oq0 = signed_char_clamp(qs0 - filter1) ^ 0x80; 103 *op0 = signed_char_clamp(ps0 + filter2) ^ 0x80; 104 105 // outer tap adjustments 106 filter = ROUND_POWER_OF_TWO(filter1, 1) & ~hev; 107 108 *oq1 = signed_char_clamp(qs1 - filter) ^ 0x80; 109 *op1 = signed_char_clamp(ps1 + filter) ^ 0x80; 110 } 111 112 void vpx_lpf_horizontal_4_c(uint8_t *s, int p /* pitch */, 113 const uint8_t *blimit, const uint8_t *limit, 114 const uint8_t *thresh) { 115 int i; 116 117 // loop filter designed to work using chars so that we can make maximum use 118 // of 8 bit simd instructions. 119 for (i = 0; i < 8; ++i) { 120 const uint8_t p3 = s[-4 * p], p2 = s[-3 * p], p1 = s[-2 * p], p0 = s[-p]; 121 const uint8_t q0 = s[0 * p], q1 = s[1 * p], q2 = s[2 * p], q3 = s[3 * p]; 122 const int8_t mask = 123 filter_mask(*limit, *blimit, p3, p2, p1, p0, q0, q1, q2, q3); 124 filter4(mask, *thresh, s - 2 * p, s - 1 * p, s, s + 1 * p); 125 ++s; 126 } 127 } 128 129 void vpx_lpf_horizontal_4_dual_c(uint8_t *s, int p, const uint8_t *blimit0, 130 const uint8_t *limit0, const uint8_t *thresh0, 131 const uint8_t *blimit1, const uint8_t *limit1, 132 const uint8_t *thresh1) { 133 vpx_lpf_horizontal_4_c(s, p, blimit0, limit0, thresh0); 134 vpx_lpf_horizontal_4_c(s + 8, p, blimit1, limit1, thresh1); 135 } 136 137 void vpx_lpf_vertical_4_c(uint8_t *s, int pitch, const uint8_t *blimit, 138 const uint8_t *limit, const uint8_t *thresh) { 139 int i; 140 141 // loop filter designed to work using chars so that we can make maximum use 142 // of 8 bit simd instructions. 143 for (i = 0; i < 8; ++i) { 144 const uint8_t p3 = s[-4], p2 = s[-3], p1 = s[-2], p0 = s[-1]; 145 const uint8_t q0 = s[0], q1 = s[1], q2 = s[2], q3 = s[3]; 146 const int8_t mask = 147 filter_mask(*limit, *blimit, p3, p2, p1, p0, q0, q1, q2, q3); 148 filter4(mask, *thresh, s - 2, s - 1, s, s + 1); 149 s += pitch; 150 } 151 } 152 153 void vpx_lpf_vertical_4_dual_c(uint8_t *s, int pitch, const uint8_t *blimit0, 154 const uint8_t *limit0, const uint8_t *thresh0, 155 const uint8_t *blimit1, const uint8_t *limit1, 156 const uint8_t *thresh1) { 157 vpx_lpf_vertical_4_c(s, pitch, blimit0, limit0, thresh0); 158 vpx_lpf_vertical_4_c(s + 8 * pitch, pitch, blimit1, limit1, thresh1); 159 } 160 161 static INLINE void filter8(int8_t mask, uint8_t thresh, uint8_t flat, 162 uint8_t *op3, uint8_t *op2, uint8_t *op1, 163 uint8_t *op0, uint8_t *oq0, uint8_t *oq1, 164 uint8_t *oq2, uint8_t *oq3) { 165 if (flat && mask) { 166 const uint8_t p3 = *op3, p2 = *op2, p1 = *op1, p0 = *op0; 167 const uint8_t q0 = *oq0, q1 = *oq1, q2 = *oq2, q3 = *oq3; 168 169 // 7-tap filter [1, 1, 1, 2, 1, 1, 1] 170 *op2 = ROUND_POWER_OF_TWO(p3 + p3 + p3 + 2 * p2 + p1 + p0 + q0, 3); 171 *op1 = ROUND_POWER_OF_TWO(p3 + p3 + p2 + 2 * p1 + p0 + q0 + q1, 3); 172 *op0 = ROUND_POWER_OF_TWO(p3 + p2 + p1 + 2 * p0 + q0 + q1 + q2, 3); 173 *oq0 = ROUND_POWER_OF_TWO(p2 + p1 + p0 + 2 * q0 + q1 + q2 + q3, 3); 174 *oq1 = ROUND_POWER_OF_TWO(p1 + p0 + q0 + 2 * q1 + q2 + q3 + q3, 3); 175 *oq2 = ROUND_POWER_OF_TWO(p0 + q0 + q1 + 2 * q2 + q3 + q3 + q3, 3); 176 } else { 177 filter4(mask, thresh, op1, op0, oq0, oq1); 178 } 179 } 180 181 void vpx_lpf_horizontal_8_c(uint8_t *s, int p, const uint8_t *blimit, 182 const uint8_t *limit, const uint8_t *thresh) { 183 int i; 184 185 // loop filter designed to work using chars so that we can make maximum use 186 // of 8 bit simd instructions. 187 for (i = 0; i < 8; ++i) { 188 const uint8_t p3 = s[-4 * p], p2 = s[-3 * p], p1 = s[-2 * p], p0 = s[-p]; 189 const uint8_t q0 = s[0 * p], q1 = s[1 * p], q2 = s[2 * p], q3 = s[3 * p]; 190 191 const int8_t mask = 192 filter_mask(*limit, *blimit, p3, p2, p1, p0, q0, q1, q2, q3); 193 const int8_t flat = flat_mask4(1, p3, p2, p1, p0, q0, q1, q2, q3); 194 filter8(mask, *thresh, flat, s - 4 * p, s - 3 * p, s - 2 * p, s - 1 * p, s, 195 s + 1 * p, s + 2 * p, s + 3 * p); 196 ++s; 197 } 198 } 199 200 void vpx_lpf_horizontal_8_dual_c(uint8_t *s, int p, const uint8_t *blimit0, 201 const uint8_t *limit0, const uint8_t *thresh0, 202 const uint8_t *blimit1, const uint8_t *limit1, 203 const uint8_t *thresh1) { 204 vpx_lpf_horizontal_8_c(s, p, blimit0, limit0, thresh0); 205 vpx_lpf_horizontal_8_c(s + 8, p, blimit1, limit1, thresh1); 206 } 207 208 void vpx_lpf_vertical_8_c(uint8_t *s, int pitch, const uint8_t *blimit, 209 const uint8_t *limit, const uint8_t *thresh) { 210 int i; 211 212 for (i = 0; i < 8; ++i) { 213 const uint8_t p3 = s[-4], p2 = s[-3], p1 = s[-2], p0 = s[-1]; 214 const uint8_t q0 = s[0], q1 = s[1], q2 = s[2], q3 = s[3]; 215 const int8_t mask = 216 filter_mask(*limit, *blimit, p3, p2, p1, p0, q0, q1, q2, q3); 217 const int8_t flat = flat_mask4(1, p3, p2, p1, p0, q0, q1, q2, q3); 218 filter8(mask, *thresh, flat, s - 4, s - 3, s - 2, s - 1, s, s + 1, s + 2, 219 s + 3); 220 s += pitch; 221 } 222 } 223 224 void vpx_lpf_vertical_8_dual_c(uint8_t *s, int pitch, const uint8_t *blimit0, 225 const uint8_t *limit0, const uint8_t *thresh0, 226 const uint8_t *blimit1, const uint8_t *limit1, 227 const uint8_t *thresh1) { 228 vpx_lpf_vertical_8_c(s, pitch, blimit0, limit0, thresh0); 229 vpx_lpf_vertical_8_c(s + 8 * pitch, pitch, blimit1, limit1, thresh1); 230 } 231 232 static INLINE void filter16(int8_t mask, uint8_t thresh, uint8_t flat, 233 uint8_t flat2, uint8_t *op7, uint8_t *op6, 234 uint8_t *op5, uint8_t *op4, uint8_t *op3, 235 uint8_t *op2, uint8_t *op1, uint8_t *op0, 236 uint8_t *oq0, uint8_t *oq1, uint8_t *oq2, 237 uint8_t *oq3, uint8_t *oq4, uint8_t *oq5, 238 uint8_t *oq6, uint8_t *oq7) { 239 if (flat2 && flat && mask) { 240 const uint8_t p7 = *op7, p6 = *op6, p5 = *op5, p4 = *op4, p3 = *op3, 241 p2 = *op2, p1 = *op1, p0 = *op0; 242 243 const uint8_t q0 = *oq0, q1 = *oq1, q2 = *oq2, q3 = *oq3, q4 = *oq4, 244 q5 = *oq5, q6 = *oq6, q7 = *oq7; 245 246 // 15-tap filter [1, 1, 1, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1] 247 *op6 = ROUND_POWER_OF_TWO( 248 p7 * 7 + p6 * 2 + p5 + p4 + p3 + p2 + p1 + p0 + q0, 4); 249 *op5 = ROUND_POWER_OF_TWO( 250 p7 * 6 + p6 + p5 * 2 + p4 + p3 + p2 + p1 + p0 + q0 + q1, 4); 251 *op4 = ROUND_POWER_OF_TWO( 252 p7 * 5 + p6 + p5 + p4 * 2 + p3 + p2 + p1 + p0 + q0 + q1 + q2, 4); 253 *op3 = ROUND_POWER_OF_TWO( 254 p7 * 4 + p6 + p5 + p4 + p3 * 2 + p2 + p1 + p0 + q0 + q1 + q2 + q3, 4); 255 *op2 = ROUND_POWER_OF_TWO( 256 p7 * 3 + p6 + p5 + p4 + p3 + p2 * 2 + p1 + p0 + q0 + q1 + q2 + q3 + q4, 257 4); 258 *op1 = ROUND_POWER_OF_TWO(p7 * 2 + p6 + p5 + p4 + p3 + p2 + p1 * 2 + p0 + 259 q0 + q1 + q2 + q3 + q4 + q5, 260 4); 261 *op0 = ROUND_POWER_OF_TWO(p7 + p6 + p5 + p4 + p3 + p2 + p1 + p0 * 2 + q0 + 262 q1 + q2 + q3 + q4 + q5 + q6, 263 4); 264 *oq0 = ROUND_POWER_OF_TWO(p6 + p5 + p4 + p3 + p2 + p1 + p0 + q0 * 2 + q1 + 265 q2 + q3 + q4 + q5 + q6 + q7, 266 4); 267 *oq1 = ROUND_POWER_OF_TWO(p5 + p4 + p3 + p2 + p1 + p0 + q0 + q1 * 2 + q2 + 268 q3 + q4 + q5 + q6 + q7 * 2, 269 4); 270 *oq2 = ROUND_POWER_OF_TWO( 271 p4 + p3 + p2 + p1 + p0 + q0 + q1 + q2 * 2 + q3 + q4 + q5 + q6 + q7 * 3, 272 4); 273 *oq3 = ROUND_POWER_OF_TWO( 274 p3 + p2 + p1 + p0 + q0 + q1 + q2 + q3 * 2 + q4 + q5 + q6 + q7 * 4, 4); 275 *oq4 = ROUND_POWER_OF_TWO( 276 p2 + p1 + p0 + q0 + q1 + q2 + q3 + q4 * 2 + q5 + q6 + q7 * 5, 4); 277 *oq5 = ROUND_POWER_OF_TWO( 278 p1 + p0 + q0 + q1 + q2 + q3 + q4 + q5 * 2 + q6 + q7 * 6, 4); 279 *oq6 = ROUND_POWER_OF_TWO( 280 p0 + q0 + q1 + q2 + q3 + q4 + q5 + q6 * 2 + q7 * 7, 4); 281 } else { 282 filter8(mask, thresh, flat, op3, op2, op1, op0, oq0, oq1, oq2, oq3); 283 } 284 } 285 286 static void mb_lpf_horizontal_edge_w(uint8_t *s, int p, const uint8_t *blimit, 287 const uint8_t *limit, 288 const uint8_t *thresh, int count) { 289 int i; 290 291 // loop filter designed to work using chars so that we can make maximum use 292 // of 8 bit simd instructions. 293 for (i = 0; i < 8 * count; ++i) { 294 const uint8_t p3 = s[-4 * p], p2 = s[-3 * p], p1 = s[-2 * p], p0 = s[-p]; 295 const uint8_t q0 = s[0 * p], q1 = s[1 * p], q2 = s[2 * p], q3 = s[3 * p]; 296 const int8_t mask = 297 filter_mask(*limit, *blimit, p3, p2, p1, p0, q0, q1, q2, q3); 298 const int8_t flat = flat_mask4(1, p3, p2, p1, p0, q0, q1, q2, q3); 299 const int8_t flat2 = 300 flat_mask5(1, s[-8 * p], s[-7 * p], s[-6 * p], s[-5 * p], p0, q0, 301 s[4 * p], s[5 * p], s[6 * p], s[7 * p]); 302 303 filter16(mask, *thresh, flat, flat2, s - 8 * p, s - 7 * p, s - 6 * p, 304 s - 5 * p, s - 4 * p, s - 3 * p, s - 2 * p, s - 1 * p, s, 305 s + 1 * p, s + 2 * p, s + 3 * p, s + 4 * p, s + 5 * p, s + 6 * p, 306 s + 7 * p); 307 ++s; 308 } 309 } 310 311 void vpx_lpf_horizontal_16_c(uint8_t *s, int p, const uint8_t *blimit, 312 const uint8_t *limit, const uint8_t *thresh) { 313 mb_lpf_horizontal_edge_w(s, p, blimit, limit, thresh, 1); 314 } 315 316 void vpx_lpf_horizontal_16_dual_c(uint8_t *s, int p, const uint8_t *blimit, 317 const uint8_t *limit, const uint8_t *thresh) { 318 mb_lpf_horizontal_edge_w(s, p, blimit, limit, thresh, 2); 319 } 320 321 static void mb_lpf_vertical_edge_w(uint8_t *s, int p, const uint8_t *blimit, 322 const uint8_t *limit, const uint8_t *thresh, 323 int count) { 324 int i; 325 326 for (i = 0; i < count; ++i) { 327 const uint8_t p3 = s[-4], p2 = s[-3], p1 = s[-2], p0 = s[-1]; 328 const uint8_t q0 = s[0], q1 = s[1], q2 = s[2], q3 = s[3]; 329 const int8_t mask = 330 filter_mask(*limit, *blimit, p3, p2, p1, p0, q0, q1, q2, q3); 331 const int8_t flat = flat_mask4(1, p3, p2, p1, p0, q0, q1, q2, q3); 332 const int8_t flat2 = flat_mask5(1, s[-8], s[-7], s[-6], s[-5], p0, q0, s[4], 333 s[5], s[6], s[7]); 334 335 filter16(mask, *thresh, flat, flat2, s - 8, s - 7, s - 6, s - 5, s - 4, 336 s - 3, s - 2, s - 1, s, s + 1, s + 2, s + 3, s + 4, s + 5, s + 6, 337 s + 7); 338 s += p; 339 } 340 } 341 342 void vpx_lpf_vertical_16_c(uint8_t *s, int p, const uint8_t *blimit, 343 const uint8_t *limit, const uint8_t *thresh) { 344 mb_lpf_vertical_edge_w(s, p, blimit, limit, thresh, 8); 345 } 346 347 void vpx_lpf_vertical_16_dual_c(uint8_t *s, int p, const uint8_t *blimit, 348 const uint8_t *limit, const uint8_t *thresh) { 349 mb_lpf_vertical_edge_w(s, p, blimit, limit, thresh, 16); 350 } 351 352 #if CONFIG_VP9_HIGHBITDEPTH 353 // Should we apply any filter at all: 11111111 yes, 00000000 no ? 354 static INLINE int8_t highbd_filter_mask(uint8_t limit, uint8_t blimit, 355 uint16_t p3, uint16_t p2, uint16_t p1, 356 uint16_t p0, uint16_t q0, uint16_t q1, 357 uint16_t q2, uint16_t q3, int bd) { 358 int8_t mask = 0; 359 int16_t limit16 = (uint16_t)limit << (bd - 8); 360 int16_t blimit16 = (uint16_t)blimit << (bd - 8); 361 mask |= (abs(p3 - p2) > limit16) * -1; 362 mask |= (abs(p2 - p1) > limit16) * -1; 363 mask |= (abs(p1 - p0) > limit16) * -1; 364 mask |= (abs(q1 - q0) > limit16) * -1; 365 mask |= (abs(q2 - q1) > limit16) * -1; 366 mask |= (abs(q3 - q2) > limit16) * -1; 367 mask |= (abs(p0 - q0) * 2 + abs(p1 - q1) / 2 > blimit16) * -1; 368 return ~mask; 369 } 370 371 static INLINE int8_t highbd_flat_mask4(uint8_t thresh, uint16_t p3, uint16_t p2, 372 uint16_t p1, uint16_t p0, uint16_t q0, 373 uint16_t q1, uint16_t q2, uint16_t q3, 374 int bd) { 375 int8_t mask = 0; 376 int16_t thresh16 = (uint16_t)thresh << (bd - 8); 377 mask |= (abs(p1 - p0) > thresh16) * -1; 378 mask |= (abs(q1 - q0) > thresh16) * -1; 379 mask |= (abs(p2 - p0) > thresh16) * -1; 380 mask |= (abs(q2 - q0) > thresh16) * -1; 381 mask |= (abs(p3 - p0) > thresh16) * -1; 382 mask |= (abs(q3 - q0) > thresh16) * -1; 383 return ~mask; 384 } 385 386 static INLINE int8_t highbd_flat_mask5(uint8_t thresh, uint16_t p4, uint16_t p3, 387 uint16_t p2, uint16_t p1, uint16_t p0, 388 uint16_t q0, uint16_t q1, uint16_t q2, 389 uint16_t q3, uint16_t q4, int bd) { 390 int8_t mask = ~highbd_flat_mask4(thresh, p3, p2, p1, p0, q0, q1, q2, q3, bd); 391 int16_t thresh16 = (uint16_t)thresh << (bd - 8); 392 mask |= (abs(p4 - p0) > thresh16) * -1; 393 mask |= (abs(q4 - q0) > thresh16) * -1; 394 return ~mask; 395 } 396 397 // Is there high edge variance internal edge: 398 // 11111111_11111111 yes, 00000000_00000000 no ? 399 static INLINE int16_t highbd_hev_mask(uint8_t thresh, uint16_t p1, uint16_t p0, 400 uint16_t q0, uint16_t q1, int bd) { 401 int16_t hev = 0; 402 int16_t thresh16 = (uint16_t)thresh << (bd - 8); 403 hev |= (abs(p1 - p0) > thresh16) * -1; 404 hev |= (abs(q1 - q0) > thresh16) * -1; 405 return hev; 406 } 407 408 static INLINE void highbd_filter4(int8_t mask, uint8_t thresh, uint16_t *op1, 409 uint16_t *op0, uint16_t *oq0, uint16_t *oq1, 410 int bd) { 411 int16_t filter1, filter2; 412 // ^0x80 equivalent to subtracting 0x80 from the values to turn them 413 // into -128 to +127 instead of 0 to 255. 414 int shift = bd - 8; 415 const int16_t ps1 = (int16_t)*op1 - (0x80 << shift); 416 const int16_t ps0 = (int16_t)*op0 - (0x80 << shift); 417 const int16_t qs0 = (int16_t)*oq0 - (0x80 << shift); 418 const int16_t qs1 = (int16_t)*oq1 - (0x80 << shift); 419 const uint16_t hev = highbd_hev_mask(thresh, *op1, *op0, *oq0, *oq1, bd); 420 421 // Add outer taps if we have high edge variance. 422 int16_t filter = signed_char_clamp_high(ps1 - qs1, bd) & hev; 423 424 // Inner taps. 425 filter = signed_char_clamp_high(filter + 3 * (qs0 - ps0), bd) & mask; 426 427 // Save bottom 3 bits so that we round one side +4 and the other +3 428 // if it equals 4 we'll set it to adjust by -1 to account for the fact 429 // we'd round it by 3 the other way. 430 filter1 = signed_char_clamp_high(filter + 4, bd) >> 3; 431 filter2 = signed_char_clamp_high(filter + 3, bd) >> 3; 432 433 *oq0 = signed_char_clamp_high(qs0 - filter1, bd) + (0x80 << shift); 434 *op0 = signed_char_clamp_high(ps0 + filter2, bd) + (0x80 << shift); 435 436 // Outer tap adjustments. 437 filter = ROUND_POWER_OF_TWO(filter1, 1) & ~hev; 438 439 *oq1 = signed_char_clamp_high(qs1 - filter, bd) + (0x80 << shift); 440 *op1 = signed_char_clamp_high(ps1 + filter, bd) + (0x80 << shift); 441 } 442 443 void vpx_highbd_lpf_horizontal_4_c(uint16_t *s, int p /* pitch */, 444 const uint8_t *blimit, const uint8_t *limit, 445 const uint8_t *thresh, int bd) { 446 int i; 447 448 // loop filter designed to work using chars so that we can make maximum use 449 // of 8 bit simd instructions. 450 for (i = 0; i < 8; ++i) { 451 const uint16_t p3 = s[-4 * p]; 452 const uint16_t p2 = s[-3 * p]; 453 const uint16_t p1 = s[-2 * p]; 454 const uint16_t p0 = s[-p]; 455 const uint16_t q0 = s[0 * p]; 456 const uint16_t q1 = s[1 * p]; 457 const uint16_t q2 = s[2 * p]; 458 const uint16_t q3 = s[3 * p]; 459 const int8_t mask = 460 highbd_filter_mask(*limit, *blimit, p3, p2, p1, p0, q0, q1, q2, q3, bd); 461 highbd_filter4(mask, *thresh, s - 2 * p, s - 1 * p, s, s + 1 * p, bd); 462 ++s; 463 } 464 } 465 466 void vpx_highbd_lpf_horizontal_4_dual_c( 467 uint16_t *s, int p, const uint8_t *blimit0, const uint8_t *limit0, 468 const uint8_t *thresh0, const uint8_t *blimit1, const uint8_t *limit1, 469 const uint8_t *thresh1, int bd) { 470 vpx_highbd_lpf_horizontal_4_c(s, p, blimit0, limit0, thresh0, bd); 471 vpx_highbd_lpf_horizontal_4_c(s + 8, p, blimit1, limit1, thresh1, bd); 472 } 473 474 void vpx_highbd_lpf_vertical_4_c(uint16_t *s, int pitch, const uint8_t *blimit, 475 const uint8_t *limit, const uint8_t *thresh, 476 int bd) { 477 int i; 478 479 // loop filter designed to work using chars so that we can make maximum use 480 // of 8 bit simd instructions. 481 for (i = 0; i < 8; ++i) { 482 const uint16_t p3 = s[-4], p2 = s[-3], p1 = s[-2], p0 = s[-1]; 483 const uint16_t q0 = s[0], q1 = s[1], q2 = s[2], q3 = s[3]; 484 const int8_t mask = 485 highbd_filter_mask(*limit, *blimit, p3, p2, p1, p0, q0, q1, q2, q3, bd); 486 highbd_filter4(mask, *thresh, s - 2, s - 1, s, s + 1, bd); 487 s += pitch; 488 } 489 } 490 491 void vpx_highbd_lpf_vertical_4_dual_c( 492 uint16_t *s, int pitch, const uint8_t *blimit0, const uint8_t *limit0, 493 const uint8_t *thresh0, const uint8_t *blimit1, const uint8_t *limit1, 494 const uint8_t *thresh1, int bd) { 495 vpx_highbd_lpf_vertical_4_c(s, pitch, blimit0, limit0, thresh0, bd); 496 vpx_highbd_lpf_vertical_4_c(s + 8 * pitch, pitch, blimit1, limit1, thresh1, 497 bd); 498 } 499 500 static INLINE void highbd_filter8(int8_t mask, uint8_t thresh, uint8_t flat, 501 uint16_t *op3, uint16_t *op2, uint16_t *op1, 502 uint16_t *op0, uint16_t *oq0, uint16_t *oq1, 503 uint16_t *oq2, uint16_t *oq3, int bd) { 504 if (flat && mask) { 505 const uint16_t p3 = *op3, p2 = *op2, p1 = *op1, p0 = *op0; 506 const uint16_t q0 = *oq0, q1 = *oq1, q2 = *oq2, q3 = *oq3; 507 508 // 7-tap filter [1, 1, 1, 2, 1, 1, 1] 509 *op2 = ROUND_POWER_OF_TWO(p3 + p3 + p3 + 2 * p2 + p1 + p0 + q0, 3); 510 *op1 = ROUND_POWER_OF_TWO(p3 + p3 + p2 + 2 * p1 + p0 + q0 + q1, 3); 511 *op0 = ROUND_POWER_OF_TWO(p3 + p2 + p1 + 2 * p0 + q0 + q1 + q2, 3); 512 *oq0 = ROUND_POWER_OF_TWO(p2 + p1 + p0 + 2 * q0 + q1 + q2 + q3, 3); 513 *oq1 = ROUND_POWER_OF_TWO(p1 + p0 + q0 + 2 * q1 + q2 + q3 + q3, 3); 514 *oq2 = ROUND_POWER_OF_TWO(p0 + q0 + q1 + 2 * q2 + q3 + q3 + q3, 3); 515 } else { 516 highbd_filter4(mask, thresh, op1, op0, oq0, oq1, bd); 517 } 518 } 519 520 void vpx_highbd_lpf_horizontal_8_c(uint16_t *s, int p, const uint8_t *blimit, 521 const uint8_t *limit, const uint8_t *thresh, 522 int bd) { 523 int i; 524 525 // loop filter designed to work using chars so that we can make maximum use 526 // of 8 bit simd instructions. 527 for (i = 0; i < 8; ++i) { 528 const uint16_t p3 = s[-4 * p], p2 = s[-3 * p], p1 = s[-2 * p], p0 = s[-p]; 529 const uint16_t q0 = s[0 * p], q1 = s[1 * p], q2 = s[2 * p], q3 = s[3 * p]; 530 531 const int8_t mask = 532 highbd_filter_mask(*limit, *blimit, p3, p2, p1, p0, q0, q1, q2, q3, bd); 533 const int8_t flat = 534 highbd_flat_mask4(1, p3, p2, p1, p0, q0, q1, q2, q3, bd); 535 highbd_filter8(mask, *thresh, flat, s - 4 * p, s - 3 * p, s - 2 * p, 536 s - 1 * p, s, s + 1 * p, s + 2 * p, s + 3 * p, bd); 537 ++s; 538 } 539 } 540 541 void vpx_highbd_lpf_horizontal_8_dual_c( 542 uint16_t *s, int p, const uint8_t *blimit0, const uint8_t *limit0, 543 const uint8_t *thresh0, const uint8_t *blimit1, const uint8_t *limit1, 544 const uint8_t *thresh1, int bd) { 545 vpx_highbd_lpf_horizontal_8_c(s, p, blimit0, limit0, thresh0, bd); 546 vpx_highbd_lpf_horizontal_8_c(s + 8, p, blimit1, limit1, thresh1, bd); 547 } 548 549 void vpx_highbd_lpf_vertical_8_c(uint16_t *s, int pitch, const uint8_t *blimit, 550 const uint8_t *limit, const uint8_t *thresh, 551 int bd) { 552 int i; 553 554 for (i = 0; i < 8; ++i) { 555 const uint16_t p3 = s[-4], p2 = s[-3], p1 = s[-2], p0 = s[-1]; 556 const uint16_t q0 = s[0], q1 = s[1], q2 = s[2], q3 = s[3]; 557 const int8_t mask = 558 highbd_filter_mask(*limit, *blimit, p3, p2, p1, p0, q0, q1, q2, q3, bd); 559 const int8_t flat = 560 highbd_flat_mask4(1, p3, p2, p1, p0, q0, q1, q2, q3, bd); 561 highbd_filter8(mask, *thresh, flat, s - 4, s - 3, s - 2, s - 1, s, s + 1, 562 s + 2, s + 3, bd); 563 s += pitch; 564 } 565 } 566 567 void vpx_highbd_lpf_vertical_8_dual_c( 568 uint16_t *s, int pitch, const uint8_t *blimit0, const uint8_t *limit0, 569 const uint8_t *thresh0, const uint8_t *blimit1, const uint8_t *limit1, 570 const uint8_t *thresh1, int bd) { 571 vpx_highbd_lpf_vertical_8_c(s, pitch, blimit0, limit0, thresh0, bd); 572 vpx_highbd_lpf_vertical_8_c(s + 8 * pitch, pitch, blimit1, limit1, thresh1, 573 bd); 574 } 575 576 static INLINE void highbd_filter16(int8_t mask, uint8_t thresh, uint8_t flat, 577 uint8_t flat2, uint16_t *op7, uint16_t *op6, 578 uint16_t *op5, uint16_t *op4, uint16_t *op3, 579 uint16_t *op2, uint16_t *op1, uint16_t *op0, 580 uint16_t *oq0, uint16_t *oq1, uint16_t *oq2, 581 uint16_t *oq3, uint16_t *oq4, uint16_t *oq5, 582 uint16_t *oq6, uint16_t *oq7, int bd) { 583 if (flat2 && flat && mask) { 584 const uint16_t p7 = *op7; 585 const uint16_t p6 = *op6; 586 const uint16_t p5 = *op5; 587 const uint16_t p4 = *op4; 588 const uint16_t p3 = *op3; 589 const uint16_t p2 = *op2; 590 const uint16_t p1 = *op1; 591 const uint16_t p0 = *op0; 592 const uint16_t q0 = *oq0; 593 const uint16_t q1 = *oq1; 594 const uint16_t q2 = *oq2; 595 const uint16_t q3 = *oq3; 596 const uint16_t q4 = *oq4; 597 const uint16_t q5 = *oq5; 598 const uint16_t q6 = *oq6; 599 const uint16_t q7 = *oq7; 600 601 // 15-tap filter [1, 1, 1, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1] 602 *op6 = ROUND_POWER_OF_TWO( 603 p7 * 7 + p6 * 2 + p5 + p4 + p3 + p2 + p1 + p0 + q0, 4); 604 *op5 = ROUND_POWER_OF_TWO( 605 p7 * 6 + p6 + p5 * 2 + p4 + p3 + p2 + p1 + p0 + q0 + q1, 4); 606 *op4 = ROUND_POWER_OF_TWO( 607 p7 * 5 + p6 + p5 + p4 * 2 + p3 + p2 + p1 + p0 + q0 + q1 + q2, 4); 608 *op3 = ROUND_POWER_OF_TWO( 609 p7 * 4 + p6 + p5 + p4 + p3 * 2 + p2 + p1 + p0 + q0 + q1 + q2 + q3, 4); 610 *op2 = ROUND_POWER_OF_TWO( 611 p7 * 3 + p6 + p5 + p4 + p3 + p2 * 2 + p1 + p0 + q0 + q1 + q2 + q3 + q4, 612 4); 613 *op1 = ROUND_POWER_OF_TWO(p7 * 2 + p6 + p5 + p4 + p3 + p2 + p1 * 2 + p0 + 614 q0 + q1 + q2 + q3 + q4 + q5, 615 4); 616 *op0 = ROUND_POWER_OF_TWO(p7 + p6 + p5 + p4 + p3 + p2 + p1 + p0 * 2 + q0 + 617 q1 + q2 + q3 + q4 + q5 + q6, 618 4); 619 *oq0 = ROUND_POWER_OF_TWO(p6 + p5 + p4 + p3 + p2 + p1 + p0 + q0 * 2 + q1 + 620 q2 + q3 + q4 + q5 + q6 + q7, 621 4); 622 *oq1 = ROUND_POWER_OF_TWO(p5 + p4 + p3 + p2 + p1 + p0 + q0 + q1 * 2 + q2 + 623 q3 + q4 + q5 + q6 + q7 * 2, 624 4); 625 *oq2 = ROUND_POWER_OF_TWO( 626 p4 + p3 + p2 + p1 + p0 + q0 + q1 + q2 * 2 + q3 + q4 + q5 + q6 + q7 * 3, 627 4); 628 *oq3 = ROUND_POWER_OF_TWO( 629 p3 + p2 + p1 + p0 + q0 + q1 + q2 + q3 * 2 + q4 + q5 + q6 + q7 * 4, 4); 630 *oq4 = ROUND_POWER_OF_TWO( 631 p2 + p1 + p0 + q0 + q1 + q2 + q3 + q4 * 2 + q5 + q6 + q7 * 5, 4); 632 *oq5 = ROUND_POWER_OF_TWO( 633 p1 + p0 + q0 + q1 + q2 + q3 + q4 + q5 * 2 + q6 + q7 * 6, 4); 634 *oq6 = ROUND_POWER_OF_TWO( 635 p0 + q0 + q1 + q2 + q3 + q4 + q5 + q6 * 2 + q7 * 7, 4); 636 } else { 637 highbd_filter8(mask, thresh, flat, op3, op2, op1, op0, oq0, oq1, oq2, oq3, 638 bd); 639 } 640 } 641 642 static void highbd_mb_lpf_horizontal_edge_w(uint16_t *s, int p, 643 const uint8_t *blimit, 644 const uint8_t *limit, 645 const uint8_t *thresh, int count, 646 int bd) { 647 int i; 648 649 // loop filter designed to work using chars so that we can make maximum use 650 // of 8 bit simd instructions. 651 for (i = 0; i < 8 * count; ++i) { 652 const uint16_t p3 = s[-4 * p]; 653 const uint16_t p2 = s[-3 * p]; 654 const uint16_t p1 = s[-2 * p]; 655 const uint16_t p0 = s[-p]; 656 const uint16_t q0 = s[0 * p]; 657 const uint16_t q1 = s[1 * p]; 658 const uint16_t q2 = s[2 * p]; 659 const uint16_t q3 = s[3 * p]; 660 const int8_t mask = 661 highbd_filter_mask(*limit, *blimit, p3, p2, p1, p0, q0, q1, q2, q3, bd); 662 const int8_t flat = 663 highbd_flat_mask4(1, p3, p2, p1, p0, q0, q1, q2, q3, bd); 664 const int8_t flat2 = 665 highbd_flat_mask5(1, s[-8 * p], s[-7 * p], s[-6 * p], s[-5 * p], p0, q0, 666 s[4 * p], s[5 * p], s[6 * p], s[7 * p], bd); 667 668 highbd_filter16(mask, *thresh, flat, flat2, s - 8 * p, s - 7 * p, s - 6 * p, 669 s - 5 * p, s - 4 * p, s - 3 * p, s - 2 * p, s - 1 * p, s, 670 s + 1 * p, s + 2 * p, s + 3 * p, s + 4 * p, s + 5 * p, 671 s + 6 * p, s + 7 * p, bd); 672 ++s; 673 } 674 } 675 676 void vpx_highbd_lpf_horizontal_16_c(uint16_t *s, int p, const uint8_t *blimit, 677 const uint8_t *limit, const uint8_t *thresh, 678 int bd) { 679 highbd_mb_lpf_horizontal_edge_w(s, p, blimit, limit, thresh, 1, bd); 680 } 681 682 void vpx_highbd_lpf_horizontal_16_dual_c(uint16_t *s, int p, 683 const uint8_t *blimit, 684 const uint8_t *limit, 685 const uint8_t *thresh, int bd) { 686 highbd_mb_lpf_horizontal_edge_w(s, p, blimit, limit, thresh, 2, bd); 687 } 688 689 static void highbd_mb_lpf_vertical_edge_w(uint16_t *s, int p, 690 const uint8_t *blimit, 691 const uint8_t *limit, 692 const uint8_t *thresh, int count, 693 int bd) { 694 int i; 695 696 for (i = 0; i < count; ++i) { 697 const uint16_t p3 = s[-4]; 698 const uint16_t p2 = s[-3]; 699 const uint16_t p1 = s[-2]; 700 const uint16_t p0 = s[-1]; 701 const uint16_t q0 = s[0]; 702 const uint16_t q1 = s[1]; 703 const uint16_t q2 = s[2]; 704 const uint16_t q3 = s[3]; 705 const int8_t mask = 706 highbd_filter_mask(*limit, *blimit, p3, p2, p1, p0, q0, q1, q2, q3, bd); 707 const int8_t flat = 708 highbd_flat_mask4(1, p3, p2, p1, p0, q0, q1, q2, q3, bd); 709 const int8_t flat2 = highbd_flat_mask5(1, s[-8], s[-7], s[-6], s[-5], p0, 710 q0, s[4], s[5], s[6], s[7], bd); 711 712 highbd_filter16(mask, *thresh, flat, flat2, s - 8, s - 7, s - 6, s - 5, 713 s - 4, s - 3, s - 2, s - 1, s, s + 1, s + 2, s + 3, s + 4, 714 s + 5, s + 6, s + 7, bd); 715 s += p; 716 } 717 } 718 719 void vpx_highbd_lpf_vertical_16_c(uint16_t *s, int p, const uint8_t *blimit, 720 const uint8_t *limit, const uint8_t *thresh, 721 int bd) { 722 highbd_mb_lpf_vertical_edge_w(s, p, blimit, limit, thresh, 8, bd); 723 } 724 725 void vpx_highbd_lpf_vertical_16_dual_c(uint16_t *s, int p, 726 const uint8_t *blimit, 727 const uint8_t *limit, 728 const uint8_t *thresh, int bd) { 729 highbd_mb_lpf_vertical_edge_w(s, p, blimit, limit, thresh, 16, bd); 730 } 731 #endif // CONFIG_VP9_HIGHBITDEPTH 732
