1 // Copyright 2018 Google Inc. All Rights Reserved. 2 // 3 // Use of this source code is governed by a BSD-style license 4 // that can be found in the COPYING file in the root of the source 5 // tree. An additional intellectual property rights grant can be found 6 // in the file PATENTS. All contributing project authors may 7 // be found in the AUTHORS file in the root of the source tree. 8 // ----------------------------------------------------------------------------- 9 // 10 // ARM NEON version of cost functions 11 12 #include "src/dsp/dsp.h" 13 14 #if defined(WEBP_USE_NEON) 15 16 #include "src/dsp/neon.h" 17 #include "src/enc/cost_enc.h" 18 19 static const uint8_t position[16] = { 1, 2, 3, 4, 5, 6, 7, 8, 20 9, 10, 11, 12, 13, 14, 15, 16 }; 21 22 static void SetResidualCoeffs_NEON(const int16_t* const coeffs, 23 VP8Residual* const res) { 24 const int16x8_t minus_one = vdupq_n_s16(-1); 25 const int16x8_t coeffs_0 = vld1q_s16(coeffs); 26 const int16x8_t coeffs_1 = vld1q_s16(coeffs + 8); 27 const uint16x8_t eob_0 = vtstq_s16(coeffs_0, minus_one); 28 const uint16x8_t eob_1 = vtstq_s16(coeffs_1, minus_one); 29 const uint8x16_t eob = vcombine_u8(vqmovn_u16(eob_0), vqmovn_u16(eob_1)); 30 const uint8x16_t masked = vandq_u8(eob, vld1q_u8(position)); 31 32 #ifdef __aarch64__ 33 res->last = vmaxvq_u8(masked) - 1; 34 #else 35 const uint8x8_t eob_8x8 = vmax_u8(vget_low_u8(masked), vget_high_u8(masked)); 36 const uint16x8_t eob_16x8 = vmovl_u8(eob_8x8); 37 const uint16x4_t eob_16x4 = 38 vmax_u16(vget_low_u16(eob_16x8), vget_high_u16(eob_16x8)); 39 const uint32x4_t eob_32x4 = vmovl_u16(eob_16x4); 40 uint32x2_t eob_32x2 = 41 vmax_u32(vget_low_u32(eob_32x4), vget_high_u32(eob_32x4)); 42 eob_32x2 = vpmax_u32(eob_32x2, eob_32x2); 43 44 vst1_lane_s32(&res->last, vreinterpret_s32_u32(eob_32x2), 0); 45 --res->last; 46 #endif // __aarch64__ 47 48 res->coeffs = coeffs; 49 } 50 51 static int GetResidualCost_NEON(int ctx0, const VP8Residual* const res) { 52 uint8_t levels[16], ctxs[16]; 53 uint16_t abs_levels[16]; 54 int n = res->first; 55 // should be prob[VP8EncBands[n]], but it's equivalent for n=0 or 1 56 const int p0 = res->prob[n][ctx0][0]; 57 CostArrayPtr const costs = res->costs; 58 const uint16_t* t = costs[n][ctx0]; 59 // bit_cost(1, p0) is already incorporated in t[] tables, but only if ctx != 0 60 // (as required by the syntax). For ctx0 == 0, we need to add it here or it'll 61 // be missing during the loop. 62 int cost = (ctx0 == 0) ? VP8BitCost(1, p0) : 0; 63 64 if (res->last < 0) { 65 return VP8BitCost(0, p0); 66 } 67 68 { // precompute clamped levels and contexts, packed to 8b. 69 const uint8x16_t kCst2 = vdupq_n_u8(2); 70 const uint8x16_t kCst67 = vdupq_n_u8(MAX_VARIABLE_LEVEL); 71 const int16x8_t c0 = vld1q_s16(res->coeffs); 72 const int16x8_t c1 = vld1q_s16(res->coeffs + 8); 73 const uint16x8_t E0 = vreinterpretq_u16_s16(vabsq_s16(c0)); 74 const uint16x8_t E1 = vreinterpretq_u16_s16(vabsq_s16(c1)); 75 const uint8x16_t F = vcombine_u8(vqmovn_u16(E0), vqmovn_u16(E1)); 76 const uint8x16_t G = vminq_u8(F, kCst2); // context = 0,1,2 77 const uint8x16_t H = vminq_u8(F, kCst67); // clamp_level in [0..67] 78 79 vst1q_u8(ctxs, G); 80 vst1q_u8(levels, H); 81 82 vst1q_u16(abs_levels, E0); 83 vst1q_u16(abs_levels + 8, E1); 84 } 85 for (; n < res->last; ++n) { 86 const int ctx = ctxs[n]; 87 const int level = levels[n]; 88 const int flevel = abs_levels[n]; // full level 89 cost += VP8LevelFixedCosts[flevel] + t[level]; // simplified VP8LevelCost() 90 t = costs[n + 1][ctx]; 91 } 92 // Last coefficient is always non-zero 93 { 94 const int level = levels[n]; 95 const int flevel = abs_levels[n]; 96 assert(flevel != 0); 97 cost += VP8LevelFixedCosts[flevel] + t[level]; 98 if (n < 15) { 99 const int b = VP8EncBands[n + 1]; 100 const int ctx = ctxs[n]; 101 const int last_p0 = res->prob[b][ctx][0]; 102 cost += VP8BitCost(0, last_p0); 103 } 104 } 105 return cost; 106 } 107 108 //------------------------------------------------------------------------------ 109 // Entry point 110 111 extern void VP8EncDspCostInitNEON(void); 112 113 WEBP_TSAN_IGNORE_FUNCTION void VP8EncDspCostInitNEON(void) { 114 VP8SetResidualCoeffs = SetResidualCoeffs_NEON; 115 VP8GetResidualCost = GetResidualCost_NEON; 116 } 117 118 #else // !WEBP_USE_NEON 119 120 WEBP_DSP_INIT_STUB(VP8EncDspCostInitNEON) 121 122 #endif // WEBP_USE_NEON 123
