1 // Copyright 2011 Google Inc. All Rights Reserved. 2 // 3 // Use of this source code is governed by a BSD-style license 4 // that can be found in the COPYING file in the root of the source 5 // tree. An additional intellectual property rights grant can be found 6 // in the file PATENTS. All contributing project authors may 7 // be found in the AUTHORS file in the root of the source tree. 8 // ----------------------------------------------------------------------------- 9 // 10 // NEON version of YUV to RGB upsampling functions. 11 // 12 // Author: mans (at) mansr.com (Mans Rullgard) 13 // Based on SSE code by: somnath (at) google.com (Somnath Banerjee) 14 15 #include "./dsp.h" 16 17 #if defined(WEBP_USE_NEON) 18 19 #include <assert.h> 20 #include <arm_neon.h> 21 #include <string.h> 22 #include "./neon.h" 23 #include "./yuv.h" 24 25 #ifdef FANCY_UPSAMPLING 26 27 //----------------------------------------------------------------------------- 28 // U/V upsampling 29 30 // Loads 9 pixels each from rows r1 and r2 and generates 16 pixels. 31 #define UPSAMPLE_16PIXELS(r1, r2, out) { \ 32 uint8x8_t a = vld1_u8(r1); \ 33 uint8x8_t b = vld1_u8(r1 + 1); \ 34 uint8x8_t c = vld1_u8(r2); \ 35 uint8x8_t d = vld1_u8(r2 + 1); \ 36 \ 37 uint16x8_t al = vshll_n_u8(a, 1); \ 38 uint16x8_t bl = vshll_n_u8(b, 1); \ 39 uint16x8_t cl = vshll_n_u8(c, 1); \ 40 uint16x8_t dl = vshll_n_u8(d, 1); \ 41 \ 42 uint8x8_t diag1, diag2; \ 43 uint16x8_t sl; \ 44 \ 45 /* a + b + c + d */ \ 46 sl = vaddl_u8(a, b); \ 47 sl = vaddw_u8(sl, c); \ 48 sl = vaddw_u8(sl, d); \ 49 \ 50 al = vaddq_u16(sl, al); /* 3a + b + c + d */ \ 51 bl = vaddq_u16(sl, bl); /* a + 3b + c + d */ \ 52 \ 53 al = vaddq_u16(al, dl); /* 3a + b + c + 3d */ \ 54 bl = vaddq_u16(bl, cl); /* a + 3b + 3c + d */ \ 55 \ 56 diag2 = vshrn_n_u16(al, 3); \ 57 diag1 = vshrn_n_u16(bl, 3); \ 58 \ 59 a = vrhadd_u8(a, diag1); \ 60 b = vrhadd_u8(b, diag2); \ 61 c = vrhadd_u8(c, diag2); \ 62 d = vrhadd_u8(d, diag1); \ 63 \ 64 { \ 65 uint8x8x2_t a_b, c_d; \ 66 INIT_VECTOR2(a_b, a, b); \ 67 INIT_VECTOR2(c_d, c, d); \ 68 vst2_u8(out, a_b); \ 69 vst2_u8(out + 32, c_d); \ 70 } \ 71 } 72 73 // Turn the macro into a function for reducing code-size when non-critical 74 static void Upsample16Pixels(const uint8_t *r1, const uint8_t *r2, 75 uint8_t *out) { 76 UPSAMPLE_16PIXELS(r1, r2, out); 77 } 78 79 #define UPSAMPLE_LAST_BLOCK(tb, bb, num_pixels, out) { \ 80 uint8_t r1[9], r2[9]; \ 81 memcpy(r1, (tb), (num_pixels)); \ 82 memcpy(r2, (bb), (num_pixels)); \ 83 /* replicate last byte */ \ 84 memset(r1 + (num_pixels), r1[(num_pixels) - 1], 9 - (num_pixels)); \ 85 memset(r2 + (num_pixels), r2[(num_pixels) - 1], 9 - (num_pixels)); \ 86 Upsample16Pixels(r1, r2, out); \ 87 } 88 89 //----------------------------------------------------------------------------- 90 // YUV->RGB conversion 91 92 // note: we represent the 33050 large constant as 32768 + 282 93 static const int16_t kCoeffs1[4] = { 19077, 26149, 6419, 13320 }; 94 95 #define v255 vdup_n_u8(255) 96 #define v_0x0f vdup_n_u8(15) 97 98 #define STORE_Rgb(out, r, g, b) do { \ 99 uint8x8x3_t r_g_b; \ 100 INIT_VECTOR3(r_g_b, r, g, b); \ 101 vst3_u8(out, r_g_b); \ 102 } while (0) 103 104 #define STORE_Bgr(out, r, g, b) do { \ 105 uint8x8x3_t b_g_r; \ 106 INIT_VECTOR3(b_g_r, b, g, r); \ 107 vst3_u8(out, b_g_r); \ 108 } while (0) 109 110 #define STORE_Rgba(out, r, g, b) do { \ 111 uint8x8x4_t r_g_b_v255; \ 112 INIT_VECTOR4(r_g_b_v255, r, g, b, v255); \ 113 vst4_u8(out, r_g_b_v255); \ 114 } while (0) 115 116 #define STORE_Bgra(out, r, g, b) do { \ 117 uint8x8x4_t b_g_r_v255; \ 118 INIT_VECTOR4(b_g_r_v255, b, g, r, v255); \ 119 vst4_u8(out, b_g_r_v255); \ 120 } while (0) 121 122 #define STORE_Argb(out, r, g, b) do { \ 123 uint8x8x4_t v255_r_g_b; \ 124 INIT_VECTOR4(v255_r_g_b, v255, r, g, b); \ 125 vst4_u8(out, v255_r_g_b); \ 126 } while (0) 127 128 #if !defined(WEBP_SWAP_16BIT_CSP) 129 #define ZIP_U8(lo, hi) vzip_u8((lo), (hi)) 130 #else 131 #define ZIP_U8(lo, hi) vzip_u8((hi), (lo)) 132 #endif 133 134 #define STORE_Rgba4444(out, r, g, b) do { \ 135 const uint8x8_t r1 = vshl_n_u8(vshr_n_u8(r, 4), 4); /* 4bits */ \ 136 const uint8x8_t g1 = vshr_n_u8(g, 4); \ 137 const uint8x8_t ba = vorr_u8(b, v_0x0f); \ 138 const uint8x8_t rg = vorr_u8(r1, g1); \ 139 const uint8x8x2_t rgba4444 = ZIP_U8(rg, ba); \ 140 vst1q_u8(out, vcombine_u8(rgba4444.val[0], rgba4444.val[1])); \ 141 } while (0) 142 143 #define STORE_Rgb565(out, r, g, b) do { \ 144 const uint8x8_t r1 = vshl_n_u8(vshr_n_u8(r, 3), 3); /* 5bits */ \ 145 const uint8x8_t g1 = vshr_n_u8(g, 5); /* upper 3bits */\ 146 const uint8x8_t g2 = vshl_n_u8(vshr_n_u8(g, 2), 5); /* lower 3bits */\ 147 const uint8x8_t b1 = vshr_n_u8(b, 3); /* 5bits */ \ 148 const uint8x8_t rg = vorr_u8(r1, g1); \ 149 const uint8x8_t gb = vorr_u8(g2, b1); \ 150 const uint8x8x2_t rgb565 = ZIP_U8(rg, gb); \ 151 vst1q_u8(out, vcombine_u8(rgb565.val[0], rgb565.val[1])); \ 152 } while (0) 153 154 #define CONVERT8(FMT, XSTEP, N, src_y, src_uv, out, cur_x) do { \ 155 int i; \ 156 for (i = 0; i < N; i += 8) { \ 157 const int off = ((cur_x) + i) * XSTEP; \ 158 const uint8x8_t y = vld1_u8((src_y) + (cur_x) + i); \ 159 const uint8x8_t u = vld1_u8((src_uv) + i + 0); \ 160 const uint8x8_t v = vld1_u8((src_uv) + i + 16); \ 161 const int16x8_t Y0 = vreinterpretq_s16_u16(vshll_n_u8(y, 7)); \ 162 const int16x8_t U0 = vreinterpretq_s16_u16(vshll_n_u8(u, 7)); \ 163 const int16x8_t V0 = vreinterpretq_s16_u16(vshll_n_u8(v, 7)); \ 164 const int16x8_t Y1 = vqdmulhq_lane_s16(Y0, coeff1, 0); \ 165 const int16x8_t R0 = vqdmulhq_lane_s16(V0, coeff1, 1); \ 166 const int16x8_t G0 = vqdmulhq_lane_s16(U0, coeff1, 2); \ 167 const int16x8_t G1 = vqdmulhq_lane_s16(V0, coeff1, 3); \ 168 const int16x8_t B0 = vqdmulhq_n_s16(U0, 282); \ 169 const int16x8_t R1 = vqaddq_s16(Y1, R_Rounder); \ 170 const int16x8_t G2 = vqaddq_s16(Y1, G_Rounder); \ 171 const int16x8_t B1 = vqaddq_s16(Y1, B_Rounder); \ 172 const int16x8_t R2 = vqaddq_s16(R0, R1); \ 173 const int16x8_t G3 = vqaddq_s16(G0, G1); \ 174 const int16x8_t B2 = vqaddq_s16(B0, B1); \ 175 const int16x8_t G4 = vqsubq_s16(G2, G3); \ 176 const int16x8_t B3 = vqaddq_s16(B2, U0); \ 177 const uint8x8_t R = vqshrun_n_s16(R2, YUV_FIX2); \ 178 const uint8x8_t G = vqshrun_n_s16(G4, YUV_FIX2); \ 179 const uint8x8_t B = vqshrun_n_s16(B3, YUV_FIX2); \ 180 STORE_ ## FMT(out + off, R, G, B); \ 181 } \ 182 } while (0) 183 184 #define CONVERT1(FUNC, XSTEP, N, src_y, src_uv, rgb, cur_x) { \ 185 int i; \ 186 for (i = 0; i < N; i++) { \ 187 const int off = ((cur_x) + i) * XSTEP; \ 188 const int y = src_y[(cur_x) + i]; \ 189 const int u = (src_uv)[i]; \ 190 const int v = (src_uv)[i + 16]; \ 191 FUNC(y, u, v, rgb + off); \ 192 } \ 193 } 194 195 #define CONVERT2RGB_8(FMT, XSTEP, top_y, bottom_y, uv, \ 196 top_dst, bottom_dst, cur_x, len) { \ 197 CONVERT8(FMT, XSTEP, len, top_y, uv, top_dst, cur_x); \ 198 if (bottom_y != NULL) { \ 199 CONVERT8(FMT, XSTEP, len, bottom_y, (uv) + 32, bottom_dst, cur_x); \ 200 } \ 201 } 202 203 #define CONVERT2RGB_1(FUNC, XSTEP, top_y, bottom_y, uv, \ 204 top_dst, bottom_dst, cur_x, len) { \ 205 CONVERT1(FUNC, XSTEP, len, top_y, uv, top_dst, cur_x); \ 206 if (bottom_y != NULL) { \ 207 CONVERT1(FUNC, XSTEP, len, bottom_y, (uv) + 32, bottom_dst, cur_x); \ 208 } \ 209 } 210 211 #define NEON_UPSAMPLE_FUNC(FUNC_NAME, FMT, XSTEP) \ 212 static void FUNC_NAME(const uint8_t *top_y, const uint8_t *bottom_y, \ 213 const uint8_t *top_u, const uint8_t *top_v, \ 214 const uint8_t *cur_u, const uint8_t *cur_v, \ 215 uint8_t *top_dst, uint8_t *bottom_dst, int len) { \ 216 int block; \ 217 /* 16 byte aligned array to cache reconstructed u and v */ \ 218 uint8_t uv_buf[2 * 32 + 15]; \ 219 uint8_t *const r_uv = (uint8_t*)((uintptr_t)(uv_buf + 15) & ~15); \ 220 const int uv_len = (len + 1) >> 1; \ 221 /* 9 pixels must be read-able for each block */ \ 222 const int num_blocks = (uv_len - 1) >> 3; \ 223 const int leftover = uv_len - num_blocks * 8; \ 224 const int last_pos = 1 + 16 * num_blocks; \ 225 \ 226 const int u_diag = ((top_u[0] + cur_u[0]) >> 1) + 1; \ 227 const int v_diag = ((top_v[0] + cur_v[0]) >> 1) + 1; \ 228 \ 229 const int16x4_t coeff1 = vld1_s16(kCoeffs1); \ 230 const int16x8_t R_Rounder = vdupq_n_s16(-14234); \ 231 const int16x8_t G_Rounder = vdupq_n_s16(8708); \ 232 const int16x8_t B_Rounder = vdupq_n_s16(-17685); \ 233 \ 234 /* Treat the first pixel in regular way */ \ 235 assert(top_y != NULL); \ 236 { \ 237 const int u0 = (top_u[0] + u_diag) >> 1; \ 238 const int v0 = (top_v[0] + v_diag) >> 1; \ 239 VP8YuvTo ## FMT(top_y[0], u0, v0, top_dst); \ 240 } \ 241 if (bottom_y != NULL) { \ 242 const int u0 = (cur_u[0] + u_diag) >> 1; \ 243 const int v0 = (cur_v[0] + v_diag) >> 1; \ 244 VP8YuvTo ## FMT(bottom_y[0], u0, v0, bottom_dst); \ 245 } \ 246 \ 247 for (block = 0; block < num_blocks; ++block) { \ 248 UPSAMPLE_16PIXELS(top_u, cur_u, r_uv); \ 249 UPSAMPLE_16PIXELS(top_v, cur_v, r_uv + 16); \ 250 CONVERT2RGB_8(FMT, XSTEP, top_y, bottom_y, r_uv, \ 251 top_dst, bottom_dst, 16 * block + 1, 16); \ 252 top_u += 8; \ 253 cur_u += 8; \ 254 top_v += 8; \ 255 cur_v += 8; \ 256 } \ 257 \ 258 UPSAMPLE_LAST_BLOCK(top_u, cur_u, leftover, r_uv); \ 259 UPSAMPLE_LAST_BLOCK(top_v, cur_v, leftover, r_uv + 16); \ 260 CONVERT2RGB_1(VP8YuvTo ## FMT, XSTEP, top_y, bottom_y, r_uv, \ 261 top_dst, bottom_dst, last_pos, len - last_pos); \ 262 } 263 264 // NEON variants of the fancy upsampler. 265 NEON_UPSAMPLE_FUNC(UpsampleRgbLinePair, Rgb, 3) 266 NEON_UPSAMPLE_FUNC(UpsampleBgrLinePair, Bgr, 3) 267 NEON_UPSAMPLE_FUNC(UpsampleRgbaLinePair, Rgba, 4) 268 NEON_UPSAMPLE_FUNC(UpsampleBgraLinePair, Bgra, 4) 269 NEON_UPSAMPLE_FUNC(UpsampleArgbLinePair, Argb, 4) 270 NEON_UPSAMPLE_FUNC(UpsampleRgba4444LinePair, Rgba4444, 2) 271 NEON_UPSAMPLE_FUNC(UpsampleRgb565LinePair, Rgb565, 2) 272 273 //------------------------------------------------------------------------------ 274 // Entry point 275 276 extern WebPUpsampleLinePairFunc WebPUpsamplers[/* MODE_LAST */]; 277 278 extern void WebPInitUpsamplersNEON(void); 279 280 WEBP_TSAN_IGNORE_FUNCTION void WebPInitUpsamplersNEON(void) { 281 WebPUpsamplers[MODE_RGB] = UpsampleRgbLinePair; 282 WebPUpsamplers[MODE_RGBA] = UpsampleRgbaLinePair; 283 WebPUpsamplers[MODE_BGR] = UpsampleBgrLinePair; 284 WebPUpsamplers[MODE_BGRA] = UpsampleBgraLinePair; 285 WebPUpsamplers[MODE_ARGB] = UpsampleArgbLinePair; 286 WebPUpsamplers[MODE_rgbA] = UpsampleRgbaLinePair; 287 WebPUpsamplers[MODE_bgrA] = UpsampleBgraLinePair; 288 WebPUpsamplers[MODE_Argb] = UpsampleArgbLinePair; 289 WebPUpsamplers[MODE_RGB_565] = UpsampleRgb565LinePair; 290 WebPUpsamplers[MODE_RGBA_4444] = UpsampleRgba4444LinePair; 291 WebPUpsamplers[MODE_rgbA_4444] = UpsampleRgba4444LinePair; 292 } 293 294 #endif // FANCY_UPSAMPLING 295 296 #endif // WEBP_USE_NEON 297 298 #if !(defined(FANCY_UPSAMPLING) && defined(WEBP_USE_NEON)) 299 WEBP_DSP_INIT_STUB(WebPInitUpsamplersNEON) 300 #endif 301
