Home | History | Annotate | Download | only in neon 
      1 /*
      2  *  Copyright (c) 2014 The WebM project authors. All Rights Reserved.
      3  *
      4  *  Use of this source code is governed by a BSD-style license
      5  *  that can be found in the LICENSE file in the root of the source
      6  *  tree. An additional intellectual property rights grant can be found
      7  *  in the file PATENTS.  All contributing project authors may
      8  *  be found in the AUTHORS file in the root of the source tree.
      9  */
     10 
     11 #include <arm_neon.h>
     12 #include "./vpx_config.h"
     13 #include "vpx_ports/arm.h"
     14 
     15 #ifdef VPX_INCOMPATIBLE_GCC
     16 static INLINE void write_2x4(unsigned char *dst, int pitch,
     17                              const uint8x8x2_t result) {
     18   /*
     19    * uint8x8x2_t result
     20   00 01 02 03 | 04 05 06 07
     21   10 11 12 13 | 14 15 16 17
     22   ---
     23   * after vtrn_u8
     24   00 10 02 12 | 04 14 06 16
     25   01 11 03 13 | 05 15 07 17
     26   */
     27   const uint8x8x2_t r01_u8 = vtrn_u8(result.val[0], result.val[1]);
     28   const uint16x4_t x_0_4 = vreinterpret_u16_u8(r01_u8.val[0]);
     29   const uint16x4_t x_1_5 = vreinterpret_u16_u8(r01_u8.val[1]);
     30   vst1_lane_u16((uint16_t *)dst, x_0_4, 0);
     31   dst += pitch;
     32   vst1_lane_u16((uint16_t *)dst, x_1_5, 0);
     33   dst += pitch;
     34   vst1_lane_u16((uint16_t *)dst, x_0_4, 1);
     35   dst += pitch;
     36   vst1_lane_u16((uint16_t *)dst, x_1_5, 1);
     37   dst += pitch;
     38   vst1_lane_u16((uint16_t *)dst, x_0_4, 2);
     39   dst += pitch;
     40   vst1_lane_u16((uint16_t *)dst, x_1_5, 2);
     41   dst += pitch;
     42   vst1_lane_u16((uint16_t *)dst, x_0_4, 3);
     43   dst += pitch;
     44   vst1_lane_u16((uint16_t *)dst, x_1_5, 3);
     45 }
     46 
     47 static INLINE void write_2x8(unsigned char *dst, int pitch,
     48                              const uint8x8x2_t result,
     49                              const uint8x8x2_t result2) {
     50   write_2x4(dst, pitch, result);
     51   dst += pitch * 8;
     52   write_2x4(dst, pitch, result2);
     53 }
     54 #else
     55 static INLINE void write_2x8(unsigned char *dst, int pitch,
     56                              const uint8x8x2_t result,
     57                              const uint8x8x2_t result2) {
     58   vst2_lane_u8(dst, result, 0);
     59   dst += pitch;
     60   vst2_lane_u8(dst, result, 1);
     61   dst += pitch;
     62   vst2_lane_u8(dst, result, 2);
     63   dst += pitch;
     64   vst2_lane_u8(dst, result, 3);
     65   dst += pitch;
     66   vst2_lane_u8(dst, result, 4);
     67   dst += pitch;
     68   vst2_lane_u8(dst, result, 5);
     69   dst += pitch;
     70   vst2_lane_u8(dst, result, 6);
     71   dst += pitch;
     72   vst2_lane_u8(dst, result, 7);
     73   dst += pitch;
     74 
     75   vst2_lane_u8(dst, result2, 0);
     76   dst += pitch;
     77   vst2_lane_u8(dst, result2, 1);
     78   dst += pitch;
     79   vst2_lane_u8(dst, result2, 2);
     80   dst += pitch;
     81   vst2_lane_u8(dst, result2, 3);
     82   dst += pitch;
     83   vst2_lane_u8(dst, result2, 4);
     84   dst += pitch;
     85   vst2_lane_u8(dst, result2, 5);
     86   dst += pitch;
     87   vst2_lane_u8(dst, result2, 6);
     88   dst += pitch;
     89   vst2_lane_u8(dst, result2, 7);
     90 }
     91 #endif  // VPX_INCOMPATIBLE_GCC
     92 
     93 #ifdef VPX_INCOMPATIBLE_GCC
     94 static INLINE uint8x8x4_t read_4x8(unsigned char *src, int pitch) {
     95   uint8x8x4_t x;
     96   const uint8x8_t a = vld1_u8(src);
     97   const uint8x8_t b = vld1_u8(src + pitch * 1);
     98   const uint8x8_t c = vld1_u8(src + pitch * 2);
     99   const uint8x8_t d = vld1_u8(src + pitch * 3);
    100   const uint8x8_t e = vld1_u8(src + pitch * 4);
    101   const uint8x8_t f = vld1_u8(src + pitch * 5);
    102   const uint8x8_t g = vld1_u8(src + pitch * 6);
    103   const uint8x8_t h = vld1_u8(src + pitch * 7);
    104   const uint32x2x2_t r04_u32 =
    105       vtrn_u32(vreinterpret_u32_u8(a), vreinterpret_u32_u8(e));
    106   const uint32x2x2_t r15_u32 =
    107       vtrn_u32(vreinterpret_u32_u8(b), vreinterpret_u32_u8(f));
    108   const uint32x2x2_t r26_u32 =
    109       vtrn_u32(vreinterpret_u32_u8(c), vreinterpret_u32_u8(g));
    110   const uint32x2x2_t r37_u32 =
    111       vtrn_u32(vreinterpret_u32_u8(d), vreinterpret_u32_u8(h));
    112   const uint16x4x2_t r02_u16 = vtrn_u16(vreinterpret_u16_u32(r04_u32.val[0]),
    113                                         vreinterpret_u16_u32(r26_u32.val[0]));
    114   const uint16x4x2_t r13_u16 = vtrn_u16(vreinterpret_u16_u32(r15_u32.val[0]),
    115                                         vreinterpret_u16_u32(r37_u32.val[0]));
    116   const uint8x8x2_t r01_u8 = vtrn_u8(vreinterpret_u8_u16(r02_u16.val[0]),
    117                                      vreinterpret_u8_u16(r13_u16.val[0]));
    118   const uint8x8x2_t r23_u8 = vtrn_u8(vreinterpret_u8_u16(r02_u16.val[1]),
    119                                      vreinterpret_u8_u16(r13_u16.val[1]));
    120   /*
    121    * after vtrn_u32
    122   00 01 02 03 | 40 41 42 43
    123   10 11 12 13 | 50 51 52 53
    124   20 21 22 23 | 60 61 62 63
    125   30 31 32 33 | 70 71 72 73
    126   ---
    127   * after vtrn_u16
    128   00 01 20 21 | 40 41 60 61
    129   02 03 22 23 | 42 43 62 63
    130   10 11 30 31 | 50 51 70 71
    131   12 13 32 33 | 52 52 72 73
    132 
    133   00 01 20 21 | 40 41 60 61
    134   10 11 30 31 | 50 51 70 71
    135   02 03 22 23 | 42 43 62 63
    136   12 13 32 33 | 52 52 72 73
    137   ---
    138   * after vtrn_u8
    139   00 10 20 30 | 40 50 60 70
    140   01 11 21 31 | 41 51 61 71
    141   02 12 22 32 | 42 52 62 72
    142   03 13 23 33 | 43 53 63 73
    143   */
    144   x.val[0] = r01_u8.val[0];
    145   x.val[1] = r01_u8.val[1];
    146   x.val[2] = r23_u8.val[0];
    147   x.val[3] = r23_u8.val[1];
    148 
    149   return x;
    150 }
    151 #else
    152 static INLINE uint8x8x4_t read_4x8(unsigned char *src, int pitch) {
    153   uint8x8x4_t x;
    154   x.val[0] = x.val[1] = x.val[2] = x.val[3] = vdup_n_u8(0);
    155   x = vld4_lane_u8(src, x, 0);
    156   src += pitch;
    157   x = vld4_lane_u8(src, x, 1);
    158   src += pitch;
    159   x = vld4_lane_u8(src, x, 2);
    160   src += pitch;
    161   x = vld4_lane_u8(src, x, 3);
    162   src += pitch;
    163   x = vld4_lane_u8(src, x, 4);
    164   src += pitch;
    165   x = vld4_lane_u8(src, x, 5);
    166   src += pitch;
    167   x = vld4_lane_u8(src, x, 6);
    168   src += pitch;
    169   x = vld4_lane_u8(src, x, 7);
    170   return x;
    171 }
    172 #endif  // VPX_INCOMPATIBLE_GCC
    173 
    174 static INLINE void vp8_loop_filter_simple_vertical_edge_neon(
    175     unsigned char *s, int p, const unsigned char *blimit) {
    176   unsigned char *src1;
    177   uint8x16_t qblimit, q0u8;
    178   uint8x16_t q3u8, q4u8, q5u8, q6u8, q7u8, q11u8, q12u8, q14u8, q15u8;
    179   int16x8_t q2s16, q13s16, q11s16;
    180   int8x8_t d28s8, d29s8;
    181   int8x16_t q2s8, q3s8, q10s8, q11s8, q14s8;
    182   uint8x8x4_t d0u8x4;  // d6, d7, d8, d9
    183   uint8x8x4_t d1u8x4;  // d10, d11, d12, d13
    184   uint8x8x2_t d2u8x2;  // d12, d13
    185   uint8x8x2_t d3u8x2;  // d14, d15
    186 
    187   qblimit = vdupq_n_u8(*blimit);
    188 
    189   src1 = s - 2;
    190   d0u8x4 = read_4x8(src1, p);
    191   src1 += p * 8;
    192   d1u8x4 = read_4x8(src1, p);
    193 
    194   q3u8 = vcombine_u8(d0u8x4.val[0], d1u8x4.val[0]);  // d6 d10
    195   q4u8 = vcombine_u8(d0u8x4.val[2], d1u8x4.val[2]);  // d8 d12
    196   q5u8 = vcombine_u8(d0u8x4.val[1], d1u8x4.val[1]);  // d7 d11
    197   q6u8 = vcombine_u8(d0u8x4.val[3], d1u8x4.val[3]);  // d9 d13
    198 
    199   q15u8 = vabdq_u8(q5u8, q4u8);
    200   q14u8 = vabdq_u8(q3u8, q6u8);
    201 
    202   q15u8 = vqaddq_u8(q15u8, q15u8);
    203   q14u8 = vshrq_n_u8(q14u8, 1);
    204   q0u8 = vdupq_n_u8(0x80);
    205   q11s16 = vdupq_n_s16(3);
    206   q15u8 = vqaddq_u8(q15u8, q14u8);
    207 
    208   q3u8 = veorq_u8(q3u8, q0u8);
    209   q4u8 = veorq_u8(q4u8, q0u8);
    210   q5u8 = veorq_u8(q5u8, q0u8);
    211   q6u8 = veorq_u8(q6u8, q0u8);
    212 
    213   q15u8 = vcgeq_u8(qblimit, q15u8);
    214 
    215   q2s16 = vsubl_s8(vget_low_s8(vreinterpretq_s8_u8(q4u8)),
    216                    vget_low_s8(vreinterpretq_s8_u8(q5u8)));
    217   q13s16 = vsubl_s8(vget_high_s8(vreinterpretq_s8_u8(q4u8)),
    218                     vget_high_s8(vreinterpretq_s8_u8(q5u8)));
    219 
    220   q14s8 = vqsubq_s8(vreinterpretq_s8_u8(q3u8), vreinterpretq_s8_u8(q6u8));
    221 
    222   q2s16 = vmulq_s16(q2s16, q11s16);
    223   q13s16 = vmulq_s16(q13s16, q11s16);
    224 
    225   q11u8 = vdupq_n_u8(3);
    226   q12u8 = vdupq_n_u8(4);
    227 
    228   q2s16 = vaddw_s8(q2s16, vget_low_s8(q14s8));
    229   q13s16 = vaddw_s8(q13s16, vget_high_s8(q14s8));
    230 
    231   d28s8 = vqmovn_s16(q2s16);
    232   d29s8 = vqmovn_s16(q13s16);
    233   q14s8 = vcombine_s8(d28s8, d29s8);
    234 
    235   q14s8 = vandq_s8(q14s8, vreinterpretq_s8_u8(q15u8));
    236 
    237   q2s8 = vqaddq_s8(q14s8, vreinterpretq_s8_u8(q11u8));
    238   q3s8 = vqaddq_s8(q14s8, vreinterpretq_s8_u8(q12u8));
    239   q2s8 = vshrq_n_s8(q2s8, 3);
    240   q14s8 = vshrq_n_s8(q3s8, 3);
    241 
    242   q11s8 = vqaddq_s8(vreinterpretq_s8_u8(q5u8), q2s8);
    243   q10s8 = vqsubq_s8(vreinterpretq_s8_u8(q4u8), q14s8);
    244 
    245   q6u8 = veorq_u8(vreinterpretq_u8_s8(q11s8), q0u8);
    246   q7u8 = veorq_u8(vreinterpretq_u8_s8(q10s8), q0u8);
    247 
    248   d2u8x2.val[0] = vget_low_u8(q6u8);   // d12
    249   d2u8x2.val[1] = vget_low_u8(q7u8);   // d14
    250   d3u8x2.val[0] = vget_high_u8(q6u8);  // d13
    251   d3u8x2.val[1] = vget_high_u8(q7u8);  // d15
    252 
    253   src1 = s - 1;
    254   write_2x8(src1, p, d2u8x2, d3u8x2);
    255 }
    256 
    257 void vp8_loop_filter_bvs_neon(unsigned char *y_ptr, int y_stride,
    258                               const unsigned char *blimit) {
    259   y_ptr += 4;
    260   vp8_loop_filter_simple_vertical_edge_neon(y_ptr, y_stride, blimit);
    261   y_ptr += 4;
    262   vp8_loop_filter_simple_vertical_edge_neon(y_ptr, y_stride, blimit);
    263   y_ptr += 4;
    264   vp8_loop_filter_simple_vertical_edge_neon(y_ptr, y_stride, blimit);
    265   return;
    266 }
    267 
    268 void vp8_loop_filter_mbvs_neon(unsigned char *y_ptr, int y_stride,
    269                                const unsigned char *blimit) {
    270   vp8_loop_filter_simple_vertical_edge_neon(y_ptr, y_stride, blimit);
    271   return;
    272 }
    273