Home | History | Annotate | Download | only in clang-include
      1 /*===---- smmintrin.h - SSE4 intrinsics ------------------------------------===
      2  *
      3  * Permission is hereby granted, free of charge, to any person obtaining a copy
      4  * of this software and associated documentation files (the "Software"), to deal
      5  * in the Software without restriction, including without limitation the rights
      6  * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
      7  * copies of the Software, and to permit persons to whom the Software is
      8  * furnished to do so, subject to the following conditions:
      9  *
     10  * The above copyright notice and this permission notice shall be included in
     11  * all copies or substantial portions of the Software.
     12  *
     13  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
     14  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
     15  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
     16  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
     17  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
     18  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
     19  * THE SOFTWARE.
     20  *
     21  *===-----------------------------------------------------------------------===
     22  */
     23 
     24 #ifndef _SMMINTRIN_H
     25 #define _SMMINTRIN_H
     26 
     27 #include <tmmintrin.h>
     28 
     29 /* Define the default attributes for the functions in this file. */
     30 #define __DEFAULT_FN_ATTRS __attribute__((__always_inline__, __nodebug__, __target__("sse4.1")))
     31 
     32 /* SSE4 Rounding macros. */
     33 #define _MM_FROUND_TO_NEAREST_INT    0x00
     34 #define _MM_FROUND_TO_NEG_INF        0x01
     35 #define _MM_FROUND_TO_POS_INF        0x02
     36 #define _MM_FROUND_TO_ZERO           0x03
     37 #define _MM_FROUND_CUR_DIRECTION     0x04
     38 
     39 #define _MM_FROUND_RAISE_EXC         0x00
     40 #define _MM_FROUND_NO_EXC            0x08
     41 
     42 #define _MM_FROUND_NINT      (_MM_FROUND_RAISE_EXC | _MM_FROUND_TO_NEAREST_INT)
     43 #define _MM_FROUND_FLOOR     (_MM_FROUND_RAISE_EXC | _MM_FROUND_TO_NEG_INF)
     44 #define _MM_FROUND_CEIL      (_MM_FROUND_RAISE_EXC | _MM_FROUND_TO_POS_INF)
     45 #define _MM_FROUND_TRUNC     (_MM_FROUND_RAISE_EXC | _MM_FROUND_TO_ZERO)
     46 #define _MM_FROUND_RINT      (_MM_FROUND_RAISE_EXC | _MM_FROUND_CUR_DIRECTION)
     47 #define _MM_FROUND_NEARBYINT (_MM_FROUND_NO_EXC | _MM_FROUND_CUR_DIRECTION)
     48 
     49 #define _mm_ceil_ps(X)       _mm_round_ps((X), _MM_FROUND_CEIL)
     50 #define _mm_ceil_pd(X)       _mm_round_pd((X), _MM_FROUND_CEIL)
     51 #define _mm_ceil_ss(X, Y)    _mm_round_ss((X), (Y), _MM_FROUND_CEIL)
     52 #define _mm_ceil_sd(X, Y)    _mm_round_sd((X), (Y), _MM_FROUND_CEIL)
     53 
     54 #define _mm_floor_ps(X)      _mm_round_ps((X), _MM_FROUND_FLOOR)
     55 #define _mm_floor_pd(X)      _mm_round_pd((X), _MM_FROUND_FLOOR)
     56 #define _mm_floor_ss(X, Y)   _mm_round_ss((X), (Y), _MM_FROUND_FLOOR)
     57 #define _mm_floor_sd(X, Y)   _mm_round_sd((X), (Y), _MM_FROUND_FLOOR)
     58 
     59 #define _mm_round_ps(X, M) __extension__ ({ \
     60   (__m128)__builtin_ia32_roundps((__v4sf)(__m128)(X), (M)); })
     61 
     62 #define _mm_round_ss(X, Y, M) __extension__ ({ \
     63   (__m128)__builtin_ia32_roundss((__v4sf)(__m128)(X), \
     64                                  (__v4sf)(__m128)(Y), (M)); })
     65 
     66 #define _mm_round_pd(X, M) __extension__ ({ \
     67   (__m128d)__builtin_ia32_roundpd((__v2df)(__m128d)(X), (M)); })
     68 
     69 #define _mm_round_sd(X, Y, M) __extension__ ({ \
     70   (__m128d)__builtin_ia32_roundsd((__v2df)(__m128d)(X), \
     71                                   (__v2df)(__m128d)(Y), (M)); })
     72 
     73 /* SSE4 Packed Blending Intrinsics.  */
     74 #define _mm_blend_pd(V1, V2, M) __extension__ ({ \
     75   (__m128d)__builtin_shufflevector((__v2df)(__m128d)(V1), \
     76                                    (__v2df)(__m128d)(V2), \
     77                                    (((M) & 0x01) ? 2 : 0), \
     78                                    (((M) & 0x02) ? 3 : 1)); })
     79 
     80 #define _mm_blend_ps(V1, V2, M) __extension__ ({ \
     81   (__m128)__builtin_shufflevector((__v4sf)(__m128)(V1), (__v4sf)(__m128)(V2), \
     82                                   (((M) & 0x01) ? 4 : 0), \
     83                                   (((M) & 0x02) ? 5 : 1), \
     84                                   (((M) & 0x04) ? 6 : 2), \
     85                                   (((M) & 0x08) ? 7 : 3)); })
     86 
     87 static __inline__ __m128d __DEFAULT_FN_ATTRS
     88 _mm_blendv_pd (__m128d __V1, __m128d __V2, __m128d __M)
     89 {
     90   return (__m128d) __builtin_ia32_blendvpd ((__v2df)__V1, (__v2df)__V2,
     91                                             (__v2df)__M);
     92 }
     93 
     94 static __inline__ __m128 __DEFAULT_FN_ATTRS
     95 _mm_blendv_ps (__m128 __V1, __m128 __V2, __m128 __M)
     96 {
     97   return (__m128) __builtin_ia32_blendvps ((__v4sf)__V1, (__v4sf)__V2,
     98                                            (__v4sf)__M);
     99 }
    100 
    101 static __inline__ __m128i __DEFAULT_FN_ATTRS
    102 _mm_blendv_epi8 (__m128i __V1, __m128i __V2, __m128i __M)
    103 {
    104   return (__m128i) __builtin_ia32_pblendvb128 ((__v16qi)__V1, (__v16qi)__V2,
    105                                                (__v16qi)__M);
    106 }
    107 
    108 #define _mm_blend_epi16(V1, V2, M) __extension__ ({ \
    109   (__m128i)__builtin_shufflevector((__v8hi)(__m128i)(V1), \
    110                                    (__v8hi)(__m128i)(V2), \
    111                                    (((M) & 0x01) ?  8 : 0), \
    112                                    (((M) & 0x02) ?  9 : 1), \
    113                                    (((M) & 0x04) ? 10 : 2), \
    114                                    (((M) & 0x08) ? 11 : 3), \
    115                                    (((M) & 0x10) ? 12 : 4), \
    116                                    (((M) & 0x20) ? 13 : 5), \
    117                                    (((M) & 0x40) ? 14 : 6), \
    118                                    (((M) & 0x80) ? 15 : 7)); })
    119 
    120 /* SSE4 Dword Multiply Instructions.  */
    121 static __inline__  __m128i __DEFAULT_FN_ATTRS
    122 _mm_mullo_epi32 (__m128i __V1, __m128i __V2)
    123 {
    124   return (__m128i) ((__v4si)__V1 * (__v4si)__V2);
    125 }
    126 
    127 static __inline__  __m128i __DEFAULT_FN_ATTRS
    128 _mm_mul_epi32 (__m128i __V1, __m128i __V2)
    129 {
    130   return (__m128i) __builtin_ia32_pmuldq128 ((__v4si)__V1, (__v4si)__V2);
    131 }
    132 
    133 /* SSE4 Floating Point Dot Product Instructions.  */
    134 #define _mm_dp_ps(X, Y, M) __extension__ ({ \
    135   (__m128) __builtin_ia32_dpps((__v4sf)(__m128)(X), \
    136                                (__v4sf)(__m128)(Y), (M)); })
    137 
    138 #define _mm_dp_pd(X, Y, M) __extension__ ({\
    139   (__m128d) __builtin_ia32_dppd((__v2df)(__m128d)(X), \
    140                                 (__v2df)(__m128d)(Y), (M)); })
    141 
    142 /* SSE4 Streaming Load Hint Instruction.  */
    143 static __inline__  __m128i __DEFAULT_FN_ATTRS
    144 _mm_stream_load_si128 (__m128i const *__V)
    145 {
    146   return (__m128i) __builtin_ia32_movntdqa ((const __v2di *) __V);
    147 }
    148 
    149 /* SSE4 Packed Integer Min/Max Instructions.  */
    150 static __inline__  __m128i __DEFAULT_FN_ATTRS
    151 _mm_min_epi8 (__m128i __V1, __m128i __V2)
    152 {
    153   return (__m128i) __builtin_ia32_pminsb128 ((__v16qi) __V1, (__v16qi) __V2);
    154 }
    155 
    156 static __inline__  __m128i __DEFAULT_FN_ATTRS
    157 _mm_max_epi8 (__m128i __V1, __m128i __V2)
    158 {
    159   return (__m128i) __builtin_ia32_pmaxsb128 ((__v16qi) __V1, (__v16qi) __V2);
    160 }
    161 
    162 static __inline__  __m128i __DEFAULT_FN_ATTRS
    163 _mm_min_epu16 (__m128i __V1, __m128i __V2)
    164 {
    165   return (__m128i) __builtin_ia32_pminuw128 ((__v8hi) __V1, (__v8hi) __V2);
    166 }
    167 
    168 static __inline__  __m128i __DEFAULT_FN_ATTRS
    169 _mm_max_epu16 (__m128i __V1, __m128i __V2)
    170 {
    171   return (__m128i) __builtin_ia32_pmaxuw128 ((__v8hi) __V1, (__v8hi) __V2);
    172 }
    173 
    174 static __inline__  __m128i __DEFAULT_FN_ATTRS
    175 _mm_min_epi32 (__m128i __V1, __m128i __V2)
    176 {
    177   return (__m128i) __builtin_ia32_pminsd128 ((__v4si) __V1, (__v4si) __V2);
    178 }
    179 
    180 static __inline__  __m128i __DEFAULT_FN_ATTRS
    181 _mm_max_epi32 (__m128i __V1, __m128i __V2)
    182 {
    183   return (__m128i) __builtin_ia32_pmaxsd128 ((__v4si) __V1, (__v4si) __V2);
    184 }
    185 
    186 static __inline__  __m128i __DEFAULT_FN_ATTRS
    187 _mm_min_epu32 (__m128i __V1, __m128i __V2)
    188 {
    189   return (__m128i) __builtin_ia32_pminud128((__v4si) __V1, (__v4si) __V2);
    190 }
    191 
    192 static __inline__  __m128i __DEFAULT_FN_ATTRS
    193 _mm_max_epu32 (__m128i __V1, __m128i __V2)
    194 {
    195   return (__m128i) __builtin_ia32_pmaxud128((__v4si) __V1, (__v4si) __V2);
    196 }
    197 
    198 /* SSE4 Insertion and Extraction from XMM Register Instructions.  */
    199 #define _mm_insert_ps(X, Y, N) __builtin_ia32_insertps128((X), (Y), (N))
    200 #define _mm_extract_ps(X, N) (__extension__                      \
    201                               ({ union { int __i; float __f; } __t;  \
    202                                  __v4sf __a = (__v4sf)(__m128)(X);       \
    203                                  __t.__f = __a[(N) & 3];                 \
    204                                  __t.__i;}))
    205 
    206 /* Miscellaneous insert and extract macros.  */
    207 /* Extract a single-precision float from X at index N into D.  */
    208 #define _MM_EXTRACT_FLOAT(D, X, N) (__extension__ ({ __v4sf __a = (__v4sf)(X); \
    209                                                     (D) = __a[N]; }))
    210 
    211 /* Or together 2 sets of indexes (X and Y) with the zeroing bits (Z) to create
    212    an index suitable for _mm_insert_ps.  */
    213 #define _MM_MK_INSERTPS_NDX(X, Y, Z) (((X) << 6) | ((Y) << 4) | (Z))
    214 
    215 /* Extract a float from X at index N into the first index of the return.  */
    216 #define _MM_PICK_OUT_PS(X, N) _mm_insert_ps (_mm_setzero_ps(), (X),   \
    217                                              _MM_MK_INSERTPS_NDX((N), 0, 0x0e))
    218 
    219 /* Insert int into packed integer array at index.  */
    220 #define _mm_insert_epi8(X, I, N) (__extension__                           \
    221                                   ({ __v16qi __a = (__v16qi)(__m128i)(X); \
    222                                      __a[(N) & 15] = (I);                 \
    223                                      __a;}))
    224 #define _mm_insert_epi32(X, I, N) (__extension__                         \
    225                                    ({ __v4si __a = (__v4si)(__m128i)(X); \
    226                                       __a[(N) & 3] = (I);                \
    227                                       __a;}))
    228 #ifdef __x86_64__
    229 #define _mm_insert_epi64(X, I, N) (__extension__                         \
    230                                    ({ __v2di __a = (__v2di)(__m128i)(X); \
    231                                       __a[(N) & 1] = (I);                \
    232                                       __a;}))
    233 #endif /* __x86_64__ */
    234 
    235 /* Extract int from packed integer array at index.  This returns the element
    236  * as a zero extended value, so it is unsigned.
    237  */
    238 #define _mm_extract_epi8(X, N) (__extension__                           \
    239                                 ({ __v16qi __a = (__v16qi)(__m128i)(X); \
    240                                    (int)(unsigned char) __a[(N) & 15];}))
    241 #define _mm_extract_epi32(X, N) (__extension__                         \
    242                                  ({ __v4si __a = (__v4si)(__m128i)(X); \
    243                                     (int)__a[(N) & 3];}))
    244 #ifdef __x86_64__
    245 #define _mm_extract_epi64(X, N) (__extension__                         \
    246                                  ({ __v2di __a = (__v2di)(__m128i)(X); \
    247                                     (long long)__a[(N) & 1];}))
    248 #endif /* __x86_64 */
    249 
    250 /* SSE4 128-bit Packed Integer Comparisons.  */
    251 static __inline__ int __DEFAULT_FN_ATTRS
    252 _mm_testz_si128(__m128i __M, __m128i __V)
    253 {
    254   return __builtin_ia32_ptestz128((__v2di)__M, (__v2di)__V);
    255 }
    256 
    257 static __inline__ int __DEFAULT_FN_ATTRS
    258 _mm_testc_si128(__m128i __M, __m128i __V)
    259 {
    260   return __builtin_ia32_ptestc128((__v2di)__M, (__v2di)__V);
    261 }
    262 
    263 static __inline__ int __DEFAULT_FN_ATTRS
    264 _mm_testnzc_si128(__m128i __M, __m128i __V)
    265 {
    266   return __builtin_ia32_ptestnzc128((__v2di)__M, (__v2di)__V);
    267 }
    268 
    269 #define _mm_test_all_ones(V) _mm_testc_si128((V), _mm_cmpeq_epi32((V), (V)))
    270 #define _mm_test_mix_ones_zeros(M, V) _mm_testnzc_si128((M), (V))
    271 #define _mm_test_all_zeros(M, V) _mm_testz_si128 ((M), (V))
    272 
    273 /* SSE4 64-bit Packed Integer Comparisons.  */
    274 static __inline__ __m128i __DEFAULT_FN_ATTRS
    275 _mm_cmpeq_epi64(__m128i __V1, __m128i __V2)
    276 {
    277   return (__m128i)((__v2di)__V1 == (__v2di)__V2);
    278 }
    279 
    280 /* SSE4 Packed Integer Sign-Extension.  */
    281 static __inline__ __m128i __DEFAULT_FN_ATTRS
    282 _mm_cvtepi8_epi16(__m128i __V)
    283 {
    284   /* This function always performs a signed extension, but __v16qi is a char
    285      which may be signed or unsigned, so use __v16qs. */
    286   return (__m128i)__builtin_convertvector(__builtin_shufflevector((__v16qs)__V, (__v16qs)__V, 0, 1, 2, 3, 4, 5, 6, 7), __v8hi);
    287 }
    288 
    289 static __inline__ __m128i __DEFAULT_FN_ATTRS
    290 _mm_cvtepi8_epi32(__m128i __V)
    291 {
    292   /* This function always performs a signed extension, but __v16qi is a char
    293      which may be signed or unsigned, so use __v16qs. */
    294   return (__m128i)__builtin_convertvector(__builtin_shufflevector((__v16qs)__V, (__v16qs)__V, 0, 1, 2, 3), __v4si);
    295 }
    296 
    297 static __inline__ __m128i __DEFAULT_FN_ATTRS
    298 _mm_cvtepi8_epi64(__m128i __V)
    299 {
    300   /* This function always performs a signed extension, but __v16qi is a char
    301      which may be signed or unsigned, so use __v16qs. */
    302   typedef signed char __v16qs __attribute__((__vector_size__(16)));
    303   return (__m128i)__builtin_convertvector(__builtin_shufflevector((__v16qs)__V, (__v16qs)__V, 0, 1), __v2di);
    304 }
    305 
    306 static __inline__ __m128i __DEFAULT_FN_ATTRS
    307 _mm_cvtepi16_epi32(__m128i __V)
    308 {
    309   return (__m128i)__builtin_convertvector(__builtin_shufflevector((__v8hi)__V, (__v8hi)__V, 0, 1, 2, 3), __v4si);
    310 }
    311 
    312 static __inline__ __m128i __DEFAULT_FN_ATTRS
    313 _mm_cvtepi16_epi64(__m128i __V)
    314 {
    315   return (__m128i)__builtin_convertvector(__builtin_shufflevector((__v8hi)__V, (__v8hi)__V, 0, 1), __v2di);
    316 }
    317 
    318 static __inline__ __m128i __DEFAULT_FN_ATTRS
    319 _mm_cvtepi32_epi64(__m128i __V)
    320 {
    321   return (__m128i)__builtin_convertvector(__builtin_shufflevector((__v4si)__V, (__v4si)__V, 0, 1), __v2di);
    322 }
    323 
    324 /* SSE4 Packed Integer Zero-Extension.  */
    325 static __inline__ __m128i __DEFAULT_FN_ATTRS
    326 _mm_cvtepu8_epi16(__m128i __V)
    327 {
    328   return (__m128i) __builtin_ia32_pmovzxbw128((__v16qi) __V);
    329 }
    330 
    331 static __inline__ __m128i __DEFAULT_FN_ATTRS
    332 _mm_cvtepu8_epi32(__m128i __V)
    333 {
    334   return (__m128i) __builtin_ia32_pmovzxbd128((__v16qi)__V);
    335 }
    336 
    337 static __inline__ __m128i __DEFAULT_FN_ATTRS
    338 _mm_cvtepu8_epi64(__m128i __V)
    339 {
    340   return (__m128i) __builtin_ia32_pmovzxbq128((__v16qi)__V);
    341 }
    342 
    343 static __inline__ __m128i __DEFAULT_FN_ATTRS
    344 _mm_cvtepu16_epi32(__m128i __V)
    345 {
    346   return (__m128i) __builtin_ia32_pmovzxwd128((__v8hi)__V);
    347 }
    348 
    349 static __inline__ __m128i __DEFAULT_FN_ATTRS
    350 _mm_cvtepu16_epi64(__m128i __V)
    351 {
    352   return (__m128i) __builtin_ia32_pmovzxwq128((__v8hi)__V);
    353 }
    354 
    355 static __inline__ __m128i __DEFAULT_FN_ATTRS
    356 _mm_cvtepu32_epi64(__m128i __V)
    357 {
    358   return (__m128i) __builtin_ia32_pmovzxdq128((__v4si)__V);
    359 }
    360 
    361 /* SSE4 Pack with Unsigned Saturation.  */
    362 static __inline__ __m128i __DEFAULT_FN_ATTRS
    363 _mm_packus_epi32(__m128i __V1, __m128i __V2)
    364 {
    365   return (__m128i) __builtin_ia32_packusdw128((__v4si)__V1, (__v4si)__V2);
    366 }
    367 
    368 /* SSE4 Multiple Packed Sums of Absolute Difference.  */
    369 #define _mm_mpsadbw_epu8(X, Y, M) __extension__ ({ \
    370   (__m128i) __builtin_ia32_mpsadbw128((__v16qi)(__m128i)(X), \
    371                                       (__v16qi)(__m128i)(Y), (M)); })
    372 
    373 static __inline__ __m128i __DEFAULT_FN_ATTRS
    374 _mm_minpos_epu16(__m128i __V)
    375 {
    376   return (__m128i) __builtin_ia32_phminposuw128((__v8hi)__V);
    377 }
    378 
    379 /* Handle the sse4.2 definitions here. */
    380 
    381 /* These definitions are normally in nmmintrin.h, but gcc puts them in here
    382    so we'll do the same.  */
    383 
    384 #undef __DEFAULT_FN_ATTRS
    385 #define __DEFAULT_FN_ATTRS __attribute__((__always_inline__, __nodebug__, __target__("sse4.2")))
    386 
    387 /* These specify the type of data that we're comparing.  */
    388 #define _SIDD_UBYTE_OPS                 0x00
    389 #define _SIDD_UWORD_OPS                 0x01
    390 #define _SIDD_SBYTE_OPS                 0x02
    391 #define _SIDD_SWORD_OPS                 0x03
    392 
    393 /* These specify the type of comparison operation.  */
    394 #define _SIDD_CMP_EQUAL_ANY             0x00
    395 #define _SIDD_CMP_RANGES                0x04
    396 #define _SIDD_CMP_EQUAL_EACH            0x08
    397 #define _SIDD_CMP_EQUAL_ORDERED         0x0c
    398 
    399 /* These macros specify the polarity of the operation.  */
    400 #define _SIDD_POSITIVE_POLARITY         0x00
    401 #define _SIDD_NEGATIVE_POLARITY         0x10
    402 #define _SIDD_MASKED_POSITIVE_POLARITY  0x20
    403 #define _SIDD_MASKED_NEGATIVE_POLARITY  0x30
    404 
    405 /* These macros are used in _mm_cmpXstri() to specify the return.  */
    406 #define _SIDD_LEAST_SIGNIFICANT         0x00
    407 #define _SIDD_MOST_SIGNIFICANT          0x40
    408 
    409 /* These macros are used in _mm_cmpXstri() to specify the return.  */
    410 #define _SIDD_BIT_MASK                  0x00
    411 #define _SIDD_UNIT_MASK                 0x40
    412 
    413 /* SSE4.2 Packed Comparison Intrinsics.  */
    414 #define _mm_cmpistrm(A, B, M) \
    415   (__m128i)__builtin_ia32_pcmpistrm128((__v16qi)(__m128i)(A), \
    416                                        (__v16qi)(__m128i)(B), (int)(M))
    417 #define _mm_cmpistri(A, B, M) \
    418   (int)__builtin_ia32_pcmpistri128((__v16qi)(__m128i)(A), \
    419                                    (__v16qi)(__m128i)(B), (int)(M))
    420 
    421 #define _mm_cmpestrm(A, LA, B, LB, M) \
    422   (__m128i)__builtin_ia32_pcmpestrm128((__v16qi)(__m128i)(A), (int)(LA), \
    423                                        (__v16qi)(__m128i)(B), (int)(LB), \
    424                                        (int)(M))
    425 #define _mm_cmpestri(A, LA, B, LB, M) \
    426   (int)__builtin_ia32_pcmpestri128((__v16qi)(__m128i)(A), (int)(LA), \
    427                                    (__v16qi)(__m128i)(B), (int)(LB), \
    428                                    (int)(M))
    429 
    430 /* SSE4.2 Packed Comparison Intrinsics and EFlag Reading.  */
    431 #define _mm_cmpistra(A, B, M) \
    432   (int)__builtin_ia32_pcmpistria128((__v16qi)(__m128i)(A), \
    433                                     (__v16qi)(__m128i)(B), (int)(M))
    434 #define _mm_cmpistrc(A, B, M) \
    435   (int)__builtin_ia32_pcmpistric128((__v16qi)(__m128i)(A), \
    436                                     (__v16qi)(__m128i)(B), (int)(M))
    437 #define _mm_cmpistro(A, B, M) \
    438   (int)__builtin_ia32_pcmpistrio128((__v16qi)(__m128i)(A), \
    439                                     (__v16qi)(__m128i)(B), (int)(M))
    440 #define _mm_cmpistrs(A, B, M) \
    441   (int)__builtin_ia32_pcmpistris128((__v16qi)(__m128i)(A), \
    442                                     (__v16qi)(__m128i)(B), (int)(M))
    443 #define _mm_cmpistrz(A, B, M) \
    444   (int)__builtin_ia32_pcmpistriz128((__v16qi)(__m128i)(A), \
    445                                     (__v16qi)(__m128i)(B), (int)(M))
    446 
    447 #define _mm_cmpestra(A, LA, B, LB, M) \
    448   (int)__builtin_ia32_pcmpestria128((__v16qi)(__m128i)(A), (int)(LA), \
    449                                     (__v16qi)(__m128i)(B), (int)(LB), \
    450                                     (int)(M))
    451 #define _mm_cmpestrc(A, LA, B, LB, M) \
    452   (int)__builtin_ia32_pcmpestric128((__v16qi)(__m128i)(A), (int)(LA), \
    453                                     (__v16qi)(__m128i)(B), (int)(LB), \
    454                                     (int)(M))
    455 #define _mm_cmpestro(A, LA, B, LB, M) \
    456   (int)__builtin_ia32_pcmpestrio128((__v16qi)(__m128i)(A), (int)(LA), \
    457                                     (__v16qi)(__m128i)(B), (int)(LB), \
    458                                     (int)(M))
    459 #define _mm_cmpestrs(A, LA, B, LB, M) \
    460   (int)__builtin_ia32_pcmpestris128((__v16qi)(__m128i)(A), (int)(LA), \
    461                                     (__v16qi)(__m128i)(B), (int)(LB), \
    462                                     (int)(M))
    463 #define _mm_cmpestrz(A, LA, B, LB, M) \
    464   (int)__builtin_ia32_pcmpestriz128((__v16qi)(__m128i)(A), (int)(LA), \
    465                                     (__v16qi)(__m128i)(B), (int)(LB), \
    466                                     (int)(M))
    467 
    468 /* SSE4.2 Compare Packed Data -- Greater Than.  */
    469 static __inline__ __m128i __DEFAULT_FN_ATTRS
    470 _mm_cmpgt_epi64(__m128i __V1, __m128i __V2)
    471 {
    472   return (__m128i)((__v2di)__V1 > (__v2di)__V2);
    473 }
    474 
    475 /* SSE4.2 Accumulate CRC32.  */
    476 static __inline__ unsigned int __DEFAULT_FN_ATTRS
    477 _mm_crc32_u8(unsigned int __C, unsigned char __D)
    478 {
    479   return __builtin_ia32_crc32qi(__C, __D);
    480 }
    481 
    482 static __inline__ unsigned int __DEFAULT_FN_ATTRS
    483 _mm_crc32_u16(unsigned int __C, unsigned short __D)
    484 {
    485   return __builtin_ia32_crc32hi(__C, __D);
    486 }
    487 
    488 static __inline__ unsigned int __DEFAULT_FN_ATTRS
    489 _mm_crc32_u32(unsigned int __C, unsigned int __D)
    490 {
    491   return __builtin_ia32_crc32si(__C, __D);
    492 }
    493 
    494 #ifdef __x86_64__
    495 static __inline__ unsigned long long __DEFAULT_FN_ATTRS
    496 _mm_crc32_u64(unsigned long long __C, unsigned long long __D)
    497 {
    498   return __builtin_ia32_crc32di(__C, __D);
    499 }
    500 #endif /* __x86_64__ */
    501 
    502 #undef __DEFAULT_FN_ATTRS
    503 
    504 #ifdef __POPCNT__
    505 #include <popcntintrin.h>
    506 #endif
    507 
    508 #endif /* _SMMINTRIN_H */
    509