1 /**************************************************************************** 2 * Copyright (C) 2017 Intel Corporation. All Rights Reserved. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice (including the next 12 * paragraph) shall be included in all copies or substantial portions of the 13 * Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 21 * IN THE SOFTWARE. 22 ****************************************************************************/ 23 #if !defined(__SIMD_LIB_AVX512_HPP__) 24 #error Do not include this file directly, use "simdlib.hpp" instead. 25 #endif 26 27 //============================================================================ 28 // SIMD16 AVX512 (F) implementation for Core processors 29 // 30 //============================================================================ 31 32 #define SIMD_WRAPPER_1_(op, intrin) \ 33 static SIMDINLINE Float SIMDCALL op(Float a) \ 34 {\ 35 return intrin(a);\ 36 } 37 38 #define SIMD_WRAPPER_1(op) \ 39 SIMD_WRAPPER_1_(op, _mm512_##op) 40 41 #define SIMD_WRAPPER_2_(op, intrin) \ 42 static SIMDINLINE Float SIMDCALL op(Float a, Float b) \ 43 {\ 44 return _mm512_##intrin(a, b);\ 45 } 46 #define SIMD_WRAPPER_2(op) SIMD_WRAPPER_2_(op, op) 47 48 #define SIMD_WRAPPERI_2_(op, intrin) \ 49 static SIMDINLINE Float SIMDCALL op(Float a, Float b) \ 50 {\ 51 return _mm512_castsi512_ps(_mm512_##intrin(\ 52 _mm512_castps_si512(a), _mm512_castps_si512(b)));\ 53 } 54 55 #define SIMD_DWRAPPER_2(op) \ 56 static SIMDINLINE Double SIMDCALL op(Double a, Double b) \ 57 {\ 58 return _mm512_##op(a, b);\ 59 } 60 61 #define SIMD_WRAPPER_2I_(op, intrin) \ 62 template<int ImmT>\ 63 static SIMDINLINE Float SIMDCALL op(Float a, Float b) \ 64 {\ 65 return _mm512_##intrin(a, b, ImmT);\ 66 } 67 #define SIMD_WRAPPER_2I(op) SIMD_WRAPPER_2I_(op, op) 68 69 #define SIMD_DWRAPPER_2I_(op, intrin) \ 70 template<int ImmT>\ 71 static SIMDINLINE Double SIMDCALL op(Double a, Double b) \ 72 {\ 73 return _mm512_##intrin(a, b, ImmT);\ 74 } 75 #define SIMD_DWRAPPER_2I(op) SIMD_DWRAPPER_2I_(op, op) 76 77 #define SIMD_WRAPPER_3(op) \ 78 static SIMDINLINE Float SIMDCALL op(Float a, Float b, Float c) \ 79 {\ 80 return _mm512_##op(a, b, c);\ 81 } 82 83 #define SIMD_IWRAPPER_1(op) \ 84 static SIMDINLINE Integer SIMDCALL op(Integer a) \ 85 {\ 86 return _mm512_##op(a);\ 87 } 88 #define SIMD_IWRAPPER_1_8(op) \ 89 static SIMDINLINE Integer SIMDCALL op(SIMD256Impl::Integer a) \ 90 {\ 91 return _mm512_##op(a);\ 92 } 93 94 #define SIMD_IWRAPPER_1_4(op) \ 95 static SIMDINLINE Integer SIMDCALL op(SIMD128Impl::Integer a) \ 96 {\ 97 return _mm512_##op(a);\ 98 } 99 100 #define SIMD_IWRAPPER_1I_(op, intrin) \ 101 template<int ImmT> \ 102 static SIMDINLINE Integer SIMDCALL op(Integer a) \ 103 {\ 104 return intrin(a, ImmT);\ 105 } 106 #define SIMD_IWRAPPER_1I(op) SIMD_IWRAPPER_1I_(op, _mm512_##op) 107 108 #define SIMD_IWRAPPER_2_(op, intrin) \ 109 static SIMDINLINE Integer SIMDCALL op(Integer a, Integer b) \ 110 {\ 111 return _mm512_##intrin(a, b);\ 112 } 113 #define SIMD_IWRAPPER_2(op) SIMD_IWRAPPER_2_(op, op) 114 115 #define SIMD_IWRAPPER_2_CMP(op, cmp) \ 116 static SIMDINLINE Integer SIMDCALL op(Integer a, Integer b) \ 117 {\ 118 return cmp(a, b);\ 119 } 120 121 #define SIMD_IFWRAPPER_2(op, intrin) \ 122 static SIMDINLINE Integer SIMDCALL op(Integer a, Integer b) \ 123 {\ 124 return castps_si(_mm512_##intrin(castsi_ps(a), castsi_ps(b)) );\ 125 } 126 127 #define SIMD_IWRAPPER_2I_(op, intrin) \ 128 template<int ImmT>\ 129 static SIMDINLINE Integer SIMDCALL op(Integer a, Integer b) \ 130 {\ 131 return _mm512_##intrin(a, b, ImmT);\ 132 } 133 #define SIMD_IWRAPPER_2I(op) SIMD_IWRAPPER_2I_(op, op) 134 135 private: 136 static SIMDINLINE Integer vmask(__mmask32 m) 137 { 138 return _mm512_maskz_set1_epi16(m, -1); 139 } 140 static SIMDINLINE Integer vmask(__mmask64 m) 141 { 142 return _mm512_maskz_set1_epi8(m, -1); 143 } 144 public: 145 146 SIMD_IWRAPPER_2(add_epi8); // return a + b (int8) 147 SIMD_IWRAPPER_2(adds_epu8); // return ((a + b) > 0xff) ? 0xff : (a + b) (uint8) 148 SIMD_IWRAPPER_2(subs_epu8); // return (b > a) ? 0 : (a - b) (uint8) 149 150 SIMD_WRAPPER_2(and_ps); // return a & b (float treated as int) 151 SIMD_WRAPPER_2(andnot_ps); // return (~a) & b (float treated as int) 152 SIMD_WRAPPER_2(or_ps); // return a | b (float treated as int) 153 SIMD_WRAPPER_2(xor_ps); // return a ^ b (float treated as int) 154 155 SIMD_IWRAPPER_1_8(cvtepu8_epi16); // return (int16)a (uint8 --> int16) 156 157 template<CompareTypeInt CmpTypeT> 158 static SIMDINLINE Integer SIMDCALL cmp_epi8(Integer a, Integer b) 159 { 160 // Legacy vector mask generator 161 __mmask64 result = _mm512_cmp_epi8_mask(a, b, static_cast<const int>(CmpTypeT)); 162 return vmask(result); 163 } 164 template<CompareTypeInt CmpTypeT> 165 static SIMDINLINE Integer SIMDCALL cmp_epi16(Integer a, Integer b) 166 { 167 // Legacy vector mask generator 168 __mmask32 result = _mm512_cmp_epi16_mask(a, b, static_cast<const int>(CmpTypeT)); 169 return vmask(result); 170 } 171 172 SIMD_IWRAPPER_2_CMP(cmpeq_epi8, cmp_epi8<CompareTypeInt::EQ>); // return a == b (int8) 173 SIMD_IWRAPPER_2_CMP(cmpeq_epi16, cmp_epi16<CompareTypeInt::EQ>); // return a == b (int16) 174 SIMD_IWRAPPER_2_CMP(cmpgt_epi8, cmp_epi8<CompareTypeInt::GT>); // return a > b (int8) 175 SIMD_IWRAPPER_2_CMP(cmpgt_epi16, cmp_epi16<CompareTypeInt::GT>); // return a > b (int16) 176 177 SIMD_IWRAPPER_2(packs_epi16); // See documentation for _mm512_packs_epi16 178 SIMD_IWRAPPER_2(packs_epi32); // See documentation for _mm512_packs_epi32 179 SIMD_IWRAPPER_2(packus_epi16); // See documentation for _mm512_packus_epi16 180 SIMD_IWRAPPER_2(packus_epi32); // See documentation for _mm512_packus_epi32 181 182 SIMD_IWRAPPER_2(unpackhi_epi8); // See documentation for _mm512_unpackhi_epi8 183 SIMD_IWRAPPER_2(unpacklo_epi16); // See documentation for _mm512_unpacklo_epi16 184 SIMD_IWRAPPER_2(unpacklo_epi8); // See documentation for _mm512_unpacklo_epi8 185 186 SIMD_IWRAPPER_2(shuffle_epi8); 187 188 static SIMDINLINE uint64_t SIMDCALL movemask_epi8(Integer a) 189 { 190 __mmask64 m = _mm512_cmplt_epi8_mask(a, setzero_si()); 191 return static_cast<uint64_t>(m); 192 } 193 194 195 196 #undef SIMD_WRAPPER_1_ 197 #undef SIMD_WRAPPER_1 198 #undef SIMD_WRAPPER_2 199 #undef SIMD_WRAPPER_2_ 200 #undef SIMD_WRAPPERI_2_ 201 #undef SIMD_DWRAPPER_2 202 #undef SIMD_DWRAPPER_2I 203 #undef SIMD_WRAPPER_2I_ 204 #undef SIMD_WRAPPER_3_ 205 #undef SIMD_WRAPPER_2I 206 #undef SIMD_WRAPPER_3 207 #undef SIMD_IWRAPPER_1 208 #undef SIMD_IWRAPPER_2 209 #undef SIMD_IFWRAPPER_2 210 #undef SIMD_IWRAPPER_2I 211 #undef SIMD_IWRAPPER_1 212 #undef SIMD_IWRAPPER_1I 213 #undef SIMD_IWRAPPER_1I_ 214 #undef SIMD_IWRAPPER_2 215 #undef SIMD_IWRAPPER_2_ 216 #undef SIMD_IWRAPPER_2I 217 218
