1 /************************************************************************** 2 * 3 * Copyright 2009 VMware, Inc. 4 * All Rights Reserved. 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining a 7 * copy of this software and associated documentation files (the 8 * "Software"), to deal in the Software without restriction, including 9 * without limitation the rights to use, copy, modify, merge, publish, 10 * distribute, sub license, and/or sell copies of the Software, and to 11 * permit persons to whom the Software is furnished to do so, subject to 12 * the following conditions: 13 * 14 * The above copyright notice and this permission notice (including the 15 * next paragraph) shall be included in all copies or substantial portions 16 * of the Software. 17 * 18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 19 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 20 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. 21 * IN NO EVENT SHALL VMWARE AND/OR ITS SUPPLIERS BE LIABLE FOR 22 * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, 23 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE 24 * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 25 * 26 **************************************************************************/ 27 28 29 /** 30 * @file 31 * Helpers for emiting intrinsic calls. 32 * 33 * LLVM vanilla IR doesn't represent all basic arithmetic operations we care 34 * about, and it is often necessary to resort target-specific intrinsics for 35 * performance, convenience. 36 * 37 * Ideally we would like to stay away from target specific intrinsics and 38 * move all the instruction selection logic into upstream LLVM where it belongs. 39 * 40 * These functions are also used for calling C functions provided by us from 41 * generated LLVM code. 42 * 43 * @author Jose Fonseca <jfonseca (at) vmware.com> 44 */ 45 46 47 #include "util/u_debug.h" 48 #include "util/u_string.h" 49 #include "util/bitscan.h" 50 51 #include "lp_bld_const.h" 52 #include "lp_bld_intr.h" 53 #include "lp_bld_type.h" 54 #include "lp_bld_pack.h" 55 #include "lp_bld_debug.h" 56 57 58 void 59 lp_format_intrinsic(char *name, 60 size_t size, 61 const char *name_root, 62 LLVMTypeRef type) 63 { 64 unsigned length = 0; 65 unsigned width; 66 char c; 67 68 LLVMTypeKind kind = LLVMGetTypeKind(type); 69 if (kind == LLVMVectorTypeKind) { 70 length = LLVMGetVectorSize(type); 71 type = LLVMGetElementType(type); 72 kind = LLVMGetTypeKind(type); 73 } 74 75 switch (kind) { 76 case LLVMIntegerTypeKind: 77 c = 'i'; 78 width = LLVMGetIntTypeWidth(type); 79 break; 80 case LLVMFloatTypeKind: 81 c = 'f'; 82 width = 32; 83 break; 84 case LLVMDoubleTypeKind: 85 c = 'f'; 86 width = 64; 87 break; 88 default: 89 unreachable("unexpected LLVMTypeKind"); 90 } 91 92 if (length) { 93 util_snprintf(name, size, "%s.v%u%c%u", name_root, length, c, width); 94 } else { 95 util_snprintf(name, size, "%s.%c%u", name_root, c, width); 96 } 97 } 98 99 100 LLVMValueRef 101 lp_declare_intrinsic(LLVMModuleRef module, 102 const char *name, 103 LLVMTypeRef ret_type, 104 LLVMTypeRef *arg_types, 105 unsigned num_args) 106 { 107 LLVMTypeRef function_type; 108 LLVMValueRef function; 109 110 assert(!LLVMGetNamedFunction(module, name)); 111 112 function_type = LLVMFunctionType(ret_type, arg_types, num_args, 0); 113 function = LLVMAddFunction(module, name, function_type); 114 115 LLVMSetFunctionCallConv(function, LLVMCCallConv); 116 LLVMSetLinkage(function, LLVMExternalLinkage); 117 118 assert(LLVMIsDeclaration(function)); 119 120 return function; 121 } 122 123 124 #if HAVE_LLVM < 0x0400 125 static LLVMAttribute lp_attr_to_llvm_attr(enum lp_func_attr attr) 126 { 127 switch (attr) { 128 case LP_FUNC_ATTR_ALWAYSINLINE: return LLVMAlwaysInlineAttribute; 129 case LP_FUNC_ATTR_BYVAL: return LLVMByValAttribute; 130 case LP_FUNC_ATTR_INREG: return LLVMInRegAttribute; 131 case LP_FUNC_ATTR_NOALIAS: return LLVMNoAliasAttribute; 132 case LP_FUNC_ATTR_NOUNWIND: return LLVMNoUnwindAttribute; 133 case LP_FUNC_ATTR_READNONE: return LLVMReadNoneAttribute; 134 case LP_FUNC_ATTR_READONLY: return LLVMReadOnlyAttribute; 135 default: 136 _debug_printf("Unhandled function attribute: %x\n", attr); 137 return 0; 138 } 139 } 140 141 #else 142 143 static const char *attr_to_str(enum lp_func_attr attr) 144 { 145 switch (attr) { 146 case LP_FUNC_ATTR_ALWAYSINLINE: return "alwaysinline"; 147 case LP_FUNC_ATTR_BYVAL: return "byval"; 148 case LP_FUNC_ATTR_INREG: return "inreg"; 149 case LP_FUNC_ATTR_NOALIAS: return "noalias"; 150 case LP_FUNC_ATTR_NOUNWIND: return "nounwind"; 151 case LP_FUNC_ATTR_READNONE: return "readnone"; 152 case LP_FUNC_ATTR_READONLY: return "readonly"; 153 default: 154 _debug_printf("Unhandled function attribute: %x\n", attr); 155 return 0; 156 } 157 } 158 159 #endif 160 161 void 162 lp_add_function_attr(LLVMValueRef function, 163 int attr_idx, 164 enum lp_func_attr attr) 165 { 166 167 #if HAVE_LLVM < 0x0400 168 LLVMAttribute llvm_attr = lp_attr_to_llvm_attr(attr); 169 if (attr_idx == -1) { 170 LLVMAddFunctionAttr(function, llvm_attr); 171 } else { 172 LLVMAddAttribute(LLVMGetParam(function, attr_idx - 1), llvm_attr); 173 } 174 #else 175 LLVMContextRef context = LLVMGetModuleContext(LLVMGetGlobalParent(function)); 176 const char *attr_name = attr_to_str(attr); 177 unsigned kind_id = LLVMGetEnumAttributeKindForName(attr_name, 178 strlen(attr_name)); 179 LLVMAttributeRef llvm_attr = LLVMCreateEnumAttribute(context, kind_id, 0); 180 LLVMAddAttributeAtIndex(function, attr_idx, llvm_attr); 181 #endif 182 } 183 184 LLVMValueRef 185 lp_build_intrinsic(LLVMBuilderRef builder, 186 const char *name, 187 LLVMTypeRef ret_type, 188 LLVMValueRef *args, 189 unsigned num_args, 190 unsigned attr_mask) 191 { 192 LLVMModuleRef module = LLVMGetGlobalParent(LLVMGetBasicBlockParent(LLVMGetInsertBlock(builder))); 193 LLVMValueRef function; 194 195 function = LLVMGetNamedFunction(module, name); 196 if(!function) { 197 LLVMTypeRef arg_types[LP_MAX_FUNC_ARGS]; 198 unsigned i; 199 200 assert(num_args <= LP_MAX_FUNC_ARGS); 201 202 for(i = 0; i < num_args; ++i) { 203 assert(args[i]); 204 arg_types[i] = LLVMTypeOf(args[i]); 205 } 206 207 function = lp_declare_intrinsic(module, name, ret_type, arg_types, num_args); 208 209 /* NoUnwind indicates that the intrinsic never raises a C++ exception. 210 * Set it for all intrinsics. 211 */ 212 attr_mask |= LP_FUNC_ATTR_NOUNWIND; 213 214 while (attr_mask) { 215 enum lp_func_attr attr = 1 << u_bit_scan(&attr_mask); 216 lp_add_function_attr(function, -1, attr); 217 } 218 219 if (gallivm_debug & GALLIVM_DEBUG_IR) { 220 lp_debug_dump_value(function); 221 } 222 } 223 224 return LLVMBuildCall(builder, function, args, num_args, ""); 225 } 226 227 228 LLVMValueRef 229 lp_build_intrinsic_unary(LLVMBuilderRef builder, 230 const char *name, 231 LLVMTypeRef ret_type, 232 LLVMValueRef a) 233 { 234 return lp_build_intrinsic(builder, name, ret_type, &a, 1, 0); 235 } 236 237 238 LLVMValueRef 239 lp_build_intrinsic_binary(LLVMBuilderRef builder, 240 const char *name, 241 LLVMTypeRef ret_type, 242 LLVMValueRef a, 243 LLVMValueRef b) 244 { 245 LLVMValueRef args[2]; 246 247 args[0] = a; 248 args[1] = b; 249 250 return lp_build_intrinsic(builder, name, ret_type, args, 2, 0); 251 } 252 253 254 /** 255 * Call intrinsic with arguments adapted to intrinsic vector length. 256 * 257 * Split vectors which are too large for the hw, or expand them if they 258 * are too small, so a caller calling a function which might use intrinsics 259 * doesn't need to do splitting/expansion on its own. 260 * This only supports intrinsics where src and dst types match. 261 */ 262 LLVMValueRef 263 lp_build_intrinsic_binary_anylength(struct gallivm_state *gallivm, 264 const char *name, 265 struct lp_type src_type, 266 unsigned intr_size, 267 LLVMValueRef a, 268 LLVMValueRef b) 269 { 270 unsigned i; 271 struct lp_type intrin_type = src_type; 272 LLVMBuilderRef builder = gallivm->builder; 273 LLVMValueRef i32undef = LLVMGetUndef(LLVMInt32TypeInContext(gallivm->context)); 274 LLVMValueRef anative, bnative; 275 unsigned intrin_length = intr_size / src_type.width; 276 277 intrin_type.length = intrin_length; 278 279 if (intrin_length > src_type.length) { 280 LLVMValueRef elems[LP_MAX_VECTOR_LENGTH]; 281 LLVMValueRef constvec, tmp; 282 283 for (i = 0; i < src_type.length; i++) { 284 elems[i] = lp_build_const_int32(gallivm, i); 285 } 286 for (; i < intrin_length; i++) { 287 elems[i] = i32undef; 288 } 289 if (src_type.length == 1) { 290 LLVMTypeRef elem_type = lp_build_elem_type(gallivm, intrin_type); 291 a = LLVMBuildBitCast(builder, a, LLVMVectorType(elem_type, 1), ""); 292 b = LLVMBuildBitCast(builder, b, LLVMVectorType(elem_type, 1), ""); 293 } 294 constvec = LLVMConstVector(elems, intrin_length); 295 anative = LLVMBuildShuffleVector(builder, a, a, constvec, ""); 296 bnative = LLVMBuildShuffleVector(builder, b, b, constvec, ""); 297 tmp = lp_build_intrinsic_binary(builder, name, 298 lp_build_vec_type(gallivm, intrin_type), 299 anative, bnative); 300 if (src_type.length > 1) { 301 constvec = LLVMConstVector(elems, src_type.length); 302 return LLVMBuildShuffleVector(builder, tmp, tmp, constvec, ""); 303 } 304 else { 305 return LLVMBuildExtractElement(builder, tmp, elems[0], ""); 306 } 307 } 308 else if (intrin_length < src_type.length) { 309 unsigned num_vec = src_type.length / intrin_length; 310 LLVMValueRef tmp[LP_MAX_VECTOR_LENGTH]; 311 312 /*don't support arbitrary size here as this is so yuck */ 313 if (src_type.length % intrin_length) { 314 /*FIXME: This is something which should be supported 315 * but there doesn't seem to be any need for it currently 316 * so crash and burn. 317 */ 318 debug_printf("%s: should handle arbitrary vector size\n", 319 __FUNCTION__); 320 assert(0); 321 return NULL; 322 } 323 324 for (i = 0; i < num_vec; i++) { 325 anative = lp_build_extract_range(gallivm, a, i*intrin_length, 326 intrin_length); 327 bnative = lp_build_extract_range(gallivm, b, i*intrin_length, 328 intrin_length); 329 tmp[i] = lp_build_intrinsic_binary(builder, name, 330 lp_build_vec_type(gallivm, intrin_type), 331 anative, bnative); 332 } 333 return lp_build_concat(gallivm, tmp, intrin_type, num_vec); 334 } 335 else { 336 return lp_build_intrinsic_binary(builder, name, 337 lp_build_vec_type(gallivm, src_type), 338 a, b); 339 } 340 } 341 342 343 LLVMValueRef 344 lp_build_intrinsic_map(struct gallivm_state *gallivm, 345 const char *name, 346 LLVMTypeRef ret_type, 347 LLVMValueRef *args, 348 unsigned num_args) 349 { 350 LLVMBuilderRef builder = gallivm->builder; 351 LLVMTypeRef ret_elem_type = LLVMGetElementType(ret_type); 352 unsigned n = LLVMGetVectorSize(ret_type); 353 unsigned i, j; 354 LLVMValueRef res; 355 356 assert(num_args <= LP_MAX_FUNC_ARGS); 357 358 res = LLVMGetUndef(ret_type); 359 for(i = 0; i < n; ++i) { 360 LLVMValueRef index = lp_build_const_int32(gallivm, i); 361 LLVMValueRef arg_elems[LP_MAX_FUNC_ARGS]; 362 LLVMValueRef res_elem; 363 for(j = 0; j < num_args; ++j) 364 arg_elems[j] = LLVMBuildExtractElement(builder, args[j], index, ""); 365 res_elem = lp_build_intrinsic(builder, name, ret_elem_type, arg_elems, num_args, 0); 366 res = LLVMBuildInsertElement(builder, res, res_elem, index, ""); 367 } 368 369 return res; 370 } 371 372 373 LLVMValueRef 374 lp_build_intrinsic_map_unary(struct gallivm_state *gallivm, 375 const char *name, 376 LLVMTypeRef ret_type, 377 LLVMValueRef a) 378 { 379 return lp_build_intrinsic_map(gallivm, name, ret_type, &a, 1); 380 } 381 382 383 LLVMValueRef 384 lp_build_intrinsic_map_binary(struct gallivm_state *gallivm, 385 const char *name, 386 LLVMTypeRef ret_type, 387 LLVMValueRef a, 388 LLVMValueRef b) 389 { 390 LLVMValueRef args[2]; 391 392 args[0] = a; 393 args[1] = b; 394 395 return lp_build_intrinsic_map(gallivm, name, ret_type, args, 2); 396 } 397 398 399
