1 /************************************************************************** 2 * 3 * Copyright 2009 VMware, Inc. 4 * All Rights Reserved. 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining a 7 * copy of this software and associated documentation files (the 8 * "Software"), to deal in the Software without restriction, including 9 * without limitation the rights to use, copy, modify, merge, publish, 10 * distribute, sub license, and/or sell copies of the Software, and to 11 * permit persons to whom the Software is furnished to do so, subject to 12 * the following conditions: 13 * 14 * The above copyright notice and this permission notice (including the 15 * next paragraph) shall be included in all copies or substantial portions 16 * of the Software. 17 * 18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 19 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 20 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. 21 * IN NO EVENT SHALL VMWARE AND/OR ITS SUPPLIERS BE LIABLE FOR 22 * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, 23 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE 24 * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 25 * 26 **************************************************************************/ 27 28 29 /** 30 * @file 31 * Helpers for emiting intrinsic calls. 32 * 33 * LLVM vanilla IR doesn't represent all basic arithmetic operations we care 34 * about, and it is often necessary to resort target-specific intrinsics for 35 * performance, convenience. 36 * 37 * Ideally we would like to stay away from target specific intrinsics and 38 * move all the instruction selection logic into upstream LLVM where it belongs. 39 * 40 * These functions are also used for calling C functions provided by us from 41 * generated LLVM code. 42 * 43 * @author Jose Fonseca <jfonseca (at) vmware.com> 44 */ 45 46 47 #include "util/u_debug.h" 48 49 #include "lp_bld_const.h" 50 #include "lp_bld_intr.h" 51 #include "lp_bld_type.h" 52 #include "lp_bld_pack.h" 53 54 55 LLVMValueRef 56 lp_declare_intrinsic(LLVMModuleRef module, 57 const char *name, 58 LLVMTypeRef ret_type, 59 LLVMTypeRef *arg_types, 60 unsigned num_args) 61 { 62 LLVMTypeRef function_type; 63 LLVMValueRef function; 64 65 assert(!LLVMGetNamedFunction(module, name)); 66 67 function_type = LLVMFunctionType(ret_type, arg_types, num_args, 0); 68 function = LLVMAddFunction(module, name, function_type); 69 70 LLVMSetFunctionCallConv(function, LLVMCCallConv); 71 LLVMSetLinkage(function, LLVMExternalLinkage); 72 73 assert(LLVMIsDeclaration(function)); 74 75 return function; 76 } 77 78 79 LLVMValueRef 80 lp_build_intrinsic(LLVMBuilderRef builder, 81 const char *name, 82 LLVMTypeRef ret_type, 83 LLVMValueRef *args, 84 unsigned num_args) 85 { 86 LLVMModuleRef module = LLVMGetGlobalParent(LLVMGetBasicBlockParent(LLVMGetInsertBlock(builder))); 87 LLVMValueRef function; 88 89 function = LLVMGetNamedFunction(module, name); 90 if(!function) { 91 LLVMTypeRef arg_types[LP_MAX_FUNC_ARGS]; 92 unsigned i; 93 94 assert(num_args <= LP_MAX_FUNC_ARGS); 95 96 for(i = 0; i < num_args; ++i) { 97 assert(args[i]); 98 arg_types[i] = LLVMTypeOf(args[i]); 99 } 100 101 function = lp_declare_intrinsic(module, name, ret_type, arg_types, num_args); 102 } 103 104 return LLVMBuildCall(builder, function, args, num_args, ""); 105 } 106 107 108 LLVMValueRef 109 lp_build_intrinsic_unary(LLVMBuilderRef builder, 110 const char *name, 111 LLVMTypeRef ret_type, 112 LLVMValueRef a) 113 { 114 return lp_build_intrinsic(builder, name, ret_type, &a, 1); 115 } 116 117 118 LLVMValueRef 119 lp_build_intrinsic_binary(LLVMBuilderRef builder, 120 const char *name, 121 LLVMTypeRef ret_type, 122 LLVMValueRef a, 123 LLVMValueRef b) 124 { 125 LLVMValueRef args[2]; 126 127 args[0] = a; 128 args[1] = b; 129 130 return lp_build_intrinsic(builder, name, ret_type, args, 2); 131 } 132 133 134 /** 135 * Call intrinsic with arguments adapted to intrinsic vector length. 136 * 137 * Split vectors which are too large for the hw, or expand them if they 138 * are too small, so a caller calling a function which might use intrinsics 139 * doesn't need to do splitting/expansion on its own. 140 * This only supports intrinsics where src and dst types match. 141 */ 142 LLVMValueRef 143 lp_build_intrinsic_binary_anylength(struct gallivm_state *gallivm, 144 const char *name, 145 struct lp_type src_type, 146 unsigned intr_size, 147 LLVMValueRef a, 148 LLVMValueRef b) 149 { 150 unsigned i; 151 struct lp_type intrin_type = src_type; 152 LLVMBuilderRef builder = gallivm->builder; 153 LLVMValueRef i32undef = LLVMGetUndef(LLVMInt32TypeInContext(gallivm->context)); 154 LLVMValueRef anative, bnative; 155 unsigned intrin_length = intr_size / src_type.width; 156 157 intrin_type.length = intrin_length; 158 159 if (intrin_length > src_type.length) { 160 LLVMValueRef elems[LP_MAX_VECTOR_LENGTH]; 161 LLVMValueRef constvec, tmp; 162 163 for (i = 0; i < src_type.length; i++) { 164 elems[i] = lp_build_const_int32(gallivm, i); 165 } 166 for (; i < intrin_length; i++) { 167 elems[i] = i32undef; 168 } 169 if (src_type.length == 1) { 170 LLVMTypeRef elem_type = lp_build_elem_type(gallivm, intrin_type); 171 a = LLVMBuildBitCast(builder, a, LLVMVectorType(elem_type, 1), ""); 172 b = LLVMBuildBitCast(builder, b, LLVMVectorType(elem_type, 1), ""); 173 } 174 constvec = LLVMConstVector(elems, intrin_length); 175 anative = LLVMBuildShuffleVector(builder, a, a, constvec, ""); 176 bnative = LLVMBuildShuffleVector(builder, b, b, constvec, ""); 177 tmp = lp_build_intrinsic_binary(builder, name, 178 lp_build_vec_type(gallivm, intrin_type), 179 anative, bnative); 180 if (src_type.length > 1) { 181 constvec = LLVMConstVector(elems, src_type.length); 182 return LLVMBuildShuffleVector(builder, tmp, tmp, constvec, ""); 183 } 184 else { 185 return LLVMBuildExtractElement(builder, tmp, elems[0], ""); 186 } 187 } 188 else if (intrin_length < src_type.length) { 189 unsigned num_vec = src_type.length / intrin_length; 190 LLVMValueRef tmp[LP_MAX_VECTOR_LENGTH]; 191 192 /*don't support arbitrary size here as this is so yuck */ 193 if (src_type.length % intrin_length) { 194 /*FIXME: This is something which should be supported 195 * but there doesn't seem to be any need for it currently 196 * so crash and burn. 197 */ 198 debug_printf("%s: should handle arbitrary vector size\n", 199 __FUNCTION__); 200 assert(0); 201 return NULL; 202 } 203 204 for (i = 0; i < num_vec; i++) { 205 anative = lp_build_extract_range(gallivm, a, i*intrin_length, 206 intrin_length); 207 bnative = lp_build_extract_range(gallivm, b, i*intrin_length, 208 intrin_length); 209 tmp[i] = lp_build_intrinsic_binary(builder, name, 210 lp_build_vec_type(gallivm, intrin_type), 211 anative, bnative); 212 } 213 return lp_build_concat(gallivm, tmp, intrin_type, num_vec); 214 } 215 else { 216 return lp_build_intrinsic_binary(builder, name, 217 lp_build_vec_type(gallivm, src_type), 218 a, b); 219 } 220 } 221 222 223 LLVMValueRef 224 lp_build_intrinsic_map(struct gallivm_state *gallivm, 225 const char *name, 226 LLVMTypeRef ret_type, 227 LLVMValueRef *args, 228 unsigned num_args) 229 { 230 LLVMBuilderRef builder = gallivm->builder; 231 LLVMTypeRef ret_elem_type = LLVMGetElementType(ret_type); 232 unsigned n = LLVMGetVectorSize(ret_type); 233 unsigned i, j; 234 LLVMValueRef res; 235 236 assert(num_args <= LP_MAX_FUNC_ARGS); 237 238 res = LLVMGetUndef(ret_type); 239 for(i = 0; i < n; ++i) { 240 LLVMValueRef index = lp_build_const_int32(gallivm, i); 241 LLVMValueRef arg_elems[LP_MAX_FUNC_ARGS]; 242 LLVMValueRef res_elem; 243 for(j = 0; j < num_args; ++j) 244 arg_elems[j] = LLVMBuildExtractElement(builder, args[j], index, ""); 245 res_elem = lp_build_intrinsic(builder, name, ret_elem_type, arg_elems, num_args); 246 res = LLVMBuildInsertElement(builder, res, res_elem, index, ""); 247 } 248 249 return res; 250 } 251 252 253 LLVMValueRef 254 lp_build_intrinsic_map_unary(struct gallivm_state *gallivm, 255 const char *name, 256 LLVMTypeRef ret_type, 257 LLVMValueRef a) 258 { 259 return lp_build_intrinsic_map(gallivm, name, ret_type, &a, 1); 260 } 261 262 263 LLVMValueRef 264 lp_build_intrinsic_map_binary(struct gallivm_state *gallivm, 265 const char *name, 266 LLVMTypeRef ret_type, 267 LLVMValueRef a, 268 LLVMValueRef b) 269 { 270 LLVMValueRef args[2]; 271 272 args[0] = a; 273 args[1] = b; 274 275 return lp_build_intrinsic_map(gallivm, name, ret_type, args, 2); 276 } 277 278 279
