1 //===-------- LegalizeTypesGeneric.cpp - Generic type legalization --------===// 2 // 3 // The LLVM Compiler Infrastructure 4 // 5 // This file is distributed under the University of Illinois Open Source 6 // License. See LICENSE.TXT for details. 7 // 8 //===----------------------------------------------------------------------===// 9 // 10 // This file implements generic type expansion and splitting for LegalizeTypes. 11 // The routines here perform legalization when the details of the type (such as 12 // whether it is an integer or a float) do not matter. 13 // Expansion is the act of changing a computation in an illegal type to be a 14 // computation in two identical registers of a smaller type. The Lo/Hi part 15 // is required to be stored first in memory on little/big-endian machines. 16 // Splitting is the act of changing a computation in an illegal type to be a 17 // computation in two not necessarily identical registers of a smaller type. 18 // There are no requirements on how the type is represented in memory. 19 // 20 //===----------------------------------------------------------------------===// 21 22 #include "LegalizeTypes.h" 23 #include "llvm/Target/TargetData.h" 24 #include "llvm/CodeGen/PseudoSourceValue.h" 25 using namespace llvm; 26 27 //===----------------------------------------------------------------------===// 28 // Generic Result Expansion. 29 //===----------------------------------------------------------------------===// 30 31 // These routines assume that the Lo/Hi part is stored first in memory on 32 // little/big-endian machines, followed by the Hi/Lo part. This means that 33 // they cannot be used as is on vectors, for which Lo is always stored first. 34 void DAGTypeLegalizer::ExpandRes_MERGE_VALUES(SDNode *N, unsigned ResNo, 35 SDValue &Lo, SDValue &Hi) { 36 SDValue Op = DisintegrateMERGE_VALUES(N, ResNo); 37 GetExpandedOp(Op, Lo, Hi); 38 } 39 40 void DAGTypeLegalizer::ExpandRes_BITCAST(SDNode *N, SDValue &Lo, SDValue &Hi) { 41 EVT OutVT = N->getValueType(0); 42 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT); 43 SDValue InOp = N->getOperand(0); 44 EVT InVT = InOp.getValueType(); 45 DebugLoc dl = N->getDebugLoc(); 46 47 // Handle some special cases efficiently. 48 switch (getTypeAction(InVT)) { 49 default: 50 assert(false && "Unknown type action!"); 51 case TargetLowering::TypeLegal: 52 case TargetLowering::TypePromoteInteger: 53 break; 54 case TargetLowering::TypeSoftenFloat: 55 // Convert the integer operand instead. 56 SplitInteger(GetSoftenedFloat(InOp), Lo, Hi); 57 Lo = DAG.getNode(ISD::BITCAST, dl, NOutVT, Lo); 58 Hi = DAG.getNode(ISD::BITCAST, dl, NOutVT, Hi); 59 return; 60 case TargetLowering::TypeExpandInteger: 61 case TargetLowering::TypeExpandFloat: 62 // Convert the expanded pieces of the input. 63 GetExpandedOp(InOp, Lo, Hi); 64 Lo = DAG.getNode(ISD::BITCAST, dl, NOutVT, Lo); 65 Hi = DAG.getNode(ISD::BITCAST, dl, NOutVT, Hi); 66 return; 67 case TargetLowering::TypeSplitVector: 68 GetSplitVector(InOp, Lo, Hi); 69 if (TLI.isBigEndian()) 70 std::swap(Lo, Hi); 71 Lo = DAG.getNode(ISD::BITCAST, dl, NOutVT, Lo); 72 Hi = DAG.getNode(ISD::BITCAST, dl, NOutVT, Hi); 73 return; 74 case TargetLowering::TypeScalarizeVector: 75 // Convert the element instead. 76 SplitInteger(BitConvertToInteger(GetScalarizedVector(InOp)), Lo, Hi); 77 Lo = DAG.getNode(ISD::BITCAST, dl, NOutVT, Lo); 78 Hi = DAG.getNode(ISD::BITCAST, dl, NOutVT, Hi); 79 return; 80 case TargetLowering::TypeWidenVector: { 81 assert(!(InVT.getVectorNumElements() & 1) && "Unsupported BITCAST"); 82 InOp = GetWidenedVector(InOp); 83 EVT InNVT = EVT::getVectorVT(*DAG.getContext(), InVT.getVectorElementType(), 84 InVT.getVectorNumElements()/2); 85 Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, InNVT, InOp, 86 DAG.getIntPtrConstant(0)); 87 Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, InNVT, InOp, 88 DAG.getIntPtrConstant(InNVT.getVectorNumElements())); 89 if (TLI.isBigEndian()) 90 std::swap(Lo, Hi); 91 Lo = DAG.getNode(ISD::BITCAST, dl, NOutVT, Lo); 92 Hi = DAG.getNode(ISD::BITCAST, dl, NOutVT, Hi); 93 return; 94 } 95 } 96 97 if (InVT.isVector() && OutVT.isInteger()) { 98 // Handle cases like i64 = BITCAST v1i64 on x86, where the operand 99 // is legal but the result is not. 100 EVT NVT = EVT::getVectorVT(*DAG.getContext(), NOutVT, 2); 101 102 if (isTypeLegal(NVT)) { 103 SDValue CastInOp = DAG.getNode(ISD::BITCAST, dl, NVT, InOp); 104 Lo = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, NOutVT, CastInOp, 105 DAG.getIntPtrConstant(0)); 106 Hi = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, NOutVT, CastInOp, 107 DAG.getIntPtrConstant(1)); 108 109 if (TLI.isBigEndian()) 110 std::swap(Lo, Hi); 111 112 return; 113 } 114 } 115 116 // Lower the bit-convert to a store/load from the stack. 117 assert(NOutVT.isByteSized() && "Expanded type not byte sized!"); 118 119 // Create the stack frame object. Make sure it is aligned for both 120 // the source and expanded destination types. 121 unsigned Alignment = 122 TLI.getTargetData()->getPrefTypeAlignment(NOutVT. 123 getTypeForEVT(*DAG.getContext())); 124 SDValue StackPtr = DAG.CreateStackTemporary(InVT, Alignment); 125 int SPFI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex(); 126 MachinePointerInfo PtrInfo = MachinePointerInfo::getFixedStack(SPFI); 127 128 // Emit a store to the stack slot. 129 SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, InOp, StackPtr, PtrInfo, 130 false, false, 0); 131 132 // Load the first half from the stack slot. 133 Lo = DAG.getLoad(NOutVT, dl, Store, StackPtr, PtrInfo, false, false, 0); 134 135 // Increment the pointer to the other half. 136 unsigned IncrementSize = NOutVT.getSizeInBits() / 8; 137 StackPtr = DAG.getNode(ISD::ADD, dl, StackPtr.getValueType(), StackPtr, 138 DAG.getIntPtrConstant(IncrementSize)); 139 140 // Load the second half from the stack slot. 141 Hi = DAG.getLoad(NOutVT, dl, Store, StackPtr, 142 PtrInfo.getWithOffset(IncrementSize), false, 143 false, MinAlign(Alignment, IncrementSize)); 144 145 // Handle endianness of the load. 146 if (TLI.isBigEndian()) 147 std::swap(Lo, Hi); 148 } 149 150 void DAGTypeLegalizer::ExpandRes_BUILD_PAIR(SDNode *N, SDValue &Lo, 151 SDValue &Hi) { 152 // Return the operands. 153 Lo = N->getOperand(0); 154 Hi = N->getOperand(1); 155 } 156 157 void DAGTypeLegalizer::ExpandRes_EXTRACT_ELEMENT(SDNode *N, SDValue &Lo, 158 SDValue &Hi) { 159 GetExpandedOp(N->getOperand(0), Lo, Hi); 160 SDValue Part = cast<ConstantSDNode>(N->getOperand(1))->getZExtValue() ? 161 Hi : Lo; 162 163 assert(Part.getValueType() == N->getValueType(0) && 164 "Type twice as big as expanded type not itself expanded!"); 165 166 GetPairElements(Part, Lo, Hi); 167 } 168 169 void DAGTypeLegalizer::ExpandRes_EXTRACT_VECTOR_ELT(SDNode *N, SDValue &Lo, 170 SDValue &Hi) { 171 SDValue OldVec = N->getOperand(0); 172 unsigned OldElts = OldVec.getValueType().getVectorNumElements(); 173 DebugLoc dl = N->getDebugLoc(); 174 175 // Convert to a vector of the expanded element type, for example 176 // <3 x i64> -> <6 x i32>. 177 EVT OldVT = N->getValueType(0); 178 EVT NewVT = TLI.getTypeToTransformTo(*DAG.getContext(), OldVT); 179 180 SDValue NewVec = DAG.getNode(ISD::BITCAST, dl, 181 EVT::getVectorVT(*DAG.getContext(), 182 NewVT, 2*OldElts), 183 OldVec); 184 185 // Extract the elements at 2 * Idx and 2 * Idx + 1 from the new vector. 186 SDValue Idx = N->getOperand(1); 187 188 // Make sure the type of Idx is big enough to hold the new values. 189 if (Idx.getValueType().bitsLT(TLI.getPointerTy())) 190 Idx = DAG.getNode(ISD::ZERO_EXTEND, dl, TLI.getPointerTy(), Idx); 191 192 Idx = DAG.getNode(ISD::ADD, dl, Idx.getValueType(), Idx, Idx); 193 Lo = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, NewVT, NewVec, Idx); 194 195 Idx = DAG.getNode(ISD::ADD, dl, Idx.getValueType(), Idx, 196 DAG.getConstant(1, Idx.getValueType())); 197 Hi = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, NewVT, NewVec, Idx); 198 199 if (TLI.isBigEndian()) 200 std::swap(Lo, Hi); 201 } 202 203 void DAGTypeLegalizer::ExpandRes_NormalLoad(SDNode *N, SDValue &Lo, 204 SDValue &Hi) { 205 assert(ISD::isNormalLoad(N) && "This routine only for normal loads!"); 206 DebugLoc dl = N->getDebugLoc(); 207 208 LoadSDNode *LD = cast<LoadSDNode>(N); 209 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), LD->getValueType(0)); 210 SDValue Chain = LD->getChain(); 211 SDValue Ptr = LD->getBasePtr(); 212 unsigned Alignment = LD->getAlignment(); 213 bool isVolatile = LD->isVolatile(); 214 bool isNonTemporal = LD->isNonTemporal(); 215 216 assert(NVT.isByteSized() && "Expanded type not byte sized!"); 217 218 Lo = DAG.getLoad(NVT, dl, Chain, Ptr, LD->getPointerInfo(), 219 isVolatile, isNonTemporal, Alignment); 220 221 // Increment the pointer to the other half. 222 unsigned IncrementSize = NVT.getSizeInBits() / 8; 223 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, 224 DAG.getIntPtrConstant(IncrementSize)); 225 Hi = DAG.getLoad(NVT, dl, Chain, Ptr, 226 LD->getPointerInfo().getWithOffset(IncrementSize), 227 isVolatile, isNonTemporal, 228 MinAlign(Alignment, IncrementSize)); 229 230 // Build a factor node to remember that this load is independent of the 231 // other one. 232 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1), 233 Hi.getValue(1)); 234 235 // Handle endianness of the load. 236 if (TLI.isBigEndian()) 237 std::swap(Lo, Hi); 238 239 // Modified the chain - switch anything that used the old chain to use 240 // the new one. 241 ReplaceValueWith(SDValue(N, 1), Chain); 242 } 243 244 void DAGTypeLegalizer::ExpandRes_VAARG(SDNode *N, SDValue &Lo, SDValue &Hi) { 245 EVT OVT = N->getValueType(0); 246 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT); 247 SDValue Chain = N->getOperand(0); 248 SDValue Ptr = N->getOperand(1); 249 DebugLoc dl = N->getDebugLoc(); 250 const unsigned Align = N->getConstantOperandVal(3); 251 252 Lo = DAG.getVAArg(NVT, dl, Chain, Ptr, N->getOperand(2), Align); 253 Hi = DAG.getVAArg(NVT, dl, Lo.getValue(1), Ptr, N->getOperand(2), 0); 254 255 // Handle endianness of the load. 256 if (TLI.isBigEndian()) 257 std::swap(Lo, Hi); 258 259 // Modified the chain - switch anything that used the old chain to use 260 // the new one. 261 ReplaceValueWith(SDValue(N, 1), Hi.getValue(1)); 262 } 263 264 265 //===--------------------------------------------------------------------===// 266 // Generic Operand Expansion. 267 //===--------------------------------------------------------------------===// 268 269 SDValue DAGTypeLegalizer::ExpandOp_BITCAST(SDNode *N) { 270 DebugLoc dl = N->getDebugLoc(); 271 if (N->getValueType(0).isVector()) { 272 // An illegal expanding type is being converted to a legal vector type. 273 // Make a two element vector out of the expanded parts and convert that 274 // instead, but only if the new vector type is legal (otherwise there 275 // is no point, and it might create expansion loops). For example, on 276 // x86 this turns v1i64 = BITCAST i64 into v1i64 = BITCAST v2i32. 277 EVT OVT = N->getOperand(0).getValueType(); 278 EVT NVT = EVT::getVectorVT(*DAG.getContext(), 279 TLI.getTypeToTransformTo(*DAG.getContext(), OVT), 280 2); 281 282 if (isTypeLegal(NVT)) { 283 SDValue Parts[2]; 284 GetExpandedOp(N->getOperand(0), Parts[0], Parts[1]); 285 286 if (TLI.isBigEndian()) 287 std::swap(Parts[0], Parts[1]); 288 289 SDValue Vec = DAG.getNode(ISD::BUILD_VECTOR, dl, NVT, Parts, 2); 290 return DAG.getNode(ISD::BITCAST, dl, N->getValueType(0), Vec); 291 } 292 } 293 294 // Otherwise, store to a temporary and load out again as the new type. 295 return CreateStackStoreLoad(N->getOperand(0), N->getValueType(0)); 296 } 297 298 SDValue DAGTypeLegalizer::ExpandOp_BUILD_VECTOR(SDNode *N) { 299 // The vector type is legal but the element type needs expansion. 300 EVT VecVT = N->getValueType(0); 301 unsigned NumElts = VecVT.getVectorNumElements(); 302 EVT OldVT = N->getOperand(0).getValueType(); 303 EVT NewVT = TLI.getTypeToTransformTo(*DAG.getContext(), OldVT); 304 DebugLoc dl = N->getDebugLoc(); 305 306 assert(OldVT == VecVT.getVectorElementType() && 307 "BUILD_VECTOR operand type doesn't match vector element type!"); 308 309 // Build a vector of twice the length out of the expanded elements. 310 // For example <3 x i64> -> <6 x i32>. 311 std::vector<SDValue> NewElts; 312 NewElts.reserve(NumElts*2); 313 314 for (unsigned i = 0; i < NumElts; ++i) { 315 SDValue Lo, Hi; 316 GetExpandedOp(N->getOperand(i), Lo, Hi); 317 if (TLI.isBigEndian()) 318 std::swap(Lo, Hi); 319 NewElts.push_back(Lo); 320 NewElts.push_back(Hi); 321 } 322 323 SDValue NewVec = DAG.getNode(ISD::BUILD_VECTOR, dl, 324 EVT::getVectorVT(*DAG.getContext(), 325 NewVT, NewElts.size()), 326 &NewElts[0], NewElts.size()); 327 328 // Convert the new vector to the old vector type. 329 return DAG.getNode(ISD::BITCAST, dl, VecVT, NewVec); 330 } 331 332 SDValue DAGTypeLegalizer::ExpandOp_EXTRACT_ELEMENT(SDNode *N) { 333 SDValue Lo, Hi; 334 GetExpandedOp(N->getOperand(0), Lo, Hi); 335 return cast<ConstantSDNode>(N->getOperand(1))->getZExtValue() ? Hi : Lo; 336 } 337 338 SDValue DAGTypeLegalizer::ExpandOp_INSERT_VECTOR_ELT(SDNode *N) { 339 // The vector type is legal but the element type needs expansion. 340 EVT VecVT = N->getValueType(0); 341 unsigned NumElts = VecVT.getVectorNumElements(); 342 DebugLoc dl = N->getDebugLoc(); 343 344 SDValue Val = N->getOperand(1); 345 EVT OldEVT = Val.getValueType(); 346 EVT NewEVT = TLI.getTypeToTransformTo(*DAG.getContext(), OldEVT); 347 348 assert(OldEVT == VecVT.getVectorElementType() && 349 "Inserted element type doesn't match vector element type!"); 350 351 // Bitconvert to a vector of twice the length with elements of the expanded 352 // type, insert the expanded vector elements, and then convert back. 353 EVT NewVecVT = EVT::getVectorVT(*DAG.getContext(), NewEVT, NumElts*2); 354 SDValue NewVec = DAG.getNode(ISD::BITCAST, dl, 355 NewVecVT, N->getOperand(0)); 356 357 SDValue Lo, Hi; 358 GetExpandedOp(Val, Lo, Hi); 359 if (TLI.isBigEndian()) 360 std::swap(Lo, Hi); 361 362 SDValue Idx = N->getOperand(2); 363 Idx = DAG.getNode(ISD::ADD, dl, Idx.getValueType(), Idx, Idx); 364 NewVec = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, NewVecVT, NewVec, Lo, Idx); 365 Idx = DAG.getNode(ISD::ADD, dl, 366 Idx.getValueType(), Idx, DAG.getIntPtrConstant(1)); 367 NewVec = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, NewVecVT, NewVec, Hi, Idx); 368 369 // Convert the new vector to the old vector type. 370 return DAG.getNode(ISD::BITCAST, dl, VecVT, NewVec); 371 } 372 373 SDValue DAGTypeLegalizer::ExpandOp_SCALAR_TO_VECTOR(SDNode *N) { 374 DebugLoc dl = N->getDebugLoc(); 375 EVT VT = N->getValueType(0); 376 assert(VT.getVectorElementType() == N->getOperand(0).getValueType() && 377 "SCALAR_TO_VECTOR operand type doesn't match vector element type!"); 378 unsigned NumElts = VT.getVectorNumElements(); 379 SmallVector<SDValue, 16> Ops(NumElts); 380 Ops[0] = N->getOperand(0); 381 SDValue UndefVal = DAG.getUNDEF(Ops[0].getValueType()); 382 for (unsigned i = 1; i < NumElts; ++i) 383 Ops[i] = UndefVal; 384 return DAG.getNode(ISD::BUILD_VECTOR, dl, VT, &Ops[0], NumElts); 385 } 386 387 SDValue DAGTypeLegalizer::ExpandOp_NormalStore(SDNode *N, unsigned OpNo) { 388 assert(ISD::isNormalStore(N) && "This routine only for normal stores!"); 389 assert(OpNo == 1 && "Can only expand the stored value so far"); 390 DebugLoc dl = N->getDebugLoc(); 391 392 StoreSDNode *St = cast<StoreSDNode>(N); 393 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), 394 St->getValue().getValueType()); 395 SDValue Chain = St->getChain(); 396 SDValue Ptr = St->getBasePtr(); 397 unsigned Alignment = St->getAlignment(); 398 bool isVolatile = St->isVolatile(); 399 bool isNonTemporal = St->isNonTemporal(); 400 401 assert(NVT.isByteSized() && "Expanded type not byte sized!"); 402 unsigned IncrementSize = NVT.getSizeInBits() / 8; 403 404 SDValue Lo, Hi; 405 GetExpandedOp(St->getValue(), Lo, Hi); 406 407 if (TLI.isBigEndian()) 408 std::swap(Lo, Hi); 409 410 Lo = DAG.getStore(Chain, dl, Lo, Ptr, St->getPointerInfo(), 411 isVolatile, isNonTemporal, Alignment); 412 413 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, 414 DAG.getIntPtrConstant(IncrementSize)); 415 assert(isTypeLegal(Ptr.getValueType()) && "Pointers must be legal!"); 416 Hi = DAG.getStore(Chain, dl, Hi, Ptr, 417 St->getPointerInfo().getWithOffset(IncrementSize), 418 isVolatile, isNonTemporal, 419 MinAlign(Alignment, IncrementSize)); 420 421 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi); 422 } 423 424 425 //===--------------------------------------------------------------------===// 426 // Generic Result Splitting. 427 //===--------------------------------------------------------------------===// 428 429 // Be careful to make no assumptions about which of Lo/Hi is stored first in 430 // memory (for vectors it is always Lo first followed by Hi in the following 431 // bytes; for integers and floats it is Lo first if and only if the machine is 432 // little-endian). 433 434 void DAGTypeLegalizer::SplitRes_MERGE_VALUES(SDNode *N, unsigned ResNo, 435 SDValue &Lo, SDValue &Hi) { 436 SDValue Op = DisintegrateMERGE_VALUES(N, ResNo); 437 GetSplitOp(Op, Lo, Hi); 438 } 439 440 void DAGTypeLegalizer::SplitRes_SELECT(SDNode *N, SDValue &Lo, 441 SDValue &Hi) { 442 SDValue LL, LH, RL, RH, CL, CH; 443 DebugLoc dl = N->getDebugLoc(); 444 GetSplitOp(N->getOperand(1), LL, LH); 445 GetSplitOp(N->getOperand(2), RL, RH); 446 447 SDValue Cond = N->getOperand(0); 448 CL = CH = Cond; 449 if (Cond.getValueType().isVector()) { 450 assert(Cond.getValueType().getVectorElementType() == MVT::i1 && 451 "Condition legalized before result?"); 452 unsigned NumElements = Cond.getValueType().getVectorNumElements(); 453 EVT VCondTy = EVT::getVectorVT(*DAG.getContext(), MVT::i1, NumElements / 2); 454 CL = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VCondTy, Cond, 455 DAG.getIntPtrConstant(0)); 456 CH = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VCondTy, Cond, 457 DAG.getIntPtrConstant(NumElements / 2)); 458 } 459 460 Lo = DAG.getNode(N->getOpcode(), dl, LL.getValueType(), CL, LL, RL); 461 Hi = DAG.getNode(N->getOpcode(), dl, LH.getValueType(), CH, LH, RH); 462 } 463 464 void DAGTypeLegalizer::SplitRes_SELECT_CC(SDNode *N, SDValue &Lo, 465 SDValue &Hi) { 466 SDValue LL, LH, RL, RH; 467 DebugLoc dl = N->getDebugLoc(); 468 GetSplitOp(N->getOperand(2), LL, LH); 469 GetSplitOp(N->getOperand(3), RL, RH); 470 471 Lo = DAG.getNode(ISD::SELECT_CC, dl, LL.getValueType(), N->getOperand(0), 472 N->getOperand(1), LL, RL, N->getOperand(4)); 473 Hi = DAG.getNode(ISD::SELECT_CC, dl, LH.getValueType(), N->getOperand(0), 474 N->getOperand(1), LH, RH, N->getOperand(4)); 475 } 476 477 void DAGTypeLegalizer::SplitRes_UNDEF(SDNode *N, SDValue &Lo, SDValue &Hi) { 478 EVT LoVT, HiVT; 479 GetSplitDestVTs(N->getValueType(0), LoVT, HiVT); 480 Lo = DAG.getUNDEF(LoVT); 481 Hi = DAG.getUNDEF(HiVT); 482 } 483