1 //===----- LegalizeIntegerTypes.cpp - Legalization of integer types -------===// 2 // 3 // The LLVM Compiler Infrastructure 4 // 5 // This file is distributed under the University of Illinois Open Source 6 // License. See LICENSE.TXT for details. 7 // 8 //===----------------------------------------------------------------------===// 9 // 10 // This file implements integer type expansion and promotion for LegalizeTypes. 11 // Promotion is the act of changing a computation in an illegal type into a 12 // computation in a larger type. For example, implementing i8 arithmetic in an 13 // i32 register (often needed on powerpc). 14 // Expansion is the act of changing a computation in an illegal type into a 15 // computation in two identical registers of a smaller type. For example, 16 // implementing i64 arithmetic in two i32 registers (often needed on 32-bit 17 // targets). 18 // 19 //===----------------------------------------------------------------------===// 20 21 #include "LegalizeTypes.h" 22 #include "llvm/IR/DerivedTypes.h" 23 #include "llvm/Support/ErrorHandling.h" 24 #include "llvm/Support/raw_ostream.h" 25 using namespace llvm; 26 27 #define DEBUG_TYPE "legalize-types" 28 29 //===----------------------------------------------------------------------===// 30 // Integer Result Promotion 31 //===----------------------------------------------------------------------===// 32 33 /// PromoteIntegerResult - This method is called when a result of a node is 34 /// found to be in need of promotion to a larger type. At this point, the node 35 /// may also have invalid operands or may have other results that need 36 /// expansion, we just know that (at least) one result needs promotion. 37 void DAGTypeLegalizer::PromoteIntegerResult(SDNode *N, unsigned ResNo) { 38 DEBUG(dbgs() << "Promote integer result: "; N->dump(&DAG); dbgs() << "\n"); 39 SDValue Res = SDValue(); 40 41 // See if the target wants to custom expand this node. 42 if (CustomLowerNode(N, N->getValueType(ResNo), true)) 43 return; 44 45 switch (N->getOpcode()) { 46 default: 47 #ifndef NDEBUG 48 dbgs() << "PromoteIntegerResult #" << ResNo << ": "; 49 N->dump(&DAG); dbgs() << "\n"; 50 #endif 51 llvm_unreachable("Do not know how to promote this operator!"); 52 case ISD::MERGE_VALUES:Res = PromoteIntRes_MERGE_VALUES(N, ResNo); break; 53 case ISD::AssertSext: Res = PromoteIntRes_AssertSext(N); break; 54 case ISD::AssertZext: Res = PromoteIntRes_AssertZext(N); break; 55 case ISD::BITCAST: Res = PromoteIntRes_BITCAST(N); break; 56 case ISD::BITREVERSE: Res = PromoteIntRes_BITREVERSE(N); break; 57 case ISD::BSWAP: Res = PromoteIntRes_BSWAP(N); break; 58 case ISD::BUILD_PAIR: Res = PromoteIntRes_BUILD_PAIR(N); break; 59 case ISD::Constant: Res = PromoteIntRes_Constant(N); break; 60 case ISD::CONVERT_RNDSAT: 61 Res = PromoteIntRes_CONVERT_RNDSAT(N); break; 62 case ISD::CTLZ_ZERO_UNDEF: 63 case ISD::CTLZ: Res = PromoteIntRes_CTLZ(N); break; 64 case ISD::CTPOP: Res = PromoteIntRes_CTPOP(N); break; 65 case ISD::CTTZ_ZERO_UNDEF: 66 case ISD::CTTZ: Res = PromoteIntRes_CTTZ(N); break; 67 case ISD::EXTRACT_VECTOR_ELT: 68 Res = PromoteIntRes_EXTRACT_VECTOR_ELT(N); break; 69 case ISD::LOAD: Res = PromoteIntRes_LOAD(cast<LoadSDNode>(N)); break; 70 case ISD::MLOAD: Res = PromoteIntRes_MLOAD(cast<MaskedLoadSDNode>(N)); 71 break; 72 case ISD::MGATHER: Res = PromoteIntRes_MGATHER(cast<MaskedGatherSDNode>(N)); 73 break; 74 case ISD::SELECT: Res = PromoteIntRes_SELECT(N); break; 75 case ISD::VSELECT: Res = PromoteIntRes_VSELECT(N); break; 76 case ISD::SELECT_CC: Res = PromoteIntRes_SELECT_CC(N); break; 77 case ISD::SETCC: Res = PromoteIntRes_SETCC(N); break; 78 case ISD::SMIN: 79 case ISD::SMAX: Res = PromoteIntRes_SExtIntBinOp(N); break; 80 case ISD::UMIN: 81 case ISD::UMAX: Res = PromoteIntRes_ZExtIntBinOp(N); break; 82 83 case ISD::SHL: Res = PromoteIntRes_SHL(N); break; 84 case ISD::SIGN_EXTEND_INREG: 85 Res = PromoteIntRes_SIGN_EXTEND_INREG(N); break; 86 case ISD::SRA: Res = PromoteIntRes_SRA(N); break; 87 case ISD::SRL: Res = PromoteIntRes_SRL(N); break; 88 case ISD::TRUNCATE: Res = PromoteIntRes_TRUNCATE(N); break; 89 case ISD::UNDEF: Res = PromoteIntRes_UNDEF(N); break; 90 case ISD::VAARG: Res = PromoteIntRes_VAARG(N); break; 91 92 case ISD::EXTRACT_SUBVECTOR: 93 Res = PromoteIntRes_EXTRACT_SUBVECTOR(N); break; 94 case ISD::VECTOR_SHUFFLE: 95 Res = PromoteIntRes_VECTOR_SHUFFLE(N); break; 96 case ISD::INSERT_VECTOR_ELT: 97 Res = PromoteIntRes_INSERT_VECTOR_ELT(N); break; 98 case ISD::BUILD_VECTOR: 99 Res = PromoteIntRes_BUILD_VECTOR(N); break; 100 case ISD::SCALAR_TO_VECTOR: 101 Res = PromoteIntRes_SCALAR_TO_VECTOR(N); break; 102 case ISD::CONCAT_VECTORS: 103 Res = PromoteIntRes_CONCAT_VECTORS(N); break; 104 105 case ISD::ANY_EXTEND_VECTOR_INREG: 106 case ISD::SIGN_EXTEND_VECTOR_INREG: 107 case ISD::ZERO_EXTEND_VECTOR_INREG: 108 Res = PromoteIntRes_EXTEND_VECTOR_INREG(N); break; 109 110 case ISD::SIGN_EXTEND: 111 case ISD::ZERO_EXTEND: 112 case ISD::ANY_EXTEND: Res = PromoteIntRes_INT_EXTEND(N); break; 113 114 case ISD::FP_TO_SINT: 115 case ISD::FP_TO_UINT: Res = PromoteIntRes_FP_TO_XINT(N); break; 116 117 case ISD::FP_TO_FP16: Res = PromoteIntRes_FP_TO_FP16(N); break; 118 119 case ISD::AND: 120 case ISD::OR: 121 case ISD::XOR: 122 case ISD::ADD: 123 case ISD::SUB: 124 case ISD::MUL: Res = PromoteIntRes_SimpleIntBinOp(N); break; 125 126 case ISD::SDIV: 127 case ISD::SREM: Res = PromoteIntRes_SExtIntBinOp(N); break; 128 129 case ISD::UDIV: 130 case ISD::UREM: Res = PromoteIntRes_ZExtIntBinOp(N); break; 131 132 case ISD::SADDO: 133 case ISD::SSUBO: Res = PromoteIntRes_SADDSUBO(N, ResNo); break; 134 case ISD::UADDO: 135 case ISD::USUBO: Res = PromoteIntRes_UADDSUBO(N, ResNo); break; 136 case ISD::SMULO: 137 case ISD::UMULO: Res = PromoteIntRes_XMULO(N, ResNo); break; 138 139 case ISD::ATOMIC_LOAD: 140 Res = PromoteIntRes_Atomic0(cast<AtomicSDNode>(N)); break; 141 142 case ISD::ATOMIC_LOAD_ADD: 143 case ISD::ATOMIC_LOAD_SUB: 144 case ISD::ATOMIC_LOAD_AND: 145 case ISD::ATOMIC_LOAD_OR: 146 case ISD::ATOMIC_LOAD_XOR: 147 case ISD::ATOMIC_LOAD_NAND: 148 case ISD::ATOMIC_LOAD_MIN: 149 case ISD::ATOMIC_LOAD_MAX: 150 case ISD::ATOMIC_LOAD_UMIN: 151 case ISD::ATOMIC_LOAD_UMAX: 152 case ISD::ATOMIC_SWAP: 153 Res = PromoteIntRes_Atomic1(cast<AtomicSDNode>(N)); break; 154 155 case ISD::ATOMIC_CMP_SWAP: 156 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS: 157 Res = PromoteIntRes_AtomicCmpSwap(cast<AtomicSDNode>(N), ResNo); 158 break; 159 } 160 161 // If the result is null then the sub-method took care of registering it. 162 if (Res.getNode()) 163 SetPromotedInteger(SDValue(N, ResNo), Res); 164 } 165 166 SDValue DAGTypeLegalizer::PromoteIntRes_MERGE_VALUES(SDNode *N, 167 unsigned ResNo) { 168 SDValue Op = DisintegrateMERGE_VALUES(N, ResNo); 169 return GetPromotedInteger(Op); 170 } 171 172 SDValue DAGTypeLegalizer::PromoteIntRes_AssertSext(SDNode *N) { 173 // Sign-extend the new bits, and continue the assertion. 174 SDValue Op = SExtPromotedInteger(N->getOperand(0)); 175 return DAG.getNode(ISD::AssertSext, SDLoc(N), 176 Op.getValueType(), Op, N->getOperand(1)); 177 } 178 179 SDValue DAGTypeLegalizer::PromoteIntRes_AssertZext(SDNode *N) { 180 // Zero the new bits, and continue the assertion. 181 SDValue Op = ZExtPromotedInteger(N->getOperand(0)); 182 return DAG.getNode(ISD::AssertZext, SDLoc(N), 183 Op.getValueType(), Op, N->getOperand(1)); 184 } 185 186 SDValue DAGTypeLegalizer::PromoteIntRes_Atomic0(AtomicSDNode *N) { 187 EVT ResVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 188 SDValue Res = DAG.getAtomic(N->getOpcode(), SDLoc(N), 189 N->getMemoryVT(), ResVT, 190 N->getChain(), N->getBasePtr(), 191 N->getMemOperand(), N->getOrdering(), 192 N->getSynchScope()); 193 // Legalize the chain result - switch anything that used the old chain to 194 // use the new one. 195 ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); 196 return Res; 197 } 198 199 SDValue DAGTypeLegalizer::PromoteIntRes_Atomic1(AtomicSDNode *N) { 200 SDValue Op2 = GetPromotedInteger(N->getOperand(2)); 201 SDValue Res = DAG.getAtomic(N->getOpcode(), SDLoc(N), 202 N->getMemoryVT(), 203 N->getChain(), N->getBasePtr(), 204 Op2, N->getMemOperand(), N->getOrdering(), 205 N->getSynchScope()); 206 // Legalize the chain result - switch anything that used the old chain to 207 // use the new one. 208 ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); 209 return Res; 210 } 211 212 SDValue DAGTypeLegalizer::PromoteIntRes_AtomicCmpSwap(AtomicSDNode *N, 213 unsigned ResNo) { 214 if (ResNo == 1) { 215 assert(N->getOpcode() == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS); 216 EVT SVT = getSetCCResultType(N->getOperand(2).getValueType()); 217 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(1)); 218 219 // Only use the result of getSetCCResultType if it is legal, 220 // otherwise just use the promoted result type (NVT). 221 if (!TLI.isTypeLegal(SVT)) 222 SVT = NVT; 223 224 SDVTList VTs = DAG.getVTList(N->getValueType(0), SVT, MVT::Other); 225 SDValue Res = DAG.getAtomicCmpSwap( 226 ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, SDLoc(N), N->getMemoryVT(), VTs, 227 N->getChain(), N->getBasePtr(), N->getOperand(2), N->getOperand(3), 228 N->getMemOperand(), N->getSuccessOrdering(), N->getFailureOrdering(), 229 N->getSynchScope()); 230 ReplaceValueWith(SDValue(N, 0), Res.getValue(0)); 231 ReplaceValueWith(SDValue(N, 2), Res.getValue(2)); 232 return Res.getValue(1); 233 } 234 235 SDValue Op2 = GetPromotedInteger(N->getOperand(2)); 236 SDValue Op3 = GetPromotedInteger(N->getOperand(3)); 237 SDVTList VTs = 238 DAG.getVTList(Op2.getValueType(), N->getValueType(1), MVT::Other); 239 SDValue Res = DAG.getAtomicCmpSwap( 240 N->getOpcode(), SDLoc(N), N->getMemoryVT(), VTs, N->getChain(), 241 N->getBasePtr(), Op2, Op3, N->getMemOperand(), N->getSuccessOrdering(), 242 N->getFailureOrdering(), N->getSynchScope()); 243 // Update the use to N with the newly created Res. 244 for (unsigned i = 1, NumResults = N->getNumValues(); i < NumResults; ++i) 245 ReplaceValueWith(SDValue(N, i), Res.getValue(i)); 246 return Res; 247 } 248 249 SDValue DAGTypeLegalizer::PromoteIntRes_BITCAST(SDNode *N) { 250 SDValue InOp = N->getOperand(0); 251 EVT InVT = InOp.getValueType(); 252 EVT NInVT = TLI.getTypeToTransformTo(*DAG.getContext(), InVT); 253 EVT OutVT = N->getValueType(0); 254 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT); 255 SDLoc dl(N); 256 257 switch (getTypeAction(InVT)) { 258 case TargetLowering::TypeLegal: 259 break; 260 case TargetLowering::TypePromoteInteger: 261 if (NOutVT.bitsEq(NInVT) && !NOutVT.isVector() && !NInVT.isVector()) 262 // The input promotes to the same size. Convert the promoted value. 263 return DAG.getNode(ISD::BITCAST, dl, NOutVT, GetPromotedInteger(InOp)); 264 break; 265 case TargetLowering::TypeSoftenFloat: 266 // Promote the integer operand by hand. 267 return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, GetSoftenedFloat(InOp)); 268 case TargetLowering::TypePromoteFloat: { 269 // Convert the promoted float by hand. 270 SDValue PromotedOp = GetPromotedFloat(InOp); 271 return DAG.getNode(ISD::FP_TO_FP16, dl, NOutVT, PromotedOp); 272 break; 273 } 274 case TargetLowering::TypeExpandInteger: 275 case TargetLowering::TypeExpandFloat: 276 break; 277 case TargetLowering::TypeScalarizeVector: 278 // Convert the element to an integer and promote it by hand. 279 if (!NOutVT.isVector()) 280 return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, 281 BitConvertToInteger(GetScalarizedVector(InOp))); 282 break; 283 case TargetLowering::TypeSplitVector: { 284 // For example, i32 = BITCAST v2i16 on alpha. Convert the split 285 // pieces of the input into integers and reassemble in the final type. 286 SDValue Lo, Hi; 287 GetSplitVector(N->getOperand(0), Lo, Hi); 288 Lo = BitConvertToInteger(Lo); 289 Hi = BitConvertToInteger(Hi); 290 291 if (DAG.getDataLayout().isBigEndian()) 292 std::swap(Lo, Hi); 293 294 InOp = DAG.getNode(ISD::ANY_EXTEND, dl, 295 EVT::getIntegerVT(*DAG.getContext(), 296 NOutVT.getSizeInBits()), 297 JoinIntegers(Lo, Hi)); 298 return DAG.getNode(ISD::BITCAST, dl, NOutVT, InOp); 299 } 300 case TargetLowering::TypeWidenVector: 301 // The input is widened to the same size. Convert to the widened value. 302 // Make sure that the outgoing value is not a vector, because this would 303 // make us bitcast between two vectors which are legalized in different ways. 304 if (NOutVT.bitsEq(NInVT) && !NOutVT.isVector()) 305 return DAG.getNode(ISD::BITCAST, dl, NOutVT, GetWidenedVector(InOp)); 306 } 307 308 return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, 309 CreateStackStoreLoad(InOp, OutVT)); 310 } 311 312 SDValue DAGTypeLegalizer::PromoteIntRes_BSWAP(SDNode *N) { 313 SDValue Op = GetPromotedInteger(N->getOperand(0)); 314 EVT OVT = N->getValueType(0); 315 EVT NVT = Op.getValueType(); 316 SDLoc dl(N); 317 318 unsigned DiffBits = NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits(); 319 return DAG.getNode( 320 ISD::SRL, dl, NVT, DAG.getNode(ISD::BSWAP, dl, NVT, Op), 321 DAG.getConstant(DiffBits, dl, 322 TLI.getShiftAmountTy(NVT, DAG.getDataLayout()))); 323 } 324 325 SDValue DAGTypeLegalizer::PromoteIntRes_BITREVERSE(SDNode *N) { 326 SDValue Op = GetPromotedInteger(N->getOperand(0)); 327 EVT OVT = N->getValueType(0); 328 EVT NVT = Op.getValueType(); 329 SDLoc dl(N); 330 331 unsigned DiffBits = NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits(); 332 return DAG.getNode( 333 ISD::SRL, dl, NVT, DAG.getNode(ISD::BITREVERSE, dl, NVT, Op), 334 DAG.getConstant(DiffBits, dl, 335 TLI.getShiftAmountTy(NVT, DAG.getDataLayout()))); 336 } 337 338 SDValue DAGTypeLegalizer::PromoteIntRes_BUILD_PAIR(SDNode *N) { 339 // The pair element type may be legal, or may not promote to the same type as 340 // the result, for example i14 = BUILD_PAIR (i7, i7). Handle all cases. 341 return DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), 342 TLI.getTypeToTransformTo(*DAG.getContext(), 343 N->getValueType(0)), JoinIntegers(N->getOperand(0), 344 N->getOperand(1))); 345 } 346 347 SDValue DAGTypeLegalizer::PromoteIntRes_Constant(SDNode *N) { 348 EVT VT = N->getValueType(0); 349 // FIXME there is no actual debug info here 350 SDLoc dl(N); 351 // Zero extend things like i1, sign extend everything else. It shouldn't 352 // matter in theory which one we pick, but this tends to give better code? 353 unsigned Opc = VT.isByteSized() ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND; 354 SDValue Result = DAG.getNode(Opc, dl, 355 TLI.getTypeToTransformTo(*DAG.getContext(), VT), 356 SDValue(N, 0)); 357 assert(isa<ConstantSDNode>(Result) && "Didn't constant fold ext?"); 358 return Result; 359 } 360 361 SDValue DAGTypeLegalizer::PromoteIntRes_CONVERT_RNDSAT(SDNode *N) { 362 ISD::CvtCode CvtCode = cast<CvtRndSatSDNode>(N)->getCvtCode(); 363 assert ((CvtCode == ISD::CVT_SS || CvtCode == ISD::CVT_SU || 364 CvtCode == ISD::CVT_US || CvtCode == ISD::CVT_UU || 365 CvtCode == ISD::CVT_SF || CvtCode == ISD::CVT_UF) && 366 "can only promote integers"); 367 EVT OutVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 368 return DAG.getConvertRndSat(OutVT, SDLoc(N), N->getOperand(0), 369 N->getOperand(1), N->getOperand(2), 370 N->getOperand(3), N->getOperand(4), CvtCode); 371 } 372 373 SDValue DAGTypeLegalizer::PromoteIntRes_CTLZ(SDNode *N) { 374 // Zero extend to the promoted type and do the count there. 375 SDValue Op = ZExtPromotedInteger(N->getOperand(0)); 376 SDLoc dl(N); 377 EVT OVT = N->getValueType(0); 378 EVT NVT = Op.getValueType(); 379 Op = DAG.getNode(N->getOpcode(), dl, NVT, Op); 380 // Subtract off the extra leading bits in the bigger type. 381 return DAG.getNode( 382 ISD::SUB, dl, NVT, Op, 383 DAG.getConstant(NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits(), dl, 384 NVT)); 385 } 386 387 SDValue DAGTypeLegalizer::PromoteIntRes_CTPOP(SDNode *N) { 388 // Zero extend to the promoted type and do the count there. 389 SDValue Op = ZExtPromotedInteger(N->getOperand(0)); 390 return DAG.getNode(ISD::CTPOP, SDLoc(N), Op.getValueType(), Op); 391 } 392 393 SDValue DAGTypeLegalizer::PromoteIntRes_CTTZ(SDNode *N) { 394 SDValue Op = GetPromotedInteger(N->getOperand(0)); 395 EVT OVT = N->getValueType(0); 396 EVT NVT = Op.getValueType(); 397 SDLoc dl(N); 398 if (N->getOpcode() == ISD::CTTZ) { 399 // The count is the same in the promoted type except if the original 400 // value was zero. This can be handled by setting the bit just off 401 // the top of the original type. 402 auto TopBit = APInt::getOneBitSet(NVT.getScalarSizeInBits(), 403 OVT.getScalarSizeInBits()); 404 Op = DAG.getNode(ISD::OR, dl, NVT, Op, DAG.getConstant(TopBit, dl, NVT)); 405 } 406 return DAG.getNode(N->getOpcode(), dl, NVT, Op); 407 } 408 409 SDValue DAGTypeLegalizer::PromoteIntRes_EXTRACT_VECTOR_ELT(SDNode *N) { 410 SDLoc dl(N); 411 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 412 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, NVT, N->getOperand(0), 413 N->getOperand(1)); 414 } 415 416 SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_XINT(SDNode *N) { 417 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 418 unsigned NewOpc = N->getOpcode(); 419 SDLoc dl(N); 420 421 // If we're promoting a UINT to a larger size and the larger FP_TO_UINT is 422 // not Legal, check to see if we can use FP_TO_SINT instead. (If both UINT 423 // and SINT conversions are Custom, there is no way to tell which is 424 // preferable. We choose SINT because that's the right thing on PPC.) 425 if (N->getOpcode() == ISD::FP_TO_UINT && 426 !TLI.isOperationLegal(ISD::FP_TO_UINT, NVT) && 427 TLI.isOperationLegalOrCustom(ISD::FP_TO_SINT, NVT)) 428 NewOpc = ISD::FP_TO_SINT; 429 430 SDValue Res = DAG.getNode(NewOpc, dl, NVT, N->getOperand(0)); 431 432 // Assert that the converted value fits in the original type. If it doesn't 433 // (eg: because the value being converted is too big), then the result of the 434 // original operation was undefined anyway, so the assert is still correct. 435 return DAG.getNode(N->getOpcode() == ISD::FP_TO_UINT ? 436 ISD::AssertZext : ISD::AssertSext, dl, NVT, Res, 437 DAG.getValueType(N->getValueType(0).getScalarType())); 438 } 439 440 SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_FP16(SDNode *N) { 441 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 442 SDLoc dl(N); 443 444 return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0)); 445 } 446 447 SDValue DAGTypeLegalizer::PromoteIntRes_INT_EXTEND(SDNode *N) { 448 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 449 SDLoc dl(N); 450 451 if (getTypeAction(N->getOperand(0).getValueType()) 452 == TargetLowering::TypePromoteInteger) { 453 SDValue Res = GetPromotedInteger(N->getOperand(0)); 454 assert(Res.getValueType().bitsLE(NVT) && "Extension doesn't make sense!"); 455 456 // If the result and operand types are the same after promotion, simplify 457 // to an in-register extension. 458 if (NVT == Res.getValueType()) { 459 // The high bits are not guaranteed to be anything. Insert an extend. 460 if (N->getOpcode() == ISD::SIGN_EXTEND) 461 return DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NVT, Res, 462 DAG.getValueType(N->getOperand(0).getValueType())); 463 if (N->getOpcode() == ISD::ZERO_EXTEND) 464 return DAG.getZeroExtendInReg(Res, dl, 465 N->getOperand(0).getValueType().getScalarType()); 466 assert(N->getOpcode() == ISD::ANY_EXTEND && "Unknown integer extension!"); 467 return Res; 468 } 469 } 470 471 // Otherwise, just extend the original operand all the way to the larger type. 472 return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0)); 473 } 474 475 SDValue DAGTypeLegalizer::PromoteIntRes_LOAD(LoadSDNode *N) { 476 assert(ISD::isUNINDEXEDLoad(N) && "Indexed load during type legalization!"); 477 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 478 ISD::LoadExtType ExtType = 479 ISD::isNON_EXTLoad(N) ? ISD::EXTLOAD : N->getExtensionType(); 480 SDLoc dl(N); 481 SDValue Res = DAG.getExtLoad(ExtType, dl, NVT, N->getChain(), N->getBasePtr(), 482 N->getMemoryVT(), N->getMemOperand()); 483 484 // Legalize the chain result - switch anything that used the old chain to 485 // use the new one. 486 ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); 487 return Res; 488 } 489 490 SDValue DAGTypeLegalizer::PromoteIntRes_MLOAD(MaskedLoadSDNode *N) { 491 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 492 SDValue ExtSrc0 = GetPromotedInteger(N->getSrc0()); 493 494 SDLoc dl(N); 495 SDValue Res = DAG.getMaskedLoad(NVT, dl, N->getChain(), N->getBasePtr(), 496 N->getMask(), ExtSrc0, N->getMemoryVT(), 497 N->getMemOperand(), ISD::SEXTLOAD); 498 // Legalize the chain result - switch anything that used the old chain to 499 // use the new one. 500 ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); 501 return Res; 502 } 503 504 SDValue DAGTypeLegalizer::PromoteIntRes_MGATHER(MaskedGatherSDNode *N) { 505 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 506 SDValue ExtSrc0 = GetPromotedInteger(N->getValue()); 507 assert(NVT == ExtSrc0.getValueType() && 508 "Gather result type and the passThru agrument type should be the same"); 509 510 SDLoc dl(N); 511 SDValue Ops[] = {N->getChain(), ExtSrc0, N->getMask(), N->getBasePtr(), 512 N->getIndex()}; 513 SDValue Res = DAG.getMaskedGather(DAG.getVTList(NVT, MVT::Other), 514 N->getMemoryVT(), dl, Ops, 515 N->getMemOperand()); 516 // Legalize the chain result - switch anything that used the old chain to 517 // use the new one. 518 ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); 519 return Res; 520 } 521 522 /// Promote the overflow flag of an overflowing arithmetic node. 523 SDValue DAGTypeLegalizer::PromoteIntRes_Overflow(SDNode *N) { 524 // Simply change the return type of the boolean result. 525 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(1)); 526 EVT ValueVTs[] = { N->getValueType(0), NVT }; 527 SDValue Ops[] = { N->getOperand(0), N->getOperand(1) }; 528 SDValue Res = DAG.getNode(N->getOpcode(), SDLoc(N), 529 DAG.getVTList(ValueVTs), Ops); 530 531 // Modified the sum result - switch anything that used the old sum to use 532 // the new one. 533 ReplaceValueWith(SDValue(N, 0), Res); 534 535 return SDValue(Res.getNode(), 1); 536 } 537 538 SDValue DAGTypeLegalizer::PromoteIntRes_SADDSUBO(SDNode *N, unsigned ResNo) { 539 if (ResNo == 1) 540 return PromoteIntRes_Overflow(N); 541 542 // The operation overflowed iff the result in the larger type is not the 543 // sign extension of its truncation to the original type. 544 SDValue LHS = SExtPromotedInteger(N->getOperand(0)); 545 SDValue RHS = SExtPromotedInteger(N->getOperand(1)); 546 EVT OVT = N->getOperand(0).getValueType(); 547 EVT NVT = LHS.getValueType(); 548 SDLoc dl(N); 549 550 // Do the arithmetic in the larger type. 551 unsigned Opcode = N->getOpcode() == ISD::SADDO ? ISD::ADD : ISD::SUB; 552 SDValue Res = DAG.getNode(Opcode, dl, NVT, LHS, RHS); 553 554 // Calculate the overflow flag: sign extend the arithmetic result from 555 // the original type. 556 SDValue Ofl = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NVT, Res, 557 DAG.getValueType(OVT)); 558 // Overflowed if and only if this is not equal to Res. 559 Ofl = DAG.getSetCC(dl, N->getValueType(1), Ofl, Res, ISD::SETNE); 560 561 // Use the calculated overflow everywhere. 562 ReplaceValueWith(SDValue(N, 1), Ofl); 563 564 return Res; 565 } 566 567 SDValue DAGTypeLegalizer::PromoteIntRes_SELECT(SDNode *N) { 568 SDValue LHS = GetPromotedInteger(N->getOperand(1)); 569 SDValue RHS = GetPromotedInteger(N->getOperand(2)); 570 return DAG.getSelect(SDLoc(N), 571 LHS.getValueType(), N->getOperand(0), LHS, RHS); 572 } 573 574 SDValue DAGTypeLegalizer::PromoteIntRes_VSELECT(SDNode *N) { 575 SDValue Mask = N->getOperand(0); 576 EVT OpTy = N->getOperand(1).getValueType(); 577 578 // Promote all the way up to the canonical SetCC type. 579 Mask = PromoteTargetBoolean(Mask, OpTy); 580 SDValue LHS = GetPromotedInteger(N->getOperand(1)); 581 SDValue RHS = GetPromotedInteger(N->getOperand(2)); 582 return DAG.getNode(ISD::VSELECT, SDLoc(N), 583 LHS.getValueType(), Mask, LHS, RHS); 584 } 585 586 SDValue DAGTypeLegalizer::PromoteIntRes_SELECT_CC(SDNode *N) { 587 SDValue LHS = GetPromotedInteger(N->getOperand(2)); 588 SDValue RHS = GetPromotedInteger(N->getOperand(3)); 589 return DAG.getNode(ISD::SELECT_CC, SDLoc(N), 590 LHS.getValueType(), N->getOperand(0), 591 N->getOperand(1), LHS, RHS, N->getOperand(4)); 592 } 593 594 SDValue DAGTypeLegalizer::PromoteIntRes_SETCC(SDNode *N) { 595 EVT SVT = getSetCCResultType(N->getOperand(0).getValueType()); 596 597 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 598 599 // Only use the result of getSetCCResultType if it is legal, 600 // otherwise just use the promoted result type (NVT). 601 if (!TLI.isTypeLegal(SVT)) 602 SVT = NVT; 603 604 SDLoc dl(N); 605 assert(SVT.isVector() == N->getOperand(0).getValueType().isVector() && 606 "Vector compare must return a vector result!"); 607 608 SDValue LHS = N->getOperand(0); 609 SDValue RHS = N->getOperand(1); 610 if (LHS.getValueType() != RHS.getValueType()) { 611 if (getTypeAction(LHS.getValueType()) == TargetLowering::TypePromoteInteger && 612 !LHS.getValueType().isVector()) 613 LHS = GetPromotedInteger(LHS); 614 if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger && 615 !RHS.getValueType().isVector()) 616 RHS = GetPromotedInteger(RHS); 617 } 618 619 // Get the SETCC result using the canonical SETCC type. 620 SDValue SetCC = DAG.getNode(N->getOpcode(), dl, SVT, LHS, RHS, 621 N->getOperand(2)); 622 623 assert(NVT.bitsLE(SVT) && "Integer type overpromoted?"); 624 // Convert to the expected type. 625 return DAG.getNode(ISD::TRUNCATE, dl, NVT, SetCC); 626 } 627 628 SDValue DAGTypeLegalizer::PromoteIntRes_SHL(SDNode *N) { 629 SDValue LHS = N->getOperand(0); 630 SDValue RHS = N->getOperand(1); 631 if (getTypeAction(LHS.getValueType()) == TargetLowering::TypePromoteInteger) 632 LHS = GetPromotedInteger(LHS); 633 if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger) 634 RHS = ZExtPromotedInteger(RHS); 635 return DAG.getNode(ISD::SHL, SDLoc(N), LHS.getValueType(), LHS, RHS); 636 } 637 638 SDValue DAGTypeLegalizer::PromoteIntRes_SIGN_EXTEND_INREG(SDNode *N) { 639 SDValue Op = GetPromotedInteger(N->getOperand(0)); 640 return DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N), 641 Op.getValueType(), Op, N->getOperand(1)); 642 } 643 644 SDValue DAGTypeLegalizer::PromoteIntRes_SimpleIntBinOp(SDNode *N) { 645 // The input may have strange things in the top bits of the registers, but 646 // these operations don't care. They may have weird bits going out, but 647 // that too is okay if they are integer operations. 648 SDValue LHS = GetPromotedInteger(N->getOperand(0)); 649 SDValue RHS = GetPromotedInteger(N->getOperand(1)); 650 return DAG.getNode(N->getOpcode(), SDLoc(N), 651 LHS.getValueType(), LHS, RHS); 652 } 653 654 SDValue DAGTypeLegalizer::PromoteIntRes_SExtIntBinOp(SDNode *N) { 655 // Sign extend the input. 656 SDValue LHS = SExtPromotedInteger(N->getOperand(0)); 657 SDValue RHS = SExtPromotedInteger(N->getOperand(1)); 658 return DAG.getNode(N->getOpcode(), SDLoc(N), 659 LHS.getValueType(), LHS, RHS); 660 } 661 662 SDValue DAGTypeLegalizer::PromoteIntRes_ZExtIntBinOp(SDNode *N) { 663 // Zero extend the input. 664 SDValue LHS = ZExtPromotedInteger(N->getOperand(0)); 665 SDValue RHS = ZExtPromotedInteger(N->getOperand(1)); 666 return DAG.getNode(N->getOpcode(), SDLoc(N), 667 LHS.getValueType(), LHS, RHS); 668 } 669 670 SDValue DAGTypeLegalizer::PromoteIntRes_SRA(SDNode *N) { 671 SDValue LHS = N->getOperand(0); 672 SDValue RHS = N->getOperand(1); 673 // The input value must be properly sign extended. 674 if (getTypeAction(LHS.getValueType()) == TargetLowering::TypePromoteInteger) 675 LHS = SExtPromotedInteger(LHS); 676 if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger) 677 RHS = ZExtPromotedInteger(RHS); 678 return DAG.getNode(ISD::SRA, SDLoc(N), LHS.getValueType(), LHS, RHS); 679 } 680 681 SDValue DAGTypeLegalizer::PromoteIntRes_SRL(SDNode *N) { 682 SDValue LHS = N->getOperand(0); 683 SDValue RHS = N->getOperand(1); 684 // The input value must be properly zero extended. 685 if (getTypeAction(LHS.getValueType()) == TargetLowering::TypePromoteInteger) 686 LHS = ZExtPromotedInteger(LHS); 687 if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger) 688 RHS = ZExtPromotedInteger(RHS); 689 return DAG.getNode(ISD::SRL, SDLoc(N), LHS.getValueType(), LHS, RHS); 690 } 691 692 SDValue DAGTypeLegalizer::PromoteIntRes_TRUNCATE(SDNode *N) { 693 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 694 SDValue Res; 695 SDValue InOp = N->getOperand(0); 696 SDLoc dl(N); 697 698 switch (getTypeAction(InOp.getValueType())) { 699 default: llvm_unreachable("Unknown type action!"); 700 case TargetLowering::TypeLegal: 701 case TargetLowering::TypeExpandInteger: 702 Res = InOp; 703 break; 704 case TargetLowering::TypePromoteInteger: 705 Res = GetPromotedInteger(InOp); 706 break; 707 case TargetLowering::TypeSplitVector: 708 EVT InVT = InOp.getValueType(); 709 assert(InVT.isVector() && "Cannot split scalar types"); 710 unsigned NumElts = InVT.getVectorNumElements(); 711 assert(NumElts == NVT.getVectorNumElements() && 712 "Dst and Src must have the same number of elements"); 713 assert(isPowerOf2_32(NumElts) && 714 "Promoted vector type must be a power of two"); 715 716 SDValue EOp1, EOp2; 717 GetSplitVector(InOp, EOp1, EOp2); 718 719 EVT HalfNVT = EVT::getVectorVT(*DAG.getContext(), NVT.getScalarType(), 720 NumElts/2); 721 EOp1 = DAG.getNode(ISD::TRUNCATE, dl, HalfNVT, EOp1); 722 EOp2 = DAG.getNode(ISD::TRUNCATE, dl, HalfNVT, EOp2); 723 724 return DAG.getNode(ISD::CONCAT_VECTORS, dl, NVT, EOp1, EOp2); 725 } 726 727 // Truncate to NVT instead of VT 728 return DAG.getNode(ISD::TRUNCATE, dl, NVT, Res); 729 } 730 731 SDValue DAGTypeLegalizer::PromoteIntRes_UADDSUBO(SDNode *N, unsigned ResNo) { 732 if (ResNo == 1) 733 return PromoteIntRes_Overflow(N); 734 735 // The operation overflowed iff the result in the larger type is not the 736 // zero extension of its truncation to the original type. 737 SDValue LHS = ZExtPromotedInteger(N->getOperand(0)); 738 SDValue RHS = ZExtPromotedInteger(N->getOperand(1)); 739 EVT OVT = N->getOperand(0).getValueType(); 740 EVT NVT = LHS.getValueType(); 741 SDLoc dl(N); 742 743 // Do the arithmetic in the larger type. 744 unsigned Opcode = N->getOpcode() == ISD::UADDO ? ISD::ADD : ISD::SUB; 745 SDValue Res = DAG.getNode(Opcode, dl, NVT, LHS, RHS); 746 747 // Calculate the overflow flag: zero extend the arithmetic result from 748 // the original type. 749 SDValue Ofl = DAG.getZeroExtendInReg(Res, dl, OVT); 750 // Overflowed if and only if this is not equal to Res. 751 Ofl = DAG.getSetCC(dl, N->getValueType(1), Ofl, Res, ISD::SETNE); 752 753 // Use the calculated overflow everywhere. 754 ReplaceValueWith(SDValue(N, 1), Ofl); 755 756 return Res; 757 } 758 759 SDValue DAGTypeLegalizer::PromoteIntRes_XMULO(SDNode *N, unsigned ResNo) { 760 // Promote the overflow bit trivially. 761 if (ResNo == 1) 762 return PromoteIntRes_Overflow(N); 763 764 SDValue LHS = N->getOperand(0), RHS = N->getOperand(1); 765 SDLoc DL(N); 766 EVT SmallVT = LHS.getValueType(); 767 768 // To determine if the result overflowed in a larger type, we extend the 769 // input to the larger type, do the multiply (checking if it overflows), 770 // then also check the high bits of the result to see if overflow happened 771 // there. 772 if (N->getOpcode() == ISD::SMULO) { 773 LHS = SExtPromotedInteger(LHS); 774 RHS = SExtPromotedInteger(RHS); 775 } else { 776 LHS = ZExtPromotedInteger(LHS); 777 RHS = ZExtPromotedInteger(RHS); 778 } 779 SDVTList VTs = DAG.getVTList(LHS.getValueType(), N->getValueType(1)); 780 SDValue Mul = DAG.getNode(N->getOpcode(), DL, VTs, LHS, RHS); 781 782 // Overflow occurred if it occurred in the larger type, or if the high part 783 // of the result does not zero/sign-extend the low part. Check this second 784 // possibility first. 785 SDValue Overflow; 786 if (N->getOpcode() == ISD::UMULO) { 787 // Unsigned overflow occurred if the high part is non-zero. 788 SDValue Hi = DAG.getNode(ISD::SRL, DL, Mul.getValueType(), Mul, 789 DAG.getIntPtrConstant(SmallVT.getSizeInBits(), 790 DL)); 791 Overflow = DAG.getSetCC(DL, N->getValueType(1), Hi, 792 DAG.getConstant(0, DL, Hi.getValueType()), 793 ISD::SETNE); 794 } else { 795 // Signed overflow occurred if the high part does not sign extend the low. 796 SDValue SExt = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, Mul.getValueType(), 797 Mul, DAG.getValueType(SmallVT)); 798 Overflow = DAG.getSetCC(DL, N->getValueType(1), SExt, Mul, ISD::SETNE); 799 } 800 801 // The only other way for overflow to occur is if the multiplication in the 802 // larger type itself overflowed. 803 Overflow = DAG.getNode(ISD::OR, DL, N->getValueType(1), Overflow, 804 SDValue(Mul.getNode(), 1)); 805 806 // Use the calculated overflow everywhere. 807 ReplaceValueWith(SDValue(N, 1), Overflow); 808 return Mul; 809 } 810 811 SDValue DAGTypeLegalizer::PromoteIntRes_UNDEF(SDNode *N) { 812 return DAG.getUNDEF(TLI.getTypeToTransformTo(*DAG.getContext(), 813 N->getValueType(0))); 814 } 815 816 SDValue DAGTypeLegalizer::PromoteIntRes_VAARG(SDNode *N) { 817 SDValue Chain = N->getOperand(0); // Get the chain. 818 SDValue Ptr = N->getOperand(1); // Get the pointer. 819 EVT VT = N->getValueType(0); 820 SDLoc dl(N); 821 822 MVT RegVT = TLI.getRegisterType(*DAG.getContext(), VT); 823 unsigned NumRegs = TLI.getNumRegisters(*DAG.getContext(), VT); 824 // The argument is passed as NumRegs registers of type RegVT. 825 826 SmallVector<SDValue, 8> Parts(NumRegs); 827 for (unsigned i = 0; i < NumRegs; ++i) { 828 Parts[i] = DAG.getVAArg(RegVT, dl, Chain, Ptr, N->getOperand(2), 829 N->getConstantOperandVal(3)); 830 Chain = Parts[i].getValue(1); 831 } 832 833 // Handle endianness of the load. 834 if (DAG.getDataLayout().isBigEndian()) 835 std::reverse(Parts.begin(), Parts.end()); 836 837 // Assemble the parts in the promoted type. 838 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 839 SDValue Res = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Parts[0]); 840 for (unsigned i = 1; i < NumRegs; ++i) { 841 SDValue Part = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Parts[i]); 842 // Shift it to the right position and "or" it in. 843 Part = DAG.getNode(ISD::SHL, dl, NVT, Part, 844 DAG.getConstant(i * RegVT.getSizeInBits(), dl, 845 TLI.getPointerTy(DAG.getDataLayout()))); 846 Res = DAG.getNode(ISD::OR, dl, NVT, Res, Part); 847 } 848 849 // Modified the chain result - switch anything that used the old chain to 850 // use the new one. 851 ReplaceValueWith(SDValue(N, 1), Chain); 852 853 return Res; 854 } 855 856 //===----------------------------------------------------------------------===// 857 // Integer Operand Promotion 858 //===----------------------------------------------------------------------===// 859 860 /// PromoteIntegerOperand - This method is called when the specified operand of 861 /// the specified node is found to need promotion. At this point, all of the 862 /// result types of the node are known to be legal, but other operands of the 863 /// node may need promotion or expansion as well as the specified one. 864 bool DAGTypeLegalizer::PromoteIntegerOperand(SDNode *N, unsigned OpNo) { 865 DEBUG(dbgs() << "Promote integer operand: "; N->dump(&DAG); dbgs() << "\n"); 866 SDValue Res = SDValue(); 867 868 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false)) 869 return false; 870 871 switch (N->getOpcode()) { 872 default: 873 #ifndef NDEBUG 874 dbgs() << "PromoteIntegerOperand Op #" << OpNo << ": "; 875 N->dump(&DAG); dbgs() << "\n"; 876 #endif 877 llvm_unreachable("Do not know how to promote this operator's operand!"); 878 879 case ISD::ANY_EXTEND: Res = PromoteIntOp_ANY_EXTEND(N); break; 880 case ISD::ATOMIC_STORE: 881 Res = PromoteIntOp_ATOMIC_STORE(cast<AtomicSDNode>(N)); 882 break; 883 case ISD::BITCAST: Res = PromoteIntOp_BITCAST(N); break; 884 case ISD::BR_CC: Res = PromoteIntOp_BR_CC(N, OpNo); break; 885 case ISD::BRCOND: Res = PromoteIntOp_BRCOND(N, OpNo); break; 886 case ISD::BUILD_PAIR: Res = PromoteIntOp_BUILD_PAIR(N); break; 887 case ISD::BUILD_VECTOR: Res = PromoteIntOp_BUILD_VECTOR(N); break; 888 case ISD::CONCAT_VECTORS: Res = PromoteIntOp_CONCAT_VECTORS(N); break; 889 case ISD::EXTRACT_VECTOR_ELT: Res = PromoteIntOp_EXTRACT_VECTOR_ELT(N); break; 890 case ISD::CONVERT_RNDSAT: 891 Res = PromoteIntOp_CONVERT_RNDSAT(N); break; 892 case ISD::INSERT_VECTOR_ELT: 893 Res = PromoteIntOp_INSERT_VECTOR_ELT(N, OpNo);break; 894 case ISD::SCALAR_TO_VECTOR: 895 Res = PromoteIntOp_SCALAR_TO_VECTOR(N); break; 896 case ISD::VSELECT: 897 case ISD::SELECT: Res = PromoteIntOp_SELECT(N, OpNo); break; 898 case ISD::SELECT_CC: Res = PromoteIntOp_SELECT_CC(N, OpNo); break; 899 case ISD::SETCC: Res = PromoteIntOp_SETCC(N, OpNo); break; 900 case ISD::SIGN_EXTEND: Res = PromoteIntOp_SIGN_EXTEND(N); break; 901 case ISD::SINT_TO_FP: Res = PromoteIntOp_SINT_TO_FP(N); break; 902 case ISD::STORE: Res = PromoteIntOp_STORE(cast<StoreSDNode>(N), 903 OpNo); break; 904 case ISD::MSTORE: Res = PromoteIntOp_MSTORE(cast<MaskedStoreSDNode>(N), 905 OpNo); break; 906 case ISD::MLOAD: Res = PromoteIntOp_MLOAD(cast<MaskedLoadSDNode>(N), 907 OpNo); break; 908 case ISD::MGATHER: Res = PromoteIntOp_MGATHER(cast<MaskedGatherSDNode>(N), 909 OpNo); break; 910 case ISD::MSCATTER: Res = PromoteIntOp_MSCATTER(cast<MaskedScatterSDNode>(N), 911 OpNo); break; 912 case ISD::TRUNCATE: Res = PromoteIntOp_TRUNCATE(N); break; 913 case ISD::FP16_TO_FP: 914 case ISD::UINT_TO_FP: Res = PromoteIntOp_UINT_TO_FP(N); break; 915 case ISD::ZERO_EXTEND: Res = PromoteIntOp_ZERO_EXTEND(N); break; 916 case ISD::EXTRACT_SUBVECTOR: Res = PromoteIntOp_EXTRACT_SUBVECTOR(N); break; 917 918 case ISD::SHL: 919 case ISD::SRA: 920 case ISD::SRL: 921 case ISD::ROTL: 922 case ISD::ROTR: Res = PromoteIntOp_Shift(N); break; 923 } 924 925 // If the result is null, the sub-method took care of registering results etc. 926 if (!Res.getNode()) return false; 927 928 // If the result is N, the sub-method updated N in place. Tell the legalizer 929 // core about this. 930 if (Res.getNode() == N) 931 return true; 932 933 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 && 934 "Invalid operand expansion"); 935 936 ReplaceValueWith(SDValue(N, 0), Res); 937 return false; 938 } 939 940 /// PromoteSetCCOperands - Promote the operands of a comparison. This code is 941 /// shared among BR_CC, SELECT_CC, and SETCC handlers. 942 void DAGTypeLegalizer::PromoteSetCCOperands(SDValue &NewLHS,SDValue &NewRHS, 943 ISD::CondCode CCCode) { 944 // We have to insert explicit sign or zero extends. Note that we could 945 // insert sign extends for ALL conditions, but zero extend is cheaper on 946 // many machines (an AND instead of two shifts), so prefer it. 947 switch (CCCode) { 948 default: llvm_unreachable("Unknown integer comparison!"); 949 case ISD::SETEQ: 950 case ISD::SETNE: { 951 SDValue OpL = GetPromotedInteger(NewLHS); 952 SDValue OpR = GetPromotedInteger(NewRHS); 953 954 // We would prefer to promote the comparison operand with sign extension, 955 // if we find the operand is actually to truncate an AssertSext. With this 956 // optimization, we can avoid inserting real truncate instruction, which 957 // is redudant eventually. 958 if (OpL->getOpcode() == ISD::AssertSext && 959 cast<VTSDNode>(OpL->getOperand(1))->getVT() == NewLHS.getValueType() && 960 OpR->getOpcode() == ISD::AssertSext && 961 cast<VTSDNode>(OpR->getOperand(1))->getVT() == NewRHS.getValueType()) { 962 NewLHS = OpL; 963 NewRHS = OpR; 964 } else { 965 NewLHS = ZExtPromotedInteger(NewLHS); 966 NewRHS = ZExtPromotedInteger(NewRHS); 967 } 968 break; 969 } 970 case ISD::SETUGE: 971 case ISD::SETUGT: 972 case ISD::SETULE: 973 case ISD::SETULT: 974 // ALL of these operations will work if we either sign or zero extend 975 // the operands (including the unsigned comparisons!). Zero extend is 976 // usually a simpler/cheaper operation, so prefer it. 977 NewLHS = ZExtPromotedInteger(NewLHS); 978 NewRHS = ZExtPromotedInteger(NewRHS); 979 break; 980 case ISD::SETGE: 981 case ISD::SETGT: 982 case ISD::SETLT: 983 case ISD::SETLE: 984 NewLHS = SExtPromotedInteger(NewLHS); 985 NewRHS = SExtPromotedInteger(NewRHS); 986 break; 987 } 988 } 989 990 SDValue DAGTypeLegalizer::PromoteIntOp_ANY_EXTEND(SDNode *N) { 991 SDValue Op = GetPromotedInteger(N->getOperand(0)); 992 return DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), N->getValueType(0), Op); 993 } 994 995 SDValue DAGTypeLegalizer::PromoteIntOp_ATOMIC_STORE(AtomicSDNode *N) { 996 SDValue Op2 = GetPromotedInteger(N->getOperand(2)); 997 return DAG.getAtomic(N->getOpcode(), SDLoc(N), N->getMemoryVT(), 998 N->getChain(), N->getBasePtr(), Op2, N->getMemOperand(), 999 N->getOrdering(), N->getSynchScope()); 1000 } 1001 1002 SDValue DAGTypeLegalizer::PromoteIntOp_BITCAST(SDNode *N) { 1003 // This should only occur in unusual situations like bitcasting to an 1004 // x86_fp80, so just turn it into a store+load 1005 return CreateStackStoreLoad(N->getOperand(0), N->getValueType(0)); 1006 } 1007 1008 SDValue DAGTypeLegalizer::PromoteIntOp_BR_CC(SDNode *N, unsigned OpNo) { 1009 assert(OpNo == 2 && "Don't know how to promote this operand!"); 1010 1011 SDValue LHS = N->getOperand(2); 1012 SDValue RHS = N->getOperand(3); 1013 PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(1))->get()); 1014 1015 // The chain (Op#0), CC (#1) and basic block destination (Op#4) are always 1016 // legal types. 1017 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), 1018 N->getOperand(1), LHS, RHS, N->getOperand(4)), 1019 0); 1020 } 1021 1022 SDValue DAGTypeLegalizer::PromoteIntOp_BRCOND(SDNode *N, unsigned OpNo) { 1023 assert(OpNo == 1 && "only know how to promote condition"); 1024 1025 // Promote all the way up to the canonical SetCC type. 1026 SDValue Cond = PromoteTargetBoolean(N->getOperand(1), MVT::Other); 1027 1028 // The chain (Op#0) and basic block destination (Op#2) are always legal types. 1029 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), Cond, 1030 N->getOperand(2)), 0); 1031 } 1032 1033 SDValue DAGTypeLegalizer::PromoteIntOp_BUILD_PAIR(SDNode *N) { 1034 // Since the result type is legal, the operands must promote to it. 1035 EVT OVT = N->getOperand(0).getValueType(); 1036 SDValue Lo = ZExtPromotedInteger(N->getOperand(0)); 1037 SDValue Hi = GetPromotedInteger(N->getOperand(1)); 1038 assert(Lo.getValueType() == N->getValueType(0) && "Operand over promoted?"); 1039 SDLoc dl(N); 1040 1041 Hi = DAG.getNode(ISD::SHL, dl, N->getValueType(0), Hi, 1042 DAG.getConstant(OVT.getSizeInBits(), dl, 1043 TLI.getPointerTy(DAG.getDataLayout()))); 1044 return DAG.getNode(ISD::OR, dl, N->getValueType(0), Lo, Hi); 1045 } 1046 1047 SDValue DAGTypeLegalizer::PromoteIntOp_BUILD_VECTOR(SDNode *N) { 1048 // The vector type is legal but the element type is not. This implies 1049 // that the vector is a power-of-two in length and that the element 1050 // type does not have a strange size (eg: it is not i1). 1051 EVT VecVT = N->getValueType(0); 1052 unsigned NumElts = VecVT.getVectorNumElements(); 1053 assert(!((NumElts & 1) && (!TLI.isTypeLegal(VecVT))) && 1054 "Legal vector of one illegal element?"); 1055 1056 // Promote the inserted value. The type does not need to match the 1057 // vector element type. Check that any extra bits introduced will be 1058 // truncated away. 1059 assert(N->getOperand(0).getValueType().getSizeInBits() >= 1060 N->getValueType(0).getVectorElementType().getSizeInBits() && 1061 "Type of inserted value narrower than vector element type!"); 1062 1063 SmallVector<SDValue, 16> NewOps; 1064 for (unsigned i = 0; i < NumElts; ++i) 1065 NewOps.push_back(GetPromotedInteger(N->getOperand(i))); 1066 1067 return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0); 1068 } 1069 1070 SDValue DAGTypeLegalizer::PromoteIntOp_CONVERT_RNDSAT(SDNode *N) { 1071 ISD::CvtCode CvtCode = cast<CvtRndSatSDNode>(N)->getCvtCode(); 1072 assert ((CvtCode == ISD::CVT_SS || CvtCode == ISD::CVT_SU || 1073 CvtCode == ISD::CVT_US || CvtCode == ISD::CVT_UU || 1074 CvtCode == ISD::CVT_FS || CvtCode == ISD::CVT_FU) && 1075 "can only promote integer arguments"); 1076 SDValue InOp = GetPromotedInteger(N->getOperand(0)); 1077 return DAG.getConvertRndSat(N->getValueType(0), SDLoc(N), InOp, 1078 N->getOperand(1), N->getOperand(2), 1079 N->getOperand(3), N->getOperand(4), CvtCode); 1080 } 1081 1082 SDValue DAGTypeLegalizer::PromoteIntOp_INSERT_VECTOR_ELT(SDNode *N, 1083 unsigned OpNo) { 1084 if (OpNo == 1) { 1085 // Promote the inserted value. This is valid because the type does not 1086 // have to match the vector element type. 1087 1088 // Check that any extra bits introduced will be truncated away. 1089 assert(N->getOperand(1).getValueType().getSizeInBits() >= 1090 N->getValueType(0).getVectorElementType().getSizeInBits() && 1091 "Type of inserted value narrower than vector element type!"); 1092 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), 1093 GetPromotedInteger(N->getOperand(1)), 1094 N->getOperand(2)), 1095 0); 1096 } 1097 1098 assert(OpNo == 2 && "Different operand and result vector types?"); 1099 1100 // Promote the index. 1101 SDValue Idx = DAG.getZExtOrTrunc(N->getOperand(2), SDLoc(N), 1102 TLI.getVectorIdxTy(DAG.getDataLayout())); 1103 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), 1104 N->getOperand(1), Idx), 0); 1105 } 1106 1107 SDValue DAGTypeLegalizer::PromoteIntOp_SCALAR_TO_VECTOR(SDNode *N) { 1108 // Integer SCALAR_TO_VECTOR operands are implicitly truncated, so just promote 1109 // the operand in place. 1110 return SDValue(DAG.UpdateNodeOperands(N, 1111 GetPromotedInteger(N->getOperand(0))), 0); 1112 } 1113 1114 SDValue DAGTypeLegalizer::PromoteIntOp_SELECT(SDNode *N, unsigned OpNo) { 1115 assert(OpNo == 0 && "Only know how to promote the condition!"); 1116 SDValue Cond = N->getOperand(0); 1117 EVT OpTy = N->getOperand(1).getValueType(); 1118 1119 // Promote all the way up to the canonical SetCC type. 1120 EVT OpVT = N->getOpcode() == ISD::SELECT ? OpTy.getScalarType() : OpTy; 1121 Cond = PromoteTargetBoolean(Cond, OpVT); 1122 1123 return SDValue(DAG.UpdateNodeOperands(N, Cond, N->getOperand(1), 1124 N->getOperand(2)), 0); 1125 } 1126 1127 SDValue DAGTypeLegalizer::PromoteIntOp_SELECT_CC(SDNode *N, unsigned OpNo) { 1128 assert(OpNo == 0 && "Don't know how to promote this operand!"); 1129 1130 SDValue LHS = N->getOperand(0); 1131 SDValue RHS = N->getOperand(1); 1132 PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(4))->get()); 1133 1134 // The CC (#4) and the possible return values (#2 and #3) have legal types. 1135 return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, N->getOperand(2), 1136 N->getOperand(3), N->getOperand(4)), 0); 1137 } 1138 1139 SDValue DAGTypeLegalizer::PromoteIntOp_SETCC(SDNode *N, unsigned OpNo) { 1140 assert(OpNo == 0 && "Don't know how to promote this operand!"); 1141 1142 SDValue LHS = N->getOperand(0); 1143 SDValue RHS = N->getOperand(1); 1144 PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(2))->get()); 1145 1146 // The CC (#2) is always legal. 1147 return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, N->getOperand(2)), 0); 1148 } 1149 1150 SDValue DAGTypeLegalizer::PromoteIntOp_Shift(SDNode *N) { 1151 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), 1152 ZExtPromotedInteger(N->getOperand(1))), 0); 1153 } 1154 1155 SDValue DAGTypeLegalizer::PromoteIntOp_SIGN_EXTEND(SDNode *N) { 1156 SDValue Op = GetPromotedInteger(N->getOperand(0)); 1157 SDLoc dl(N); 1158 Op = DAG.getNode(ISD::ANY_EXTEND, dl, N->getValueType(0), Op); 1159 return DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Op.getValueType(), 1160 Op, DAG.getValueType(N->getOperand(0).getValueType())); 1161 } 1162 1163 SDValue DAGTypeLegalizer::PromoteIntOp_SINT_TO_FP(SDNode *N) { 1164 return SDValue(DAG.UpdateNodeOperands(N, 1165 SExtPromotedInteger(N->getOperand(0))), 0); 1166 } 1167 1168 SDValue DAGTypeLegalizer::PromoteIntOp_STORE(StoreSDNode *N, unsigned OpNo){ 1169 assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!"); 1170 SDValue Ch = N->getChain(), Ptr = N->getBasePtr(); 1171 SDLoc dl(N); 1172 1173 SDValue Val = GetPromotedInteger(N->getValue()); // Get promoted value. 1174 1175 // Truncate the value and store the result. 1176 return DAG.getTruncStore(Ch, dl, Val, Ptr, 1177 N->getMemoryVT(), N->getMemOperand()); 1178 } 1179 1180 SDValue DAGTypeLegalizer::PromoteIntOp_MSTORE(MaskedStoreSDNode *N, 1181 unsigned OpNo) { 1182 1183 SDValue DataOp = N->getValue(); 1184 EVT DataVT = DataOp.getValueType(); 1185 SDValue Mask = N->getMask(); 1186 SDLoc dl(N); 1187 1188 bool TruncateStore = false; 1189 if (OpNo == 2) { 1190 // Mask comes before the data operand. If the data operand is legal, we just 1191 // promote the mask. 1192 // When the data operand has illegal type, we should legalize the data 1193 // operand first. The mask will be promoted/splitted/widened according to 1194 // the data operand type. 1195 if (TLI.isTypeLegal(DataVT)) 1196 Mask = PromoteTargetBoolean(Mask, DataVT); 1197 else { 1198 if (getTypeAction(DataVT) == TargetLowering::TypePromoteInteger) 1199 return PromoteIntOp_MSTORE(N, 3); 1200 1201 else if (getTypeAction(DataVT) == TargetLowering::TypeWidenVector) 1202 return WidenVecOp_MSTORE(N, 3); 1203 1204 else { 1205 assert (getTypeAction(DataVT) == TargetLowering::TypeSplitVector); 1206 return SplitVecOp_MSTORE(N, 3); 1207 } 1208 } 1209 } else { // Data operand 1210 assert(OpNo == 3 && "Unexpected operand for promotion"); 1211 DataOp = GetPromotedInteger(DataOp); 1212 Mask = PromoteTargetBoolean(Mask, DataOp.getValueType()); 1213 TruncateStore = true; 1214 } 1215 1216 return DAG.getMaskedStore(N->getChain(), dl, DataOp, N->getBasePtr(), Mask, 1217 N->getMemoryVT(), N->getMemOperand(), 1218 TruncateStore); 1219 } 1220 1221 SDValue DAGTypeLegalizer::PromoteIntOp_MLOAD(MaskedLoadSDNode *N, 1222 unsigned OpNo) { 1223 assert(OpNo == 2 && "Only know how to promote the mask!"); 1224 EVT DataVT = N->getValueType(0); 1225 SDValue Mask = PromoteTargetBoolean(N->getOperand(OpNo), DataVT); 1226 SmallVector<SDValue, 4> NewOps(N->op_begin(), N->op_end()); 1227 NewOps[OpNo] = Mask; 1228 return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0); 1229 } 1230 1231 SDValue DAGTypeLegalizer::PromoteIntOp_MGATHER(MaskedGatherSDNode *N, 1232 unsigned OpNo) { 1233 1234 SmallVector<SDValue, 5> NewOps(N->op_begin(), N->op_end()); 1235 if (OpNo == 2) { 1236 // The Mask 1237 EVT DataVT = N->getValueType(0); 1238 NewOps[OpNo] = PromoteTargetBoolean(N->getOperand(OpNo), DataVT); 1239 } else 1240 NewOps[OpNo] = GetPromotedInteger(N->getOperand(OpNo)); 1241 return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0); 1242 } 1243 1244 SDValue DAGTypeLegalizer::PromoteIntOp_MSCATTER(MaskedScatterSDNode *N, 1245 unsigned OpNo) { 1246 SmallVector<SDValue, 5> NewOps(N->op_begin(), N->op_end()); 1247 if (OpNo == 2) { 1248 // The Mask 1249 EVT DataVT = N->getValue().getValueType(); 1250 NewOps[OpNo] = PromoteTargetBoolean(N->getOperand(OpNo), DataVT); 1251 } else 1252 NewOps[OpNo] = GetPromotedInteger(N->getOperand(OpNo)); 1253 return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0); 1254 } 1255 1256 SDValue DAGTypeLegalizer::PromoteIntOp_TRUNCATE(SDNode *N) { 1257 SDValue Op = GetPromotedInteger(N->getOperand(0)); 1258 return DAG.getNode(ISD::TRUNCATE, SDLoc(N), N->getValueType(0), Op); 1259 } 1260 1261 SDValue DAGTypeLegalizer::PromoteIntOp_UINT_TO_FP(SDNode *N) { 1262 return SDValue(DAG.UpdateNodeOperands(N, 1263 ZExtPromotedInteger(N->getOperand(0))), 0); 1264 } 1265 1266 SDValue DAGTypeLegalizer::PromoteIntOp_ZERO_EXTEND(SDNode *N) { 1267 SDLoc dl(N); 1268 SDValue Op = GetPromotedInteger(N->getOperand(0)); 1269 Op = DAG.getNode(ISD::ANY_EXTEND, dl, N->getValueType(0), Op); 1270 return DAG.getZeroExtendInReg(Op, dl, 1271 N->getOperand(0).getValueType().getScalarType()); 1272 } 1273 1274 1275 //===----------------------------------------------------------------------===// 1276 // Integer Result Expansion 1277 //===----------------------------------------------------------------------===// 1278 1279 /// ExpandIntegerResult - This method is called when the specified result of the 1280 /// specified node is found to need expansion. At this point, the node may also 1281 /// have invalid operands or may have other results that need promotion, we just 1282 /// know that (at least) one result needs expansion. 1283 void DAGTypeLegalizer::ExpandIntegerResult(SDNode *N, unsigned ResNo) { 1284 DEBUG(dbgs() << "Expand integer result: "; N->dump(&DAG); dbgs() << "\n"); 1285 SDValue Lo, Hi; 1286 Lo = Hi = SDValue(); 1287 1288 // See if the target wants to custom expand this node. 1289 if (CustomLowerNode(N, N->getValueType(ResNo), true)) 1290 return; 1291 1292 switch (N->getOpcode()) { 1293 default: 1294 #ifndef NDEBUG 1295 dbgs() << "ExpandIntegerResult #" << ResNo << ": "; 1296 N->dump(&DAG); dbgs() << "\n"; 1297 #endif 1298 llvm_unreachable("Do not know how to expand the result of this operator!"); 1299 1300 case ISD::MERGE_VALUES: SplitRes_MERGE_VALUES(N, ResNo, Lo, Hi); break; 1301 case ISD::SELECT: SplitRes_SELECT(N, Lo, Hi); break; 1302 case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break; 1303 case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break; 1304 1305 case ISD::BITCAST: ExpandRes_BITCAST(N, Lo, Hi); break; 1306 case ISD::BUILD_PAIR: ExpandRes_BUILD_PAIR(N, Lo, Hi); break; 1307 case ISD::EXTRACT_ELEMENT: ExpandRes_EXTRACT_ELEMENT(N, Lo, Hi); break; 1308 case ISD::EXTRACT_VECTOR_ELT: ExpandRes_EXTRACT_VECTOR_ELT(N, Lo, Hi); break; 1309 case ISD::VAARG: ExpandRes_VAARG(N, Lo, Hi); break; 1310 1311 case ISD::ANY_EXTEND: ExpandIntRes_ANY_EXTEND(N, Lo, Hi); break; 1312 case ISD::AssertSext: ExpandIntRes_AssertSext(N, Lo, Hi); break; 1313 case ISD::AssertZext: ExpandIntRes_AssertZext(N, Lo, Hi); break; 1314 case ISD::BITREVERSE: ExpandIntRes_BITREVERSE(N, Lo, Hi); break; 1315 case ISD::BSWAP: ExpandIntRes_BSWAP(N, Lo, Hi); break; 1316 case ISD::Constant: ExpandIntRes_Constant(N, Lo, Hi); break; 1317 case ISD::CTLZ_ZERO_UNDEF: 1318 case ISD::CTLZ: ExpandIntRes_CTLZ(N, Lo, Hi); break; 1319 case ISD::CTPOP: ExpandIntRes_CTPOP(N, Lo, Hi); break; 1320 case ISD::CTTZ_ZERO_UNDEF: 1321 case ISD::CTTZ: ExpandIntRes_CTTZ(N, Lo, Hi); break; 1322 case ISD::FP_TO_SINT: ExpandIntRes_FP_TO_SINT(N, Lo, Hi); break; 1323 case ISD::FP_TO_UINT: ExpandIntRes_FP_TO_UINT(N, Lo, Hi); break; 1324 case ISD::LOAD: ExpandIntRes_LOAD(cast<LoadSDNode>(N), Lo, Hi); break; 1325 case ISD::MUL: ExpandIntRes_MUL(N, Lo, Hi); break; 1326 case ISD::READCYCLECOUNTER: ExpandIntRes_READCYCLECOUNTER(N, Lo, Hi); break; 1327 case ISD::SDIV: ExpandIntRes_SDIV(N, Lo, Hi); break; 1328 case ISD::SIGN_EXTEND: ExpandIntRes_SIGN_EXTEND(N, Lo, Hi); break; 1329 case ISD::SIGN_EXTEND_INREG: ExpandIntRes_SIGN_EXTEND_INREG(N, Lo, Hi); break; 1330 case ISD::SREM: ExpandIntRes_SREM(N, Lo, Hi); break; 1331 case ISD::TRUNCATE: ExpandIntRes_TRUNCATE(N, Lo, Hi); break; 1332 case ISD::UDIV: ExpandIntRes_UDIV(N, Lo, Hi); break; 1333 case ISD::UREM: ExpandIntRes_UREM(N, Lo, Hi); break; 1334 case ISD::ZERO_EXTEND: ExpandIntRes_ZERO_EXTEND(N, Lo, Hi); break; 1335 case ISD::ATOMIC_LOAD: ExpandIntRes_ATOMIC_LOAD(N, Lo, Hi); break; 1336 1337 case ISD::ATOMIC_LOAD_ADD: 1338 case ISD::ATOMIC_LOAD_SUB: 1339 case ISD::ATOMIC_LOAD_AND: 1340 case ISD::ATOMIC_LOAD_OR: 1341 case ISD::ATOMIC_LOAD_XOR: 1342 case ISD::ATOMIC_LOAD_NAND: 1343 case ISD::ATOMIC_LOAD_MIN: 1344 case ISD::ATOMIC_LOAD_MAX: 1345 case ISD::ATOMIC_LOAD_UMIN: 1346 case ISD::ATOMIC_LOAD_UMAX: 1347 case ISD::ATOMIC_SWAP: 1348 case ISD::ATOMIC_CMP_SWAP: { 1349 std::pair<SDValue, SDValue> Tmp = ExpandAtomic(N); 1350 SplitInteger(Tmp.first, Lo, Hi); 1351 ReplaceValueWith(SDValue(N, 1), Tmp.second); 1352 break; 1353 } 1354 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS: { 1355 AtomicSDNode *AN = cast<AtomicSDNode>(N); 1356 SDVTList VTs = DAG.getVTList(N->getValueType(0), MVT::Other); 1357 SDValue Tmp = DAG.getAtomicCmpSwap( 1358 ISD::ATOMIC_CMP_SWAP, SDLoc(N), AN->getMemoryVT(), VTs, 1359 N->getOperand(0), N->getOperand(1), N->getOperand(2), N->getOperand(3), 1360 AN->getMemOperand(), AN->getSuccessOrdering(), AN->getFailureOrdering(), 1361 AN->getSynchScope()); 1362 1363 // Expanding to the strong ATOMIC_CMP_SWAP node means we can determine 1364 // success simply by comparing the loaded value against the ingoing 1365 // comparison. 1366 SDValue Success = DAG.getSetCC(SDLoc(N), N->getValueType(1), Tmp, 1367 N->getOperand(2), ISD::SETEQ); 1368 1369 SplitInteger(Tmp, Lo, Hi); 1370 ReplaceValueWith(SDValue(N, 1), Success); 1371 ReplaceValueWith(SDValue(N, 2), Tmp.getValue(1)); 1372 break; 1373 } 1374 1375 case ISD::AND: 1376 case ISD::OR: 1377 case ISD::XOR: ExpandIntRes_Logical(N, Lo, Hi); break; 1378 1379 case ISD::UMAX: 1380 case ISD::SMAX: 1381 case ISD::UMIN: 1382 case ISD::SMIN: ExpandIntRes_MINMAX(N, Lo, Hi); break; 1383 1384 case ISD::ADD: 1385 case ISD::SUB: ExpandIntRes_ADDSUB(N, Lo, Hi); break; 1386 1387 case ISD::ADDC: 1388 case ISD::SUBC: ExpandIntRes_ADDSUBC(N, Lo, Hi); break; 1389 1390 case ISD::ADDE: 1391 case ISD::SUBE: ExpandIntRes_ADDSUBE(N, Lo, Hi); break; 1392 1393 case ISD::SHL: 1394 case ISD::SRA: 1395 case ISD::SRL: ExpandIntRes_Shift(N, Lo, Hi); break; 1396 1397 case ISD::SADDO: 1398 case ISD::SSUBO: ExpandIntRes_SADDSUBO(N, Lo, Hi); break; 1399 case ISD::UADDO: 1400 case ISD::USUBO: ExpandIntRes_UADDSUBO(N, Lo, Hi); break; 1401 case ISD::UMULO: 1402 case ISD::SMULO: ExpandIntRes_XMULO(N, Lo, Hi); break; 1403 } 1404 1405 // If Lo/Hi is null, the sub-method took care of registering results etc. 1406 if (Lo.getNode()) 1407 SetExpandedInteger(SDValue(N, ResNo), Lo, Hi); 1408 } 1409 1410 /// Lower an atomic node to the appropriate builtin call. 1411 std::pair <SDValue, SDValue> DAGTypeLegalizer::ExpandAtomic(SDNode *Node) { 1412 unsigned Opc = Node->getOpcode(); 1413 MVT VT = cast<AtomicSDNode>(Node)->getMemoryVT().getSimpleVT(); 1414 RTLIB::Libcall LC = RTLIB::getSYNC(Opc, VT); 1415 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected atomic op or value type!"); 1416 1417 return ExpandChainLibCall(LC, Node, false); 1418 } 1419 1420 /// N is a shift by a value that needs to be expanded, 1421 /// and the shift amount is a constant 'Amt'. Expand the operation. 1422 void DAGTypeLegalizer::ExpandShiftByConstant(SDNode *N, const APInt &Amt, 1423 SDValue &Lo, SDValue &Hi) { 1424 SDLoc DL(N); 1425 // Expand the incoming operand to be shifted, so that we have its parts 1426 SDValue InL, InH; 1427 GetExpandedInteger(N->getOperand(0), InL, InH); 1428 1429 // Though Amt shouldn't usually be 0, it's possible. E.g. when legalization 1430 // splitted a vector shift, like this: <op1, op2> SHL <0, 2>. 1431 if (!Amt) { 1432 Lo = InL; 1433 Hi = InH; 1434 return; 1435 } 1436 1437 EVT NVT = InL.getValueType(); 1438 unsigned VTBits = N->getValueType(0).getSizeInBits(); 1439 unsigned NVTBits = NVT.getSizeInBits(); 1440 EVT ShTy = N->getOperand(1).getValueType(); 1441 1442 if (N->getOpcode() == ISD::SHL) { 1443 if (Amt.ugt(VTBits)) { 1444 Lo = Hi = DAG.getConstant(0, DL, NVT); 1445 } else if (Amt.ugt(NVTBits)) { 1446 Lo = DAG.getConstant(0, DL, NVT); 1447 Hi = DAG.getNode(ISD::SHL, DL, 1448 NVT, InL, DAG.getConstant(Amt - NVTBits, DL, ShTy)); 1449 } else if (Amt == NVTBits) { 1450 Lo = DAG.getConstant(0, DL, NVT); 1451 Hi = InL; 1452 } else { 1453 Lo = DAG.getNode(ISD::SHL, DL, NVT, InL, DAG.getConstant(Amt, DL, ShTy)); 1454 Hi = DAG.getNode(ISD::OR, DL, NVT, 1455 DAG.getNode(ISD::SHL, DL, NVT, InH, 1456 DAG.getConstant(Amt, DL, ShTy)), 1457 DAG.getNode(ISD::SRL, DL, NVT, InL, 1458 DAG.getConstant(-Amt + NVTBits, DL, ShTy))); 1459 } 1460 return; 1461 } 1462 1463 if (N->getOpcode() == ISD::SRL) { 1464 if (Amt.ugt(VTBits)) { 1465 Lo = Hi = DAG.getConstant(0, DL, NVT); 1466 } else if (Amt.ugt(NVTBits)) { 1467 Lo = DAG.getNode(ISD::SRL, DL, 1468 NVT, InH, DAG.getConstant(Amt - NVTBits, DL, ShTy)); 1469 Hi = DAG.getConstant(0, DL, NVT); 1470 } else if (Amt == NVTBits) { 1471 Lo = InH; 1472 Hi = DAG.getConstant(0, DL, NVT); 1473 } else { 1474 Lo = DAG.getNode(ISD::OR, DL, NVT, 1475 DAG.getNode(ISD::SRL, DL, NVT, InL, 1476 DAG.getConstant(Amt, DL, ShTy)), 1477 DAG.getNode(ISD::SHL, DL, NVT, InH, 1478 DAG.getConstant(-Amt + NVTBits, DL, ShTy))); 1479 Hi = DAG.getNode(ISD::SRL, DL, NVT, InH, DAG.getConstant(Amt, DL, ShTy)); 1480 } 1481 return; 1482 } 1483 1484 assert(N->getOpcode() == ISD::SRA && "Unknown shift!"); 1485 if (Amt.ugt(VTBits)) { 1486 Hi = Lo = DAG.getNode(ISD::SRA, DL, NVT, InH, 1487 DAG.getConstant(NVTBits - 1, DL, ShTy)); 1488 } else if (Amt.ugt(NVTBits)) { 1489 Lo = DAG.getNode(ISD::SRA, DL, NVT, InH, 1490 DAG.getConstant(Amt - NVTBits, DL, ShTy)); 1491 Hi = DAG.getNode(ISD::SRA, DL, NVT, InH, 1492 DAG.getConstant(NVTBits - 1, DL, ShTy)); 1493 } else if (Amt == NVTBits) { 1494 Lo = InH; 1495 Hi = DAG.getNode(ISD::SRA, DL, NVT, InH, 1496 DAG.getConstant(NVTBits - 1, DL, ShTy)); 1497 } else { 1498 Lo = DAG.getNode(ISD::OR, DL, NVT, 1499 DAG.getNode(ISD::SRL, DL, NVT, InL, 1500 DAG.getConstant(Amt, DL, ShTy)), 1501 DAG.getNode(ISD::SHL, DL, NVT, InH, 1502 DAG.getConstant(-Amt + NVTBits, DL, ShTy))); 1503 Hi = DAG.getNode(ISD::SRA, DL, NVT, InH, DAG.getConstant(Amt, DL, ShTy)); 1504 } 1505 } 1506 1507 /// ExpandShiftWithKnownAmountBit - Try to determine whether we can simplify 1508 /// this shift based on knowledge of the high bit of the shift amount. If we 1509 /// can tell this, we know that it is >= 32 or < 32, without knowing the actual 1510 /// shift amount. 1511 bool DAGTypeLegalizer:: 1512 ExpandShiftWithKnownAmountBit(SDNode *N, SDValue &Lo, SDValue &Hi) { 1513 SDValue Amt = N->getOperand(1); 1514 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1515 EVT ShTy = Amt.getValueType(); 1516 unsigned ShBits = ShTy.getScalarType().getSizeInBits(); 1517 unsigned NVTBits = NVT.getScalarType().getSizeInBits(); 1518 assert(isPowerOf2_32(NVTBits) && 1519 "Expanded integer type size not a power of two!"); 1520 SDLoc dl(N); 1521 1522 APInt HighBitMask = APInt::getHighBitsSet(ShBits, ShBits - Log2_32(NVTBits)); 1523 APInt KnownZero, KnownOne; 1524 DAG.computeKnownBits(N->getOperand(1), KnownZero, KnownOne); 1525 1526 // If we don't know anything about the high bits, exit. 1527 if (((KnownZero|KnownOne) & HighBitMask) == 0) 1528 return false; 1529 1530 // Get the incoming operand to be shifted. 1531 SDValue InL, InH; 1532 GetExpandedInteger(N->getOperand(0), InL, InH); 1533 1534 // If we know that any of the high bits of the shift amount are one, then we 1535 // can do this as a couple of simple shifts. 1536 if (KnownOne.intersects(HighBitMask)) { 1537 // Mask out the high bit, which we know is set. 1538 Amt = DAG.getNode(ISD::AND, dl, ShTy, Amt, 1539 DAG.getConstant(~HighBitMask, dl, ShTy)); 1540 1541 switch (N->getOpcode()) { 1542 default: llvm_unreachable("Unknown shift"); 1543 case ISD::SHL: 1544 Lo = DAG.getConstant(0, dl, NVT); // Low part is zero. 1545 Hi = DAG.getNode(ISD::SHL, dl, NVT, InL, Amt); // High part from Lo part. 1546 return true; 1547 case ISD::SRL: 1548 Hi = DAG.getConstant(0, dl, NVT); // Hi part is zero. 1549 Lo = DAG.getNode(ISD::SRL, dl, NVT, InH, Amt); // Lo part from Hi part. 1550 return true; 1551 case ISD::SRA: 1552 Hi = DAG.getNode(ISD::SRA, dl, NVT, InH, // Sign extend high part. 1553 DAG.getConstant(NVTBits - 1, dl, ShTy)); 1554 Lo = DAG.getNode(ISD::SRA, dl, NVT, InH, Amt); // Lo part from Hi part. 1555 return true; 1556 } 1557 } 1558 1559 // If we know that all of the high bits of the shift amount are zero, then we 1560 // can do this as a couple of simple shifts. 1561 if ((KnownZero & HighBitMask) == HighBitMask) { 1562 // Calculate 31-x. 31 is used instead of 32 to avoid creating an undefined 1563 // shift if x is zero. We can use XOR here because x is known to be smaller 1564 // than 32. 1565 SDValue Amt2 = DAG.getNode(ISD::XOR, dl, ShTy, Amt, 1566 DAG.getConstant(NVTBits - 1, dl, ShTy)); 1567 1568 unsigned Op1, Op2; 1569 switch (N->getOpcode()) { 1570 default: llvm_unreachable("Unknown shift"); 1571 case ISD::SHL: Op1 = ISD::SHL; Op2 = ISD::SRL; break; 1572 case ISD::SRL: 1573 case ISD::SRA: Op1 = ISD::SRL; Op2 = ISD::SHL; break; 1574 } 1575 1576 // When shifting right the arithmetic for Lo and Hi is swapped. 1577 if (N->getOpcode() != ISD::SHL) 1578 std::swap(InL, InH); 1579 1580 // Use a little trick to get the bits that move from Lo to Hi. First 1581 // shift by one bit. 1582 SDValue Sh1 = DAG.getNode(Op2, dl, NVT, InL, DAG.getConstant(1, dl, ShTy)); 1583 // Then compute the remaining shift with amount-1. 1584 SDValue Sh2 = DAG.getNode(Op2, dl, NVT, Sh1, Amt2); 1585 1586 Lo = DAG.getNode(N->getOpcode(), dl, NVT, InL, Amt); 1587 Hi = DAG.getNode(ISD::OR, dl, NVT, DAG.getNode(Op1, dl, NVT, InH, Amt),Sh2); 1588 1589 if (N->getOpcode() != ISD::SHL) 1590 std::swap(Hi, Lo); 1591 return true; 1592 } 1593 1594 return false; 1595 } 1596 1597 /// ExpandShiftWithUnknownAmountBit - Fully general expansion of integer shift 1598 /// of any size. 1599 bool DAGTypeLegalizer:: 1600 ExpandShiftWithUnknownAmountBit(SDNode *N, SDValue &Lo, SDValue &Hi) { 1601 SDValue Amt = N->getOperand(1); 1602 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1603 EVT ShTy = Amt.getValueType(); 1604 unsigned NVTBits = NVT.getSizeInBits(); 1605 assert(isPowerOf2_32(NVTBits) && 1606 "Expanded integer type size not a power of two!"); 1607 SDLoc dl(N); 1608 1609 // Get the incoming operand to be shifted. 1610 SDValue InL, InH; 1611 GetExpandedInteger(N->getOperand(0), InL, InH); 1612 1613 SDValue NVBitsNode = DAG.getConstant(NVTBits, dl, ShTy); 1614 SDValue AmtExcess = DAG.getNode(ISD::SUB, dl, ShTy, Amt, NVBitsNode); 1615 SDValue AmtLack = DAG.getNode(ISD::SUB, dl, ShTy, NVBitsNode, Amt); 1616 SDValue isShort = DAG.getSetCC(dl, getSetCCResultType(ShTy), 1617 Amt, NVBitsNode, ISD::SETULT); 1618 SDValue isZero = DAG.getSetCC(dl, getSetCCResultType(ShTy), 1619 Amt, DAG.getConstant(0, dl, ShTy), 1620 ISD::SETEQ); 1621 1622 SDValue LoS, HiS, LoL, HiL; 1623 switch (N->getOpcode()) { 1624 default: llvm_unreachable("Unknown shift"); 1625 case ISD::SHL: 1626 // Short: ShAmt < NVTBits 1627 LoS = DAG.getNode(ISD::SHL, dl, NVT, InL, Amt); 1628 HiS = DAG.getNode(ISD::OR, dl, NVT, 1629 DAG.getNode(ISD::SHL, dl, NVT, InH, Amt), 1630 DAG.getNode(ISD::SRL, dl, NVT, InL, AmtLack)); 1631 1632 // Long: ShAmt >= NVTBits 1633 LoL = DAG.getConstant(0, dl, NVT); // Lo part is zero. 1634 HiL = DAG.getNode(ISD::SHL, dl, NVT, InL, AmtExcess); // Hi from Lo part. 1635 1636 Lo = DAG.getSelect(dl, NVT, isShort, LoS, LoL); 1637 Hi = DAG.getSelect(dl, NVT, isZero, InH, 1638 DAG.getSelect(dl, NVT, isShort, HiS, HiL)); 1639 return true; 1640 case ISD::SRL: 1641 // Short: ShAmt < NVTBits 1642 HiS = DAG.getNode(ISD::SRL, dl, NVT, InH, Amt); 1643 LoS = DAG.getNode(ISD::OR, dl, NVT, 1644 DAG.getNode(ISD::SRL, dl, NVT, InL, Amt), 1645 // FIXME: If Amt is zero, the following shift generates an undefined result 1646 // on some architectures. 1647 DAG.getNode(ISD::SHL, dl, NVT, InH, AmtLack)); 1648 1649 // Long: ShAmt >= NVTBits 1650 HiL = DAG.getConstant(0, dl, NVT); // Hi part is zero. 1651 LoL = DAG.getNode(ISD::SRL, dl, NVT, InH, AmtExcess); // Lo from Hi part. 1652 1653 Lo = DAG.getSelect(dl, NVT, isZero, InL, 1654 DAG.getSelect(dl, NVT, isShort, LoS, LoL)); 1655 Hi = DAG.getSelect(dl, NVT, isShort, HiS, HiL); 1656 return true; 1657 case ISD::SRA: 1658 // Short: ShAmt < NVTBits 1659 HiS = DAG.getNode(ISD::SRA, dl, NVT, InH, Amt); 1660 LoS = DAG.getNode(ISD::OR, dl, NVT, 1661 DAG.getNode(ISD::SRL, dl, NVT, InL, Amt), 1662 DAG.getNode(ISD::SHL, dl, NVT, InH, AmtLack)); 1663 1664 // Long: ShAmt >= NVTBits 1665 HiL = DAG.getNode(ISD::SRA, dl, NVT, InH, // Sign of Hi part. 1666 DAG.getConstant(NVTBits - 1, dl, ShTy)); 1667 LoL = DAG.getNode(ISD::SRA, dl, NVT, InH, AmtExcess); // Lo from Hi part. 1668 1669 Lo = DAG.getSelect(dl, NVT, isZero, InL, 1670 DAG.getSelect(dl, NVT, isShort, LoS, LoL)); 1671 Hi = DAG.getSelect(dl, NVT, isShort, HiS, HiL); 1672 return true; 1673 } 1674 } 1675 1676 static std::pair<ISD::CondCode, ISD::NodeType> getExpandedMinMaxOps(int Op) { 1677 1678 switch (Op) { 1679 default: llvm_unreachable("invalid min/max opcode"); 1680 case ISD::SMAX: 1681 return std::make_pair(ISD::SETGT, ISD::UMAX); 1682 case ISD::UMAX: 1683 return std::make_pair(ISD::SETUGT, ISD::UMAX); 1684 case ISD::SMIN: 1685 return std::make_pair(ISD::SETLT, ISD::UMIN); 1686 case ISD::UMIN: 1687 return std::make_pair(ISD::SETULT, ISD::UMIN); 1688 } 1689 } 1690 1691 void DAGTypeLegalizer::ExpandIntRes_MINMAX(SDNode *N, 1692 SDValue &Lo, SDValue &Hi) { 1693 SDLoc DL(N); 1694 ISD::NodeType LoOpc; 1695 ISD::CondCode CondC; 1696 std::tie(CondC, LoOpc) = getExpandedMinMaxOps(N->getOpcode()); 1697 1698 // Expand the subcomponents. 1699 SDValue LHSL, LHSH, RHSL, RHSH; 1700 GetExpandedInteger(N->getOperand(0), LHSL, LHSH); 1701 GetExpandedInteger(N->getOperand(1), RHSL, RHSH); 1702 1703 // Value types 1704 EVT NVT = LHSL.getValueType(); 1705 EVT CCT = getSetCCResultType(NVT); 1706 1707 // Hi part is always the same op 1708 Hi = DAG.getNode(N->getOpcode(), DL, {NVT, NVT}, {LHSH, RHSH}); 1709 1710 // We need to know whether to select Lo part that corresponds to 'winning' 1711 // Hi part or if Hi parts are equal. 1712 SDValue IsHiLeft = DAG.getSetCC(DL, CCT, LHSH, RHSH, CondC); 1713 SDValue IsHiEq = DAG.getSetCC(DL, CCT, LHSH, RHSH, ISD::SETEQ); 1714 1715 // Lo part corresponding to the 'winning' Hi part 1716 SDValue LoCmp = DAG.getSelect(DL, NVT, IsHiLeft, LHSL, RHSL); 1717 1718 // Recursed Lo part if Hi parts are equal, this uses unsigned version 1719 SDValue LoMinMax = DAG.getNode(LoOpc, DL, {NVT, NVT}, {LHSL, RHSL}); 1720 1721 Lo = DAG.getSelect(DL, NVT, IsHiEq, LoMinMax, LoCmp); 1722 } 1723 1724 void DAGTypeLegalizer::ExpandIntRes_ADDSUB(SDNode *N, 1725 SDValue &Lo, SDValue &Hi) { 1726 SDLoc dl(N); 1727 // Expand the subcomponents. 1728 SDValue LHSL, LHSH, RHSL, RHSH; 1729 GetExpandedInteger(N->getOperand(0), LHSL, LHSH); 1730 GetExpandedInteger(N->getOperand(1), RHSL, RHSH); 1731 1732 EVT NVT = LHSL.getValueType(); 1733 SDValue LoOps[2] = { LHSL, RHSL }; 1734 SDValue HiOps[3] = { LHSH, RHSH }; 1735 1736 // Do not generate ADDC/ADDE or SUBC/SUBE if the target does not support 1737 // them. TODO: Teach operation legalization how to expand unsupported 1738 // ADDC/ADDE/SUBC/SUBE. The problem is that these operations generate 1739 // a carry of type MVT::Glue, but there doesn't seem to be any way to 1740 // generate a value of this type in the expanded code sequence. 1741 bool hasCarry = 1742 TLI.isOperationLegalOrCustom(N->getOpcode() == ISD::ADD ? 1743 ISD::ADDC : ISD::SUBC, 1744 TLI.getTypeToExpandTo(*DAG.getContext(), NVT)); 1745 1746 if (hasCarry) { 1747 SDVTList VTList = DAG.getVTList(NVT, MVT::Glue); 1748 if (N->getOpcode() == ISD::ADD) { 1749 Lo = DAG.getNode(ISD::ADDC, dl, VTList, LoOps); 1750 HiOps[2] = Lo.getValue(1); 1751 Hi = DAG.getNode(ISD::ADDE, dl, VTList, HiOps); 1752 } else { 1753 Lo = DAG.getNode(ISD::SUBC, dl, VTList, LoOps); 1754 HiOps[2] = Lo.getValue(1); 1755 Hi = DAG.getNode(ISD::SUBE, dl, VTList, HiOps); 1756 } 1757 return; 1758 } 1759 1760 bool hasOVF = 1761 TLI.isOperationLegalOrCustom(N->getOpcode() == ISD::ADD ? 1762 ISD::UADDO : ISD::USUBO, 1763 TLI.getTypeToExpandTo(*DAG.getContext(), NVT)); 1764 if (hasOVF) { 1765 SDVTList VTList = DAG.getVTList(NVT, NVT); 1766 TargetLoweringBase::BooleanContent BoolType = TLI.getBooleanContents(NVT); 1767 int RevOpc; 1768 if (N->getOpcode() == ISD::ADD) { 1769 RevOpc = ISD::SUB; 1770 Lo = DAG.getNode(ISD::UADDO, dl, VTList, LoOps); 1771 Hi = DAG.getNode(ISD::ADD, dl, NVT, makeArrayRef(HiOps, 2)); 1772 } else { 1773 RevOpc = ISD::ADD; 1774 Lo = DAG.getNode(ISD::USUBO, dl, VTList, LoOps); 1775 Hi = DAG.getNode(ISD::SUB, dl, NVT, makeArrayRef(HiOps, 2)); 1776 } 1777 SDValue OVF = Lo.getValue(1); 1778 1779 switch (BoolType) { 1780 case TargetLoweringBase::UndefinedBooleanContent: 1781 OVF = DAG.getNode(ISD::AND, dl, NVT, DAG.getConstant(1, dl, NVT), OVF); 1782 // Fallthrough 1783 case TargetLoweringBase::ZeroOrOneBooleanContent: 1784 Hi = DAG.getNode(N->getOpcode(), dl, NVT, Hi, OVF); 1785 break; 1786 case TargetLoweringBase::ZeroOrNegativeOneBooleanContent: 1787 Hi = DAG.getNode(RevOpc, dl, NVT, Hi, OVF); 1788 } 1789 return; 1790 } 1791 1792 if (N->getOpcode() == ISD::ADD) { 1793 Lo = DAG.getNode(ISD::ADD, dl, NVT, LoOps); 1794 Hi = DAG.getNode(ISD::ADD, dl, NVT, makeArrayRef(HiOps, 2)); 1795 SDValue Cmp1 = DAG.getSetCC(dl, getSetCCResultType(NVT), Lo, LoOps[0], 1796 ISD::SETULT); 1797 SDValue Carry1 = DAG.getSelect(dl, NVT, Cmp1, 1798 DAG.getConstant(1, dl, NVT), 1799 DAG.getConstant(0, dl, NVT)); 1800 SDValue Cmp2 = DAG.getSetCC(dl, getSetCCResultType(NVT), Lo, LoOps[1], 1801 ISD::SETULT); 1802 SDValue Carry2 = DAG.getSelect(dl, NVT, Cmp2, 1803 DAG.getConstant(1, dl, NVT), Carry1); 1804 Hi = DAG.getNode(ISD::ADD, dl, NVT, Hi, Carry2); 1805 } else { 1806 Lo = DAG.getNode(ISD::SUB, dl, NVT, LoOps); 1807 Hi = DAG.getNode(ISD::SUB, dl, NVT, makeArrayRef(HiOps, 2)); 1808 SDValue Cmp = 1809 DAG.getSetCC(dl, getSetCCResultType(LoOps[0].getValueType()), 1810 LoOps[0], LoOps[1], ISD::SETULT); 1811 SDValue Borrow = DAG.getSelect(dl, NVT, Cmp, 1812 DAG.getConstant(1, dl, NVT), 1813 DAG.getConstant(0, dl, NVT)); 1814 Hi = DAG.getNode(ISD::SUB, dl, NVT, Hi, Borrow); 1815 } 1816 } 1817 1818 void DAGTypeLegalizer::ExpandIntRes_ADDSUBC(SDNode *N, 1819 SDValue &Lo, SDValue &Hi) { 1820 // Expand the subcomponents. 1821 SDValue LHSL, LHSH, RHSL, RHSH; 1822 SDLoc dl(N); 1823 GetExpandedInteger(N->getOperand(0), LHSL, LHSH); 1824 GetExpandedInteger(N->getOperand(1), RHSL, RHSH); 1825 SDVTList VTList = DAG.getVTList(LHSL.getValueType(), MVT::Glue); 1826 SDValue LoOps[2] = { LHSL, RHSL }; 1827 SDValue HiOps[3] = { LHSH, RHSH }; 1828 1829 if (N->getOpcode() == ISD::ADDC) { 1830 Lo = DAG.getNode(ISD::ADDC, dl, VTList, LoOps); 1831 HiOps[2] = Lo.getValue(1); 1832 Hi = DAG.getNode(ISD::ADDE, dl, VTList, HiOps); 1833 } else { 1834 Lo = DAG.getNode(ISD::SUBC, dl, VTList, LoOps); 1835 HiOps[2] = Lo.getValue(1); 1836 Hi = DAG.getNode(ISD::SUBE, dl, VTList, HiOps); 1837 } 1838 1839 // Legalized the flag result - switch anything that used the old flag to 1840 // use the new one. 1841 ReplaceValueWith(SDValue(N, 1), Hi.getValue(1)); 1842 } 1843 1844 void DAGTypeLegalizer::ExpandIntRes_ADDSUBE(SDNode *N, 1845 SDValue &Lo, SDValue &Hi) { 1846 // Expand the subcomponents. 1847 SDValue LHSL, LHSH, RHSL, RHSH; 1848 SDLoc dl(N); 1849 GetExpandedInteger(N->getOperand(0), LHSL, LHSH); 1850 GetExpandedInteger(N->getOperand(1), RHSL, RHSH); 1851 SDVTList VTList = DAG.getVTList(LHSL.getValueType(), MVT::Glue); 1852 SDValue LoOps[3] = { LHSL, RHSL, N->getOperand(2) }; 1853 SDValue HiOps[3] = { LHSH, RHSH }; 1854 1855 Lo = DAG.getNode(N->getOpcode(), dl, VTList, LoOps); 1856 HiOps[2] = Lo.getValue(1); 1857 Hi = DAG.getNode(N->getOpcode(), dl, VTList, HiOps); 1858 1859 // Legalized the flag result - switch anything that used the old flag to 1860 // use the new one. 1861 ReplaceValueWith(SDValue(N, 1), Hi.getValue(1)); 1862 } 1863 1864 void DAGTypeLegalizer::ExpandIntRes_ANY_EXTEND(SDNode *N, 1865 SDValue &Lo, SDValue &Hi) { 1866 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1867 SDLoc dl(N); 1868 SDValue Op = N->getOperand(0); 1869 if (Op.getValueType().bitsLE(NVT)) { 1870 // The low part is any extension of the input (which degenerates to a copy). 1871 Lo = DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Op); 1872 Hi = DAG.getUNDEF(NVT); // The high part is undefined. 1873 } else { 1874 // For example, extension of an i48 to an i64. The operand type necessarily 1875 // promotes to the result type, so will end up being expanded too. 1876 assert(getTypeAction(Op.getValueType()) == 1877 TargetLowering::TypePromoteInteger && 1878 "Only know how to promote this result!"); 1879 SDValue Res = GetPromotedInteger(Op); 1880 assert(Res.getValueType() == N->getValueType(0) && 1881 "Operand over promoted?"); 1882 // Split the promoted operand. This will simplify when it is expanded. 1883 SplitInteger(Res, Lo, Hi); 1884 } 1885 } 1886 1887 void DAGTypeLegalizer::ExpandIntRes_AssertSext(SDNode *N, 1888 SDValue &Lo, SDValue &Hi) { 1889 SDLoc dl(N); 1890 GetExpandedInteger(N->getOperand(0), Lo, Hi); 1891 EVT NVT = Lo.getValueType(); 1892 EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT(); 1893 unsigned NVTBits = NVT.getSizeInBits(); 1894 unsigned EVTBits = EVT.getSizeInBits(); 1895 1896 if (NVTBits < EVTBits) { 1897 Hi = DAG.getNode(ISD::AssertSext, dl, NVT, Hi, 1898 DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(), 1899 EVTBits - NVTBits))); 1900 } else { 1901 Lo = DAG.getNode(ISD::AssertSext, dl, NVT, Lo, DAG.getValueType(EVT)); 1902 // The high part replicates the sign bit of Lo, make it explicit. 1903 Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo, 1904 DAG.getConstant(NVTBits - 1, dl, 1905 TLI.getPointerTy(DAG.getDataLayout()))); 1906 } 1907 } 1908 1909 void DAGTypeLegalizer::ExpandIntRes_AssertZext(SDNode *N, 1910 SDValue &Lo, SDValue &Hi) { 1911 SDLoc dl(N); 1912 GetExpandedInteger(N->getOperand(0), Lo, Hi); 1913 EVT NVT = Lo.getValueType(); 1914 EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT(); 1915 unsigned NVTBits = NVT.getSizeInBits(); 1916 unsigned EVTBits = EVT.getSizeInBits(); 1917 1918 if (NVTBits < EVTBits) { 1919 Hi = DAG.getNode(ISD::AssertZext, dl, NVT, Hi, 1920 DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(), 1921 EVTBits - NVTBits))); 1922 } else { 1923 Lo = DAG.getNode(ISD::AssertZext, dl, NVT, Lo, DAG.getValueType(EVT)); 1924 // The high part must be zero, make it explicit. 1925 Hi = DAG.getConstant(0, dl, NVT); 1926 } 1927 } 1928 1929 void DAGTypeLegalizer::ExpandIntRes_BITREVERSE(SDNode *N, 1930 SDValue &Lo, SDValue &Hi) { 1931 SDLoc dl(N); 1932 GetExpandedInteger(N->getOperand(0), Hi, Lo); // Note swapped operands. 1933 Lo = DAG.getNode(ISD::BITREVERSE, dl, Lo.getValueType(), Lo); 1934 Hi = DAG.getNode(ISD::BITREVERSE, dl, Hi.getValueType(), Hi); 1935 } 1936 1937 void DAGTypeLegalizer::ExpandIntRes_BSWAP(SDNode *N, 1938 SDValue &Lo, SDValue &Hi) { 1939 SDLoc dl(N); 1940 GetExpandedInteger(N->getOperand(0), Hi, Lo); // Note swapped operands. 1941 Lo = DAG.getNode(ISD::BSWAP, dl, Lo.getValueType(), Lo); 1942 Hi = DAG.getNode(ISD::BSWAP, dl, Hi.getValueType(), Hi); 1943 } 1944 1945 void DAGTypeLegalizer::ExpandIntRes_Constant(SDNode *N, 1946 SDValue &Lo, SDValue &Hi) { 1947 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1948 unsigned NBitWidth = NVT.getSizeInBits(); 1949 auto Constant = cast<ConstantSDNode>(N); 1950 const APInt &Cst = Constant->getAPIntValue(); 1951 bool IsTarget = Constant->isTargetOpcode(); 1952 bool IsOpaque = Constant->isOpaque(); 1953 SDLoc dl(N); 1954 Lo = DAG.getConstant(Cst.trunc(NBitWidth), dl, NVT, IsTarget, IsOpaque); 1955 Hi = DAG.getConstant(Cst.lshr(NBitWidth).trunc(NBitWidth), dl, NVT, IsTarget, 1956 IsOpaque); 1957 } 1958 1959 void DAGTypeLegalizer::ExpandIntRes_CTLZ(SDNode *N, 1960 SDValue &Lo, SDValue &Hi) { 1961 SDLoc dl(N); 1962 // ctlz (HiLo) -> Hi != 0 ? ctlz(Hi) : (ctlz(Lo)+32) 1963 GetExpandedInteger(N->getOperand(0), Lo, Hi); 1964 EVT NVT = Lo.getValueType(); 1965 1966 SDValue HiNotZero = DAG.getSetCC(dl, getSetCCResultType(NVT), Hi, 1967 DAG.getConstant(0, dl, NVT), ISD::SETNE); 1968 1969 SDValue LoLZ = DAG.getNode(N->getOpcode(), dl, NVT, Lo); 1970 SDValue HiLZ = DAG.getNode(ISD::CTLZ_ZERO_UNDEF, dl, NVT, Hi); 1971 1972 Lo = DAG.getSelect(dl, NVT, HiNotZero, HiLZ, 1973 DAG.getNode(ISD::ADD, dl, NVT, LoLZ, 1974 DAG.getConstant(NVT.getSizeInBits(), dl, 1975 NVT))); 1976 Hi = DAG.getConstant(0, dl, NVT); 1977 } 1978 1979 void DAGTypeLegalizer::ExpandIntRes_CTPOP(SDNode *N, 1980 SDValue &Lo, SDValue &Hi) { 1981 SDLoc dl(N); 1982 // ctpop(HiLo) -> ctpop(Hi)+ctpop(Lo) 1983 GetExpandedInteger(N->getOperand(0), Lo, Hi); 1984 EVT NVT = Lo.getValueType(); 1985 Lo = DAG.getNode(ISD::ADD, dl, NVT, DAG.getNode(ISD::CTPOP, dl, NVT, Lo), 1986 DAG.getNode(ISD::CTPOP, dl, NVT, Hi)); 1987 Hi = DAG.getConstant(0, dl, NVT); 1988 } 1989 1990 void DAGTypeLegalizer::ExpandIntRes_CTTZ(SDNode *N, 1991 SDValue &Lo, SDValue &Hi) { 1992 SDLoc dl(N); 1993 // cttz (HiLo) -> Lo != 0 ? cttz(Lo) : (cttz(Hi)+32) 1994 GetExpandedInteger(N->getOperand(0), Lo, Hi); 1995 EVT NVT = Lo.getValueType(); 1996 1997 SDValue LoNotZero = DAG.getSetCC(dl, getSetCCResultType(NVT), Lo, 1998 DAG.getConstant(0, dl, NVT), ISD::SETNE); 1999 2000 SDValue LoLZ = DAG.getNode(ISD::CTTZ_ZERO_UNDEF, dl, NVT, Lo); 2001 SDValue HiLZ = DAG.getNode(N->getOpcode(), dl, NVT, Hi); 2002 2003 Lo = DAG.getSelect(dl, NVT, LoNotZero, LoLZ, 2004 DAG.getNode(ISD::ADD, dl, NVT, HiLZ, 2005 DAG.getConstant(NVT.getSizeInBits(), dl, 2006 NVT))); 2007 Hi = DAG.getConstant(0, dl, NVT); 2008 } 2009 2010 void DAGTypeLegalizer::ExpandIntRes_FP_TO_SINT(SDNode *N, SDValue &Lo, 2011 SDValue &Hi) { 2012 SDLoc dl(N); 2013 EVT VT = N->getValueType(0); 2014 2015 SDValue Op = N->getOperand(0); 2016 if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteFloat) 2017 Op = GetPromotedFloat(Op); 2018 2019 RTLIB::Libcall LC = RTLIB::getFPTOSINT(Op.getValueType(), VT); 2020 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fp-to-sint conversion!"); 2021 SplitInteger(TLI.makeLibCall(DAG, LC, VT, Op, true/*irrelevant*/, dl).first, 2022 Lo, Hi); 2023 } 2024 2025 void DAGTypeLegalizer::ExpandIntRes_FP_TO_UINT(SDNode *N, SDValue &Lo, 2026 SDValue &Hi) { 2027 SDLoc dl(N); 2028 EVT VT = N->getValueType(0); 2029 2030 SDValue Op = N->getOperand(0); 2031 if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteFloat) 2032 Op = GetPromotedFloat(Op); 2033 2034 RTLIB::Libcall LC = RTLIB::getFPTOUINT(Op.getValueType(), VT); 2035 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fp-to-uint conversion!"); 2036 SplitInteger(TLI.makeLibCall(DAG, LC, VT, Op, false/*irrelevant*/, dl).first, 2037 Lo, Hi); 2038 } 2039 2040 void DAGTypeLegalizer::ExpandIntRes_LOAD(LoadSDNode *N, 2041 SDValue &Lo, SDValue &Hi) { 2042 if (ISD::isNormalLoad(N)) { 2043 ExpandRes_NormalLoad(N, Lo, Hi); 2044 return; 2045 } 2046 2047 assert(ISD::isUNINDEXEDLoad(N) && "Indexed load during type legalization!"); 2048 2049 EVT VT = N->getValueType(0); 2050 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 2051 SDValue Ch = N->getChain(); 2052 SDValue Ptr = N->getBasePtr(); 2053 ISD::LoadExtType ExtType = N->getExtensionType(); 2054 unsigned Alignment = N->getAlignment(); 2055 bool isVolatile = N->isVolatile(); 2056 bool isNonTemporal = N->isNonTemporal(); 2057 bool isInvariant = N->isInvariant(); 2058 AAMDNodes AAInfo = N->getAAInfo(); 2059 SDLoc dl(N); 2060 2061 assert(NVT.isByteSized() && "Expanded type not byte sized!"); 2062 2063 if (N->getMemoryVT().bitsLE(NVT)) { 2064 EVT MemVT = N->getMemoryVT(); 2065 2066 Lo = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr, N->getPointerInfo(), 2067 MemVT, isVolatile, isNonTemporal, isInvariant, 2068 Alignment, AAInfo); 2069 2070 // Remember the chain. 2071 Ch = Lo.getValue(1); 2072 2073 if (ExtType == ISD::SEXTLOAD) { 2074 // The high part is obtained by SRA'ing all but one of the bits of the 2075 // lo part. 2076 unsigned LoSize = Lo.getValueType().getSizeInBits(); 2077 Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo, 2078 DAG.getConstant(LoSize - 1, dl, 2079 TLI.getPointerTy(DAG.getDataLayout()))); 2080 } else if (ExtType == ISD::ZEXTLOAD) { 2081 // The high part is just a zero. 2082 Hi = DAG.getConstant(0, dl, NVT); 2083 } else { 2084 assert(ExtType == ISD::EXTLOAD && "Unknown extload!"); 2085 // The high part is undefined. 2086 Hi = DAG.getUNDEF(NVT); 2087 } 2088 } else if (DAG.getDataLayout().isLittleEndian()) { 2089 // Little-endian - low bits are at low addresses. 2090 Lo = DAG.getLoad(NVT, dl, Ch, Ptr, N->getPointerInfo(), 2091 isVolatile, isNonTemporal, isInvariant, Alignment, 2092 AAInfo); 2093 2094 unsigned ExcessBits = 2095 N->getMemoryVT().getSizeInBits() - NVT.getSizeInBits(); 2096 EVT NEVT = EVT::getIntegerVT(*DAG.getContext(), ExcessBits); 2097 2098 // Increment the pointer to the other half. 2099 unsigned IncrementSize = NVT.getSizeInBits()/8; 2100 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, 2101 DAG.getConstant(IncrementSize, dl, Ptr.getValueType())); 2102 Hi = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr, 2103 N->getPointerInfo().getWithOffset(IncrementSize), NEVT, 2104 isVolatile, isNonTemporal, isInvariant, 2105 MinAlign(Alignment, IncrementSize), AAInfo); 2106 2107 // Build a factor node to remember that this load is independent of the 2108 // other one. 2109 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1), 2110 Hi.getValue(1)); 2111 } else { 2112 // Big-endian - high bits are at low addresses. Favor aligned loads at 2113 // the cost of some bit-fiddling. 2114 EVT MemVT = N->getMemoryVT(); 2115 unsigned EBytes = MemVT.getStoreSize(); 2116 unsigned IncrementSize = NVT.getSizeInBits()/8; 2117 unsigned ExcessBits = (EBytes - IncrementSize)*8; 2118 2119 // Load both the high bits and maybe some of the low bits. 2120 Hi = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr, N->getPointerInfo(), 2121 EVT::getIntegerVT(*DAG.getContext(), 2122 MemVT.getSizeInBits() - ExcessBits), 2123 isVolatile, isNonTemporal, isInvariant, Alignment, 2124 AAInfo); 2125 2126 // Increment the pointer to the other half. 2127 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, 2128 DAG.getConstant(IncrementSize, dl, Ptr.getValueType())); 2129 // Load the rest of the low bits. 2130 Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, NVT, Ch, Ptr, 2131 N->getPointerInfo().getWithOffset(IncrementSize), 2132 EVT::getIntegerVT(*DAG.getContext(), ExcessBits), 2133 isVolatile, isNonTemporal, isInvariant, 2134 MinAlign(Alignment, IncrementSize), AAInfo); 2135 2136 // Build a factor node to remember that this load is independent of the 2137 // other one. 2138 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1), 2139 Hi.getValue(1)); 2140 2141 if (ExcessBits < NVT.getSizeInBits()) { 2142 // Transfer low bits from the bottom of Hi to the top of Lo. 2143 Lo = DAG.getNode( 2144 ISD::OR, dl, NVT, Lo, 2145 DAG.getNode(ISD::SHL, dl, NVT, Hi, 2146 DAG.getConstant(ExcessBits, dl, 2147 TLI.getPointerTy(DAG.getDataLayout())))); 2148 // Move high bits to the right position in Hi. 2149 Hi = DAG.getNode(ExtType == ISD::SEXTLOAD ? ISD::SRA : ISD::SRL, dl, NVT, 2150 Hi, 2151 DAG.getConstant(NVT.getSizeInBits() - ExcessBits, dl, 2152 TLI.getPointerTy(DAG.getDataLayout()))); 2153 } 2154 } 2155 2156 // Legalize the chain result - switch anything that used the old chain to 2157 // use the new one. 2158 ReplaceValueWith(SDValue(N, 1), Ch); 2159 } 2160 2161 void DAGTypeLegalizer::ExpandIntRes_Logical(SDNode *N, 2162 SDValue &Lo, SDValue &Hi) { 2163 SDLoc dl(N); 2164 SDValue LL, LH, RL, RH; 2165 GetExpandedInteger(N->getOperand(0), LL, LH); 2166 GetExpandedInteger(N->getOperand(1), RL, RH); 2167 Lo = DAG.getNode(N->getOpcode(), dl, LL.getValueType(), LL, RL); 2168 Hi = DAG.getNode(N->getOpcode(), dl, LL.getValueType(), LH, RH); 2169 } 2170 2171 void DAGTypeLegalizer::ExpandIntRes_MUL(SDNode *N, 2172 SDValue &Lo, SDValue &Hi) { 2173 EVT VT = N->getValueType(0); 2174 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 2175 SDLoc dl(N); 2176 2177 SDValue LL, LH, RL, RH; 2178 GetExpandedInteger(N->getOperand(0), LL, LH); 2179 GetExpandedInteger(N->getOperand(1), RL, RH); 2180 2181 if (TLI.expandMUL(N, Lo, Hi, NVT, DAG, LL, LH, RL, RH)) 2182 return; 2183 2184 // If nothing else, we can make a libcall. 2185 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 2186 if (VT == MVT::i16) 2187 LC = RTLIB::MUL_I16; 2188 else if (VT == MVT::i32) 2189 LC = RTLIB::MUL_I32; 2190 else if (VT == MVT::i64) 2191 LC = RTLIB::MUL_I64; 2192 else if (VT == MVT::i128) 2193 LC = RTLIB::MUL_I128; 2194 2195 if (LC == RTLIB::UNKNOWN_LIBCALL) { 2196 // We'll expand the multiplication by brute force because we have no other 2197 // options. This is a trivially-generalized version of the code from 2198 // Hacker's Delight (itself derived from Knuth's Algorithm M from section 2199 // 4.3.1). 2200 SDValue Mask = 2201 DAG.getConstant(APInt::getLowBitsSet(NVT.getSizeInBits(), 2202 NVT.getSizeInBits() >> 1), dl, NVT); 2203 SDValue LLL = DAG.getNode(ISD::AND, dl, NVT, LL, Mask); 2204 SDValue RLL = DAG.getNode(ISD::AND, dl, NVT, RL, Mask); 2205 2206 SDValue T = DAG.getNode(ISD::MUL, dl, NVT, LLL, RLL); 2207 SDValue TL = DAG.getNode(ISD::AND, dl, NVT, T, Mask); 2208 2209 SDValue Shift = 2210 DAG.getConstant(NVT.getSizeInBits() >> 1, dl, 2211 TLI.getShiftAmountTy(NVT, DAG.getDataLayout())); 2212 SDValue TH = DAG.getNode(ISD::SRL, dl, NVT, T, Shift); 2213 SDValue LLH = DAG.getNode(ISD::SRL, dl, NVT, LL, Shift); 2214 SDValue RLH = DAG.getNode(ISD::SRL, dl, NVT, RL, Shift); 2215 2216 SDValue U = DAG.getNode(ISD::ADD, dl, NVT, 2217 DAG.getNode(ISD::MUL, dl, NVT, LLH, RLL), TL); 2218 SDValue UL = DAG.getNode(ISD::AND, dl, NVT, U, Mask); 2219 SDValue UH = DAG.getNode(ISD::SRL, dl, NVT, U, Shift); 2220 2221 SDValue V = DAG.getNode(ISD::ADD, dl, NVT, 2222 DAG.getNode(ISD::MUL, dl, NVT, LLL, RLH), UL); 2223 SDValue VH = DAG.getNode(ISD::SRL, dl, NVT, V, Shift); 2224 2225 SDValue W = DAG.getNode(ISD::ADD, dl, NVT, 2226 DAG.getNode(ISD::MUL, dl, NVT, LL, RL), 2227 DAG.getNode(ISD::ADD, dl, NVT, UH, VH)); 2228 Lo = DAG.getNode(ISD::ADD, dl, NVT, TH, 2229 DAG.getNode(ISD::SHL, dl, NVT, V, Shift)); 2230 2231 Hi = DAG.getNode(ISD::ADD, dl, NVT, W, 2232 DAG.getNode(ISD::ADD, dl, NVT, 2233 DAG.getNode(ISD::MUL, dl, NVT, RH, LL), 2234 DAG.getNode(ISD::MUL, dl, NVT, RL, LH))); 2235 return; 2236 } 2237 2238 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) }; 2239 SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, true/*irrelevant*/, dl).first, 2240 Lo, Hi); 2241 } 2242 2243 void DAGTypeLegalizer::ExpandIntRes_READCYCLECOUNTER(SDNode *N, SDValue &Lo, 2244 SDValue &Hi) { 2245 SDLoc DL(N); 2246 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 2247 SDVTList VTs = DAG.getVTList(NVT, NVT, MVT::Other); 2248 SDValue R = DAG.getNode(N->getOpcode(), DL, VTs, N->getOperand(0)); 2249 Lo = R.getValue(0); 2250 Hi = R.getValue(1); 2251 ReplaceValueWith(SDValue(N, 1), R.getValue(2)); 2252 } 2253 2254 void DAGTypeLegalizer::ExpandIntRes_SADDSUBO(SDNode *Node, 2255 SDValue &Lo, SDValue &Hi) { 2256 SDValue LHS = Node->getOperand(0); 2257 SDValue RHS = Node->getOperand(1); 2258 SDLoc dl(Node); 2259 2260 // Expand the result by simply replacing it with the equivalent 2261 // non-overflow-checking operation. 2262 SDValue Sum = DAG.getNode(Node->getOpcode() == ISD::SADDO ? 2263 ISD::ADD : ISD::SUB, dl, LHS.getValueType(), 2264 LHS, RHS); 2265 SplitInteger(Sum, Lo, Hi); 2266 2267 // Compute the overflow. 2268 // 2269 // LHSSign -> LHS >= 0 2270 // RHSSign -> RHS >= 0 2271 // SumSign -> Sum >= 0 2272 // 2273 // Add: 2274 // Overflow -> (LHSSign == RHSSign) && (LHSSign != SumSign) 2275 // Sub: 2276 // Overflow -> (LHSSign != RHSSign) && (LHSSign != SumSign) 2277 // 2278 EVT OType = Node->getValueType(1); 2279 SDValue Zero = DAG.getConstant(0, dl, LHS.getValueType()); 2280 2281 SDValue LHSSign = DAG.getSetCC(dl, OType, LHS, Zero, ISD::SETGE); 2282 SDValue RHSSign = DAG.getSetCC(dl, OType, RHS, Zero, ISD::SETGE); 2283 SDValue SignsMatch = DAG.getSetCC(dl, OType, LHSSign, RHSSign, 2284 Node->getOpcode() == ISD::SADDO ? 2285 ISD::SETEQ : ISD::SETNE); 2286 2287 SDValue SumSign = DAG.getSetCC(dl, OType, Sum, Zero, ISD::SETGE); 2288 SDValue SumSignNE = DAG.getSetCC(dl, OType, LHSSign, SumSign, ISD::SETNE); 2289 2290 SDValue Cmp = DAG.getNode(ISD::AND, dl, OType, SignsMatch, SumSignNE); 2291 2292 // Use the calculated overflow everywhere. 2293 ReplaceValueWith(SDValue(Node, 1), Cmp); 2294 } 2295 2296 void DAGTypeLegalizer::ExpandIntRes_SDIV(SDNode *N, 2297 SDValue &Lo, SDValue &Hi) { 2298 EVT VT = N->getValueType(0); 2299 SDLoc dl(N); 2300 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) }; 2301 2302 if (TLI.getOperationAction(ISD::SDIVREM, VT) == TargetLowering::Custom) { 2303 SDValue Res = DAG.getNode(ISD::SDIVREM, dl, DAG.getVTList(VT, VT), Ops); 2304 SplitInteger(Res.getValue(0), Lo, Hi); 2305 return; 2306 } 2307 2308 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 2309 if (VT == MVT::i16) 2310 LC = RTLIB::SDIV_I16; 2311 else if (VT == MVT::i32) 2312 LC = RTLIB::SDIV_I32; 2313 else if (VT == MVT::i64) 2314 LC = RTLIB::SDIV_I64; 2315 else if (VT == MVT::i128) 2316 LC = RTLIB::SDIV_I128; 2317 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported SDIV!"); 2318 2319 SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, true, dl).first, Lo, Hi); 2320 } 2321 2322 void DAGTypeLegalizer::ExpandIntRes_Shift(SDNode *N, 2323 SDValue &Lo, SDValue &Hi) { 2324 EVT VT = N->getValueType(0); 2325 SDLoc dl(N); 2326 2327 // If we can emit an efficient shift operation, do so now. Check to see if 2328 // the RHS is a constant. 2329 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N->getOperand(1))) 2330 return ExpandShiftByConstant(N, CN->getAPIntValue(), Lo, Hi); 2331 2332 // If we can determine that the high bit of the shift is zero or one, even if 2333 // the low bits are variable, emit this shift in an optimized form. 2334 if (ExpandShiftWithKnownAmountBit(N, Lo, Hi)) 2335 return; 2336 2337 // If this target supports shift_PARTS, use it. First, map to the _PARTS opc. 2338 unsigned PartsOpc; 2339 if (N->getOpcode() == ISD::SHL) { 2340 PartsOpc = ISD::SHL_PARTS; 2341 } else if (N->getOpcode() == ISD::SRL) { 2342 PartsOpc = ISD::SRL_PARTS; 2343 } else { 2344 assert(N->getOpcode() == ISD::SRA && "Unknown shift!"); 2345 PartsOpc = ISD::SRA_PARTS; 2346 } 2347 2348 // Next check to see if the target supports this SHL_PARTS operation or if it 2349 // will custom expand it. 2350 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 2351 TargetLowering::LegalizeAction Action = TLI.getOperationAction(PartsOpc, NVT); 2352 if ((Action == TargetLowering::Legal && TLI.isTypeLegal(NVT)) || 2353 Action == TargetLowering::Custom) { 2354 // Expand the subcomponents. 2355 SDValue LHSL, LHSH; 2356 GetExpandedInteger(N->getOperand(0), LHSL, LHSH); 2357 EVT VT = LHSL.getValueType(); 2358 2359 // If the shift amount operand is coming from a vector legalization it may 2360 // have an illegal type. Fix that first by casting the operand, otherwise 2361 // the new SHL_PARTS operation would need further legalization. 2362 SDValue ShiftOp = N->getOperand(1); 2363 EVT ShiftTy = TLI.getShiftAmountTy(VT, DAG.getDataLayout()); 2364 assert(ShiftTy.getScalarType().getSizeInBits() >= 2365 Log2_32_Ceil(VT.getScalarType().getSizeInBits()) && 2366 "ShiftAmountTy is too small to cover the range of this type!"); 2367 if (ShiftOp.getValueType() != ShiftTy) 2368 ShiftOp = DAG.getZExtOrTrunc(ShiftOp, dl, ShiftTy); 2369 2370 SDValue Ops[] = { LHSL, LHSH, ShiftOp }; 2371 Lo = DAG.getNode(PartsOpc, dl, DAG.getVTList(VT, VT), Ops); 2372 Hi = Lo.getValue(1); 2373 return; 2374 } 2375 2376 // Otherwise, emit a libcall. 2377 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 2378 bool isSigned; 2379 if (N->getOpcode() == ISD::SHL) { 2380 isSigned = false; /*sign irrelevant*/ 2381 if (VT == MVT::i16) 2382 LC = RTLIB::SHL_I16; 2383 else if (VT == MVT::i32) 2384 LC = RTLIB::SHL_I32; 2385 else if (VT == MVT::i64) 2386 LC = RTLIB::SHL_I64; 2387 else if (VT == MVT::i128) 2388 LC = RTLIB::SHL_I128; 2389 } else if (N->getOpcode() == ISD::SRL) { 2390 isSigned = false; 2391 if (VT == MVT::i16) 2392 LC = RTLIB::SRL_I16; 2393 else if (VT == MVT::i32) 2394 LC = RTLIB::SRL_I32; 2395 else if (VT == MVT::i64) 2396 LC = RTLIB::SRL_I64; 2397 else if (VT == MVT::i128) 2398 LC = RTLIB::SRL_I128; 2399 } else { 2400 assert(N->getOpcode() == ISD::SRA && "Unknown shift!"); 2401 isSigned = true; 2402 if (VT == MVT::i16) 2403 LC = RTLIB::SRA_I16; 2404 else if (VT == MVT::i32) 2405 LC = RTLIB::SRA_I32; 2406 else if (VT == MVT::i64) 2407 LC = RTLIB::SRA_I64; 2408 else if (VT == MVT::i128) 2409 LC = RTLIB::SRA_I128; 2410 } 2411 2412 if (LC != RTLIB::UNKNOWN_LIBCALL && TLI.getLibcallName(LC)) { 2413 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) }; 2414 SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, isSigned, dl).first, Lo, Hi); 2415 return; 2416 } 2417 2418 if (!ExpandShiftWithUnknownAmountBit(N, Lo, Hi)) 2419 llvm_unreachable("Unsupported shift!"); 2420 } 2421 2422 void DAGTypeLegalizer::ExpandIntRes_SIGN_EXTEND(SDNode *N, 2423 SDValue &Lo, SDValue &Hi) { 2424 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 2425 SDLoc dl(N); 2426 SDValue Op = N->getOperand(0); 2427 if (Op.getValueType().bitsLE(NVT)) { 2428 // The low part is sign extension of the input (degenerates to a copy). 2429 Lo = DAG.getNode(ISD::SIGN_EXTEND, dl, NVT, N->getOperand(0)); 2430 // The high part is obtained by SRA'ing all but one of the bits of low part. 2431 unsigned LoSize = NVT.getSizeInBits(); 2432 Hi = DAG.getNode( 2433 ISD::SRA, dl, NVT, Lo, 2434 DAG.getConstant(LoSize - 1, dl, TLI.getPointerTy(DAG.getDataLayout()))); 2435 } else { 2436 // For example, extension of an i48 to an i64. The operand type necessarily 2437 // promotes to the result type, so will end up being expanded too. 2438 assert(getTypeAction(Op.getValueType()) == 2439 TargetLowering::TypePromoteInteger && 2440 "Only know how to promote this result!"); 2441 SDValue Res = GetPromotedInteger(Op); 2442 assert(Res.getValueType() == N->getValueType(0) && 2443 "Operand over promoted?"); 2444 // Split the promoted operand. This will simplify when it is expanded. 2445 SplitInteger(Res, Lo, Hi); 2446 unsigned ExcessBits = 2447 Op.getValueType().getSizeInBits() - NVT.getSizeInBits(); 2448 Hi = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Hi.getValueType(), Hi, 2449 DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(), 2450 ExcessBits))); 2451 } 2452 } 2453 2454 void DAGTypeLegalizer:: 2455 ExpandIntRes_SIGN_EXTEND_INREG(SDNode *N, SDValue &Lo, SDValue &Hi) { 2456 SDLoc dl(N); 2457 GetExpandedInteger(N->getOperand(0), Lo, Hi); 2458 EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT(); 2459 2460 if (EVT.bitsLE(Lo.getValueType())) { 2461 // sext_inreg the low part if needed. 2462 Lo = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Lo.getValueType(), Lo, 2463 N->getOperand(1)); 2464 2465 // The high part gets the sign extension from the lo-part. This handles 2466 // things like sextinreg V:i64 from i8. 2467 Hi = DAG.getNode(ISD::SRA, dl, Hi.getValueType(), Lo, 2468 DAG.getConstant(Hi.getValueType().getSizeInBits() - 1, dl, 2469 TLI.getPointerTy(DAG.getDataLayout()))); 2470 } else { 2471 // For example, extension of an i48 to an i64. Leave the low part alone, 2472 // sext_inreg the high part. 2473 unsigned ExcessBits = 2474 EVT.getSizeInBits() - Lo.getValueType().getSizeInBits(); 2475 Hi = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Hi.getValueType(), Hi, 2476 DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(), 2477 ExcessBits))); 2478 } 2479 } 2480 2481 void DAGTypeLegalizer::ExpandIntRes_SREM(SDNode *N, 2482 SDValue &Lo, SDValue &Hi) { 2483 EVT VT = N->getValueType(0); 2484 SDLoc dl(N); 2485 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) }; 2486 2487 if (TLI.getOperationAction(ISD::SDIVREM, VT) == TargetLowering::Custom) { 2488 SDValue Res = DAG.getNode(ISD::SDIVREM, dl, DAG.getVTList(VT, VT), Ops); 2489 SplitInteger(Res.getValue(1), Lo, Hi); 2490 return; 2491 } 2492 2493 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 2494 if (VT == MVT::i16) 2495 LC = RTLIB::SREM_I16; 2496 else if (VT == MVT::i32) 2497 LC = RTLIB::SREM_I32; 2498 else if (VT == MVT::i64) 2499 LC = RTLIB::SREM_I64; 2500 else if (VT == MVT::i128) 2501 LC = RTLIB::SREM_I128; 2502 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported SREM!"); 2503 2504 SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, true, dl).first, Lo, Hi); 2505 } 2506 2507 void DAGTypeLegalizer::ExpandIntRes_TRUNCATE(SDNode *N, 2508 SDValue &Lo, SDValue &Hi) { 2509 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 2510 SDLoc dl(N); 2511 Lo = DAG.getNode(ISD::TRUNCATE, dl, NVT, N->getOperand(0)); 2512 Hi = DAG.getNode(ISD::SRL, dl, N->getOperand(0).getValueType(), 2513 N->getOperand(0), 2514 DAG.getConstant(NVT.getSizeInBits(), dl, 2515 TLI.getPointerTy(DAG.getDataLayout()))); 2516 Hi = DAG.getNode(ISD::TRUNCATE, dl, NVT, Hi); 2517 } 2518 2519 void DAGTypeLegalizer::ExpandIntRes_UADDSUBO(SDNode *N, 2520 SDValue &Lo, SDValue &Hi) { 2521 SDValue LHS = N->getOperand(0); 2522 SDValue RHS = N->getOperand(1); 2523 SDLoc dl(N); 2524 2525 // Expand the result by simply replacing it with the equivalent 2526 // non-overflow-checking operation. 2527 SDValue Sum = DAG.getNode(N->getOpcode() == ISD::UADDO ? 2528 ISD::ADD : ISD::SUB, dl, LHS.getValueType(), 2529 LHS, RHS); 2530 SplitInteger(Sum, Lo, Hi); 2531 2532 // Calculate the overflow: addition overflows iff a + b < a, and subtraction 2533 // overflows iff a - b > a. 2534 SDValue Ofl = DAG.getSetCC(dl, N->getValueType(1), Sum, LHS, 2535 N->getOpcode () == ISD::UADDO ? 2536 ISD::SETULT : ISD::SETUGT); 2537 2538 // Use the calculated overflow everywhere. 2539 ReplaceValueWith(SDValue(N, 1), Ofl); 2540 } 2541 2542 void DAGTypeLegalizer::ExpandIntRes_XMULO(SDNode *N, 2543 SDValue &Lo, SDValue &Hi) { 2544 EVT VT = N->getValueType(0); 2545 SDLoc dl(N); 2546 2547 // A divide for UMULO should be faster than a function call. 2548 if (N->getOpcode() == ISD::UMULO) { 2549 SDValue LHS = N->getOperand(0), RHS = N->getOperand(1); 2550 2551 SDValue MUL = DAG.getNode(ISD::MUL, dl, LHS.getValueType(), LHS, RHS); 2552 SplitInteger(MUL, Lo, Hi); 2553 2554 // A divide for UMULO will be faster than a function call. Select to 2555 // make sure we aren't using 0. 2556 SDValue isZero = DAG.getSetCC(dl, getSetCCResultType(VT), 2557 RHS, DAG.getConstant(0, dl, VT), ISD::SETEQ); 2558 SDValue NotZero = DAG.getSelect(dl, VT, isZero, 2559 DAG.getConstant(1, dl, VT), RHS); 2560 SDValue DIV = DAG.getNode(ISD::UDIV, dl, VT, MUL, NotZero); 2561 SDValue Overflow = DAG.getSetCC(dl, N->getValueType(1), DIV, LHS, 2562 ISD::SETNE); 2563 Overflow = DAG.getSelect(dl, N->getValueType(1), isZero, 2564 DAG.getConstant(0, dl, N->getValueType(1)), 2565 Overflow); 2566 ReplaceValueWith(SDValue(N, 1), Overflow); 2567 return; 2568 } 2569 2570 Type *RetTy = VT.getTypeForEVT(*DAG.getContext()); 2571 EVT PtrVT = TLI.getPointerTy(DAG.getDataLayout()); 2572 Type *PtrTy = PtrVT.getTypeForEVT(*DAG.getContext()); 2573 2574 // Replace this with a libcall that will check overflow. 2575 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 2576 if (VT == MVT::i32) 2577 LC = RTLIB::MULO_I32; 2578 else if (VT == MVT::i64) 2579 LC = RTLIB::MULO_I64; 2580 else if (VT == MVT::i128) 2581 LC = RTLIB::MULO_I128; 2582 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported XMULO!"); 2583 2584 SDValue Temp = DAG.CreateStackTemporary(PtrVT); 2585 // Temporary for the overflow value, default it to zero. 2586 SDValue Chain = DAG.getStore(DAG.getEntryNode(), dl, 2587 DAG.getConstant(0, dl, PtrVT), Temp, 2588 MachinePointerInfo(), false, false, 0); 2589 2590 TargetLowering::ArgListTy Args; 2591 TargetLowering::ArgListEntry Entry; 2592 for (const SDValue &Op : N->op_values()) { 2593 EVT ArgVT = Op.getValueType(); 2594 Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext()); 2595 Entry.Node = Op; 2596 Entry.Ty = ArgTy; 2597 Entry.isSExt = true; 2598 Entry.isZExt = false; 2599 Args.push_back(Entry); 2600 } 2601 2602 // Also pass the address of the overflow check. 2603 Entry.Node = Temp; 2604 Entry.Ty = PtrTy->getPointerTo(); 2605 Entry.isSExt = true; 2606 Entry.isZExt = false; 2607 Args.push_back(Entry); 2608 2609 SDValue Func = DAG.getExternalSymbol(TLI.getLibcallName(LC), PtrVT); 2610 2611 TargetLowering::CallLoweringInfo CLI(DAG); 2612 CLI.setDebugLoc(dl).setChain(Chain) 2613 .setCallee(TLI.getLibcallCallingConv(LC), RetTy, Func, std::move(Args)) 2614 .setSExtResult(); 2615 2616 std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI); 2617 2618 SplitInteger(CallInfo.first, Lo, Hi); 2619 SDValue Temp2 = DAG.getLoad(PtrVT, dl, CallInfo.second, Temp, 2620 MachinePointerInfo(), false, false, false, 0); 2621 SDValue Ofl = DAG.getSetCC(dl, N->getValueType(1), Temp2, 2622 DAG.getConstant(0, dl, PtrVT), 2623 ISD::SETNE); 2624 // Use the overflow from the libcall everywhere. 2625 ReplaceValueWith(SDValue(N, 1), Ofl); 2626 } 2627 2628 void DAGTypeLegalizer::ExpandIntRes_UDIV(SDNode *N, 2629 SDValue &Lo, SDValue &Hi) { 2630 EVT VT = N->getValueType(0); 2631 SDLoc dl(N); 2632 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) }; 2633 2634 if (TLI.getOperationAction(ISD::UDIVREM, VT) == TargetLowering::Custom) { 2635 SDValue Res = DAG.getNode(ISD::UDIVREM, dl, DAG.getVTList(VT, VT), Ops); 2636 SplitInteger(Res.getValue(0), Lo, Hi); 2637 return; 2638 } 2639 2640 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 2641 if (VT == MVT::i16) 2642 LC = RTLIB::UDIV_I16; 2643 else if (VT == MVT::i32) 2644 LC = RTLIB::UDIV_I32; 2645 else if (VT == MVT::i64) 2646 LC = RTLIB::UDIV_I64; 2647 else if (VT == MVT::i128) 2648 LC = RTLIB::UDIV_I128; 2649 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported UDIV!"); 2650 2651 SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, false, dl).first, Lo, Hi); 2652 } 2653 2654 void DAGTypeLegalizer::ExpandIntRes_UREM(SDNode *N, 2655 SDValue &Lo, SDValue &Hi) { 2656 EVT VT = N->getValueType(0); 2657 SDLoc dl(N); 2658 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) }; 2659 2660 if (TLI.getOperationAction(ISD::UDIVREM, VT) == TargetLowering::Custom) { 2661 SDValue Res = DAG.getNode(ISD::UDIVREM, dl, DAG.getVTList(VT, VT), Ops); 2662 SplitInteger(Res.getValue(1), Lo, Hi); 2663 return; 2664 } 2665 2666 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 2667 if (VT == MVT::i16) 2668 LC = RTLIB::UREM_I16; 2669 else if (VT == MVT::i32) 2670 LC = RTLIB::UREM_I32; 2671 else if (VT == MVT::i64) 2672 LC = RTLIB::UREM_I64; 2673 else if (VT == MVT::i128) 2674 LC = RTLIB::UREM_I128; 2675 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported UREM!"); 2676 2677 SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, false, dl).first, Lo, Hi); 2678 } 2679 2680 void DAGTypeLegalizer::ExpandIntRes_ZERO_EXTEND(SDNode *N, 2681 SDValue &Lo, SDValue &Hi) { 2682 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 2683 SDLoc dl(N); 2684 SDValue Op = N->getOperand(0); 2685 if (Op.getValueType().bitsLE(NVT)) { 2686 // The low part is zero extension of the input (degenerates to a copy). 2687 Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, N->getOperand(0)); 2688 Hi = DAG.getConstant(0, dl, NVT); // The high part is just a zero. 2689 } else { 2690 // For example, extension of an i48 to an i64. The operand type necessarily 2691 // promotes to the result type, so will end up being expanded too. 2692 assert(getTypeAction(Op.getValueType()) == 2693 TargetLowering::TypePromoteInteger && 2694 "Only know how to promote this result!"); 2695 SDValue Res = GetPromotedInteger(Op); 2696 assert(Res.getValueType() == N->getValueType(0) && 2697 "Operand over promoted?"); 2698 // Split the promoted operand. This will simplify when it is expanded. 2699 SplitInteger(Res, Lo, Hi); 2700 unsigned ExcessBits = 2701 Op.getValueType().getSizeInBits() - NVT.getSizeInBits(); 2702 Hi = DAG.getZeroExtendInReg(Hi, dl, 2703 EVT::getIntegerVT(*DAG.getContext(), 2704 ExcessBits)); 2705 } 2706 } 2707 2708 void DAGTypeLegalizer::ExpandIntRes_ATOMIC_LOAD(SDNode *N, 2709 SDValue &Lo, SDValue &Hi) { 2710 SDLoc dl(N); 2711 EVT VT = cast<AtomicSDNode>(N)->getMemoryVT(); 2712 SDVTList VTs = DAG.getVTList(VT, MVT::i1, MVT::Other); 2713 SDValue Zero = DAG.getConstant(0, dl, VT); 2714 SDValue Swap = DAG.getAtomicCmpSwap( 2715 ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, dl, 2716 cast<AtomicSDNode>(N)->getMemoryVT(), VTs, N->getOperand(0), 2717 N->getOperand(1), Zero, Zero, cast<AtomicSDNode>(N)->getMemOperand(), 2718 cast<AtomicSDNode>(N)->getOrdering(), 2719 cast<AtomicSDNode>(N)->getOrdering(), 2720 cast<AtomicSDNode>(N)->getSynchScope()); 2721 2722 ReplaceValueWith(SDValue(N, 0), Swap.getValue(0)); 2723 ReplaceValueWith(SDValue(N, 1), Swap.getValue(2)); 2724 } 2725 2726 //===----------------------------------------------------------------------===// 2727 // Integer Operand Expansion 2728 //===----------------------------------------------------------------------===// 2729 2730 /// ExpandIntegerOperand - This method is called when the specified operand of 2731 /// the specified node is found to need expansion. At this point, all of the 2732 /// result types of the node are known to be legal, but other operands of the 2733 /// node may need promotion or expansion as well as the specified one. 2734 bool DAGTypeLegalizer::ExpandIntegerOperand(SDNode *N, unsigned OpNo) { 2735 DEBUG(dbgs() << "Expand integer operand: "; N->dump(&DAG); dbgs() << "\n"); 2736 SDValue Res = SDValue(); 2737 2738 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false)) 2739 return false; 2740 2741 switch (N->getOpcode()) { 2742 default: 2743 #ifndef NDEBUG 2744 dbgs() << "ExpandIntegerOperand Op #" << OpNo << ": "; 2745 N->dump(&DAG); dbgs() << "\n"; 2746 #endif 2747 llvm_unreachable("Do not know how to expand this operator's operand!"); 2748 2749 case ISD::BITCAST: Res = ExpandOp_BITCAST(N); break; 2750 case ISD::BR_CC: Res = ExpandIntOp_BR_CC(N); break; 2751 case ISD::BUILD_VECTOR: Res = ExpandOp_BUILD_VECTOR(N); break; 2752 case ISD::EXTRACT_ELEMENT: Res = ExpandOp_EXTRACT_ELEMENT(N); break; 2753 case ISD::INSERT_VECTOR_ELT: Res = ExpandOp_INSERT_VECTOR_ELT(N); break; 2754 case ISD::SCALAR_TO_VECTOR: Res = ExpandOp_SCALAR_TO_VECTOR(N); break; 2755 case ISD::SELECT_CC: Res = ExpandIntOp_SELECT_CC(N); break; 2756 case ISD::SETCC: Res = ExpandIntOp_SETCC(N); break; 2757 case ISD::SETCCE: Res = ExpandIntOp_SETCCE(N); break; 2758 case ISD::SINT_TO_FP: Res = ExpandIntOp_SINT_TO_FP(N); break; 2759 case ISD::STORE: Res = ExpandIntOp_STORE(cast<StoreSDNode>(N), OpNo); break; 2760 case ISD::TRUNCATE: Res = ExpandIntOp_TRUNCATE(N); break; 2761 case ISD::UINT_TO_FP: Res = ExpandIntOp_UINT_TO_FP(N); break; 2762 2763 case ISD::SHL: 2764 case ISD::SRA: 2765 case ISD::SRL: 2766 case ISD::ROTL: 2767 case ISD::ROTR: Res = ExpandIntOp_Shift(N); break; 2768 case ISD::RETURNADDR: 2769 case ISD::FRAMEADDR: Res = ExpandIntOp_RETURNADDR(N); break; 2770 2771 case ISD::ATOMIC_STORE: Res = ExpandIntOp_ATOMIC_STORE(N); break; 2772 } 2773 2774 // If the result is null, the sub-method took care of registering results etc. 2775 if (!Res.getNode()) return false; 2776 2777 // If the result is N, the sub-method updated N in place. Tell the legalizer 2778 // core about this. 2779 if (Res.getNode() == N) 2780 return true; 2781 2782 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 && 2783 "Invalid operand expansion"); 2784 2785 ReplaceValueWith(SDValue(N, 0), Res); 2786 return false; 2787 } 2788 2789 /// IntegerExpandSetCCOperands - Expand the operands of a comparison. This code 2790 /// is shared among BR_CC, SELECT_CC, and SETCC handlers. 2791 void DAGTypeLegalizer::IntegerExpandSetCCOperands(SDValue &NewLHS, 2792 SDValue &NewRHS, 2793 ISD::CondCode &CCCode, 2794 const SDLoc &dl) { 2795 SDValue LHSLo, LHSHi, RHSLo, RHSHi; 2796 GetExpandedInteger(NewLHS, LHSLo, LHSHi); 2797 GetExpandedInteger(NewRHS, RHSLo, RHSHi); 2798 2799 if (CCCode == ISD::SETEQ || CCCode == ISD::SETNE) { 2800 if (RHSLo == RHSHi) { 2801 if (ConstantSDNode *RHSCST = dyn_cast<ConstantSDNode>(RHSLo)) { 2802 if (RHSCST->isAllOnesValue()) { 2803 // Equality comparison to -1. 2804 NewLHS = DAG.getNode(ISD::AND, dl, 2805 LHSLo.getValueType(), LHSLo, LHSHi); 2806 NewRHS = RHSLo; 2807 return; 2808 } 2809 } 2810 } 2811 2812 NewLHS = DAG.getNode(ISD::XOR, dl, LHSLo.getValueType(), LHSLo, RHSLo); 2813 NewRHS = DAG.getNode(ISD::XOR, dl, LHSLo.getValueType(), LHSHi, RHSHi); 2814 NewLHS = DAG.getNode(ISD::OR, dl, NewLHS.getValueType(), NewLHS, NewRHS); 2815 NewRHS = DAG.getConstant(0, dl, NewLHS.getValueType()); 2816 return; 2817 } 2818 2819 // If this is a comparison of the sign bit, just look at the top part. 2820 // X > -1, x < 0 2821 if (ConstantSDNode *CST = dyn_cast<ConstantSDNode>(NewRHS)) 2822 if ((CCCode == ISD::SETLT && CST->isNullValue()) || // X < 0 2823 (CCCode == ISD::SETGT && CST->isAllOnesValue())) { // X > -1 2824 NewLHS = LHSHi; 2825 NewRHS = RHSHi; 2826 return; 2827 } 2828 2829 // FIXME: This generated code sucks. 2830 ISD::CondCode LowCC; 2831 switch (CCCode) { 2832 default: llvm_unreachable("Unknown integer setcc!"); 2833 case ISD::SETLT: 2834 case ISD::SETULT: LowCC = ISD::SETULT; break; 2835 case ISD::SETGT: 2836 case ISD::SETUGT: LowCC = ISD::SETUGT; break; 2837 case ISD::SETLE: 2838 case ISD::SETULE: LowCC = ISD::SETULE; break; 2839 case ISD::SETGE: 2840 case ISD::SETUGE: LowCC = ISD::SETUGE; break; 2841 } 2842 2843 // Tmp1 = lo(op1) < lo(op2) // Always unsigned comparison 2844 // Tmp2 = hi(op1) < hi(op2) // Signedness depends on operands 2845 // dest = hi(op1) == hi(op2) ? Tmp1 : Tmp2; 2846 2847 // NOTE: on targets without efficient SELECT of bools, we can always use 2848 // this identity: (B1 ? B2 : B3) --> (B1 & B2)|(!B1&B3) 2849 TargetLowering::DAGCombinerInfo DagCombineInfo(DAG, AfterLegalizeTypes, true, 2850 nullptr); 2851 SDValue Tmp1, Tmp2; 2852 if (TLI.isTypeLegal(LHSLo.getValueType()) && 2853 TLI.isTypeLegal(RHSLo.getValueType())) 2854 Tmp1 = TLI.SimplifySetCC(getSetCCResultType(LHSLo.getValueType()), 2855 LHSLo, RHSLo, LowCC, false, DagCombineInfo, dl); 2856 if (!Tmp1.getNode()) 2857 Tmp1 = DAG.getSetCC(dl, getSetCCResultType(LHSLo.getValueType()), 2858 LHSLo, RHSLo, LowCC); 2859 if (TLI.isTypeLegal(LHSHi.getValueType()) && 2860 TLI.isTypeLegal(RHSHi.getValueType())) 2861 Tmp2 = TLI.SimplifySetCC(getSetCCResultType(LHSHi.getValueType()), 2862 LHSHi, RHSHi, CCCode, false, DagCombineInfo, dl); 2863 if (!Tmp2.getNode()) 2864 Tmp2 = DAG.getNode(ISD::SETCC, dl, 2865 getSetCCResultType(LHSHi.getValueType()), 2866 LHSHi, RHSHi, DAG.getCondCode(CCCode)); 2867 2868 ConstantSDNode *Tmp1C = dyn_cast<ConstantSDNode>(Tmp1.getNode()); 2869 ConstantSDNode *Tmp2C = dyn_cast<ConstantSDNode>(Tmp2.getNode()); 2870 if ((Tmp1C && Tmp1C->isNullValue()) || 2871 (Tmp2C && Tmp2C->isNullValue() && 2872 (CCCode == ISD::SETLE || CCCode == ISD::SETGE || 2873 CCCode == ISD::SETUGE || CCCode == ISD::SETULE)) || 2874 (Tmp2C && Tmp2C->getAPIntValue() == 1 && 2875 (CCCode == ISD::SETLT || CCCode == ISD::SETGT || 2876 CCCode == ISD::SETUGT || CCCode == ISD::SETULT))) { 2877 // low part is known false, returns high part. 2878 // For LE / GE, if high part is known false, ignore the low part. 2879 // For LT / GT, if high part is known true, ignore the low part. 2880 NewLHS = Tmp2; 2881 NewRHS = SDValue(); 2882 return; 2883 } 2884 2885 if (LHSHi == RHSHi) { 2886 // Comparing the low bits is enough. 2887 NewLHS = Tmp1; 2888 NewRHS = SDValue(); 2889 return; 2890 } 2891 2892 // Lower with SETCCE if the target supports it. 2893 // FIXME: Make all targets support this, then remove the other lowering. 2894 if (TLI.getOperationAction( 2895 ISD::SETCCE, 2896 TLI.getTypeToExpandTo(*DAG.getContext(), LHSLo.getValueType())) == 2897 TargetLowering::Custom) { 2898 // SETCCE can detect < and >= directly. For > and <=, flip operands and 2899 // condition code. 2900 bool FlipOperands = false; 2901 switch (CCCode) { 2902 case ISD::SETGT: CCCode = ISD::SETLT; FlipOperands = true; break; 2903 case ISD::SETUGT: CCCode = ISD::SETULT; FlipOperands = true; break; 2904 case ISD::SETLE: CCCode = ISD::SETGE; FlipOperands = true; break; 2905 case ISD::SETULE: CCCode = ISD::SETUGE; FlipOperands = true; break; 2906 default: break; 2907 } 2908 if (FlipOperands) { 2909 std::swap(LHSLo, RHSLo); 2910 std::swap(LHSHi, RHSHi); 2911 } 2912 // Perform a wide subtraction, feeding the carry from the low part into 2913 // SETCCE. The SETCCE operation is essentially looking at the high part of 2914 // the result of LHS - RHS. It is negative iff LHS < RHS. It is zero or 2915 // positive iff LHS >= RHS. 2916 SDVTList VTList = DAG.getVTList(LHSLo.getValueType(), MVT::Glue); 2917 SDValue LowCmp = DAG.getNode(ISD::SUBC, dl, VTList, LHSLo, RHSLo); 2918 SDValue Res = 2919 DAG.getNode(ISD::SETCCE, dl, getSetCCResultType(LHSLo.getValueType()), 2920 LHSHi, RHSHi, LowCmp.getValue(1), DAG.getCondCode(CCCode)); 2921 NewLHS = Res; 2922 NewRHS = SDValue(); 2923 return; 2924 } 2925 2926 NewLHS = TLI.SimplifySetCC(getSetCCResultType(LHSHi.getValueType()), 2927 LHSHi, RHSHi, ISD::SETEQ, false, 2928 DagCombineInfo, dl); 2929 if (!NewLHS.getNode()) 2930 NewLHS = DAG.getSetCC(dl, getSetCCResultType(LHSHi.getValueType()), 2931 LHSHi, RHSHi, ISD::SETEQ); 2932 NewLHS = DAG.getSelect(dl, Tmp1.getValueType(), 2933 NewLHS, Tmp1, Tmp2); 2934 NewRHS = SDValue(); 2935 } 2936 2937 SDValue DAGTypeLegalizer::ExpandIntOp_BR_CC(SDNode *N) { 2938 SDValue NewLHS = N->getOperand(2), NewRHS = N->getOperand(3); 2939 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(1))->get(); 2940 IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N)); 2941 2942 // If ExpandSetCCOperands returned a scalar, we need to compare the result 2943 // against zero to select between true and false values. 2944 if (!NewRHS.getNode()) { 2945 NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType()); 2946 CCCode = ISD::SETNE; 2947 } 2948 2949 // Update N to have the operands specified. 2950 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), 2951 DAG.getCondCode(CCCode), NewLHS, NewRHS, 2952 N->getOperand(4)), 0); 2953 } 2954 2955 SDValue DAGTypeLegalizer::ExpandIntOp_SELECT_CC(SDNode *N) { 2956 SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1); 2957 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(4))->get(); 2958 IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N)); 2959 2960 // If ExpandSetCCOperands returned a scalar, we need to compare the result 2961 // against zero to select between true and false values. 2962 if (!NewRHS.getNode()) { 2963 NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType()); 2964 CCCode = ISD::SETNE; 2965 } 2966 2967 // Update N to have the operands specified. 2968 return SDValue(DAG.UpdateNodeOperands(N, NewLHS, NewRHS, 2969 N->getOperand(2), N->getOperand(3), 2970 DAG.getCondCode(CCCode)), 0); 2971 } 2972 2973 SDValue DAGTypeLegalizer::ExpandIntOp_SETCC(SDNode *N) { 2974 SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1); 2975 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(2))->get(); 2976 IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N)); 2977 2978 // If ExpandSetCCOperands returned a scalar, use it. 2979 if (!NewRHS.getNode()) { 2980 assert(NewLHS.getValueType() == N->getValueType(0) && 2981 "Unexpected setcc expansion!"); 2982 return NewLHS; 2983 } 2984 2985 // Otherwise, update N to have the operands specified. 2986 return SDValue(DAG.UpdateNodeOperands(N, NewLHS, NewRHS, 2987 DAG.getCondCode(CCCode)), 0); 2988 } 2989 2990 SDValue DAGTypeLegalizer::ExpandIntOp_SETCCE(SDNode *N) { 2991 SDValue LHS = N->getOperand(0); 2992 SDValue RHS = N->getOperand(1); 2993 SDValue Carry = N->getOperand(2); 2994 SDValue Cond = N->getOperand(3); 2995 SDLoc dl = SDLoc(N); 2996 2997 SDValue LHSLo, LHSHi, RHSLo, RHSHi; 2998 GetExpandedInteger(LHS, LHSLo, LHSHi); 2999 GetExpandedInteger(RHS, RHSLo, RHSHi); 3000 3001 // Expand to a SUBE for the low part and a smaller SETCCE for the high. 3002 SDVTList VTList = DAG.getVTList(LHSLo.getValueType(), MVT::Glue); 3003 SDValue LowCmp = DAG.getNode(ISD::SUBE, dl, VTList, LHSLo, RHSLo, Carry); 3004 return DAG.getNode(ISD::SETCCE, dl, N->getValueType(0), LHSHi, RHSHi, 3005 LowCmp.getValue(1), Cond); 3006 } 3007 3008 SDValue DAGTypeLegalizer::ExpandIntOp_Shift(SDNode *N) { 3009 // The value being shifted is legal, but the shift amount is too big. 3010 // It follows that either the result of the shift is undefined, or the 3011 // upper half of the shift amount is zero. Just use the lower half. 3012 SDValue Lo, Hi; 3013 GetExpandedInteger(N->getOperand(1), Lo, Hi); 3014 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), Lo), 0); 3015 } 3016 3017 SDValue DAGTypeLegalizer::ExpandIntOp_RETURNADDR(SDNode *N) { 3018 // The argument of RETURNADDR / FRAMEADDR builtin is 32 bit contant. This 3019 // surely makes pretty nice problems on 8/16 bit targets. Just truncate this 3020 // constant to valid type. 3021 SDValue Lo, Hi; 3022 GetExpandedInteger(N->getOperand(0), Lo, Hi); 3023 return SDValue(DAG.UpdateNodeOperands(N, Lo), 0); 3024 } 3025 3026 SDValue DAGTypeLegalizer::ExpandIntOp_SINT_TO_FP(SDNode *N) { 3027 SDValue Op = N->getOperand(0); 3028 EVT DstVT = N->getValueType(0); 3029 RTLIB::Libcall LC = RTLIB::getSINTTOFP(Op.getValueType(), DstVT); 3030 assert(LC != RTLIB::UNKNOWN_LIBCALL && 3031 "Don't know how to expand this SINT_TO_FP!"); 3032 return TLI.makeLibCall(DAG, LC, DstVT, Op, true, SDLoc(N)).first; 3033 } 3034 3035 SDValue DAGTypeLegalizer::ExpandIntOp_STORE(StoreSDNode *N, unsigned OpNo) { 3036 if (ISD::isNormalStore(N)) 3037 return ExpandOp_NormalStore(N, OpNo); 3038 3039 assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!"); 3040 assert(OpNo == 1 && "Can only expand the stored value so far"); 3041 3042 EVT VT = N->getOperand(1).getValueType(); 3043 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 3044 SDValue Ch = N->getChain(); 3045 SDValue Ptr = N->getBasePtr(); 3046 unsigned Alignment = N->getAlignment(); 3047 bool isVolatile = N->isVolatile(); 3048 bool isNonTemporal = N->isNonTemporal(); 3049 AAMDNodes AAInfo = N->getAAInfo(); 3050 SDLoc dl(N); 3051 SDValue Lo, Hi; 3052 3053 assert(NVT.isByteSized() && "Expanded type not byte sized!"); 3054 3055 if (N->getMemoryVT().bitsLE(NVT)) { 3056 GetExpandedInteger(N->getValue(), Lo, Hi); 3057 return DAG.getTruncStore(Ch, dl, Lo, Ptr, N->getPointerInfo(), 3058 N->getMemoryVT(), isVolatile, isNonTemporal, 3059 Alignment, AAInfo); 3060 } 3061 3062 if (DAG.getDataLayout().isLittleEndian()) { 3063 // Little-endian - low bits are at low addresses. 3064 GetExpandedInteger(N->getValue(), Lo, Hi); 3065 3066 Lo = DAG.getStore(Ch, dl, Lo, Ptr, N->getPointerInfo(), 3067 isVolatile, isNonTemporal, Alignment, AAInfo); 3068 3069 unsigned ExcessBits = 3070 N->getMemoryVT().getSizeInBits() - NVT.getSizeInBits(); 3071 EVT NEVT = EVT::getIntegerVT(*DAG.getContext(), ExcessBits); 3072 3073 // Increment the pointer to the other half. 3074 unsigned IncrementSize = NVT.getSizeInBits()/8; 3075 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, 3076 DAG.getConstant(IncrementSize, dl, Ptr.getValueType())); 3077 Hi = DAG.getTruncStore(Ch, dl, Hi, Ptr, 3078 N->getPointerInfo().getWithOffset(IncrementSize), 3079 NEVT, isVolatile, isNonTemporal, 3080 MinAlign(Alignment, IncrementSize), AAInfo); 3081 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi); 3082 } 3083 3084 // Big-endian - high bits are at low addresses. Favor aligned stores at 3085 // the cost of some bit-fiddling. 3086 GetExpandedInteger(N->getValue(), Lo, Hi); 3087 3088 EVT ExtVT = N->getMemoryVT(); 3089 unsigned EBytes = ExtVT.getStoreSize(); 3090 unsigned IncrementSize = NVT.getSizeInBits()/8; 3091 unsigned ExcessBits = (EBytes - IncrementSize)*8; 3092 EVT HiVT = EVT::getIntegerVT(*DAG.getContext(), 3093 ExtVT.getSizeInBits() - ExcessBits); 3094 3095 if (ExcessBits < NVT.getSizeInBits()) { 3096 // Transfer high bits from the top of Lo to the bottom of Hi. 3097 Hi = DAG.getNode(ISD::SHL, dl, NVT, Hi, 3098 DAG.getConstant(NVT.getSizeInBits() - ExcessBits, dl, 3099 TLI.getPointerTy(DAG.getDataLayout()))); 3100 Hi = DAG.getNode( 3101 ISD::OR, dl, NVT, Hi, 3102 DAG.getNode(ISD::SRL, dl, NVT, Lo, 3103 DAG.getConstant(ExcessBits, dl, 3104 TLI.getPointerTy(DAG.getDataLayout())))); 3105 } 3106 3107 // Store both the high bits and maybe some of the low bits. 3108 Hi = DAG.getTruncStore(Ch, dl, Hi, Ptr, N->getPointerInfo(), 3109 HiVT, isVolatile, isNonTemporal, Alignment, AAInfo); 3110 3111 // Increment the pointer to the other half. 3112 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, 3113 DAG.getConstant(IncrementSize, dl, Ptr.getValueType())); 3114 // Store the lowest ExcessBits bits in the second half. 3115 Lo = DAG.getTruncStore(Ch, dl, Lo, Ptr, 3116 N->getPointerInfo().getWithOffset(IncrementSize), 3117 EVT::getIntegerVT(*DAG.getContext(), ExcessBits), 3118 isVolatile, isNonTemporal, 3119 MinAlign(Alignment, IncrementSize), AAInfo); 3120 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi); 3121 } 3122 3123 SDValue DAGTypeLegalizer::ExpandIntOp_TRUNCATE(SDNode *N) { 3124 SDValue InL, InH; 3125 GetExpandedInteger(N->getOperand(0), InL, InH); 3126 // Just truncate the low part of the source. 3127 return DAG.getNode(ISD::TRUNCATE, SDLoc(N), N->getValueType(0), InL); 3128 } 3129 3130 SDValue DAGTypeLegalizer::ExpandIntOp_UINT_TO_FP(SDNode *N) { 3131 SDValue Op = N->getOperand(0); 3132 EVT SrcVT = Op.getValueType(); 3133 EVT DstVT = N->getValueType(0); 3134 SDLoc dl(N); 3135 3136 // The following optimization is valid only if every value in SrcVT (when 3137 // treated as signed) is representable in DstVT. Check that the mantissa 3138 // size of DstVT is >= than the number of bits in SrcVT -1. 3139 const fltSemantics &sem = DAG.EVTToAPFloatSemantics(DstVT); 3140 if (APFloat::semanticsPrecision(sem) >= SrcVT.getSizeInBits()-1 && 3141 TLI.getOperationAction(ISD::SINT_TO_FP, SrcVT) == TargetLowering::Custom){ 3142 // Do a signed conversion then adjust the result. 3143 SDValue SignedConv = DAG.getNode(ISD::SINT_TO_FP, dl, DstVT, Op); 3144 SignedConv = TLI.LowerOperation(SignedConv, DAG); 3145 3146 // The result of the signed conversion needs adjusting if the 'sign bit' of 3147 // the incoming integer was set. To handle this, we dynamically test to see 3148 // if it is set, and, if so, add a fudge factor. 3149 3150 const uint64_t F32TwoE32 = 0x4F800000ULL; 3151 const uint64_t F32TwoE64 = 0x5F800000ULL; 3152 const uint64_t F32TwoE128 = 0x7F800000ULL; 3153 3154 APInt FF(32, 0); 3155 if (SrcVT == MVT::i32) 3156 FF = APInt(32, F32TwoE32); 3157 else if (SrcVT == MVT::i64) 3158 FF = APInt(32, F32TwoE64); 3159 else if (SrcVT == MVT::i128) 3160 FF = APInt(32, F32TwoE128); 3161 else 3162 llvm_unreachable("Unsupported UINT_TO_FP!"); 3163 3164 // Check whether the sign bit is set. 3165 SDValue Lo, Hi; 3166 GetExpandedInteger(Op, Lo, Hi); 3167 SDValue SignSet = DAG.getSetCC(dl, 3168 getSetCCResultType(Hi.getValueType()), 3169 Hi, 3170 DAG.getConstant(0, dl, Hi.getValueType()), 3171 ISD::SETLT); 3172 3173 // Build a 64 bit pair (0, FF) in the constant pool, with FF in the lo bits. 3174 SDValue FudgePtr = 3175 DAG.getConstantPool(ConstantInt::get(*DAG.getContext(), FF.zext(64)), 3176 TLI.getPointerTy(DAG.getDataLayout())); 3177 3178 // Get a pointer to FF if the sign bit was set, or to 0 otherwise. 3179 SDValue Zero = DAG.getIntPtrConstant(0, dl); 3180 SDValue Four = DAG.getIntPtrConstant(4, dl); 3181 if (DAG.getDataLayout().isBigEndian()) 3182 std::swap(Zero, Four); 3183 SDValue Offset = DAG.getSelect(dl, Zero.getValueType(), SignSet, 3184 Zero, Four); 3185 unsigned Alignment = cast<ConstantPoolSDNode>(FudgePtr)->getAlignment(); 3186 FudgePtr = DAG.getNode(ISD::ADD, dl, FudgePtr.getValueType(), 3187 FudgePtr, Offset); 3188 Alignment = std::min(Alignment, 4u); 3189 3190 // Load the value out, extending it from f32 to the destination float type. 3191 // FIXME: Avoid the extend by constructing the right constant pool? 3192 SDValue Fudge = DAG.getExtLoad( 3193 ISD::EXTLOAD, dl, DstVT, DAG.getEntryNode(), FudgePtr, 3194 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), MVT::f32, 3195 false, false, false, Alignment); 3196 return DAG.getNode(ISD::FADD, dl, DstVT, SignedConv, Fudge); 3197 } 3198 3199 // Otherwise, use a libcall. 3200 RTLIB::Libcall LC = RTLIB::getUINTTOFP(SrcVT, DstVT); 3201 assert(LC != RTLIB::UNKNOWN_LIBCALL && 3202 "Don't know how to expand this UINT_TO_FP!"); 3203 return TLI.makeLibCall(DAG, LC, DstVT, Op, true, dl).first; 3204 } 3205 3206 SDValue DAGTypeLegalizer::ExpandIntOp_ATOMIC_STORE(SDNode *N) { 3207 SDLoc dl(N); 3208 SDValue Swap = DAG.getAtomic(ISD::ATOMIC_SWAP, dl, 3209 cast<AtomicSDNode>(N)->getMemoryVT(), 3210 N->getOperand(0), 3211 N->getOperand(1), N->getOperand(2), 3212 cast<AtomicSDNode>(N)->getMemOperand(), 3213 cast<AtomicSDNode>(N)->getOrdering(), 3214 cast<AtomicSDNode>(N)->getSynchScope()); 3215 return Swap.getValue(1); 3216 } 3217 3218 3219 SDValue DAGTypeLegalizer::PromoteIntRes_EXTRACT_SUBVECTOR(SDNode *N) { 3220 SDValue InOp0 = N->getOperand(0); 3221 EVT InVT = InOp0.getValueType(); 3222 3223 EVT OutVT = N->getValueType(0); 3224 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT); 3225 assert(NOutVT.isVector() && "This type must be promoted to a vector type"); 3226 unsigned OutNumElems = OutVT.getVectorNumElements(); 3227 EVT NOutVTElem = NOutVT.getVectorElementType(); 3228 3229 SDLoc dl(N); 3230 SDValue BaseIdx = N->getOperand(1); 3231 3232 SmallVector<SDValue, 8> Ops; 3233 Ops.reserve(OutNumElems); 3234 for (unsigned i = 0; i != OutNumElems; ++i) { 3235 3236 // Extract the element from the original vector. 3237 SDValue Index = DAG.getNode(ISD::ADD, dl, BaseIdx.getValueType(), 3238 BaseIdx, DAG.getConstant(i, dl, BaseIdx.getValueType())); 3239 SDValue Ext = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, 3240 InVT.getVectorElementType(), N->getOperand(0), Index); 3241 3242 SDValue Op = DAG.getNode(ISD::ANY_EXTEND, dl, NOutVTElem, Ext); 3243 // Insert the converted element to the new vector. 3244 Ops.push_back(Op); 3245 } 3246 3247 return DAG.getNode(ISD::BUILD_VECTOR, dl, NOutVT, Ops); 3248 } 3249 3250 3251 SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_SHUFFLE(SDNode *N) { 3252 ShuffleVectorSDNode *SV = cast<ShuffleVectorSDNode>(N); 3253 EVT VT = N->getValueType(0); 3254 SDLoc dl(N); 3255 3256 ArrayRef<int> NewMask = SV->getMask().slice(0, VT.getVectorNumElements()); 3257 3258 SDValue V0 = GetPromotedInteger(N->getOperand(0)); 3259 SDValue V1 = GetPromotedInteger(N->getOperand(1)); 3260 EVT OutVT = V0.getValueType(); 3261 3262 return DAG.getVectorShuffle(OutVT, dl, V0, V1, NewMask); 3263 } 3264 3265 3266 SDValue DAGTypeLegalizer::PromoteIntRes_BUILD_VECTOR(SDNode *N) { 3267 EVT OutVT = N->getValueType(0); 3268 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT); 3269 assert(NOutVT.isVector() && "This type must be promoted to a vector type"); 3270 unsigned NumElems = N->getNumOperands(); 3271 EVT NOutVTElem = NOutVT.getVectorElementType(); 3272 3273 SDLoc dl(N); 3274 3275 SmallVector<SDValue, 8> Ops; 3276 Ops.reserve(NumElems); 3277 for (unsigned i = 0; i != NumElems; ++i) { 3278 SDValue Op; 3279 // BUILD_VECTOR integer operand types are allowed to be larger than the 3280 // result's element type. This may still be true after the promotion. For 3281 // example, we might be promoting (<v?i1> = BV <i32>, <i32>, ...) to 3282 // (v?i16 = BV <i32>, <i32>, ...), and we can't any_extend <i32> to <i16>. 3283 if (N->getOperand(i).getValueType().bitsLT(NOutVTElem)) 3284 Op = DAG.getNode(ISD::ANY_EXTEND, dl, NOutVTElem, N->getOperand(i)); 3285 else 3286 Op = N->getOperand(i); 3287 Ops.push_back(Op); 3288 } 3289 3290 return DAG.getNode(ISD::BUILD_VECTOR, dl, NOutVT, Ops); 3291 } 3292 3293 SDValue DAGTypeLegalizer::PromoteIntRes_SCALAR_TO_VECTOR(SDNode *N) { 3294 3295 SDLoc dl(N); 3296 3297 assert(!N->getOperand(0).getValueType().isVector() && 3298 "Input must be a scalar"); 3299 3300 EVT OutVT = N->getValueType(0); 3301 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT); 3302 assert(NOutVT.isVector() && "This type must be promoted to a vector type"); 3303 EVT NOutVTElem = NOutVT.getVectorElementType(); 3304 3305 SDValue Op = DAG.getNode(ISD::ANY_EXTEND, dl, NOutVTElem, N->getOperand(0)); 3306 3307 return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, NOutVT, Op); 3308 } 3309 3310 SDValue DAGTypeLegalizer::PromoteIntRes_CONCAT_VECTORS(SDNode *N) { 3311 SDLoc dl(N); 3312 3313 EVT OutVT = N->getValueType(0); 3314 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT); 3315 assert(NOutVT.isVector() && "This type must be promoted to a vector type"); 3316 3317 EVT InElemTy = OutVT.getVectorElementType(); 3318 EVT OutElemTy = NOutVT.getVectorElementType(); 3319 3320 unsigned NumElem = N->getOperand(0).getValueType().getVectorNumElements(); 3321 unsigned NumOutElem = NOutVT.getVectorNumElements(); 3322 unsigned NumOperands = N->getNumOperands(); 3323 assert(NumElem * NumOperands == NumOutElem && 3324 "Unexpected number of elements"); 3325 3326 // Take the elements from the first vector. 3327 SmallVector<SDValue, 8> Ops(NumOutElem); 3328 for (unsigned i = 0; i < NumOperands; ++i) { 3329 SDValue Op = N->getOperand(i); 3330 for (unsigned j = 0; j < NumElem; ++j) { 3331 SDValue Ext = DAG.getNode( 3332 ISD::EXTRACT_VECTOR_ELT, dl, InElemTy, Op, 3333 DAG.getConstant(j, dl, TLI.getVectorIdxTy(DAG.getDataLayout()))); 3334 Ops[i * NumElem + j] = DAG.getNode(ISD::ANY_EXTEND, dl, OutElemTy, Ext); 3335 } 3336 } 3337 3338 return DAG.getNode(ISD::BUILD_VECTOR, dl, NOutVT, Ops); 3339 } 3340 3341 SDValue DAGTypeLegalizer::PromoteIntRes_EXTEND_VECTOR_INREG(SDNode *N) { 3342 EVT VT = N->getValueType(0); 3343 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 3344 assert(NVT.isVector() && "This type must be promoted to a vector type"); 3345 3346 SDLoc dl(N); 3347 3348 // For operands whose TypeAction is to promote, extend the promoted node 3349 // appropriately (ZERO_EXTEND or SIGN_EXTEND) from the original pre-promotion 3350 // type, and then construct a new *_EXTEND_VECTOR_INREG node to the promote-to 3351 // type.. 3352 if (getTypeAction(N->getOperand(0).getValueType()) 3353 == TargetLowering::TypePromoteInteger) { 3354 SDValue Promoted; 3355 3356 switch(N->getOpcode()) { 3357 case ISD::SIGN_EXTEND_VECTOR_INREG: 3358 Promoted = SExtPromotedInteger(N->getOperand(0)); 3359 break; 3360 case ISD::ZERO_EXTEND_VECTOR_INREG: 3361 Promoted = ZExtPromotedInteger(N->getOperand(0)); 3362 break; 3363 case ISD::ANY_EXTEND_VECTOR_INREG: 3364 Promoted = GetPromotedInteger(N->getOperand(0)); 3365 break; 3366 default: 3367 llvm_unreachable("Node has unexpected Opcode"); 3368 } 3369 return DAG.getNode(N->getOpcode(), dl, NVT, Promoted); 3370 } 3371 3372 // Directly extend to the appropriate transform-to type. 3373 return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0)); 3374 } 3375 3376 SDValue DAGTypeLegalizer::PromoteIntRes_INSERT_VECTOR_ELT(SDNode *N) { 3377 EVT OutVT = N->getValueType(0); 3378 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT); 3379 assert(NOutVT.isVector() && "This type must be promoted to a vector type"); 3380 3381 EVT NOutVTElem = NOutVT.getVectorElementType(); 3382 3383 SDLoc dl(N); 3384 SDValue V0 = GetPromotedInteger(N->getOperand(0)); 3385 3386 SDValue ConvElem = DAG.getNode(ISD::ANY_EXTEND, dl, 3387 NOutVTElem, N->getOperand(1)); 3388 return DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, NOutVT, 3389 V0, ConvElem, N->getOperand(2)); 3390 } 3391 3392 SDValue DAGTypeLegalizer::PromoteIntOp_EXTRACT_VECTOR_ELT(SDNode *N) { 3393 SDLoc dl(N); 3394 SDValue V0 = GetPromotedInteger(N->getOperand(0)); 3395 SDValue V1 = DAG.getZExtOrTrunc(N->getOperand(1), dl, 3396 TLI.getVectorIdxTy(DAG.getDataLayout())); 3397 SDValue Ext = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, 3398 V0->getValueType(0).getScalarType(), V0, V1); 3399 3400 // EXTRACT_VECTOR_ELT can return types which are wider than the incoming 3401 // element types. If this is the case then we need to expand the outgoing 3402 // value and not truncate it. 3403 return DAG.getAnyExtOrTrunc(Ext, dl, N->getValueType(0)); 3404 } 3405 3406 SDValue DAGTypeLegalizer::PromoteIntOp_EXTRACT_SUBVECTOR(SDNode *N) { 3407 SDLoc dl(N); 3408 SDValue V0 = GetPromotedInteger(N->getOperand(0)); 3409 MVT InVT = V0.getValueType().getSimpleVT(); 3410 MVT OutVT = MVT::getVectorVT(InVT.getVectorElementType(), 3411 N->getValueType(0).getVectorNumElements()); 3412 SDValue Ext = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, OutVT, V0, N->getOperand(1)); 3413 return DAG.getNode(ISD::TRUNCATE, dl, N->getValueType(0), Ext); 3414 } 3415 3416 SDValue DAGTypeLegalizer::PromoteIntOp_CONCAT_VECTORS(SDNode *N) { 3417 SDLoc dl(N); 3418 unsigned NumElems = N->getNumOperands(); 3419 3420 EVT RetSclrTy = N->getValueType(0).getVectorElementType(); 3421 3422 SmallVector<SDValue, 8> NewOps; 3423 NewOps.reserve(NumElems); 3424 3425 // For each incoming vector 3426 for (unsigned VecIdx = 0; VecIdx != NumElems; ++VecIdx) { 3427 SDValue Incoming = GetPromotedInteger(N->getOperand(VecIdx)); 3428 EVT SclrTy = Incoming->getValueType(0).getVectorElementType(); 3429 unsigned NumElem = Incoming->getValueType(0).getVectorNumElements(); 3430 3431 for (unsigned i=0; i<NumElem; ++i) { 3432 // Extract element from incoming vector 3433 SDValue Ex = DAG.getNode( 3434 ISD::EXTRACT_VECTOR_ELT, dl, SclrTy, Incoming, 3435 DAG.getConstant(i, dl, TLI.getVectorIdxTy(DAG.getDataLayout()))); 3436 SDValue Tr = DAG.getNode(ISD::TRUNCATE, dl, RetSclrTy, Ex); 3437 NewOps.push_back(Tr); 3438 } 3439 } 3440 3441 return DAG.getNode(ISD::BUILD_VECTOR, dl, N->getValueType(0), NewOps); 3442 } 3443